diff --git a/.gitignore b/.gitignore index e833cba..2263233 100644 --- a/.gitignore +++ b/.gitignore @@ -207,6 +207,8 @@ clt_test_pythia_70m_jumprelu/ clt_smoke_output_local_wandb_batchtopk/ clt_smoke_output_remote_wandb/ wandb/ +scripts/debug +scripts/optimization # models *.pt diff --git a/clt/config/clt_config.py b/clt/config/clt_config.py index aee5728..3e0d577 100644 --- a/clt/config/clt_config.py +++ b/clt/config/clt_config.py @@ -158,6 +158,9 @@ class TrainingConfig: # Optional diagnostic metrics (can be slow) compute_sparsity_diagnostics: bool = False # Whether to compute detailed sparsity diagnostics during eval + + # Performance profiling + enable_profiling: bool = False # Whether to enable detailed performance profiling # Dead feature tracking dead_feature_window: int = 1000 # Steps until a feature is considered dead diff --git a/clt/models/activations.py b/clt/models/activations.py index ca0de1a..36b268c 100644 --- a/clt/models/activations.py +++ b/clt/models/activations.py @@ -1,5 +1,5 @@ import torch -from typing import Optional, Tuple, Dict, List +from typing import Optional, Tuple, Dict, List, Any import logging from clt.config import CLTConfig from torch.distributed import ProcessGroup @@ -26,9 +26,10 @@ def _compute_mask(x: torch.Tensor, k_per_token: int, x_for_ranking: Optional[tor if k_total_batch > 0: _, flat_indices = torch.topk(ranking_flat, k_total_batch, sorted=False) - mask_flat = torch.zeros_like(x_flat, dtype=torch.bool) - mask_flat[flat_indices] = True - mask = mask_flat.view_as(x) + # Optimized mask creation - avoid individual indexing + mask = torch.zeros(x_flat.numel(), dtype=torch.bool, device=x.device) + mask[flat_indices] = True + mask = mask.view_as(x) else: mask = torch.zeros_like(x, dtype=torch.bool) @@ -118,6 +119,7 @@ def _compute_mask(x: torch.Tensor, k_float: float, x_for_ranking: Optional[torch if k_per_token > 0: _, topk_indices_per_row = torch.topk(ranking_tensor_to_use, k_per_token, dim=-1, sorted=False) + # Use scatter_ for efficient mask creation mask = torch.zeros_like(x, dtype=torch.bool) mask.scatter_(-1, topk_indices_per_row, True) else: @@ -231,6 +233,7 @@ def _apply_batch_topk_helper( dtype: torch.dtype, rank: int, process_group: Optional[ProcessGroup], + profiler: Optional[Any] = None, ) -> Dict[int, torch.Tensor]: """Helper to apply BatchTopK globally across concatenated layer pre-activations.""" @@ -304,17 +307,42 @@ def _apply_batch_topk_helper( if world_size > 1: if rank == 0: - local_mask = BatchTopK._compute_mask( - concatenated_preactivations_original, k_val, concatenated_preactivations_normalized - ) + if profiler: + with profiler.timer("batchtopk_compute_mask") as timer: + local_mask = BatchTopK._compute_mask( + concatenated_preactivations_original, k_val, concatenated_preactivations_normalized + ) + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_compute_mask", timer.elapsed) + else: + local_mask = BatchTopK._compute_mask( + concatenated_preactivations_original, k_val, concatenated_preactivations_normalized + ) mask.copy_(local_mask) - dist_ops.broadcast(mask, src=0, group=process_group) + + if hasattr(profiler, 'dist_profiler') and profiler.dist_profiler: + with profiler.dist_profiler.profile_op("batchtopk_broadcast"): + dist_ops.broadcast(mask, src=0, group=process_group) + else: + dist_ops.broadcast(mask, src=0, group=process_group) else: - dist_ops.broadcast(mask, src=0, group=process_group) + if hasattr(profiler, 'dist_profiler') and profiler.dist_profiler: + with profiler.dist_profiler.profile_op("batchtopk_broadcast"): + dist_ops.broadcast(mask, src=0, group=process_group) + else: + dist_ops.broadcast(mask, src=0, group=process_group) else: - mask = BatchTopK._compute_mask( - concatenated_preactivations_original, k_val, concatenated_preactivations_normalized - ) + if profiler: + with profiler.timer("batchtopk_compute_mask") as timer: + mask = BatchTopK._compute_mask( + concatenated_preactivations_original, k_val, concatenated_preactivations_normalized + ) + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_compute_mask", timer.elapsed) + else: + mask = BatchTopK._compute_mask( + concatenated_preactivations_original, k_val, concatenated_preactivations_normalized + ) activated_concatenated = concatenated_preactivations_original * mask.to(dtype) @@ -336,6 +364,7 @@ def _apply_token_topk_helper( dtype: torch.dtype, rank: int, process_group: Optional[ProcessGroup], + profiler: Optional[Any] = None, ) -> Dict[int, torch.Tensor]: """Helper to apply TokenTopK globally across concatenated layer pre-activations.""" world_size = dist_ops.get_world_size(process_group) @@ -408,19 +437,46 @@ def _apply_token_topk_helper( if world_size > 1: if rank == 0: - local_mask = TokenTopK._compute_mask( - concatenated_preactivations_original, - k_val_float, - concatenated_preactivations_normalized, - ) + if profiler: + with profiler.timer("topk_compute_mask") as timer: + local_mask = TokenTopK._compute_mask( + concatenated_preactivations_original, + k_val_float, + concatenated_preactivations_normalized, + ) + if hasattr(timer, 'elapsed'): + profiler.record("topk_compute_mask", timer.elapsed) + else: + local_mask = TokenTopK._compute_mask( + concatenated_preactivations_original, + k_val_float, + concatenated_preactivations_normalized, + ) mask.copy_(local_mask) - dist_ops.broadcast(mask, src=0, group=process_group) + + if hasattr(profiler, 'dist_profiler') and profiler.dist_profiler: + with profiler.dist_profiler.profile_op("topk_broadcast"): + dist_ops.broadcast(mask, src=0, group=process_group) + else: + dist_ops.broadcast(mask, src=0, group=process_group) else: - dist_ops.broadcast(mask, src=0, group=process_group) + if hasattr(profiler, 'dist_profiler') and profiler.dist_profiler: + with profiler.dist_profiler.profile_op("topk_broadcast"): + dist_ops.broadcast(mask, src=0, group=process_group) + else: + dist_ops.broadcast(mask, src=0, group=process_group) else: - mask = TokenTopK._compute_mask( - concatenated_preactivations_original, k_val_float, concatenated_preactivations_normalized - ) + if profiler: + with profiler.timer("topk_compute_mask") as timer: + mask = TokenTopK._compute_mask( + concatenated_preactivations_original, k_val_float, concatenated_preactivations_normalized + ) + if hasattr(timer, 'elapsed'): + profiler.record("topk_compute_mask", timer.elapsed) + else: + mask = TokenTopK._compute_mask( + concatenated_preactivations_original, k_val_float, concatenated_preactivations_normalized + ) activated_concatenated = concatenated_preactivations_original * mask.to(dtype) diff --git a/clt/models/activations_distributed.py b/clt/models/activations_distributed.py new file mode 100644 index 0000000..a825b23 --- /dev/null +++ b/clt/models/activations_distributed.py @@ -0,0 +1,230 @@ +"""Distributed-optimized activation functions using local topk + allgather pattern.""" + +import torch +from typing import Optional, Dict, List, Any, Tuple +import logging +from clt.config import CLTConfig +from torch.distributed import ProcessGroup +from clt.parallel import ops as dist_ops + +logger = logging.getLogger(__name__) + + +def _apply_batch_topk_distributed( + preactivations_dict: Dict[int, torch.Tensor], + config: CLTConfig, + device: torch.device, + dtype: torch.dtype, + rank: int, + process_group: Optional[ProcessGroup], + profiler: Optional[Any] = None, +) -> Dict[int, torch.Tensor]: + """ + Optimized BatchTopK using local top-k + allgather pattern. + + Instead of computing global top-k on rank 0 and broadcasting the full mask, + each rank computes local top-k and we allgather the indices. + """ + world_size = dist_ops.get_world_size(process_group) + + if not preactivations_dict: + return {} + + # Prepare concatenated tensors as before + ordered_preactivations_original: List[torch.Tensor] = [] + ordered_preactivations_normalized: List[torch.Tensor] = [] + layer_feature_sizes: List[Tuple[int, int]] = [] + + first_valid_preact = next((p for p in preactivations_dict.values() if p.numel() > 0), None) + if first_valid_preact is None: + return { + layer_idx: torch.empty((0, config.num_features), device=device, dtype=dtype) + for layer_idx in preactivations_dict.keys() + } + + batch_tokens_dim = first_valid_preact.shape[0] + + # Collect and concatenate preactivations + for layer_idx in range(config.num_layers): + if layer_idx in preactivations_dict: + preact_orig = preactivations_dict[layer_idx] + preact_orig = preact_orig.to(device=device, dtype=dtype) + current_num_features = preact_orig.shape[1] if preact_orig.numel() > 0 else config.num_features + + if preact_orig.numel() == 0 or preact_orig.shape[0] != batch_tokens_dim: + zeros_shape = (batch_tokens_dim, current_num_features) + ordered_preactivations_original.append(torch.zeros(zeros_shape, device=device, dtype=dtype)) + ordered_preactivations_normalized.append(torch.zeros(zeros_shape, device=device, dtype=dtype)) + else: + ordered_preactivations_original.append(preact_orig) + mean = preact_orig.mean(dim=0, keepdim=True) + std = preact_orig.std(dim=0, keepdim=True) + preact_norm = (preact_orig - mean) / (std + 1e-6) + ordered_preactivations_normalized.append(preact_norm) + layer_feature_sizes.append((layer_idx, current_num_features)) + + concatenated_preactivations_original = torch.cat(ordered_preactivations_original, dim=1) + concatenated_preactivations_normalized = torch.cat(ordered_preactivations_normalized, dim=1) + + k_val = int(config.batchtopk_k) if config.batchtopk_k is not None else concatenated_preactivations_original.size(1) + total_features = concatenated_preactivations_original.size(1) + + # Create the final mask + mask = torch.zeros_like(concatenated_preactivations_original, dtype=torch.bool) + + if world_size > 1 and k_val < total_features // 2: # Use distributed only if it's beneficial + # Distributed implementation: local topk + allgather + + # Step 1: Each rank computes local top-k + k_per_rank = k_val * batch_tokens_dim // world_size + k_per_rank = max(1, k_per_rank) # At least 1 per rank + + if profiler: + with profiler.timer("batchtopk_local_topk") as timer: + # Flatten for local top-k + local_flat = concatenated_preactivations_normalized.reshape(-1) + local_values, local_indices = torch.topk( + local_flat, + min(k_per_rank, local_flat.numel()), + sorted=False + ) + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_local_topk", timer.elapsed) + else: + local_flat = concatenated_preactivations_normalized.reshape(-1) + local_values, local_indices = torch.topk( + local_flat, + min(k_per_rank, local_flat.numel()), + sorted=False + ) + + # Step 2: Allgather values and indices + gathered_values = [torch.zeros_like(local_values) for _ in range(world_size)] + gathered_indices = [torch.zeros_like(local_indices) for _ in range(world_size)] + + if hasattr(profiler, 'dist_profiler') and profiler.dist_profiler: + with profiler.dist_profiler.profile_op("batchtopk_allgather"): + dist_ops.all_gather(gathered_values, local_values, group=process_group) + dist_ops.all_gather(gathered_indices, local_indices, group=process_group) + else: + dist_ops.all_gather(gathered_values, local_values, group=process_group) + dist_ops.all_gather(gathered_indices, local_indices, group=process_group) + + # Step 3: Merge and find global top-k + if profiler: + with profiler.timer("batchtopk_merge_topk") as timer: + all_values = torch.cat(gathered_values) + all_indices = torch.cat(gathered_indices) + + # Get global top-k from merged results + final_k = min(k_val * batch_tokens_dim, all_values.numel()) + _, top_indices_of_indices = torch.topk(all_values, final_k, sorted=False) + + # Get the actual feature indices + global_top_indices = all_indices[top_indices_of_indices] + + # Create mask + mask_flat = mask.reshape(-1) + mask_flat[global_top_indices] = True + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_merge_topk", timer.elapsed) + else: + all_values = torch.cat(gathered_values) + all_indices = torch.cat(gathered_indices) + final_k = min(k_val * batch_tokens_dim, all_values.numel()) + _, top_indices_of_indices = torch.topk(all_values, final_k, sorted=False) + global_top_indices = all_indices[top_indices_of_indices] + mask_flat = mask.reshape(-1) + mask_flat[global_top_indices] = True + + else: + # Single GPU or small k: use original approach + if profiler: + with profiler.timer("batchtopk_compute_mask") as timer: + flat_ranking = concatenated_preactivations_normalized.reshape(-1) + k_total = min(k_val * batch_tokens_dim, flat_ranking.numel()) + if k_total > 0: + _, flat_indices = torch.topk(flat_ranking, k_total, sorted=False) + mask_flat = mask.reshape(-1) + mask_flat[flat_indices] = True + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_compute_mask", timer.elapsed) + else: + flat_ranking = concatenated_preactivations_normalized.reshape(-1) + k_total = min(k_val * batch_tokens_dim, flat_ranking.numel()) + if k_total > 0: + _, flat_indices = torch.topk(flat_ranking, k_total, sorted=False) + mask_flat = mask.reshape(-1) + mask_flat[flat_indices] = True + + # Apply mask + activated_concatenated = concatenated_preactivations_original * mask.to(dtype) + + # Split back into layers + activations_dict: Dict[int, torch.Tensor] = {} + current_offset = 0 + for layer_idx, num_features in layer_feature_sizes: + activated_segment = activated_concatenated[:, current_offset:current_offset + num_features] + activations_dict[layer_idx] = activated_segment + current_offset += num_features + + return activations_dict + + +def benchmark_distributed_batchtopk(): + """Benchmark the distributed implementation.""" + import time + + if not torch.cuda.is_available(): + print("CUDA not available for benchmarking") + return + + print("Benchmarking Distributed BatchTopK") + print("-" * 40) + + # Test setup + batch_size = 32 + num_layers = 12 + features_per_layer = 8192 + k = 200 + device = torch.device("cuda") + + # Create test data + preact_dict = {} + for layer in range(num_layers): + preact_dict[layer] = torch.randn(batch_size, features_per_layer, device=device) + + # Warmup + for _ in range(3): + concatenated = torch.cat([preact_dict[i] for i in range(num_layers)], dim=1) + _ = torch.topk(concatenated.reshape(-1), k * batch_size) + + # Time original approach + torch.cuda.synchronize() + start = time.perf_counter() + + concatenated = torch.cat([preact_dict[i] for i in range(num_layers)], dim=1) + flat = concatenated.reshape(-1) + _, indices = torch.topk(flat, k * batch_size, sorted=False) + mask = torch.zeros_like(flat, dtype=torch.bool) + mask[indices] = True + mask = mask.view_as(concatenated) + result = concatenated * mask + + torch.cuda.synchronize() + original_time = time.perf_counter() - start + + print(f"Original approach: {original_time*1000:.2f}ms") + print(f"Communication size: {mask.numel() * 1} bytes (bool mask)") + + # Estimate distributed approach communication + world_size = 4 # Example + k_per_rank = k * batch_size // world_size + comm_size = k_per_rank * world_size * 8 # 8 bytes per index+value + print(f"\nDistributed approach (simulated):") + print(f"Communication size: {comm_size} bytes (indices+values)") + print(f"Reduction: {mask.numel() / comm_size:.1f}x") + + +if __name__ == "__main__": + benchmark_distributed_batchtopk() \ No newline at end of file diff --git a/clt/models/activations_local_global.py b/clt/models/activations_local_global.py new file mode 100644 index 0000000..99acaeb --- /dev/null +++ b/clt/models/activations_local_global.py @@ -0,0 +1,260 @@ +"""Local-then-global BatchTopK implementation that's mathematically equivalent but more efficient.""" + +import torch +from typing import Optional, Dict, List, Any, Tuple +import logging +from clt.config import CLTConfig +from torch.distributed import ProcessGroup +from clt.parallel import ops as dist_ops + +logger = logging.getLogger(__name__) + + +def _apply_batch_topk_local_global( + preactivations_dict: Dict[int, torch.Tensor], + config: CLTConfig, + device: torch.device, + dtype: torch.dtype, + rank: int, + process_group: Optional[ProcessGroup], + profiler: Optional[Any] = None, + oversample_factor: int = 4, +) -> Dict[int, torch.Tensor]: + """ + Mathematically equivalent BatchTopK using local top-k + allgather pattern. + + This computes the exact same result as global BatchTopK but with less communication. + + Args: + oversample_factor: How many times more candidates to gather locally. + Higher = more accurate but more communication. + 4x is usually sufficient. + """ + world_size = dist_ops.get_world_size(process_group) + + if not preactivations_dict: + return {} + + # Prepare concatenated tensors (same as original) + ordered_preactivations_original: List[torch.Tensor] = [] + ordered_preactivations_normalized: List[torch.Tensor] = [] + layer_feature_sizes: List[Tuple[int, int]] = [] + + first_valid_preact = next((p for p in preactivations_dict.values() if p.numel() > 0), None) + if first_valid_preact is None: + return { + layer_idx: torch.empty((0, config.num_features), device=device, dtype=dtype) + for layer_idx in preactivations_dict.keys() + } + + batch_tokens_dim = first_valid_preact.shape[0] + + # Collect and concatenate preactivations + for layer_idx in range(config.num_layers): + if layer_idx in preactivations_dict: + preact_orig = preactivations_dict[layer_idx] + preact_orig = preact_orig.to(device=device, dtype=dtype) + current_num_features = preact_orig.shape[1] if preact_orig.numel() > 0 else config.num_features + + if preact_orig.numel() == 0 or preact_orig.shape[0] != batch_tokens_dim: + zeros_shape = (batch_tokens_dim, current_num_features) + ordered_preactivations_original.append(torch.zeros(zeros_shape, device=device, dtype=dtype)) + ordered_preactivations_normalized.append(torch.zeros(zeros_shape, device=device, dtype=dtype)) + else: + ordered_preactivations_original.append(preact_orig) + mean = preact_orig.mean(dim=0, keepdim=True) + std = preact_orig.std(dim=0, keepdim=True) + preact_norm = (preact_orig - mean) / (std + 1e-6) + ordered_preactivations_normalized.append(preact_norm) + layer_feature_sizes.append((layer_idx, current_num_features)) + + concatenated_preactivations_original = torch.cat(ordered_preactivations_original, dim=1) + concatenated_preactivations_normalized = torch.cat(ordered_preactivations_normalized, dim=1) + + k_val = int(config.batchtopk_k) if config.batchtopk_k is not None else concatenated_preactivations_original.size(1) + total_features = concatenated_preactivations_original.size(1) + + # Decide whether to use distributed approach + if world_size > 1 and k_val < total_features // (2 * world_size): + # Use local-then-global approach + + # Step 1: Local top-k with oversampling + # Each rank needs to keep enough candidates so that when combined, + # we have enough to select the global top-k + final_k = k_val * batch_tokens_dim # Total elements we want globally + local_k = min(final_k * oversample_factor // world_size, total_features) + + if profiler: + with profiler.timer("batchtopk_local_topk") as timer: + # Get normalized values for ranking + flat_normalized = concatenated_preactivations_normalized.reshape(-1) + # Get original values for later use + flat_original = concatenated_preactivations_original.reshape(-1) + + # Local top-k on normalized values + local_top_values_norm, local_top_indices = torch.topk( + flat_normalized, + min(local_k, flat_normalized.numel()), + sorted=False + ) + + # Get corresponding original values + local_top_values_orig = flat_original[local_top_indices] + + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_local_topk", timer.elapsed) + else: + flat_normalized = concatenated_preactivations_normalized.reshape(-1) + flat_original = concatenated_preactivations_original.reshape(-1) + local_top_values_norm, local_top_indices = torch.topk( + flat_normalized, + min(local_k, flat_normalized.numel()), + sorted=False + ) + local_top_values_orig = flat_original[local_top_indices] + + # Step 2: Allgather top candidates + gathered_values_norm = [torch.zeros_like(local_top_values_norm) for _ in range(world_size)] + gathered_indices = [torch.zeros_like(local_top_indices) for _ in range(world_size)] + + if hasattr(profiler, 'dist_profiler') and profiler.dist_profiler: + with profiler.dist_profiler.profile_op("batchtopk_allgather"): + dist_ops.all_gather(gathered_values_norm, local_top_values_norm, group=process_group) + dist_ops.all_gather(gathered_indices, local_top_indices, group=process_group) + else: + dist_ops.all_gather(gathered_values_norm, local_top_values_norm, group=process_group) + dist_ops.all_gather(gathered_indices, local_top_indices, group=process_group) + + # Step 3: Global top-k from candidates + if profiler: + with profiler.timer("batchtopk_global_selection") as timer: + # Concatenate all candidates + all_values_norm = torch.cat(gathered_values_norm) + all_indices = torch.cat(gathered_indices) + + # Select global top-k from candidates + global_k = min(k_val * batch_tokens_dim, all_values_norm.numel()) + _, top_indices_of_candidates = torch.topk(all_values_norm, global_k, sorted=False) + + # Get the actual global indices + global_top_indices = all_indices[top_indices_of_candidates] + + # Create mask + mask = torch.zeros(concatenated_preactivations_original.numel(), + dtype=torch.bool, device=device) + mask[global_top_indices] = True + mask = mask.view_as(concatenated_preactivations_original) + + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_global_selection", timer.elapsed) + else: + all_values_norm = torch.cat(gathered_values_norm) + all_indices = torch.cat(gathered_indices) + global_k = min(k_val * batch_tokens_dim, all_values_norm.numel()) + _, top_indices_of_candidates = torch.topk(all_values_norm, global_k, sorted=False) + global_top_indices = all_indices[top_indices_of_candidates] + mask = torch.zeros(concatenated_preactivations_original.numel(), + dtype=torch.bool, device=device) + mask[global_top_indices] = True + mask = mask.view_as(concatenated_preactivations_original) + + else: + # Single GPU or large k: use original approach + if profiler: + with profiler.timer("batchtopk_compute_mask") as timer: + from clt.models.activations import BatchTopK + mask = BatchTopK._compute_mask( + concatenated_preactivations_original, + k_val, + concatenated_preactivations_normalized + ) + if hasattr(timer, 'elapsed'): + profiler.record("batchtopk_compute_mask", timer.elapsed) + else: + from clt.models.activations import BatchTopK + mask = BatchTopK._compute_mask( + concatenated_preactivations_original, + k_val, + concatenated_preactivations_normalized + ) + + # Apply mask + activated_concatenated = concatenated_preactivations_original * mask.to(dtype) + + # Split back into layers + activations_dict: Dict[int, torch.Tensor] = {} + current_offset = 0 + for layer_idx, num_features in layer_feature_sizes: + activated_segment = activated_concatenated[:, current_offset:current_offset + num_features] + activations_dict[layer_idx] = activated_segment + current_offset += num_features + + return activations_dict + + +def validate_equivalence(): + """Validate that local-global produces same results as global BatchTopK.""" + import torch.distributed as dist + + print("Validating local-global BatchTopK equivalence...") + + # Test parameters + batch_size = 32 + num_features = 8192 + num_layers = 12 + k = 200 + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + # Create test data + torch.manual_seed(42) + preact_dict = {} + for layer in range(num_layers): + preact_dict[layer] = torch.randn(batch_size, num_features, device=device) + + # Original global approach + from clt.models.activations import BatchTopK + concatenated = torch.cat([preact_dict[i] for i in range(num_layers)], dim=1) + mask_global = BatchTopK._compute_mask(concatenated, k) + + # Simulate local-global approach (single GPU simulation) + # In real distributed setting, each GPU would have different data + flat = concatenated.reshape(-1) + + # Step 1: Local top-k with oversampling + # Need to keep enough to ensure we get all global top-k + # Since we want k*batch_size total, we need at least that many + local_k = k * batch_size * 2 # 2x oversampling of final count + local_values, local_indices = torch.topk(flat, min(local_k, flat.numel()), sorted=False) + + # Step 2: In distributed, we'd allgather here + # For single GPU, just use local results + + # Step 3: Global selection from candidates + global_k = k * batch_size + _, top_indices = torch.topk(local_values, min(global_k, local_values.numel()), sorted=False) + global_indices = local_indices[top_indices] + + # Create mask from global indices + mask_local_global = torch.zeros_like(flat, dtype=torch.bool) + mask_local_global[global_indices] = True + mask_local_global = mask_local_global.view_as(concatenated) + + # Check if masks are identical + matches = torch.equal(mask_global, mask_local_global) + num_selected_global = mask_global.sum().item() + num_selected_local = mask_local_global.sum().item() + + print(f"Masks identical: {matches}") + print(f"Global approach selected: {num_selected_global}") + print(f"Local-global approach selected: {num_selected_local}") + + if matches: + print("✓ Validation passed! Approaches are mathematically equivalent.") + else: + print("✗ Validation failed! Approaches differ.") + overlap = (mask_global & mask_local_global).sum().item() + print(f"Overlap: {overlap} ({overlap/num_selected_global*100:.1f}%)") + + +if __name__ == "__main__": + validate_equivalence() \ No newline at end of file diff --git a/clt/models/activations_optimized.py b/clt/models/activations_optimized.py new file mode 100644 index 0000000..b08c7e7 --- /dev/null +++ b/clt/models/activations_optimized.py @@ -0,0 +1,163 @@ +"""Optimized activation functions for better performance.""" + +import torch +import torch.nn.functional as F +from typing import Optional, Dict, Any +import logging + +logger = logging.getLogger(__name__) + + +class OptimizedBatchTopK(torch.autograd.Function): + """Optimized BatchTopK with fused operations and better memory usage.""" + + @staticmethod + def _compute_mask_optimized( + x: torch.Tensor, + k_per_token: int, + x_for_ranking: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """Optimized mask computation with fewer allocations.""" + B = x.size(0) + if k_per_token <= 0: + return torch.zeros_like(x, dtype=torch.bool) + + # Early exit for full selection + F_total_batch = x.numel() + if F_total_batch == 0: + return torch.zeros_like(x, dtype=torch.bool) + + k_total_batch = min(k_per_token * B, F_total_batch) + + # Use the ranking tensor if provided, otherwise use x + ranking_tensor = x_for_ranking if x_for_ranking is not None else x + + # Fused reshape and topk - avoid intermediate allocations + if k_total_batch > 0: + # Get top-k values and indices in one operation + _, flat_indices = torch.topk( + ranking_tensor.view(-1), + k_total_batch, + sorted=False, + largest=True + ) + + # Create mask directly without intermediate tensor + mask = torch.zeros(F_total_batch, dtype=torch.bool, device=x.device) + mask[flat_indices] = True + return mask.view_as(x) + else: + return torch.zeros_like(x, dtype=torch.bool) + + @staticmethod + @torch.cuda.amp.custom_fwd(cast_inputs=torch.float32) + def forward( + ctx, + x: torch.Tensor, + k: float, + straight_through: bool, + x_for_ranking: Optional[torch.Tensor] = None + ) -> torch.Tensor: + """Forward with mixed precision support.""" + k_per_token = int(k) + + # Compute mask in FP32 for accuracy + with torch.cuda.amp.autocast(enabled=False): + mask = OptimizedBatchTopK._compute_mask_optimized( + x.float(), k_per_token, + x_for_ranking.float() if x_for_ranking is not None else None + ) + + ctx.save_for_backward(mask) + ctx.straight_through = straight_through + + # Apply mask in original dtype + return x * mask.to(x.dtype) + + @staticmethod + @torch.cuda.amp.custom_bwd + def backward(ctx, grad_output: torch.Tensor) -> tuple[torch.Tensor | None, None, None, None]: + """Optimized backward pass.""" + if ctx.straight_through: + mask, = ctx.saved_tensors + # Fused multiplication + grad_input = grad_output * mask.to(grad_output.dtype) + else: + mask, = ctx.saved_tensors + grad_input = grad_output * mask.to(grad_output.dtype) + + return grad_input, None, None, None + + +def create_optimized_topk_mask_batched( + concatenated_tensor: torch.Tensor, + k_values: Dict[int, int], + layer_sizes: list[tuple[int, int]] +) -> torch.Tensor: + """Create masks for different layers in parallel when they have different k values.""" + device = concatenated_tensor.device + dtype = concatenated_tensor.dtype + batch_size, total_features = concatenated_tensor.shape + + # Pre-allocate output mask + mask = torch.zeros_like(concatenated_tensor, dtype=torch.bool) + + # Group layers by k value for batch processing + k_groups = {} + for layer_idx, (start_idx, num_features) in enumerate(layer_sizes): + k_val = k_values.get(layer_idx, 0) + if k_val not in k_groups: + k_groups[k_val] = [] + k_groups[k_val].append((layer_idx, start_idx, num_features)) + + # Process each k-value group + for k_val, layer_infos in k_groups.items(): + if k_val <= 0: + continue + + # Gather all features for this k value + indices = [] + for _, start_idx, num_features in layer_infos: + indices.extend(range(start_idx, start_idx + num_features)) + + if not indices: + continue + + # Extract relevant features + group_features = concatenated_tensor[:, indices] + + # Compute top-k for this group + k_total = min(k_val * batch_size, group_features.numel()) + if k_total > 0: + _, top_indices = torch.topk( + group_features.view(-1), + k_total, + sorted=False + ) + + # Convert back to 2D indices + row_indices = top_indices // len(indices) + col_indices = top_indices % len(indices) + + # Map back to original positions + for i, (row, col) in enumerate(zip(row_indices, col_indices)): + original_col = indices[col] + mask[row, original_col] = True + + return mask + + +# Monkey patch for torch.compile compatibility +def make_compile_compatible(): + """Make activation functions compatible with torch.compile.""" + try: + # Check if torch.compile is available (PyTorch 2.0+) + if hasattr(torch, 'compile'): + # Register custom ops for better compilation + torch.fx.wrap('OptimizedBatchTopK._compute_mask_optimized') + except Exception as e: + logger.debug(f"torch.compile compatibility setup skipped: {e}") + + +# Initialize on module load +make_compile_compatible() \ No newline at end of file diff --git a/clt/models/clt.py b/clt/models/clt.py index 3e1391c..e166547 100644 --- a/clt/models/clt.py +++ b/clt/models/clt.py @@ -1,11 +1,12 @@ import torch -from typing import Dict, Optional, Union, Tuple, List +from typing import Dict, Optional, Union, Tuple, List, Any import logging from clt.config import CLTConfig from clt.models.base import BaseTranscoder from clt.models.activations import _apply_batch_topk_helper, _apply_token_topk_helper +from clt.models.activations_local_global import _apply_batch_topk_local_global from clt.models.encoder import Encoder from clt.models.decoder import Decoder from clt.models.theta import ThetaManager @@ -31,11 +32,13 @@ def __init__( config: CLTConfig, process_group: Optional["ProcessGroup"], device: Optional[torch.device] = None, + profiler: Optional[Any] = None, ): super().__init__(config) self.process_group = process_group self.world_size = dist_ops.get_world_size(process_group) self.rank = dist_ops.get_rank(process_group) + self.profiler = profiler self.dtype = self._resolve_dtype(config.clt_dtype) if device is not None: @@ -87,16 +90,23 @@ def _apply_batch_topk( self, preactivations_dict: Dict[int, torch.Tensor], ) -> Dict[int, torch.Tensor]: - return _apply_batch_topk_helper( - preactivations_dict, self.config, self.device, self.dtype, self.rank, self.process_group - ) + # Use optimized local-global approach for multi-GPU training + if self.world_size > 1: + return _apply_batch_topk_local_global( + preactivations_dict, self.config, self.device, self.dtype, self.rank, self.process_group, self.profiler + ) + else: + # Single GPU uses original implementation + return _apply_batch_topk_helper( + preactivations_dict, self.config, self.device, self.dtype, self.rank, self.process_group, self.profiler + ) def _apply_token_topk( self, preactivations_dict: Dict[int, torch.Tensor], ) -> Dict[int, torch.Tensor]: return _apply_token_topk_helper( - preactivations_dict, self.config, self.device, self.dtype, self.rank, self.process_group + preactivations_dict, self.config, self.device, self.dtype, self.rank, self.process_group, self.profiler ) def encode(self, x: torch.Tensor, layer_idx: int) -> torch.Tensor: @@ -221,9 +231,21 @@ def get_feature_activations(self, inputs: Dict[int, torch.Tensor]) -> Dict[int, return activations if self.config.activation_fn == "batchtopk": - activations = self._apply_batch_topk(preactivations_dict) + if self.profiler: + with self.profiler.timer("batchtopk_activation") as timer: + activations = self._apply_batch_topk(preactivations_dict) + if hasattr(timer, 'elapsed'): + self.profiler.record("batchtopk_activation", timer.elapsed) + else: + activations = self._apply_batch_topk(preactivations_dict) elif self.config.activation_fn == "topk": - activations = self._apply_token_topk(preactivations_dict) + if self.profiler: + with self.profiler.timer("topk_activation") as timer: + activations = self._apply_token_topk(preactivations_dict) + if hasattr(timer, 'elapsed'): + self.profiler.record("topk_activation", timer.elapsed) + else: + activations = self._apply_token_topk(preactivations_dict) else: raise ValueError(f"Unexpected activation_fn '{self.config.activation_fn}' in BatchTopK/TokenTopK path.") return activations diff --git a/clt/training/checkpointing.py b/clt/training/checkpointing.py index 6cd05bc..660492a 100644 --- a/clt/training/checkpointing.py +++ b/clt/training/checkpointing.py @@ -36,6 +36,7 @@ def __init__( rank: int, device: torch.device, world_size: int, + keep_n_checkpoints: int = 3, # Keep only last N checkpoints to save space # Add optimizer, scheduler, scaler to be available for loading if needed # For saving, they will be passed to _save_checkpoint ): @@ -47,6 +48,7 @@ def __init__( self.rank = rank self.device = device self.world_size = world_size + self.keep_n_checkpoints = keep_n_checkpoints def _save_checkpoint( self, @@ -95,27 +97,46 @@ def _save_checkpoint( # --- Distributed Save --- checkpoint_dir = os.path.join(self.log_dir, f"step_{step}") latest_checkpoint_dir = os.path.join(self.log_dir, "latest") + + # Create directories if they don't exist (all ranks should do this) + os.makedirs(checkpoint_dir, exist_ok=True) + os.makedirs(latest_checkpoint_dir, exist_ok=True) + + # Ensure all ranks see the directories before proceeding + if self.distributed: + dist.barrier() # Save model state dict using distributed checkpointing model_state_dict_for_dist_save = self.model.state_dict() + + # Option 1: Save per-rank checkpoints separately for debugging + rank_checkpoint_path = os.path.join(checkpoint_dir, f"rank_{self.rank}_model.pt") + latest_rank_checkpoint_path = os.path.join(latest_checkpoint_dir, f"rank_{self.rank}_model.pt") + try: - save_state_dict( - state_dict=model_state_dict_for_dist_save, - storage_writer=FileSystemWriter(checkpoint_dir), - planner=DefaultSavePlanner(), - no_dist=False, - ) - save_state_dict( - state_dict=model_state_dict_for_dist_save, - storage_writer=FileSystemWriter(latest_checkpoint_dir), - planner=DefaultSavePlanner(), - no_dist=False, - ) + # Save individual rank files + torch.save(model_state_dict_for_dist_save, rank_checkpoint_path) + torch.save(model_state_dict_for_dist_save, latest_rank_checkpoint_path) + logger.info(f"Rank {self.rank}: Saved individual checkpoint to {rank_checkpoint_path}") + + # Debug: Check what we saved + enc_key = "encoder_module.encoders.0.weight" + if enc_key in model_state_dict_for_dist_save: + checksum = torch.sum(torch.abs(model_state_dict_for_dist_save[enc_key])).item() + logger.info(f"Rank {self.rank}: Saved {enc_key} with checksum {checksum:.6f}") + + # Skip saving distributed checkpoint (.distcp files) to save space + # We're using individual rank files instead due to PyTorch bug + pass except Exception as e: logger.warning( f"Rank {self.rank}: Warning: Failed to save distributed model checkpoint at step {step}: {e}" ) + # Wait for all ranks to save their individual checkpoints + if self.distributed: + dist.barrier() + if self.rank == 0: # Save activation store store_checkpoint_path = os.path.join(checkpoint_dir, "activation_store.pt") @@ -125,19 +146,50 @@ def _save_checkpoint( torch.save(self.activation_store.state_dict(), latest_store_path) except Exception as e: logger.warning(f"Rank 0: Warning: Failed to save activation store state at step {step}: {e}") - - # Save consolidated model as .safetensors - model_safetensors_path = os.path.join(checkpoint_dir, "model.safetensors") - latest_model_safetensors_path = os.path.join(latest_checkpoint_dir, "model.safetensors") + + # Merge individual rank checkpoints into consolidated model + # This is a workaround for the PyTorch distributed checkpoint bug try: - full_model_state_dict = self.model.state_dict() - save_safetensors_file(full_model_state_dict, model_safetensors_path) - save_safetensors_file(full_model_state_dict, latest_model_safetensors_path) - logger.info( - f"Rank 0: Saved consolidated model to {model_safetensors_path} and {latest_model_safetensors_path}" - ) + logger.info(f"Rank 0: Merging {self.world_size} rank checkpoints...") + + # Load all rank state dicts + state_dicts = [] + for rank in range(self.world_size): + rank_path = os.path.join(checkpoint_dir, f"rank_{rank}_model.pt") + if os.path.exists(rank_path): + state_dict = torch.load(rank_path, map_location="cpu") + state_dicts.append(state_dict) + else: + logger.error(f"Rank 0: Missing rank checkpoint: {rank_path}") + state_dicts = None + break + + if state_dicts and len(state_dicts) == self.world_size: + # Merge the state dicts + merged_state = self._merge_tensor_parallel_weights(state_dicts) + + # Save as safetensors + model_safetensors_path = os.path.join(checkpoint_dir, "model.safetensors") + latest_model_safetensors_path = os.path.join(latest_checkpoint_dir, "model.safetensors") + save_safetensors_file(merged_state, model_safetensors_path) + save_safetensors_file(merged_state, latest_model_safetensors_path) + logger.info(f"Rank 0: Saved merged model to {model_safetensors_path}") + else: + logger.error(f"Rank 0: Failed to merge rank checkpoints - missing files") + # Fall back to single rank save + model_safetensors_path = os.path.join(checkpoint_dir, "model.safetensors") + latest_model_safetensors_path = os.path.join(latest_checkpoint_dir, "model.safetensors") + try: + full_model_state_dict = self.model.state_dict() + save_safetensors_file(full_model_state_dict, model_safetensors_path) + save_safetensors_file(full_model_state_dict, latest_model_safetensors_path) + logger.info( + f"Rank 0: Saved consolidated model to {model_safetensors_path} and {latest_model_safetensors_path}" + ) + except Exception as e: + logger.warning(f"Rank 0: Warning: Failed to save fallback model at step {step}: {e}") except Exception as e: - logger.warning(f"Rank 0: Warning: Failed to save consolidated .safetensors model at step {step}: {e}") + logger.warning(f"Rank 0: Warning: Failed to merge rank checkpoints at step {step}: {e}") # Save trainer state (optimizer, scheduler, etc.) trainer_state_filepath = os.path.join(checkpoint_dir, "trainer_state.pt") @@ -151,13 +203,23 @@ def _save_checkpoint( except Exception as e: logger.warning(f"Rank 0: Warning: Failed to save trainer state at step {step}: {e}") - self.wandb_logger.log_artifact( - artifact_path=checkpoint_dir, - artifact_type="model_checkpoint", - name=f"dist_checkpoint_{step}", - ) + # Only log artifact if we successfully saved something + if os.path.exists(checkpoint_dir) and os.listdir(checkpoint_dir): + try: + self.wandb_logger.log_artifact( + artifact_path=checkpoint_dir, + artifact_type="model_checkpoint", + name=f"dist_checkpoint_{step}", + ) + except Exception as e: + logger.warning(f"Rank 0: Failed to log artifact to WandB: {e}") - dist.barrier() + if self.distributed: + dist.barrier() + + # Clean up old checkpoints to save space + if self.rank == 0 and self.keep_n_checkpoints > 0: + self._cleanup_old_checkpoints() def load_checkpoint(self, checkpoint_path: str) -> Dict[str, Any]: """Load a checkpoint for model, activation store, and trainer state. @@ -396,3 +458,74 @@ def _load_non_distributed_checkpoint(self, checkpoint_path: str, store_checkpoin logger.warning( f"Warning: Activation store checkpoint path not found or specified: {store_checkpoint_path}. Store state not loaded." ) + + def _merge_tensor_parallel_weights(self, state_dicts: list) -> Dict[str, torch.Tensor]: + """ + Merge tensor-parallel weights from multiple ranks into a single state dict. + This is a workaround for the PyTorch distributed checkpoint bug. + """ + merged_state = {} + world_size = len(state_dicts) + + # Get all parameter names from first rank + param_names = list(state_dicts[0].keys()) + + for name in param_names: + tensors = [sd[name] for sd in state_dicts] + + # Check if this is a tensor-parallel weight that needs concatenation + if "encoder_module.encoders" in name: + if "weight" in name: + # Encoder weights are sharded along dim 0 (output features) + merged_state[name] = torch.cat(tensors, dim=0) + elif "bias" in name: + # Encoder biases are also sharded along dim 0 + merged_state[name] = torch.cat(tensors, dim=0) + else: + # Other encoder parameters + merged_state[name] = tensors[0] + + elif "decoder_module.decoders" in name and "weight" in name: + # Decoder weights are sharded along dim 1 (input features) + merged_state[name] = torch.cat(tensors, dim=1) + + elif "log_threshold" in name: + # For BatchTopK threshold, concatenate the per-layer thresholds + merged_state[name] = torch.cat(tensors, dim=1) + + else: + # For replicated parameters (biases, layer norms, etc.), use rank 0's version + merged_state[name] = tensors[0] + + return merged_state + + def _cleanup_old_checkpoints(self): + """Remove old checkpoints to save disk space, keeping only the last N.""" + import shutil + from pathlib import Path + + log_path = Path(self.log_dir) + + # Find all step directories + step_dirs = [] + for item in log_path.iterdir(): + if item.is_dir() and item.name.startswith("step_"): + try: + step_num = int(item.name.replace("step_", "")) + step_dirs.append((step_num, item)) + except ValueError: + continue + + # Sort by step number + step_dirs.sort(key=lambda x: x[0]) + + # Keep only the last N checkpoints + if len(step_dirs) > self.keep_n_checkpoints: + dirs_to_remove = step_dirs[:-self.keep_n_checkpoints] + + for step_num, dir_path in dirs_to_remove: + try: + shutil.rmtree(dir_path) + logger.info(f"Removed old checkpoint: {dir_path}") + except Exception as e: + logger.warning(f"Failed to remove old checkpoint {dir_path}: {e}") diff --git a/clt/training/evaluator.py b/clt/training/evaluator.py index 25e94f6..eb77645 100644 --- a/clt/training/evaluator.py +++ b/clt/training/evaluator.py @@ -100,7 +100,7 @@ def compute_metrics( Metrics are organized into 'reconstruction', 'sparsity', 'dead_features', 'layerwise'. """ - mem_before_eval = 0 + mem_before_eval = 0.0 if torch.cuda.is_available() and self.device.type == "cuda": mem_before_eval = torch.cuda.memory_allocated(self.device) / (1024**2) elapsed_str = _format_elapsed_time(time.time() - self.start_time) @@ -267,9 +267,9 @@ def _compute_reconstruction_metrics( recon_act_denorm = recon_act * std + mean # --- End De-normalisation --- - # Ensure shapes match (flatten if necessary) - target_flat = target_act_denorm.view(-1, target_act_denorm.shape[-1]) - recon_flat = recon_act_denorm.view(-1, recon_act_denorm.shape[-1]) + # Ensure shapes match (flatten if necessary) and up-cast to float32 for numerically stable metrics + target_flat = target_act_denorm.view(-1, target_act_denorm.shape[-1]).float() + recon_flat = recon_act_denorm.view(-1, recon_act_denorm.shape[-1]).float() if target_flat.shape != recon_flat.shape or target_flat.numel() == 0: continue diff --git a/clt/training/profiler.py b/clt/training/profiler.py new file mode 100644 index 0000000..bb02ef7 --- /dev/null +++ b/clt/training/profiler.py @@ -0,0 +1,240 @@ +"""Performance profiling utilities for CLT training.""" + +import time +import torch +from typing import Dict, List, Optional, Any +from collections import defaultdict +import logging +from contextlib import contextmanager + +logger = logging.getLogger(__name__) + + +class Timer: + """Simple timer context manager for measuring execution time.""" + + def __init__(self, name: str, enabled: bool = True): + self.name = name + self.enabled = enabled + self.start_time: Optional[float] = None + self.cuda_start_event: Optional[torch.cuda.Event] = None + self.cuda_end_event: Optional[torch.cuda.Event] = None + + def __enter__(self): + if not self.enabled: + return self + + if torch.cuda.is_available(): + # Use CUDA events for more accurate GPU timing + self.cuda_start_event = torch.cuda.Event(enable_timing=True) + self.cuda_end_event = torch.cuda.Event(enable_timing=True) + self.cuda_start_event.record() + else: + self.start_time = time.perf_counter() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.enabled: + return + + if torch.cuda.is_available() and self.cuda_start_event and self.cuda_end_event: + self.cuda_end_event.record() + torch.cuda.synchronize() + self.elapsed = self.cuda_start_event.elapsed_time(self.cuda_end_event) / 1000.0 # Convert to seconds + else: + self.elapsed = time.perf_counter() - self.start_time if self.start_time else 0.0 + + +class TrainingProfiler: + """Profiler for tracking performance metrics during CLT training.""" + + def __init__(self, enabled: bool = True, log_interval: int = 100): + self.enabled = enabled + self.log_interval = log_interval + self.timings: Dict[str, List[float]] = defaultdict(list) + self.step_count = 0 + + def record(self, name: str, duration: float): + """Record a timing measurement.""" + if not self.enabled: + return + self.timings[name].append(duration) + + def timer(self, name: str) -> Timer: + """Create a timer context manager.""" + return Timer(name, enabled=self.enabled) + + def step(self): + """Increment step counter and log if needed.""" + if not self.enabled: + return + + self.step_count += 1 + + if self.step_count % self.log_interval == 0: + self.log_summary() + + def log_summary(self): + """Log timing summary and clear buffers.""" + if not self.timings: + return + + logger.info(f"\n{'='*60}") + logger.info(f"Performance Profile (last {self.log_interval} steps)") + logger.info(f"{'='*60}") + + # Calculate statistics + total_time = 0.0 + timing_stats = {} + + for name, times in self.timings.items(): + if times: + avg_time = sum(times) / len(times) + min_time = min(times) + max_time = max(times) + total_time += avg_time + timing_stats[name] = { + 'avg': avg_time, + 'min': min_time, + 'max': max_time, + 'count': len(times) + } + + # Sort by average time + sorted_stats = sorted(timing_stats.items(), key=lambda x: x[1]['avg'], reverse=True) + + # Log each timing + for name, stats in sorted_stats: + pct = (stats['avg'] / total_time * 100) if total_time > 0 else 0 + logger.info( + f"{name:.<30} " + f"avg: {stats['avg']*1000:>7.2f}ms " + f"min: {stats['min']*1000:>7.2f}ms " + f"max: {stats['max']*1000:>7.2f}ms " + f"({pct:>5.1f}%)" + ) + + logger.info(f"{'='*60}") + logger.info(f"Total average step time: {total_time*1000:.2f}ms") + logger.info(f"{'='*60}\n") + + # Clear timings for next interval + self.timings.clear() + + def get_summary(self) -> Dict[str, Any]: + """Get timing summary as a dictionary.""" + summary = {} + for name, times in self.timings.items(): + if times: + summary[name] = { + 'avg': sum(times) / len(times), + 'min': min(times), + 'max': max(times), + 'total': sum(times), + 'count': len(times) + } + return summary + + +class CUDAMemoryProfiler: + """Profiler for tracking CUDA memory usage.""" + + def __init__(self, enabled: bool = True): + self.enabled = enabled and torch.cuda.is_available() + self.peak_memory = 0.0 + self.allocated_history: List[float] = [] + + def snapshot(self, label: str = ""): + """Take a memory snapshot.""" + if not self.enabled: + return + + allocated = torch.cuda.memory_allocated() / 1024**3 # Convert to GB + reserved = torch.cuda.memory_reserved() / 1024**3 + + self.allocated_history.append(allocated) + self.peak_memory = max(self.peak_memory, allocated) + + if label: + logger.debug(f"[{label}] CUDA Memory - Allocated: {allocated:.2f}GB, Reserved: {reserved:.2f}GB") + + def log_summary(self): + """Log memory usage summary.""" + if not self.enabled or not self.allocated_history: + return + + avg_allocated = sum(self.allocated_history) / len(self.allocated_history) + logger.info(f"\nCUDA Memory Summary:") + logger.info(f" Peak allocated: {self.peak_memory:.2f}GB") + logger.info(f" Average allocated: {avg_allocated:.2f}GB") + logger.info(f" Current allocated: {torch.cuda.memory_allocated() / 1024**3:.2f}GB") + logger.info(f" Current reserved: {torch.cuda.memory_reserved() / 1024**3:.2f}GB") + + +class DistributedProfiler: + """Profiler specifically for distributed operations.""" + + def __init__(self, enabled: bool = True, rank: int = 0): + self.enabled = enabled + self.rank = rank + self.timings: Dict[str, List[float]] = defaultdict(list) + + @contextmanager + def profile_op(self, op_name: str): + """Context manager to profile a distributed operation.""" + if not self.enabled: + yield + return + + if torch.cuda.is_available(): + torch.cuda.synchronize() + + start_time = time.perf_counter() + yield + + if torch.cuda.is_available(): + torch.cuda.synchronize() + + elapsed = time.perf_counter() - start_time + self.timings[op_name].append(elapsed) + + def log_summary(self): + """Log summary of distributed operations.""" + if not self.enabled or not self.timings: + return + + logger.info(f"\n{'='*60}") + logger.info(f"Distributed Operations Profile (Rank {self.rank})") + logger.info(f"{'='*60}") + + for op_name, times in sorted(self.timings.items()): + if times: + avg_time = sum(times) / len(times) + min_time = min(times) + max_time = max(times) + total_time = sum(times) + + logger.info( + f"{op_name:.<30} " + f"avg: {avg_time*1000:>7.2f}ms " + f"min: {min_time*1000:>7.2f}ms " + f"max: {max_time*1000:>7.2f}ms " + f"total: {total_time:.2f}s " + f"calls: {len(times)}" + ) + + logger.info(f"{'='*60}") + + +@contextmanager +def profile_activation_function(profiler: Optional['TrainingProfiler'], name: str): + """Context manager to profile activation functions.""" + if profiler is None or not profiler.enabled: + yield + return + + with profiler.timer(name) as timer: + yield + + if hasattr(timer, 'elapsed'): + profiler.record(name, timer.elapsed) \ No newline at end of file diff --git a/clt/training/trainer.py b/clt/training/trainer.py index 359be85..6e971e5 100644 --- a/clt/training/trainer.py +++ b/clt/training/trainer.py @@ -30,6 +30,7 @@ from clt.training.data.activation_store_factory import create_activation_store # Add this import from .metric_utils import MetricLogger # Add this import from .diagnostics import compute_sparsity_diagnostics # Add this import +from .profiler import TrainingProfiler, CUDAMemoryProfiler, DistributedProfiler # Add profiler imports # Get logger for this module logger = logging.getLogger(__name__) @@ -157,7 +158,25 @@ def __init__( else: logger.warning(f"Rank {self.rank}: No seed provided in TrainingConfig. Using default torch seeding.") - self.model = CrossLayerTranscoder(clt_config, process_group=self.process_group, device=self.device) + # Initialize profilers early (before model creation) + self.profiler = TrainingProfiler( + enabled=self.training_config.enable_profiling, + log_interval=self.training_config.log_interval + ) + self.memory_profiler = CUDAMemoryProfiler( + enabled=self.training_config.enable_profiling and torch.cuda.is_available() + ) + self.dist_profiler = DistributedProfiler( + enabled=self.training_config.enable_profiling and self.distributed, + rank=self.rank + ) + + self.model = CrossLayerTranscoder( + clt_config, + process_group=self.process_group, + device=self.device, + profiler=self.profiler if self.training_config.enable_profiling else None + ) # --- Optionally convert model to FP16 (Step 8) --- # If precision is "fp16", GradScaler is used, which expects FP32 optimizer parameters. @@ -568,18 +587,20 @@ def train(self, eval_every: int = 1000) -> CrossLayerTranscoder: try: # Get batch directly from the iterator (handles distributed sampling internally) - batch_get_start_time = time.monotonic() - inputs, targets = next(self.activation_store) - batch_get_duration = time.monotonic() - batch_get_start_time - logger.debug(f"Rank {self.rank} Step {step}: Getting batch took {batch_get_duration:.4f}s") + with self.profiler.timer("data_loading") as timer: + inputs, targets = next(self.activation_store) + if hasattr(timer, 'elapsed'): + self.profiler.record("data_loading", timer.elapsed) + logger.debug(f"Rank {self.rank} Step {step}: Getting batch took {timer.elapsed:.4f}s") # logging to diagnose batch size mismatch tok_cnt = next(iter(inputs.values())).shape[0] # number of rows (=tokens) in this batch # Only run the all_gather diagnostic when running in distributed mode if self.distributed and self.world_size > 1 and dist.is_initialized(): - tok_cnt_t = torch.tensor([tok_cnt], device=self.device) - gathered = [torch.zeros_like(tok_cnt_t) for _ in range(self.world_size)] - dist.all_gather(gathered, tok_cnt_t) + with self.dist_profiler.profile_op("batch_size_all_gather"): + tok_cnt_t = torch.tensor([tok_cnt], device=self.device) + gathered = [torch.zeros_like(tok_cnt_t) for _ in range(self.world_size)] + dist.all_gather(gathered, tok_cnt_t) except StopIteration: # Rank 0 prints message @@ -632,50 +653,62 @@ def train(self, eval_every: int = 1000) -> CrossLayerTranscoder: with torch.autocast( device_type=self.device.type, dtype=self.autocast_dtype, enabled=self.autocast_enabled ): - feature_activations_batch = self.model.get_feature_activations(inputs) - loss, loss_dict = self.loss_manager.compute_total_loss( - self.model, - inputs, - targets, - step, - self.training_config.training_steps, - precomputed_activations=feature_activations_batch, - dead_neuron_mask=self.dead_neurons_mask, - ) + # Profile forward pass + with self.profiler.timer("forward_pass") as timer: + feature_activations_batch = self.model.get_feature_activations(inputs) + if hasattr(timer, 'elapsed'): + self.profiler.record("forward_pass", timer.elapsed) + + # Profile loss computation + with self.profiler.timer("loss_computation") as timer: + loss, loss_dict = self.loss_manager.compute_total_loss( + self.model, + inputs, + targets, + step, + self.training_config.training_steps, + precomputed_activations=feature_activations_batch, + dead_neuron_mask=self.dead_neurons_mask, + ) + if hasattr(timer, 'elapsed'): + self.profiler.record("loss_computation", timer.elapsed) # --- Update Dead Neuron Counters --- (All ranks, counter is replicated) # We need *full* feature activations *after* non-linearity if hasattr(self, "n_forward_passes_since_fired") and self.n_forward_passes_since_fired is not None: - with torch.no_grad(): - for layer_idx, layer_acts in feature_activations_batch.items(): - # Ensure layer index is within bounds of the counter tensor - if layer_idx < self.n_forward_passes_since_fired.shape[0]: - if layer_acts.numel() > 0: - # layer_acts shape: [batch_tokens, num_features] - fired_mask_per_token = layer_acts > 1e-6 - fired_features_this_layer = fired_mask_per_token.any(dim=0) - - if fired_features_this_layer.shape[0] == self.n_forward_passes_since_fired.shape[1]: - self.n_forward_passes_since_fired[layer_idx] += 1 - self.n_forward_passes_since_fired[layer_idx][fired_features_this_layer] = 0 - else: - # Log warning only on rank 0 to avoid flooding logs + with self.profiler.timer("dead_neuron_update") as timer: + with torch.no_grad(): + for layer_idx, layer_acts in feature_activations_batch.items(): + # Ensure layer index is within bounds of the counter tensor + if layer_idx < self.n_forward_passes_since_fired.shape[0]: + if layer_acts.numel() > 0: + # layer_acts shape: [batch_tokens, num_features] + fired_mask_per_token = layer_acts > 1e-6 + fired_features_this_layer = fired_mask_per_token.any(dim=0) + + if fired_features_this_layer.shape[0] == self.n_forward_passes_since_fired.shape[1]: + self.n_forward_passes_since_fired[layer_idx] += 1 + self.n_forward_passes_since_fired[layer_idx][fired_features_this_layer] = 0 + else: + # Log warning only on rank 0 to avoid flooding logs + if not self.distributed or self.rank == 0: + logger.warning( + f"Rank {self.rank}: Shape mismatch for dead neuron update at layer {layer_idx}. " + f"Acts shape: {layer_acts.shape}, Fired mask: {fired_features_this_layer.shape}, " + f"Counter: {self.n_forward_passes_since_fired.shape}" + ) + else: # layer_acts.numel() == 0 if not self.distributed or self.rank == 0: - logger.warning( - f"Rank {self.rank}: Shape mismatch for dead neuron update at layer {layer_idx}. " - f"Acts shape: {layer_acts.shape}, Fired mask: {fired_features_this_layer.shape}, " - f"Counter: {self.n_forward_passes_since_fired.shape}" + logger.debug( + f"Rank {self.rank}: Layer {layer_idx} has empty activations, skipping dead neuron update for this layer." ) - else: # layer_acts.numel() == 0 + else: # layer_idx out of bounds if not self.distributed or self.rank == 0: - logger.debug( - f"Rank {self.rank}: Layer {layer_idx} has empty activations, skipping dead neuron update for this layer." + logger.warning( + f"Rank {self.rank}: layer_idx {layer_idx} out of bounds for n_forward_passes_since_fired (shape {self.n_forward_passes_since_fired.shape}). Skipping dead neuron update." ) - else: # layer_idx out of bounds - if not self.distributed or self.rank == 0: - logger.warning( - f"Rank {self.rank}: layer_idx {layer_idx} out of bounds for n_forward_passes_since_fired (shape {self.n_forward_passes_since_fired.shape}). Skipping dead neuron update." - ) + if hasattr(timer, 'elapsed'): + self.profiler.record("dead_neuron_update", timer.elapsed) else: # n_forward_passes_since_fired does not exist or is None if not self.distributed or self.rank == 0: logger.warning( @@ -693,24 +726,37 @@ def train(self, eval_every: int = 1000) -> CrossLayerTranscoder: logger.warning(f"Rank {self.rank}: NaN Loss - Detailed loss_dict at step {step}: {loss_dict}") else: # ---- Back-prop with gradient scaling ---- - self.scaler.scale(loss).backward() + with self.profiler.timer("backward_pass") as timer: + self.scaler.scale(loss).backward() + if hasattr(timer, 'elapsed'): + self.profiler.record("backward_pass", timer.elapsed) # Unscale gradients before clipping and distributed averaging self.scaler.unscale_(self.optimizer) # --- Synchronise gradients of replicated parameters --- # if self.distributed and self.world_size > 1: - average_shared_parameter_grads(self.model, self.world_size) + with self.profiler.timer("gradient_sync") as timer: + with self.dist_profiler.profile_op("gradient_all_reduce"): + average_shared_parameter_grads(self.model, self.world_size) + if hasattr(timer, 'elapsed'): + self.profiler.record("gradient_sync", timer.elapsed) # --- Gradient clipping (operates on unscaled gradients) --- # if self.training_config.gradient_clip_val is not None: - torch.nn.utils.clip_grad_norm_( - self.model.parameters(), - self.training_config.gradient_clip_val, - ) + with self.profiler.timer("gradient_clipping") as timer: + torch.nn.utils.clip_grad_norm_( + self.model.parameters(), + self.training_config.gradient_clip_val, + ) + if hasattr(timer, 'elapsed'): + self.profiler.record("gradient_clipping", timer.elapsed) # --- Optimizer step (scaler handles scaling/unscaling) --- - self.scaler.step(self.optimizer) + with self.profiler.timer("optimizer_step") as timer: + self.scaler.step(self.optimizer) + if hasattr(timer, 'elapsed'): + self.profiler.record("optimizer_step", timer.elapsed) # --- Update scaler for next iteration --- self.scaler.update() @@ -726,7 +772,11 @@ def train(self, eval_every: int = 1000) -> CrossLayerTranscoder: and hasattr(self, "n_forward_passes_since_fired") and self.n_forward_passes_since_fired is not None ): - dist.all_reduce(self.n_forward_passes_since_fired, op=dist.ReduceOp.MIN, group=self.process_group) + with self.profiler.timer("dead_neuron_sync") as timer: + with self.dist_profiler.profile_op("dead_neuron_all_reduce"): + dist.all_reduce(self.n_forward_passes_since_fired, op=dist.ReduceOp.MIN, group=self.process_group) + if hasattr(timer, 'elapsed'): + self.profiler.record("dead_neuron_sync", timer.elapsed) # --- Scheduler step --- (All ranks) if self.scheduler: @@ -775,19 +825,23 @@ def train(self, eval_every: int = 1000) -> CrossLayerTranscoder: # We still only log / store the resulting metrics on rank 0. if run_eval_flag: if self.distributed: - dist.barrier() # Sync before evaluation starts so that all ranks enter together + with self.dist_profiler.profile_op("eval_barrier"): + dist.barrier() # Sync before evaluation starts so that all ranks enter together # Compute evaluation metrics on all ranks to keep collective ops aligned # Wrap the evaluation logic in autocast - with torch.autocast( - device_type=self.device.type, dtype=self.autocast_dtype, enabled=self.autocast_enabled - ): - current_dead_mask = self.dead_neurons_mask.detach().clone() - eval_metrics = self.evaluator.compute_metrics( - inputs, # These inputs are from the current training batch - targets, # These targets are from the current training batch - dead_neuron_mask=current_dead_mask, - ) + with self.profiler.timer("evaluation") as timer: + with torch.autocast( + device_type=self.device.type, dtype=self.autocast_dtype, enabled=self.autocast_enabled + ): + current_dead_mask = self.dead_neurons_mask.detach().clone() + eval_metrics = self.evaluator.compute_metrics( + inputs, # These inputs are from the current training batch + targets, # These targets are from the current training batch + dead_neuron_mask=current_dead_mask, + ) + if hasattr(timer, 'elapsed'): + self.profiler.record("evaluation", timer.elapsed) # --- Log per-layer standard deviation of pre-activations --- # This requires getting the pre-activations first. @@ -887,6 +941,11 @@ def train(self, eval_every: int = 1000) -> CrossLayerTranscoder: } self.checkpoint_manager._save_checkpoint(step, current_trainer_state_for_checkpoint) + # --- Profile memory and step profiler --- # + if not self.distributed or self.rank == 0: + self.memory_profiler.snapshot(f"Step {step}") + self.profiler.step() + # --- Explicitly delete tensors at the very end of the loop iteration --- # # Do this on all ranks try: @@ -1013,6 +1072,17 @@ def train(self, eval_every: int = 1000) -> CrossLayerTranscoder: if hasattr(self.activation_store, "close") and callable(getattr(self.activation_store, "close")): self.activation_store.close() + # Log final profiling summaries + if not self.distributed or self.rank == 0: + logger.info("\n" + "="*80) + logger.info("FINAL PROFILING SUMMARY") + logger.info("="*80) + self.profiler.log_summary() + self.memory_profiler.log_summary() + if self.distributed: + self.dist_profiler.log_summary() + logger.info("="*80) + # Clean up distributed process group if self.distributed: dist.destroy_process_group() diff --git a/diagnose_manifest.py b/diagnose_manifest.py deleted file mode 100644 index 6649e4a..0000000 --- a/diagnose_manifest.py +++ /dev/null @@ -1,75 +0,0 @@ -import numpy as np -import argparse -import os - -# Define the manifest dtype, matching the one in ActivationGenerator and ManifestActivationStore -MANIFEST_DTYPE = np.dtype([("chunk_id", np.int32), ("num_tokens", np.int32), ("offset", np.int64)]) - - -def diagnose_manifest_file(manifest_path): - """ - Reads an index.bin manifest file and prints statistics about its contents. - """ - if not os.path.exists(manifest_path): - print(f"Error: Manifest file not found at {manifest_path}") - return - - try: - manifest_data = np.fromfile(manifest_path, dtype=MANIFEST_DTYPE) - except Exception as e: - print(f"Error reading manifest file {manifest_path}: {e}") - return - - if manifest_data.size == 0: - print(f"Manifest file {manifest_path} is empty or not in the expected format.") - return - - num_entries = manifest_data.shape[0] - chunk_ids = manifest_data["chunk_id"] - num_tokens_values = manifest_data["num_tokens"] - offsets = manifest_data["offset"] - - print(f"--- Manifest File Diagnostics for: {manifest_path} ---") - print(f"Total entries: {num_entries}") - - if num_entries > 0: - print("\nChunk ID Statistics:") - print(f" Min chunk_id: {np.min(chunk_ids)}") - print(f" Max chunk_id: {np.max(chunk_ids)}") - if not np.all(np.diff(chunk_ids) == 1) and num_entries > 1: - print(" Warning: Chunk IDs are not strictly sequential or contain duplicates!") - else: - print(" Chunk IDs appear sequential and unique.") - - print("\nNum Tokens Statistics:") - print(f" Min num_tokens: {np.min(num_tokens_values)}") - print(f" Max num_tokens: {np.max(num_tokens_values)}") - print(f" Mean num_tokens: {np.mean(num_tokens_values):.2f}") - print(f" Median num_tokens: {np.median(num_tokens_values)}") - if np.median(num_tokens_values) < 100: # Arbitrary low threshold to flag likely issues - print(f" WARNING: Median num_tokens ({np.median(num_tokens_values)}) is very low!") - - print("\nOffset Statistics:") - print(f" Min offset: {np.min(offsets)}") - print(f" Max offset: {np.max(offsets)}") - - print("\nFirst 5 entries:") - for i in range(min(5, num_entries)): - print( - f" Entry {i}: chunk_id={manifest_data[i]['chunk_id']}, num_tokens={manifest_data[i]['num_tokens']}, offset={manifest_data[i]['offset']}" - ) - - print("\nLast 5 entries:") - for i in range(max(0, num_entries - 5), num_entries): - print( - f" Entry {i}: chunk_id={manifest_data[i]['chunk_id']}, num_tokens={manifest_data[i]['num_tokens']}, offset={manifest_data[i]['offset']}" - ) - print("--- End of Diagnostics ---") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Diagnose an index.bin manifest file.") - parser.add_argument("manifest_path", type=str, help="Path to the index.bin manifest file.") - args = parser.parse_args() - - diagnose_manifest_file(args.manifest_path) diff --git a/norm_stats.json b/norm_stats.json deleted file mode 100644 index d473ffe..0000000 --- a/norm_stats.json +++ /dev/null @@ -1 +0,0 @@ -{"0":{"inputs":{"mean":[0.011929018422961235,-0.008026998490095139,0.04543500021100044,-0.08982595801353455,0.003456737846136093,-0.037446554750204086,-0.02623085491359234,0.04845714941620827,-2.0560317039489746,0.11845869570970535,0.003981353715062141,0.02808064967393875,-0.06509637832641602,0.021235520020127296,0.02389397844672203,-0.0016556538175791502,-0.00897795706987381,0.018769536167383194,-0.03576196730136871,-0.023563789203763008,0.049127593636512756,-0.0027368615847080946,-0.01264966744929552,0.005407341290265322,0.11946438997983932,0.04818002134561539,0.024203738197684288,-0.0007617946248501539,-0.0018260288052260876,-0.025322169065475464,-0.01770292967557907,-0.008212273940443993,-0.021130109205842018,0.02382679469883442,0.04788237810134888,-0.024952419102191925,0.0019623981788754463,0.07762202620506287,-0.13632449507713318,-0.015339398756623268,0.014728214591741562,-0.019414126873016357,0.015140908770263195,-0.22609572112560272,-0.02960219420492649,0.0005785321118310094,-0.006397430319339037,0.026747090741991997,-0.05419165641069412,-0.010571751743555069,0.07093260437250137,0.018497901037335396,0.022225847467780113,-0.0006354982033371925,-0.14246605336666107,-0.0111076720058918,-0.04752563685178757,-0.16765908896923065,-0.0332634337246418,0.09852521866559982,0.04374263808131218,-0.02305992878973484,0.027244728058576584,0.06265625357627869,0.022751083597540855,0.0065856981091201305,0.0549943782389164,0.01778550259768963,0.034220945090055466,0.08748238533735275,0.04567355662584305,0.031068606302142143,0.05658824369311333,0.012153573334217072,0.0038496910128742456,-0.012245727702975273,-0.06836315244436264,0.07107619196176529,-0.005394005216658115,-0.0010918789776042104,-0.09371764212846756,-0.009424651972949505,0.02258884347975254,0.030111832544207573,0.026562213897705078,-0.027416110038757324,-0.006306961644440889,-0.025578930974006653,-0.12910659611225128,0.0051354398019611835,0.0010357069550082088,0.012300663627684116,0.03188937529921532,0.10844523459672928,0.021076852455735207,-0.009596627205610275,-0.04185517877340317,0.007625509984791279,0.014570534229278564,-0.0318288654088974,-0.0657602921128273,0.047338418662548065,0.20443707704544067,0.01420596893876791,-0.0045072901993989944,0.1383475363254547,0.00026457110652700067,0.06002531200647354,0.016500884667038918,-0.029288802295923233,0.0488666370511055,-0.02654087543487549,-0.07648703455924988,0.0011265712091699243,0.0209270678460598,-0.0865657776594162,0.09189733862876892,-0.03960637375712395,-0.04041415825486183,0.033556967973709106,0.021840736269950867,-0.005496177356690168,-0.02988024801015854,-0.0016891632694751024,0.025839699432253838,0.024240020662546158,-0.05660232901573181,-0.12237655371427536,-0.0666339099407196,-0.03900347650051117,-0.04065476357936859,0.025248868390917778,-0.0385412722826004,0.015696706250309944,0.01078720111399889,0.0247072484344244,0.034329384565353394,-0.005609653890132904,-0.1608293354511261,-0.036036647856235504,0.04235347732901573,0.020961036905646324,0.04632445052266121,0.017546026036143303,0.04302467033267021,-0.00040534406434744596,0.018065910786390305,0.016879308968782425,0.018503401428461075,0.02522117830812931,0.0019349610665813088,0.054575979709625244,0.08415588736534119,-0.012666516937315464,0.008466633968055248,0.03300955519080162,0.017295438796281815,-0.15049315989017487,0.0004944414831697941,-0.030181292444467545,-0.0013120927615091205,0.09150692820549011,0.05229198932647705,-0.01601102389395237,-0.007160020060837269,0.013180560432374477,0.026181835681200027,-0.024517357349395752,0.008601058274507523,0.024657336995005608,-0.03512899950146675,0.01805700734257698,-0.11708395928144455,-0.014755092561244965,0.7093095183372498,0.04414918273687363,0.010134280659258366,-0.02368113212287426,-0.05572924017906189,0.07421691715717316,-0.0196871068328619,0.2192450910806656,0.05767246335744858,-0.03147375211119652,-0.026618054136633873,0.0035487585701048374,-0.03801581636071205,0.09944958239793777,0.05961424112319946,0.006152811460196972,-0.0036397839430719614,-0.048531271517276764,-0.01830023154616356,0.0514695830643177,-0.026141555979847908,-0.017718996852636337,-0.08300318568944931,-0.0043565272353589535,-0.019392985850572586,0.060211431235075,-0.008348777890205383,-0.043733932077884674,0.01544087566435337,-0.009711132384836674,-0.024639884009957314,-0.011644204147160053,0.014970626682043076,-0.06637728214263916,-0.09791526198387146,0.031752075999975204,0.07140493392944336,0.03830160200595856,0.0037093376740813255,0.005181314889341593,-0.012798691168427467,-0.1162225604057312,-0.0004750211082864553,-0.04199099913239479,0.07874658703804016,-0.015027684159576893,0.04342993348836899,0.11509893834590912,-0.16508261859416962,-0.005160841159522533,-0.00805071834474802,0.042113859206438065,-0.0403570756316185,0.09191770106554031,0.021402334794402122,-0.04983104020357132,0.03409591317176819,-0.006824960466474295,-0.010681441985070705,-0.0381927527487278,0.008050675503909588,-0.027242643758654594,0.012414976954460144,-0.03990647569298744,-0.011166606098413467,0.02638144977390766,0.06959397345781326,-0.014743709936738014,0.04398707300424576,0.4901573359966278,0.003278488991782069,-0.035668522119522095,-0.013828568160533905,-0.004059212282299995,-0.009292404167354107,9.516057968139648,0.002143817488104105,0.20686031877994537,0.027222424745559692,0.02462127059698105,-0.03656549006700516,0.002323145279660821,0.06634052097797394,-0.011015716008841991,-0.024014152586460114,0.028883598744869232,-0.03392612189054489,0.009275485761463642,-0.0492284893989563,9.416664397576824e-05,0.03716205060482025,-2.2056798934936523,-0.007706462871283293,-0.028659773990511894,-0.07987723499536514,-0.05122645944356918,0.0187518447637558,-0.030923491343855858,-0.050057802349328995,-0.028643498197197914,0.050654251128435135,0.16858981549739838,-0.008907810784876347,0.05536642298102379,-0.0010655780788511038,0.010376585647463799,-0.0313047394156456,-0.03978341817855835,0.017978636547923088,-0.006454259157180786,-0.012201805599033833,0.23526768386363983,0.18393486738204956,-0.028531450778245926,0.0048179663717746735,0.01903006061911583,-0.02079521305859089,-0.11938244104385376,-0.022062046453356743,0.042374398559331894,-0.10489354282617569,-0.05503934249281883,0.061161648482084274,0.008067541755735874,0.007866142317652702,-0.0636104941368103,0.02170916646718979,-0.06609121710062027,0.03293602541089058,-0.01569730043411255,-0.02669368125498295,0.0082386564463377,-0.013441160321235657,-0.12733958661556244,0.01174900308251381,0.026062874123454094,0.03044244647026062,-0.022764597088098526,-0.2124355137348175,0.012309855781495571,-0.01458284817636013,0.0017597394762560725,0.006666985806077719,0.029503846541047096,0.010061997920274734,-0.03318523243069649,-0.02925945818424225,0.02987387403845787,0.006002927199006081,-0.995499849319458,0.01945335604250431,-0.02200588770210743,-0.012095772661268711,0.027929488569498062,0.031840190291404724,-0.0053069391287863255,0.02892494574189186,0.056250978261232376,0.06489284336566925,0.005115313455462456,-0.04036540165543556,-0.06916187703609467,-0.04964579641819,0.25116318464279175,0.018154744058847427,-0.005424580071121454,0.08849073201417923,0.018832111731171608,-0.1076090931892395,0.014917701482772827,-0.09360814839601517,0.008655700832605362,-0.03119191899895668,-0.020698122680187225,-0.014113801531493664,0.0270931888371706,-0.008023416623473167,0.02167285420000553,-0.005816590506583452,0.15242671966552734,-0.05182100087404251,0.017008135095238686,-0.014414500445127487,-0.03787437453866005,0.01948590762913227,0.0006184813100844622,0.009464172646403313,-0.012056158855557442,-0.053675584495067596,-0.13779275119304657,-0.024503089487552643,-0.02755775861442089,0.06337590515613556,0.00806325301527977,-0.011668046936392784,-0.0180219616740942,-0.016216754913330078,-0.009692823514342308,-0.038947831839323044,0.006192982662469149,0.08940165489912033,-0.054789379239082336,0.01022303756326437,0.007154943887144327,-0.4995962381362915,-0.04340451583266258,0.02491801232099533,-0.0505874939262867,-0.07859551161527634,-0.054211243987083435,-0.03162800148129463,0.01925935409963131,0.02395588532090187,-0.04471395164728165,-0.0730341374874115,-0.022238461300730705,-0.021419914439320564,0.007236219011247158,0.06909748166799545,0.012760636396706104,0.05894225835800171,0.009922517463564873,-0.03715924173593521,0.05523799732327461,-0.003578832605853677,0.07156988233327866,0.025134505704045296,0.006184391677379608,0.022172318771481514,-0.035896699875593185,-0.02206871658563614,-0.05258336290717125,0.034204039722681046,0.02973760850727558,-0.009741190820932388,-0.07181039452552795,-0.05912290886044502,0.053371042013168335,-0.01451926026493311,0.048394449055194855,-0.008101360872387886,-0.0319376066327095,-0.05693778768181801,-0.03777065873146057,-0.0004237528482917696,0.007742022629827261,0.0030827021691948175,-0.020462404936552048,5.078669548034668,-0.009275761432945728,-0.009793879464268684,0.015751797705888748,0.015776420012116432,-0.02017202228307724,-0.04056544974446297,-0.05922823026776314,0.036487460136413574,-0.0030664463993161917,-0.2092612385749817,-0.021804453805088997,-0.006506381090730429,0.12601208686828613,-0.010287166573107243,-0.008661586791276932,0.0010906981769949198,-0.4204093813896179,0.5239694714546204,0.021332373842597008,-0.0406540185213089,0.007160957902669907,-0.034321900457143784,0.14534202218055725,-0.005537732969969511,0.028828877955675125,0.030545664951205254,0.006511983927339315,0.01712091825902462,-0.028053445741534233,0.015418368391692638,0.01927892304956913,0.02014336735010147,0.008843190036714077,0.02467784471809864,0.00455763004720211,-0.02622986026108265,-0.002065784763544798,0.186508908867836,0.0028135597240179777,-0.009350957348942757,-0.0259667057543993,-0.06552848219871521,-0.14123524725437164,0.0012204181402921677,-0.12846027314662933,-0.013119373470544815,0.0036815698258578777,-0.02590620145201683,-0.012077394872903824,-0.07018467038869858,-0.01536830049008131,0.013124343939125538,0.008995379321277142,0.05642808601260185,0.005477962549775839,-0.03169452026486397,-0.0021712342277169228,-0.007761022541671991,0.0007132553146220744,-0.028842274099588394,-0.006184646859765053,-0.009555975906550884,-0.09591488540172577,0.0009011730435304344,0.06622153520584106,-0.0420524924993515,-0.015741243958473206,-0.0188214723020792,9.688154386822134e-05,0.0023345157969743013,0.004356774035841227,0.006298185791820288,-0.04072876274585724,-0.132466658949852,-0.17219503223896027,0.039405759423971176,0.026557883247733116,-0.007739020045846701,-0.04592897742986679,-0.0073172347620129585,-0.026527054607868195,0.022666621953248978,0.07568706572055817,0.05013624206185341,0.005408471915870905,0.023012155666947365,0.015014064498245716,0.447488397359848,-0.11169528216123581],"std":[0.4061743915081024,0.39863887429237366,0.4197997450828552,0.45524778962135315,0.46747061610221863,0.4411328434944153,0.3882814645767212,0.42659804224967957,3.230637550354004,0.6812256574630737,0.480202317237854,0.42641299962997437,0.4284665286540985,0.3814452588558197,0.387063592672348,0.44316309690475464,0.4079097509384155,0.44093194603919983,0.40424638986587524,0.41893503069877625,0.45331886410713196,0.3799300491809845,0.3662676215171814,0.46131083369255066,0.5007411241531372,0.4756326675415039,0.40149998664855957,0.4575328230857849,0.4440682828426361,0.4186374247074127,0.44502735137939453,0.4366346597671509,0.39791005849838257,0.4168049097061157,0.4274592995643616,0.36614319682121277,0.3901873528957367,0.5254838466644287,0.5150581002235413,0.4213409423828125,0.405021071434021,0.3911328613758087,0.4384603202342987,0.6648976802825928,0.43552258610725403,0.39777353405952454,0.49530503153800964,0.39881280064582825,0.382660448551178,0.4198063611984253,0.5052202939987183,0.4048334062099457,0.4345322251319885,0.4012673497200012,0.5061508417129517,0.45913660526275635,0.42643028497695923,0.5578755736351013,0.41004979610443115,0.49943509697914124,0.44466251134872437,0.45492038130760193,0.37904587388038635,0.39463740587234497,0.41937604546546936,0.43192097544670105,0.43545445799827576,0.413359671831131,0.4291733205318451,0.4878321588039398,0.4217163622379303,0.47393137216567993,0.6392191052436829,0.4409279525279999,0.41596269607543945,0.39578813314437866,0.3965587913990021,0.45381781458854675,0.4093397855758667,0.42310675978660583,0.4705497622489929,0.38140958547592163,0.3935718536376953,0.411238431930542,0.38591429591178894,0.4204852283000946,0.41955703496932983,0.6053135991096497,0.48006030917167664,0.47036951780319214,0.42690008878707886,0.435398131608963,0.4074626564979553,0.48162901401519775,0.4960220158100128,0.4611230790615082,0.5268700122833252,0.4025828242301941,0.4981358051300049,0.42914798855781555,0.4363592267036438,0.46251198649406433,0.6474661827087402,0.41265037655830383,0.4248198866844177,0.5109621286392212,0.41639402508735657,0.43317535519599915,0.4174826741218567,0.40838688611984253,0.4050280451774597,0.4157305955886841,0.6315788626670837,0.380979061126709,0.4074832499027252,0.5549237132072449,0.47564801573753357,0.38883551955223083,0.3805289566516876,0.4812576174736023,0.4111025929450989,0.43990358710289,0.4559427499771118,0.40810227394104004,0.4710172116756439,0.3654331862926483,0.40825510025024414,0.4535331130027771,0.531555712223053,0.39132386445999146,0.39299267530441284,0.4370250105857849,0.563068687915802,0.3741491138935089,0.38366544246673584,0.38929590582847595,0.43532052636146545,0.36879366636276245,0.9356316924095154,0.37817052006721497,0.3916339576244354,0.3555563688278198,0.43535494804382324,0.3751748204231262,0.3818291127681732,0.4137916564941406,0.4052799344062805,0.3714987337589264,0.45137572288513184,0.37336456775665283,0.41988444328308105,0.4449551999568939,0.6322922110557556,0.36423784494400024,0.5641257762908936,0.3944983184337616,0.4258038401603699,0.687183141708374,0.3919913172721863,0.39086058735847473,0.3668791353702545,0.7095126509666443,0.40977808833122253,0.4226349890232086,0.4088357388973236,0.42649078369140625,0.41963741183280945,0.37540459632873535,0.4340739846229553,0.595160722732544,0.4000144302845001,0.387137770652771,0.7798965573310852,0.46338796615600586,1.9034940004348755,0.4174584150314331,0.38366079330444336,0.6115332841873169,0.42008519172668457,0.9913524985313416,0.40117645263671875,0.7531748414039612,0.6761022210121155,0.36897388100624084,0.4115138053894043,0.48130568861961365,0.36621588468551636,0.46442267298698425,0.37406593561172485,0.41617313027381897,0.4698227345943451,0.41625258326530457,0.5426918268203735,0.4696129262447357,0.43750035762786865,0.4249647557735443,0.5667720437049866,0.4135485291481018,0.4491877555847168,0.4351537227630615,0.4322699010372162,0.379721075296402,0.46761244535446167,0.4127703607082367,0.38093090057373047,0.4146561324596405,0.46862339973449707,0.48236191272735596,0.4569858908653259,0.3944260776042938,0.4380730092525482,0.40468254685401917,0.40922245383262634,0.4373898506164551,0.4736408591270447,0.48269155621528625,0.38141822814941406,0.42161887884140015,0.45711076259613037,0.41962799429893494,0.44111448526382446,0.6122820973396301,0.5223093628883362,0.37032923102378845,0.48090702295303345,0.406029611825943,0.4712555408477783,0.45913854241371155,0.3617847263813019,0.43825483322143555,0.40760958194732666,0.4187520146369934,0.41068992018699646,0.4396568834781647,0.4053465723991394,0.43911248445510864,0.4526553750038147,0.4054655134677887,0.46645495295524597,0.4075148105621338,0.4657449424266815,0.39802122116088867,0.46179625391960144,1.6504032611846924,0.4209158718585968,0.4216329753398895,0.4386637508869171,0.41199758648872375,0.6038753986358643,10.751415252685547,0.4435659945011139,0.6870443224906921,0.4837122857570648,0.45734986662864685,0.41845637559890747,0.4163818061351776,0.43970391154289246,0.4508832097053528,0.4436371922492981,0.45428305864334106,0.44806432723999023,0.43088576197624207,0.4778399169445038,0.4117577075958252,0.42539089918136597,3.4865434169769287,0.41436144709587097,0.4206641912460327,0.45234546065330505,0.5350318551063538,0.368570476770401,0.38878491520881653,0.40679609775543213,0.36048585176467896,0.6629804372787476,0.6334996819496155,0.46029606461524963,0.42066237330436707,0.38765040040016174,0.47372809052467346,0.3882875144481659,0.40071719884872437,0.3961546719074249,0.47454971075057983,0.4202497899532318,0.7933483123779297,0.6252067685127258,0.4292134642601013,0.3834637403488159,0.4483499526977539,0.3991626501083374,0.5467010140419006,0.44479256868362427,0.4404129087924957,0.4748504161834717,0.40921658277511597,0.4141322374343872,0.40937745571136475,0.4302566647529602,0.3938407003879547,0.38698244094848633,0.5033825039863586,0.3972824215888977,0.4234379231929779,0.3730085790157318,0.37656882405281067,0.34418365359306335,0.7275245189666748,0.4260683059692383,0.4098431169986725,0.40780267119407654,0.47337037324905396,0.5983314514160156,0.43338900804519653,0.4590713679790497,0.40549400448799133,0.4143604338169098,0.3672555685043335,0.43928059935569763,0.4358408749103546,0.45969390869140625,0.47189223766326904,0.41800716519355774,1.9498370885849,0.39041298627853394,0.39310187101364136,0.38834148645401,0.39836230874061584,0.3908901810646057,0.39100903272628784,0.4258091151714325,0.9652019143104553,0.3879433572292328,0.3712487518787384,0.3981514573097229,0.47513100504875183,0.44817817211151123,0.8661476969718933,0.40528348088264465,0.4013553857803345,0.4771377742290497,0.3623066544532776,0.5042554140090942,0.41280385851860046,0.44558462500572205,0.40098702907562256,0.42242512106895447,0.4191160500049591,0.4150857925415039,0.45008862018585205,0.4103500247001648,0.37758752703666687,0.38257578015327454,0.5795143842697144,0.44258272647857666,0.5153416991233826,0.3973245918750763,0.4414137601852417,0.4045168161392212,0.3606902062892914,0.4387572109699249,0.40686678886413574,0.44048023223876953,0.5211595296859741,0.5444183945655823,0.39724260568618774,0.4187309741973877,0.45639705657958984,0.45420998334884644,0.3862430453300476,0.4980219602584839,0.4130338132381439,0.50501948595047,0.3702615797519684,0.9409966468811035,0.4375024437904358,0.42008957266807556,0.41118770837783813,1.3525031805038452,0.4547814726829529,0.4670388102531433,0.4016972780227661,0.5278817415237427,0.43205726146698,0.46861276030540466,0.3847559690475464,0.3851161003112793,1.0951942205429077,0.6123683452606201,0.4202408492565155,0.40837812423706055,0.3809940218925476,0.42621660232543945,0.3936295807361603,0.4562399387359619,0.4958469867706299,0.40794891119003296,0.5038206577301025,0.4273384213447571,0.4355752170085907,0.40680715441703796,0.44909408688545227,0.4117332100868225,0.5024251937866211,0.4006809890270233,0.38025447726249695,0.43332064151763916,0.405208557844162,0.4146054983139038,0.46601760387420654,0.485452800989151,0.40725836157798767,0.4900388717651367,0.4753240644931793,0.389994353055954,0.40686872601509094,0.48173609375953674,0.41498512029647827,0.4387125074863434,0.45810964703559875,0.37075844407081604,0.4135620892047882,6.467494487762451,0.4282827079296112,0.3800799548625946,0.41501355171203613,0.4187152683734894,0.37729164958000183,0.39355772733688354,0.4278048276901245,0.37842443585395813,0.41607654094696045,0.6321366429328918,0.4119129180908203,0.3782421946525574,1.0151845216751099,0.37854209542274475,0.5658293962478638,0.41425731778144836,1.4812142848968506,0.9437413811683655,0.42558956146240234,0.4084933400154114,0.39241671562194824,0.4271699786186218,0.5151150822639465,0.39924442768096924,0.3935650885105133,0.5640481114387512,0.3867526650428772,0.4572191536426544,0.4055192172527313,0.39445021748542786,0.386593759059906,0.43814247846603394,0.4279243052005768,0.5294395089149475,0.40135833621025085,0.3929066061973572,0.41039735078811646,0.6602067351341248,0.3757745623588562,0.4427736699581146,0.40699923038482666,0.4562699496746063,0.5088639855384827,0.49694937467575073,1.1383614540100098,0.4153707027435303,0.39776143431663513,0.4229469895362854,0.38258400559425354,0.5058088898658752,0.3735244870185852,0.41034623980522156,0.38922974467277527,0.3747682273387909,0.41018426418304443,0.3979220986366272,0.4192412197589874,0.398538738489151,0.39648929238319397,0.3945298194885254,0.4137621223926544,0.4338589012622833,0.5589735507965088,0.4222353994846344,0.5191252827644348,0.4015619158744812,0.4103478789329529,0.4307929277420044,0.3619319796562195,0.4020307958126068,0.38801881670951843,0.3932156562805176,0.4638327658176422,0.5989741683006287,0.6692301630973816,0.39220908284187317,0.37555840611457825,0.4681936502456665,0.39611944556236267,0.42741385102272034,0.39254412055015564,0.43325620889663696,0.4542832374572754,0.46510636806488037,0.402349054813385,0.3981335461139679,0.39109399914741516,1.9306507110595703,0.5151744484901428]},"targets":{"mean":[-0.03357947990298271,0.0016852098051458597,0.040277086198329926,-0.004199534188956022,-0.01922144554555416,0.0007091567968018353,-0.049493372440338135,-0.04368743300437927,0.012835306115448475,-0.01755434274673462,0.02947824075818062,0.03479260951280594,-0.016792265698313713,-0.00580286979675293,-0.005820900667458773,0.0416315533220768,-0.016931701451539993,0.04650948941707611,-0.02264080010354519,0.005312380380928516,-0.004813822451978922,-0.008747990243136883,0.008965698070824146,0.5958355069160461,0.016288723796606064,-0.007113316096365452,-0.04344606027007103,-0.055625561624765396,-0.012505597434937954,0.015213360078632832,0.03788217157125473,-0.05162445455789566,0.0249503031373024,0.00374607858248055,0.058380868285894394,-0.032538507133722305,0.019710294902324677,-0.034085262566804886,-0.02640845626592636,-0.008606838062405586,0.008497803471982479,-0.010942799970507622,-0.036420054733753204,-0.01542558055371046,-0.01616797037422657,-0.019934937357902527,-0.0007223348948173225,-0.026192249730229378,-0.011380121111869812,-0.00907983910292387,0.014905333518981934,-0.05869632586836815,-0.007148417644202709,0.006492441054433584,-0.01770097389817238,0.004547553602606058,-0.04952407628297806,-0.027935191988945007,0.027865882962942123,-0.010272382758557796,-0.03930281847715378,-0.01682393066585064,0.02648250199854374,0.0021447199396789074,-0.028255993500351906,-0.023425590246915817,-0.01408646535128355,-0.029300307855010033,0.0035016981419175863,0.0010994295589625835,-0.009540710598230362,0.035639796406030655,0.029390696436166763,-0.03838560730218887,-0.021648166701197624,-0.027394426986575127,0.0043853530660271645,-0.023828331381082535,-0.03543437272310257,0.019522033631801605,-0.0026077881921082735,0.015831805765628815,0.012545155361294746,-0.00438245153054595,0.00515799131244421,0.008502891287207603,0.020915603265166283,-0.0332287922501564,0.00570450397208333,-0.03174252435564995,-0.023092983290553093,-0.008808830752968788,-0.003535968018695712,0.0187081191688776,-0.030958885326981544,0.01432042196393013,0.0005007057916373014,0.015422942116856575,0.020489949733018875,-0.053275853395462036,-0.06186723709106445,-0.013991995714604855,0.003854044945910573,-0.074490025639534,0.0021064099855720997,-0.0328734815120697,0.022655455395579338,-0.04197468236088753,0.03800036013126373,0.006328389514237642,-0.045179516077041626,-0.014504027552902699,-0.0033967180643230677,-0.010530050843954086,-0.026372037827968597,-0.08175752311944962,0.029993589967489243,0.0026314957067370415,0.033772729337215424,-0.015466023236513138,0.013385357335209846,-0.011288869194686413,-0.02970339171588421,-0.010366901755332947,0.009274754673242569,0.0005187652423046529,-0.04786762222647667,0.012950372882187366,-0.007987963035702705,-0.11140700429677963,-0.013611137866973877,-0.043496113270521164,0.01353942696005106,-0.015205816365778446,-0.033549826592206955,0.007229247596114874,-0.0329466350376606,-0.023843834176659584,0.021553730592131615,0.05280003324151039,-0.03520834818482399,0.036004919558763504,-0.017851023003458977,-0.08269864320755005,-0.020902050659060478,0.011722992174327374,0.05352310463786125,-0.04909873381257057,-0.15785083174705505,0.00040488303056918085,0.004053168930113316,-0.0007630494656041265,0.0008453327463939786,-0.030025353655219078,0.027242882177233696,-0.04205513000488281,1.7537094354629517,-0.016316518187522888,-0.0005821942468173802,-0.0003375987580511719,0.004069244489073753,-0.040322668850421906,0.011800702661275864,-0.01655103638768196,-0.02230212092399597,0.08521797508001328,-0.010669838637113571,-0.01694864220917225,-0.002500602276995778,-0.006973696872591972,-0.011799370869994164,0.03180158510804176,0.029168320819735527,0.020826321095228195,-0.028078945353627205,-0.0166705884039402,-0.021911613643169403,0.037472136318683624,-0.02532120794057846,-0.00560151319950819,-0.006426479667425156,-0.07845686376094818,-0.006599798798561096,0.017764702439308167,0.00028194652986712754,-0.026631897315382957,-0.11113378405570984,0.02648877166211605,-0.0042754774913191795,-0.002771831350401044,0.01986580714583397,-0.012430350296199322,0.0018129716627299786,-0.04703112319111824,-0.00915081612765789,0.024455847218632698,0.0014226398197934031,0.0011119757546111941,0.007884624414145947,-0.008984431624412537,-0.0384310781955719,0.034364763647317886,-0.008207187056541443,-0.07189065217971802,0.041068099439144135,-0.0067552225664258,0.01987590081989765,0.0081704780459404,-0.0014046278083696961,0.00739245256409049,-0.023261921480298042,-0.05776840075850487,-0.007067440077662468,-0.03991406783461571,-0.016185706481337547,0.020209884271025658,-0.028752297163009644,0.05019030347466469,-0.014498135074973106,0.0019951341673731804,-0.0062249163165688515,-0.03156193718314171,-0.03162407875061035,-0.04809274524450302,0.02168790064752102,-0.012766791507601738,-0.012491603381931782,-0.022834276780486107,-0.011800220236182213,-0.023359129205346107,0.021458769217133522,-0.01053168810904026,-0.034755267202854156,-0.027243558317422867,-0.023093661293387413,-0.006189918611198664,0.08809812366962433,0.021313771605491638,0.0047753239050507545,-0.010423459112644196,-0.01805480383336544,0.010716330260038376,-0.011909866705536842,-0.05572051554918289,0.01393304392695427,-0.02353612147271633,-0.01710023172199726,0.036350712180137634,0.030589580535888672,-0.08082661777734756,-0.03205186873674393,0.009935356676578522,0.011010320857167244,0.012666847556829453,-0.018062865361571312,-0.025210879743099213,0.013082921504974365,-0.009367428719997406,-0.04790225997567177,0.03159519284963608,-0.04756207391619682,0.006403416395187378,0.00047951500164344907,0.00470371451228857,0.0024657114408910275,0.008593968115746975,-0.018910620361566544,0.007068008650094271,0.004484932869672775,-0.04194645956158638,0.0011430279118940234,0.020565349608659744,-0.006414550356566906,-0.01682524010539055,-0.00932283140718937,-0.0008335941820405424,-0.21429303288459778,-0.041916169226169586,0.031377702951431274,-0.014105048961937428,-0.029528971761465073,-0.03457127884030342,0.033671244978904724,-0.043064624071121216,0.0017735024448484182,-0.014365759678184986,-0.062323883175849915,-0.024485914036631584,-0.07876203954219818,-0.02290286310017109,0.0017368928529322147,-0.034227170050144196,0.004101867787539959,-0.003120782319456339,0.013346809893846512,0.04477732628583908,-0.047650884836912155,0.00700689060613513,-0.05333777889609337,0.0008893103804439306,0.024740347638726234,0.006874515675008297,-0.049167852848768234,1.0038728760264348e-05,0.05208415538072586,-0.01016823761165142,0.031409330666065216,-0.001604839344508946,-0.07163164019584656,-0.006059981882572174,0.0171702578663826,-0.04095345363020897,-0.008807552047073841,0.015679948031902313,-0.02854168228805065,-0.000956377771217376,0.02160501666367054,-0.0104638347402215,0.01531909964978695,-0.037257611751556396,-0.04378334805369377,-0.0023307164665311575,-0.006400357000529766,0.01328221708536148,0.022982263937592506,-0.02149149589240551,-0.00453513627871871,-0.007275203242897987,-3.763987842830829e-05,-0.05555057153105736,-0.0033257887698709965,-0.03229499235749245,0.003313502063974738,-0.012069550342857838,-0.000663343642372638,0.04529618099331856,0.024598607793450356,-0.009555312804877758,-0.005039424169808626,-0.020241495221853256,-0.030311426147818565,-0.007903856225311756,-0.03665319085121155,-0.01181588601320982,-0.030084066092967987,-0.04152819141745567,-0.0529012568295002,-0.04702085629105568,-0.0032214256934821606,0.037499621510505676,-0.00711893429979682,0.02360386960208416,0.04552968963980675,-0.035548754036426544,0.013854009099304676,-0.018570981919765472,-0.02380634844303131,-0.06174059212207794,-0.011208661831915379,0.0009897383861243725,0.01417115144431591,0.03041950985789299,0.010728061199188232,-0.046308763325214386,0.018846238031983376,0.013798723928630352,-0.01809113100171089,0.0287795290350914,0.03212355822324753,0.05712189897894859,-0.03952309489250183,0.05608407035470009,0.007553793024271727,0.029578058049082756,0.04719143360853195,-0.006427424028515816,-0.9900736212730408,-0.013978175818920135,-0.03627556189894676,0.011798789724707603,0.016842419281601906,-0.04491814970970154,-0.03727562353014946,-0.03196984529495239,-0.03606077656149864,-0.016285913065075874,0.010564225725829601,-0.02269890531897545,0.020211147144436836,-0.02562040276825428,0.04266420006752014,0.00867541879415512,-0.010094759985804558,0.0050925943069159985,-0.006757659371942282,-0.04816708713769913,0.012070848606526852,-0.016691308468580246,0.012626195326447487,-0.008787370286881924,0.02243761718273163,-0.05285504087805748,0.04503408074378967,-0.001880456111393869,-0.0008153546950779855,0.008142942562699318,-0.006947258487343788,-0.018040865659713745,-0.0016728198388591409,0.0013258514227345586,-0.016141362488269806,-0.08031140267848969,-0.013190696947276592,-0.1514526754617691,-0.04516112804412842,0.0200936459004879,-0.018720028921961784,-0.017307927832007408,0.03031124919652939,0.01783752255141735,-0.005354329943656921,-0.060394953936338425,-0.05140262469649315,-0.011414007283747196,-0.010520640760660172,0.05203285440802574,-0.019293053075671196,-0.03438965976238251,-0.04445198178291321,0.04622504115104675,0.008412317372858524,-0.013656548224389553,-0.009870200417935848,0.006055897101759911,0.020252836868166924,-0.00907309353351593,-0.021133802831172943,0.06917574256658554,0.005758719518780708,-0.0009920193115249276,0.009322271682322025,-0.008250381797552109,0.01002549845725298,-0.014626488089561462,-0.00396287627518177,0.001563280588015914,-0.013880908489227295,-0.029733842238783836,0.0015845656162127852,-0.013638733886182308,-0.03531011566519737,0.0037555983290076256,0.0003416188119444996,0.054170604795217514,-0.029149677604436874,-0.0010779734002426267,-0.028420811519026756,-0.030705584213137627,0.024900564923882484,-0.003943325486034155,0.0033445488661527634,-0.04035066068172455,0.014119142666459084,0.015298404730856419,-0.004924844950437546,-0.015406074933707714,0.005307314917445183,-0.011476623825728893,0.005312860477715731,-0.03905703127384186,0.0026260430458933115,-0.01928970031440258,-0.019958818331360817,-0.018657613545656204,0.02095843106508255,-0.0013016139855608344,-0.044397298246622086,-0.007883104495704174,-0.0649283230304718,-0.0073820785619318485,0.010454289615154266,-0.018136702477931976,-0.03949227184057236,0.04372486472129822,-0.04338939115405083,0.011834845878183842,0.027699444442987442,-0.012942647561430931,-9.50057219597511e-05,0.01578107476234436,-0.013383634388446808,-0.011054491624236107,-0.004791690967977047,0.02499490976333618,-0.01353718526661396,-0.029701070860028267,-0.018942857161164284,0.0019705663435161114,-0.05301503464579582,-0.05870980769395828,-0.027365567162632942,-0.019111478701233864,0.004229350481182337,-0.04763149470090866,-0.013193299993872643,-0.021926218643784523,0.03246558830142021,-0.22356322407722473,-0.027017416432499886,-0.023648226633667946,-0.018238520249724388,-0.022188017144799232],"std":[0.1866663694381714,0.19521218538284302,0.2310248762369156,0.19801832735538483,0.19314990937709808,0.19145289063453674,0.17169882357120514,0.187493696808815,0.20779822766780853,0.19476863741874695,0.21464741230010986,0.17787377536296844,0.17269492149353027,0.2001502364873886,0.20136027038097382,0.19682689011096954,0.1977364718914032,0.18466408550739288,0.22208020091056824,0.1899014562368393,0.22015270590782166,0.1781502217054367,0.19122980535030365,0.24763880670070648,0.18188999593257904,0.19416439533233643,0.21822583675384521,0.19484874606132507,0.20756429433822632,0.23715487122535706,0.20803013443946838,0.16796930134296417,0.1979793757200241,0.1736394464969635,0.16946625709533691,0.20649120211601257,0.1626843363046646,0.17046350240707397,0.22437609732151031,0.1670607179403305,0.1803293377161026,0.1942746341228485,0.22017531096935272,0.19634775817394257,0.18509408831596375,0.19494859874248505,0.18615694344043732,0.2067009061574936,0.16415251791477203,0.18593133985996246,0.19880150258541107,0.26195836067199707,0.2214440554380417,0.22984442114830017,0.1898164600133896,0.1845443844795227,0.3686225712299347,0.1591811180114746,0.20996952056884766,0.19769759476184845,0.19256742298603058,0.18479789793491364,0.18684698641300201,0.20085950195789337,0.18554551899433136,0.1547260582447052,0.2081485390663147,0.17118340730667114,0.19970838725566864,0.20105087757110596,0.19251234829425812,0.3066624104976654,0.2859625518321991,0.21142005920410156,0.193398118019104,0.18915203213691711,0.18208813667297363,0.20532214641571045,0.16572459042072296,0.22326436638832092,0.17628049850463867,0.24919849634170532,0.21744611859321594,0.19218459725379944,0.17656275629997253,0.19328488409519196,0.2163284868001938,0.255919486284256,0.22192972898483276,0.20390333235263824,0.2027246505022049,0.19569131731987,0.17785537242889404,0.20543065667152405,0.1826067566871643,0.22214891016483307,0.20658628642559052,0.18018238246440887,0.2451283037662506,0.20816034078598022,0.2432827353477478,0.20736263692378998,0.1727999448776245,0.2217586487531662,0.16478367149829865,0.19051428139209747,0.22530853748321533,0.20283076167106628,0.22439724206924438,0.2025011032819748,0.22766272723674774,0.2626131772994995,0.18592077493667603,0.22343601286411285,0.21231216192245483,0.38108283281326294,0.1978205293416977,0.1924777776002884,0.2102658897638321,0.2589057683944702,0.16885128617286682,0.18213602900505066,0.16861453652381897,0.17651794850826263,0.18825460970401764,0.2754390835762024,0.2126636505126953,0.18455004692077637,0.21438297629356384,0.22615116834640503,0.21896658837795258,0.19848385453224182,0.2084650844335556,0.15188881754875183,0.17327886819839478,0.21374043822288513,0.17727242410182953,0.23619112372398376,0.16328808665275574,0.22380368411540985,0.26840242743492126,0.16783159971237183,0.21388068795204163,0.18536987900733948,0.20544660091400146,0.20340682566165924,0.2080785185098648,0.1819664090871811,0.3224841058254242,0.17530931532382965,0.2140851765871048,0.22648970782756805,0.1958962231874466,0.18054641783237457,0.18414084613323212,0.1557334065437317,0.7711642384529114,0.1723102331161499,0.18676817417144775,0.18152499198913574,0.19821007549762726,0.2008521407842636,0.21908579766750336,0.2231123000383377,0.21243534982204437,0.3011966347694397,0.20088708400726318,0.16998961567878723,0.16481268405914307,0.192636176943779,0.20274204015731812,0.20995891094207764,0.22574438154697418,0.3106776177883148,0.19178038835525513,0.19669531285762787,0.1728447675704956,0.2215241640806198,0.18229195475578308,0.18250292539596558,0.18431375920772552,0.2278580218553543,0.25113070011138916,0.181270569562912,0.20343440771102905,0.19843225181102753,0.1968553215265274,0.2040618509054184,0.21818679571151733,0.2267216593027115,0.17947937548160553,0.2516781687736511,0.20934544503688812,0.21881042420864105,0.1998516470193863,0.19074612855911255,0.23539291322231293,0.18475307524204254,0.1728072464466095,0.2094791680574417,0.19801805913448334,0.170819491147995,0.23029538989067078,0.20160755515098572,0.1741243451833725,0.18155460059642792,0.19906146824359894,0.17231522500514984,0.19109393656253815,0.22150780260562897,0.20127809047698975,0.29764264822006226,0.16396096348762512,0.20761114358901978,0.2340763956308365,0.1940666288137436,0.16614888608455658,0.22811944782733917,0.17926456034183502,0.2191898226737976,0.17269642651081085,0.18910588324069977,0.17069774866104126,0.18575264513492584,0.25645703077316284,0.1893722414970398,0.16827720403671265,0.1852939873933792,0.22130125761032104,0.1710900366306305,0.18175578117370605,0.27678602933883667,0.20888449251651764,0.1979028880596161,0.17522159218788147,0.1725025326013565,0.2806183993816376,0.17828096449375153,0.19209229946136475,0.1980455368757248,0.15955856442451477,0.20666658878326416,0.17586486041545868,0.1938123106956482,0.1943771243095398,0.2406749278306961,0.20495013892650604,0.22447673976421356,0.25491151213645935,0.2631292939186096,0.2138521522283554,0.17749574780464172,0.19282923638820648,0.21592475473880768,0.17971806228160858,0.19436533749103546,0.22823622822761536,0.23315119743347168,0.1975334882736206,0.17743699252605438,0.19750262796878815,0.1795327216386795,0.18822182714939117,0.1687866747379303,0.18967176973819733,0.18813590705394745,0.18849027156829834,0.17923179268836975,0.20012266933918,0.19930607080459595,0.20391473174095154,0.1491214632987976,0.2357640117406845,0.18386609852313995,0.20533481240272522,0.1890084445476532,0.4554307162761688,0.1951860785484314,0.1981479823589325,0.2089110016822815,0.19013555347919464,0.19082753360271454,0.21450333297252655,0.18144604563713074,0.1626400649547577,0.19606198370456696,0.1700163334608078,0.18671192228794098,0.20487365126609802,0.20987340807914734,0.19130855798721313,0.16896206140518188,0.18671798706054688,0.1770189255475998,0.18178090453147888,0.29874762892723083,0.24201801419258118,0.19422179460525513,0.20069465041160583,0.20033256709575653,0.17058567702770233,0.19941678643226624,0.2046346813440323,0.2028580754995346,0.24281786382198334,0.18468908965587616,0.18594779074192047,0.19427892565727234,0.2227988839149475,0.1782538741827011,0.17547863721847534,0.18785426020622253,0.17098717391490936,0.18090373277664185,0.25276312232017517,0.1927507370710373,0.2131628841161728,0.233742356300354,0.18292050063610077,0.19161176681518555,0.17516809701919556,0.28676772117614746,0.18562409281730652,0.17388597130775452,0.19520524144172668,0.1940547078847885,0.1713985800743103,0.19935186207294464,0.19687184691429138,0.18903861939907074,0.19265519082546234,0.1990894079208374,0.2161678820848465,0.1652664840221405,0.23855115473270416,0.18853062391281128,0.17251642048358917,0.1960253119468689,0.23709571361541748,0.19444315135478973,0.18684647977352142,0.16859379410743713,0.17065119743347168,0.1946323812007904,0.2089344710111618,0.19771680235862732,0.17648722231388092,0.17771326005458832,0.22846566140651703,0.19837185740470886,0.231939896941185,0.1801472008228302,0.20521396398544312,0.1793799251317978,0.20549741387367249,0.19975225627422333,0.23560024797916412,0.1834687888622284,0.198741614818573,0.22062471508979797,0.18994560837745667,0.22879847884178162,0.21856695413589478,0.18942520022392273,0.19257116317749023,0.20988917350769043,0.20217660069465637,0.1933041512966156,0.196644589304924,0.3460426330566406,0.18050621449947357,0.2093738615512848,0.1885475367307663,0.1730356067419052,0.2190379649400711,0.21236170828342438,1.1899186372756958,0.21125581860542297,0.19285793602466583,0.18448220193386078,0.2348008006811142,0.2044861912727356,0.22077959775924683,0.17110970616340637,0.18183758854866028,0.180317223072052,0.2409386932849884,0.1939561665058136,0.18847155570983887,0.23042400181293488,0.17002639174461365,0.18452131748199463,0.19253666698932648,0.2043815553188324,0.26344820857048035,0.172274649143219,0.23215006291866302,0.2065974771976471,0.180084228515625,0.20061159133911133,0.2035730630159378,0.1848687082529068,0.19617904722690582,0.20809295773506165,0.19478018581867218,0.18225494027137756,0.19867181777954102,0.2026658058166504,0.20578016340732574,0.18989847600460052,0.2532585561275482,0.19602149724960327,0.2272973209619522,0.7416240572929382,0.18682974576950073,0.2165078967809677,0.16216710209846497,0.20797014236450195,0.17917971312999725,0.16487857699394226,0.16423696279525757,0.19501961767673492,0.20011752843856812,0.20518271625041962,0.17386482656002045,0.20821718871593475,0.17117036879062653,0.21072949469089508,0.18627288937568665,0.17580823600292206,0.19103236496448517,0.18104547262191772,0.1757465898990631,0.17709164321422577,0.19984424114227295,0.18777096271514893,0.19290858507156372,0.2534022033214569,0.27702775597572327,0.1937057375907898,0.1859217882156372,0.1718408316373825,0.1780659258365631,0.23472966253757477,0.21918587386608124,0.18818148970603943,0.2451322078704834,0.16528789699077606,0.21552307903766632,0.20118728280067444,0.21327784657478333,0.19663435220718384,0.22904415428638458,0.25891321897506714,0.20901444554328918,0.18781042098999023,0.1892685443162918,0.20548515021800995,0.22471095621585846,0.2024417370557785,0.1787925362586975,0.18218114972114563,0.20058447122573853,0.22128529846668243,0.20706483721733093,0.16946673393249512,0.24796269834041595,0.24445095658302307,0.19590184092521667,0.1618577390909195,0.19828185439109802,0.19193896651268005,0.1955050677061081,0.22393083572387695,0.206254243850708,0.17547184228897095,0.2069001942873001,0.2541658580303192,0.8228771090507507,0.19502119719982147,0.17088642716407776,0.21374867856502533,0.17291180789470673,0.18513143062591553,0.1924009919166565,0.1981130689382553,0.22249998152256012,0.1986187994480133,0.1952081173658371,0.22243554890155792,0.19518370926380157,0.18142282962799072,0.18680593371391296,0.20300792157649994,0.2163313776254654,0.21728263795375824,0.1718011200428009,0.18565048277378082,0.19690945744514465,0.18719886243343353,0.23972122371196747,0.18385948240756989,0.22840039432048798,0.22278068959712982,0.18683993816375732,0.1901404857635498,0.18239007890224457,0.4259711802005768,0.19518886506557465,0.1866055130958557,0.17139819264411926,0.18590547144412994]}},"1":{"inputs":{"mean":[0.2548553943634033,0.19536378979682922,0.17539732158184052,-0.20014895498752594,-0.1854650229215622,0.160856693983078,0.09552273154258728,0.051974907517433167,-0.152003675699234,0.001675319392234087,0.12265282869338989,0.07401847094297409,0.01382077019661665,0.24481503665447235,-0.1573258638381958,0.04899273067712784,0.13547301292419434,-0.12016671150922775,-0.06148239225149155,0.15111346542835236,0.18261492252349854,0.07275219261646271,0.06770806759595871,7.46786642074585,0.20725642144680023,0.16931329667568207,-0.12556825578212738,0.03235464170575142,-0.08914145082235336,0.0969349816441536,-0.15563863515853882,-0.19395916163921356,-0.122317373752594,-0.4686022400856018,0.22378429770469666,0.19687612354755402,-0.15746724605560303,0.04951483756303787,0.04240221157670021,-0.05920737236738205,0.1615898311138153,0.17487704753875732,0.038929857313632965,-0.16715973615646362,-0.16838420927524567,-0.16441741585731506,-0.5194688439369202,0.6498796343803406,0.13960210978984833,0.14210498332977295,0.02423040382564068,-0.13592185080051422,-0.11791122704744339,0.28836700320243835,-0.020973552018404007,0.11122025549411774,-4.5931620597839355,0.026795126497745514,0.0887475460767746,0.040104225277900696,0.04642115533351898,0.09824096411466599,-0.02252231165766716,-0.11485868692398071,-0.14009898900985718,0.2374720722436905,0.13884861767292023,-0.2298349291086197,0.15009823441505432,0.27674543857574463,0.23612427711486816,0.7153290510177612,-0.24187994003295898,0.013091884553432465,-0.15445736050605774,0.33399930596351624,-0.07892706990242004,0.009808075614273548,-0.13153468072414398,0.08952073007822037,0.003292777808383107,0.3814743757247925,0.08655277639627457,-0.0177046749740839,-0.12278258800506592,-0.2100270539522171,-0.2187599241733551,-0.10764431953430176,0.02115345187485218,0.026036808267235756,-0.2738482654094696,-0.03629428893327713,0.01080423966050148,0.09377438575029373,-0.08453620225191116,-0.053721170872449875,0.11192762851715088,0.1939775049686432,-0.4481164216995239,-0.14363470673561096,-0.17352288961410522,0.1555555760860443,0.08791306614875793,0.1664663404226303,-0.1587049961090088,0.026722531765699387,0.22004829347133636,0.15773552656173706,-0.00488731823861599,0.21582813560962677,0.047926608473062515,1.2318793535232544,-0.12313561886548996,-0.04780171439051628,-0.03603168576955795,-1.0764647722244263,0.2462703287601471,-0.014992700889706612,0.11448384821414948,-0.06146729737520218,-0.2822151482105255,-0.19164559245109558,-0.01520510669797659,0.175591841340065,0.02656504325568676,0.01983427256345749,-0.2691170573234558,0.011107418686151505,0.16173787415027618,-1.0328782796859741,0.05437329411506653,-0.2650589942932129,-0.1265478879213333,-0.04397895187139511,-0.08109617978334427,-0.10025212168693542,-0.176242858171463,-0.23008503019809723,-0.032713569700717926,-0.026240484789013863,-0.06265957653522491,0.17528806626796722,0.1368333250284195,-0.19241464138031006,0.043237362056970596,-0.0545940063893795,-0.3082315921783447,0.271208792924881,-1.1602522134780884,-0.02621489018201828,0.06272654980421066,0.2577389180660248,-0.02904510125517845,-0.10557882487773895,-0.03350014612078667,0.04751841351389885,10.868144035339355,0.8888433575630188,0.08634349703788757,-0.05283051356673241,0.18004493415355682,-0.11363106220960617,-0.09971874207258224,-0.14026989042758942,0.1062222570180893,1.2126411199569702,-0.0547901876270771,-0.0816580206155777,-0.0502428375184536,-0.35479456186294556,0.20923271775245667,-0.044946521520614624,0.05165034905076027,-0.03571203351020813,0.17013171315193176,0.26201772689819336,0.2046951949596405,0.12519142031669617,-0.14885313808918,-0.26211097836494446,-0.29221731424331665,-0.07071483135223389,0.16794973611831665,0.13411012291908264,0.14426305890083313,0.15695776045322418,-2.802170515060425,-0.011984435841441154,0.3051615059375763,0.08646738529205322,-0.06849568337202072,-0.14375168085098267,0.11350559443235397,0.10333813726902008,-0.022644156590104103,-0.1869836002588272,0.18869410455226898,0.09180178493261337,0.17843525111675262,-0.2390657216310501,0.16772136092185974,-0.13337315618991852,-0.35864925384521484,-0.0789666622877121,0.24721507728099823,0.21198442578315735,-0.12302768230438232,0.18950185179710388,-0.19192685186862946,0.24793842434883118,-0.26662665605545044,-0.37188372015953064,-0.020901327952742577,-0.016737468540668488,0.04406176134943962,0.0740746557712555,0.14820976555347443,0.03792329877614975,0.09422297775745392,-0.10724817961454391,-0.016938472166657448,0.08015070110559464,-0.07335208356380463,-0.10022882372140884,-0.3133687973022461,0.171729177236557,-0.2234477996826172,-0.0698985606431961,0.3519411087036133,0.02818123809993267,0.19092129170894623,0.1146480143070221,-0.05454506725072861,-0.112120121717453,-0.12691444158554077,0.015324259176850319,1.1074057817459106,-0.11081955581903458,-0.02271747589111328,0.16884805262088776,-0.16685445606708527,-0.028896164149045944,-0.2099960297346115,0.1055716723203659,0.16421736776828766,-0.13123275339603424,-0.0036333331372588873,0.09348481893539429,0.7513483166694641,-0.021401232108473778,0.16322512924671173,0.34454652667045593,0.1258421093225479,0.049907341599464417,0.06489311903715134,-0.03939732536673546,0.10426460206508636,-0.1188754290342331,0.0070945280604064465,0.2867646813392639,-0.14576862752437592,0.07969355583190918,-0.06241985037922859,0.09629736840724945,0.14638552069664001,-0.08613166213035583,0.037659987807273865,-0.0392005518078804,0.3381193280220032,0.0013846210204064846,0.16890162229537964,-0.13829180598258972,-0.005235679447650909,-0.23078641295433044,0.187061607837677,-0.35559356212615967,-2.6737136840820312,-0.06475938111543655,0.1903286576271057,-0.2496485412120819,-0.2609170973300934,-0.17829138040542603,-0.20979294180870056,0.11073045432567596,0.026529328897595406,-0.12040884047746658,-0.04714280739426613,-0.2607021927833557,-0.17199890315532684,0.053997717797756195,0.06465396285057068,0.21807809174060822,0.09247863292694092,-0.3108390271663666,0.01296140905469656,0.6342156529426575,-0.12315260618925095,-0.19363342225551605,-0.1236787810921669,-0.17537540197372437,0.025327468290925026,-0.05780521780252457,0.12875841557979584,0.02263767644762993,0.3581547141075134,0.2064744532108307,-0.05951947718858719,-0.07956483960151672,-0.05423082783818245,0.012467728927731514,-0.03371892869472504,0.15644969046115875,0.12156851589679718,0.14290818572044373,-0.3448309600353241,0.07146966457366943,0.007531971670687199,0.04606326296925545,-0.055418532341718674,-0.08409792184829712,-0.01015777699649334,0.1525747925043106,-0.057971760630607605,0.021744754165410995,0.25479796528816223,0.10718970000743866,-0.4003846347332001,0.13302196562290192,-0.12415526807308197,-0.04492971673607826,-0.04045978561043739,0.04601702466607094,0.22139129042625427,-0.16554078459739685,0.1277942657470703,-0.018499404191970825,0.062059346586465836,-0.04232332855463028,0.33794716000556946,-0.06373348087072372,-0.28451499342918396,0.0798836350440979,0.04762880504131317,0.21284456551074982,-0.07920442521572113,0.18395420908927917,-0.05089918151497841,-0.0842604786157608,0.2589949369430542,0.03944632411003113,0.05201607570052147,0.037123121321201324,-0.08356226235628128,-0.22435355186462402,0.1951262652873993,0.1149090975522995,0.07334426790475845,-0.09581206738948822,0.22904080152511597,-0.05886644870042801,-0.06557055562734604,-0.1092643216252327,-0.2667657136917114,-0.09492850303649902,-0.1597045212984085,-0.46526166796684265,0.04416477307677269,-0.026758622378110886,0.14118479192256927,-0.14735370874404907,0.04245670139789581,-0.06860541552305222,-0.24459169805049896,-0.02815394103527069,0.7624801397323608,-0.023185908794403076,-8.585800170898438,-0.07929033786058426,0.10365892946720123,0.04403228312730789,-0.12921880185604095,-0.19558505713939667,-0.2445623278617859,-0.10447407513856888,0.1319315880537033,0.026553213596343994,-0.11213982105255127,-0.03004707582294941,-0.08491955697536469,0.0009111375547945499,0.15490858256816864,0.12009111791849136,0.3060322105884552,-0.09677138924598694,0.2291610836982727,-0.003358030691742897,-0.1407625377178192,0.11150377988815308,-0.0475144125521183,-0.15070396661758423,-0.1425596922636032,0.11541872471570969,0.005112530663609505,0.06872712820768356,-0.04764702543616295,-0.34679487347602844,0.18740670382976532,-0.07676232606172562,0.041600100696086884,0.14452193677425385,0.26926738023757935,-0.23172052204608917,-0.05327557399868965,-2.068376064300537,-0.2031770497560501,0.20053881406784058,-0.03155357018113136,0.4013199806213379,0.024749282747507095,-0.0018872215878218412,-0.2604753077030182,0.07217489928007126,0.12140801548957825,-0.16791404783725739,-0.18348082900047302,0.7440704107284546,-0.04378115013241768,0.17106273770332336,-0.04967955872416496,-0.04108993709087372,0.0760389119386673,-0.03706435486674309,-0.15448175370693207,0.06994528323411942,-0.15791110694408417,0.06866161525249481,0.10153090953826904,0.8243804574012756,0.2999841570854187,0.2405851185321808,0.029556309804320335,0.004379401449114084,0.004998001269996166,-0.05707194283604622,-0.04754132404923439,-0.10289895534515381,0.13220687210559845,0.11441215872764587,0.1573038548231125,-0.061223339289426804,0.14227481186389923,0.19464139640331268,-0.13506342470645905,0.2983555495738983,-0.02217787690460682,0.3159846067428589,0.18892356753349304,-0.04295158013701439,-0.07862590998411179,0.07831042259931564,-0.01659773662686348,-0.04611249640583992,-0.06087116524577141,-0.33338406682014465,0.05513540282845497,0.09483153373003006,-0.06324554234743118,-0.46414047479629517,-0.19253981113433838,0.12644927203655243,-0.12827539443969727,0.05931425094604492,-0.015276886522769928,0.05240371823310852,-0.03634057193994522,-0.10510881245136261,-0.07660099864006042,0.3114420175552368,-0.7095869779586792,-0.23722809553146362,-0.03037477843463421,-0.07014250010251999,-0.17382532358169556,0.12916597723960876,-0.01552293635904789,0.21575000882148743,0.07569923251867294,0.20575770735740662,0.26492124795913696,0.17640309035778046,0.10909010469913483,-0.06924707442522049,-0.06872080266475677,-0.27992936968803406,0.22472144663333893,0.18550492823123932,0.06361699104309082,0.0005437321378849447,0.1441519558429718,-0.032256513833999634,-0.08769591897726059,0.014948831871151924,-0.11742350459098816,-0.01882164180278778,-0.13677993416786194,0.07060133665800095,0.19698740541934967,-2.1039535999298096,0.09562880545854568,-0.0013811473036184907,0.11886990815401077,0.11388096213340759],"std":[0.930320143699646,0.8524194359779358,0.9012750387191772,0.9316256642341614,0.9199747443199158,0.9081078767776489,0.8678880929946899,0.8332557082176208,0.9590573906898499,0.8192772269248962,0.895570695400238,0.8782894611358643,0.8347566723823547,0.8410556316375732,0.8192752599716187,1.001265525817871,0.882396936416626,1.2130213975906372,0.8863440752029419,0.8456913828849792,1.0129625797271729,0.968368411064148,0.8230398297309875,2.5520522594451904,0.8353081941604614,0.9030712246894836,0.9658191800117493,0.8538153171539307,0.9411614537239075,0.8370823264122009,0.8182629942893982,1.0006192922592163,0.8821244239807129,1.4330408573150635,0.912883996963501,0.9395121932029724,1.001664638519287,0.818924605846405,0.914534330368042,0.9413480758666992,0.8518490195274353,1.001704216003418,0.7944045066833496,0.8830010890960693,0.9722564220428467,0.9871535897254944,1.1650789976119995,1.394429326057434,0.8189280033111572,0.905968964099884,0.8767305016517639,1.0401198863983154,0.9105784893035889,1.1446473598480225,0.8491148948669434,0.995409369468689,3.71480655670166,0.8630695343017578,0.813676655292511,0.8321301341056824,0.8759703636169434,0.8570022583007812,1.0750343799591064,0.9621121287345886,0.9047070741653442,0.8305646777153015,0.8573758602142334,0.889851987361908,0.8200580477714539,0.9700543880462646,0.9017043113708496,2.004931926727295,1.1068438291549683,0.9161807894706726,1.037705421447754,0.8573285341262817,0.8478100299835205,0.8866404891014099,0.9227720499038696,0.9427096843719482,1.2021632194519043,0.9861122965812683,0.9856577515602112,0.8382938504219055,1.017877459526062,0.9357116222381592,0.9028761386871338,0.9175383448600769,0.9708982110023499,0.8175615072250366,0.878743588924408,0.854966938495636,0.8616081476211548,0.8873261213302612,0.8161517381668091,0.9894663691520691,0.8864203095436096,0.877173125743866,1.3448420763015747,0.9191520810127258,0.8593432903289795,0.8601173162460327,0.8902735710144043,1.0153354406356812,0.9371039271354675,0.8175790309906006,0.9223026037216187,0.9315553903579712,0.9565929770469666,0.9431353211402893,0.9085094928741455,1.8744940757751465,0.9538967609405518,0.7896849513053894,0.8475036025047302,1.7094099521636963,0.9919929504394531,0.8659325242042542,0.8680130839347839,0.9700021743774414,0.8385999202728271,0.9502379894256592,0.9229744672775269,0.8298514485359192,0.8038865923881531,1.4002513885498047,0.9659967422485352,0.8303407430648804,0.8520190119743347,2.5128071308135986,0.8861987590789795,0.9758610129356384,0.8660823702812195,0.8878465294837952,0.7909784913063049,0.8715490102767944,0.919914186000824,1.3333759307861328,0.8964674472808838,0.9017568230628967,1.0213907957077026,0.8684176802635193,0.8655615448951721,0.956520676612854,0.8362578749656677,0.9133195281028748,0.9833535552024841,0.9879752993583679,2.4331624507904053,0.8431555032730103,0.9129213690757751,1.0956367254257202,0.8094445466995239,1.1134229898452759,0.8799114227294922,0.8194505572319031,3.507472515106201,1.3352853059768677,0.8708121180534363,0.8991467952728271,0.88279789686203,0.8556587100028992,0.8056687712669373,0.9478896260261536,0.8132500052452087,1.950016975402832,0.834714412689209,0.8440872430801392,0.8430430889129639,1.1282005310058594,0.9343066811561584,0.8986460566520691,0.945812463760376,1.122641682624817,0.870151698589325,0.9185472726821899,0.8479801416397095,0.8143853545188904,0.9100196361541748,0.8776801228523254,0.874998927116394,0.9570472240447998,0.8175962567329407,0.8895696997642517,0.7995765209197998,1.015989065170288,2.689183235168457,0.8560766577720642,0.9204298257827759,0.9753379225730896,0.939731240272522,0.9960852861404419,0.8000453114509583,0.9563280344009399,0.8192422389984131,0.9653105139732361,0.9991469979286194,0.9408870935440063,0.8610249161720276,0.9287546277046204,0.8460798859596252,0.8155817985534668,0.9760220646858215,0.9019027352333069,0.926923930644989,0.9373058080673218,0.9022321701049805,0.9050571918487549,0.8531250953674316,0.9240307211875916,0.9637892842292786,1.2129801511764526,0.901678740978241,0.7827337980270386,0.8411856293678284,0.8196061253547668,0.9251851439476013,0.9490318298339844,0.9236069917678833,0.8598038554191589,0.8669959306716919,0.9435551166534424,0.9383428692817688,0.8940349817276001,1.045189619064331,0.8367491960525513,0.940019965171814,0.8949586153030396,1.0190528631210327,0.8720235824584961,0.8986297249794006,1.4686737060546875,0.8516387939453125,0.8929431438446045,0.8143439292907715,0.9441744089126587,1.8269190788269043,0.9409776329994202,0.9276837706565857,0.833873987197876,1.1751296520233154,0.9090771079063416,0.9580108523368835,0.8801360130310059,0.8313724398612976,0.8186867833137512,0.9062092900276184,0.9128254055976868,2.399574041366577,1.1587049961090088,0.8952863812446594,0.9327867031097412,0.9072445631027222,1.0804415941238403,0.8299408555030823,0.8608070611953735,0.8834388852119446,0.9506438374519348,0.9388677477836609,0.9572294354438782,0.8354294896125793,0.8611640930175781,0.9278576970100403,0.9414847493171692,0.869284987449646,0.9739387035369873,1.0137499570846558,0.7848446369171143,0.9392974376678467,0.9180513024330139,0.9193434119224548,0.9833897352218628,0.8076909184455872,0.8571066856384277,0.949621319770813,0.9915148615837097,3.74928617477417,0.8394795060157776,0.8610110878944397,0.8827232718467712,1.186188817024231,0.9368348717689514,1.1196987628936768,0.9460371136665344,0.8087848424911499,0.9380084276199341,0.8837360143661499,0.8560531735420227,0.9326997995376587,0.8748763203620911,0.9075644016265869,1.124396562576294,0.9959436655044556,0.8493534326553345,0.8712470531463623,1.316004753112793,0.9808716773986816,0.8556615710258484,0.95328289270401,0.8962101340293884,0.8151745796203613,0.8596199154853821,0.8921804428100586,0.8712258338928223,1.1074583530426025,1.0043692588806152,0.8823267817497253,0.9285749197006226,0.8652169108390808,0.8350500464439392,0.881356954574585,0.9498082995414734,0.8282704949378967,0.8756049275398254,1.1378027200698853,0.8548302054405212,0.9089056253433228,0.8632138967514038,0.8198849558830261,0.9161502122879028,0.9258759617805481,1.2194417715072632,0.8208282589912415,0.9023889303207397,0.9448148012161255,0.8216822743415833,1.2611958980560303,0.9427406191825867,0.8336164951324463,0.9739077687263489,0.8588247895240784,1.030618667602539,0.9377405643463135,0.8709296584129333,0.8777050375938416,0.9413416385650635,0.9541279077529907,0.8593270182609558,1.094297170639038,0.9900858998298645,0.8835049271583557,0.8825094699859619,0.9041745662689209,0.8587464094161987,0.9305893182754517,0.9011066555976868,0.8591523766517639,0.8280577659606934,1.0002081394195557,0.9088892340660095,0.8172777891159058,0.768377423286438,0.8847436904907227,0.8675165176391602,0.9937121272087097,0.8971160650253296,0.9083778262138367,1.0555458068847656,0.9054915308952332,0.9858145713806152,0.8705539107322693,0.8213159441947937,1.2547574043273926,0.7840637564659119,0.8324620723724365,1.155650019645691,0.9604358673095703,0.7805771827697754,0.977994441986084,1.4086631536483765,0.9152287244796753,0.8224241137504578,1.0151275396347046,0.8830733299255371,1.1843515634536743,0.7715662717819214,6.8048882484436035,0.9457034468650818,0.9118496775627136,1.008231520652771,0.9933204054832458,0.8540357947349548,0.9617068767547607,0.9395961761474609,0.9284905195236206,0.9540784955024719,0.8930385708808899,0.9786312580108643,0.8474153280258179,0.8943602442741394,0.8696233034133911,0.9487552046775818,0.9550039172172546,0.8834370374679565,1.2672040462493896,0.8858014345169067,0.7943028211593628,0.8267325758934021,0.8717387318611145,0.9048898220062256,0.9987717270851135,0.9116310477256775,0.9540486335754395,0.9341286420822144,0.9373030662536621,0.9087661504745483,0.9063556790351868,0.9091573357582092,0.9505259394645691,0.9144414663314819,1.3696503639221191,1.0209726095199585,1.0271070003509521,3.8983616828918457,0.9831792712211609,0.9354395866394043,0.90744549036026,1.1131906509399414,0.9510378837585449,0.9429876804351807,0.8727506399154663,0.9341613054275513,0.9688620567321777,0.9471896886825562,0.8830767273902893,1.2004082202911377,0.8304266929626465,1.04462468624115,0.8762477040290833,0.8938809037208557,0.939881443977356,0.8278458118438721,0.8515546321868896,0.8874946236610413,0.9534666538238525,0.890289306640625,0.930730938911438,1.572676420211792,1.2633107900619507,0.9484365582466125,1.0787049531936646,0.8402930498123169,0.8761694431304932,0.9489785432815552,0.8307865262031555,0.8044022917747498,0.7916061282157898,0.8996453285217285,0.9660149812698364,0.9641627669334412,0.8745471835136414,0.9362487196922302,0.9395245313644409,1.406485676765442,0.8819296956062317,1.0984270572662354,0.900553822517395,0.8898451924324036,1.0972657203674316,0.837422788143158,0.8959987163543701,0.9370712637901306,0.8222439885139465,0.9263616800308228,0.8844549655914307,0.9207058548927307,0.872715950012207,1.0064822435379028,0.9230608344078064,0.8940209746360779,0.9037929177284241,0.8676167726516724,0.8665869832038879,0.856041669845581,0.8941741585731506,0.8512775897979736,1.0438919067382812,1.0737123489379883,3.000275135040283,0.912236213684082,0.8336737155914307,0.8483424186706543,0.9591579437255859,0.807927668094635,0.8711639046669006,0.9178234934806824,0.842496395111084,0.8652808666229248,0.905619204044342,0.9314104318618774,0.8021201491355896,0.8453201055526733,0.8095686435699463,1.000106692314148,0.848937451839447,0.8794483542442322,0.970975399017334,0.8898838758468628,0.9738542437553406,0.9212184548377991,0.9945717453956604,0.9441538453102112,0.9029722213745117,0.8521155714988708,0.9605831503868103,0.8561961650848389,0.9584232568740845,2.863443613052368,0.960321307182312,0.9305830001831055,0.9066250920295715,0.9117123484611511]},"targets":{"mean":[0.014018561691045761,-0.012521608732640743,-0.002458378905430436,0.022535208612680435,-0.026228835806250572,-0.032946910709142685,0.04970167204737663,0.010437213815748692,-0.02810881845653057,0.09722799062728882,-0.02215934731066227,-0.007052239496260881,0.04612959548830986,0.04273620620369911,0.032020825892686844,0.06272090971469879,0.025303266942501068,-0.15621133148670197,0.06715968251228333,0.05730683356523514,0.020847221836447716,0.008101521991193295,-0.05021919682621956,-0.1843438446521759,-0.028869109228253365,0.004512808285653591,-0.011621390469372272,-0.005069746635854244,-0.024018768221139908,-0.042276497930288315,0.0021405755542218685,0.015011025592684746,-0.03961598128080368,-0.051813624799251556,0.011261461302638054,0.02590753510594368,-0.017636185511946678,0.001516814110800624,-0.03145867958664894,-0.016523733735084534,-0.04845718666911125,-0.03445965051651001,-0.0629601925611496,-0.031447045505046844,0.018048526719212532,0.03307235240936279,-0.0028916692826896906,0.01982574537396431,-0.0005180782172828913,0.021181244403123856,0.0031134551391005516,-0.009788574650883675,0.043705787509679794,-0.02380049228668213,-0.021520692855119705,0.062053363770246506,-0.22869284451007843,0.04322340711951256,0.003373454324901104,-0.03535933420062065,-0.0005094334483146667,0.024229075759649277,0.01385954674333334,-0.01587682217359543,0.032040346413850784,0.03849804028868675,0.03505297005176544,-0.06391724199056625,0.05721162259578705,0.024633396416902542,-0.03767254576086998,0.11698935925960541,-0.06588209420442581,0.03174377605319023,-0.03376476466655731,-0.004363866988569498,-0.003374720923602581,-0.034201692789793015,-0.019282953813672066,-0.048640765249729156,-0.08506830781698227,-0.02994186244904995,0.009134262800216675,-0.006701902486383915,-0.04406331852078438,0.001439083251170814,0.008699694648385048,0.017272192984819412,0.0318378321826458,0.029451904818415642,-0.011719825677573681,-0.041059013456106186,0.0165084321051836,0.012015624903142452,0.04857422038912773,0.009398618713021278,0.005381252150982618,0.02689635194838047,-0.01109863817691803,0.020945120602846146,0.0798683688044548,0.002445570658892393,-0.025970159098505974,0.03168049454689026,-0.010272057726979256,0.03902743384242058,-0.00023409967252518982,-0.060838744044303894,0.016521241515874863,0.017641576007008553,0.006181811448186636,0.1131267249584198,0.02452770061790943,0.01867968589067459,0.008955639787018299,0.0831579864025116,0.05394500866532326,-0.015989692881703377,0.006297290790826082,-0.01661049574613571,0.06149543449282646,-0.036398403346538544,0.02664399892091751,-0.02671116217970848,-0.003134594764560461,0.04988883063197136,0.02885606326162815,-0.021457459777593613,-0.013681748881936073,0.03205583244562149,-0.05903749167919159,0.017032437026500702,-0.06476208567619324,0.008022194728255272,0.11520551890134811,-0.02205076813697815,0.04784749820828438,0.010725331492722034,0.001685904455371201,0.049022410064935684,0.04073687642812729,-0.02744409255683422,-0.04303305596113205,-0.019355446100234985,-0.034660425037145615,0.06270250678062439,0.0364474318921566,-0.036898281425237656,-0.0270584337413311,-0.04142346605658531,0.049444880336523056,0.006361810956150293,-0.01287921890616417,0.09435416013002396,0.02807718515396118,0.057713426649570465,0.0337410606443882,0.1547621190547943,0.01428304798901081,0.029301151633262634,-0.016835961490869522,-0.018249671906232834,-0.060638412833213806,-0.003972244448959827,-0.0134353619068861,0.046637412160634995,-0.0313662625849247,-0.002518788445740938,-0.06213541328907013,-0.054902225732803345,-0.09479370713233948,-0.00791741069406271,-0.009237347170710564,0.003526159329339862,0.0479445606470108,-0.020522117614746094,-0.017611760646104813,0.0011268857633695006,-0.035020891577005386,-0.025940798223018646,-0.052693597972393036,0.034674011170864105,-0.013083383440971375,-0.03823667764663696,-0.03058558702468872,0.019674498587846756,-0.01710587926208973,-0.0709163248538971,-0.04619041830301285,-0.045160382986068726,0.025199318304657936,0.053182244300842285,0.006203244440257549,0.010774382390081882,0.010007183067500591,0.04330877214670181,-0.009795685298740864,-0.057612136006355286,-0.013949994929134846,-0.022117318585515022,-0.06531462073326111,0.06615351140499115,0.013363583013415337,0.0653480514883995,-0.021705061197280884,0.04060419276356697,0.01825208030641079,-0.06172984838485718,0.056195031851530075,0.0048978920094668865,-0.020391516387462616,-0.04542168974876404,0.04132017120718956,0.0012211507419124246,-0.02955360896885395,0.0001237811375176534,0.0886031985282898,-0.016588004305958748,0.013865184038877487,0.037909407168626785,0.019615035504102707,-0.004181316122412682,0.06371352821588516,0.007185926660895348,0.041532937437295914,-0.0440700426697731,0.019272951409220695,-0.011018379591405392,-0.06918347626924515,0.06627406924962997,-0.039583541452884674,-0.10693039745092392,-0.0460125170648098,0.0038842265494167805,0.08659369498491287,0.046524763107299805,-0.09298978745937347,0.042321376502513885,-0.03761007636785507,0.01866830326616764,-0.13864123821258545,0.047613441944122314,0.070489302277565,-0.07881896197795868,-0.033515721559524536,0.04090999811887741,-0.030900485813617706,0.004013834986835718,-0.010084748268127441,-0.022624697536230087,0.00045741774374619126,-0.018537234514951706,-0.06816691160202026,0.02388092875480652,0.0029549337923526764,0.02958281710743904,0.011728419922292233,-0.05298061668872833,0.02613285928964615,-0.012485562823712826,0.07693839818239212,0.022565796971321106,-0.08271591365337372,0.03991695120930672,-0.015957292169332504,-0.025524210184812546,-0.07597845792770386,0.0035573223140090704,-0.008474909700453281,-0.07487748563289642,-0.032266806811094284,-0.09296497702598572,-0.017141686752438545,-0.00903515424579382,0.026578010991215706,0.04513595625758171,-0.14807336032390594,0.10653624683618546,-0.04252069815993309,-0.01673935540020466,0.02709919400513172,-0.0032645389437675476,0.10981692373752594,0.0234314426779747,0.023930195719003677,0.005550330970436335,0.11673116683959961,0.0029210166539996862,0.04647665098309517,-0.009529882110655308,0.025747012346982956,0.03265780583024025,-0.02294941246509552,0.022060677409172058,0.014868611469864845,-0.0627174898982048,-0.01699444092810154,-0.02093243971467018,0.045461948961019516,-0.044113386422395706,-0.04793624207377434,-0.007994185201823711,0.03164496272802353,-0.04408403858542442,-0.06220318749547005,-0.05012231320142746,-0.005898619536310434,-0.044882744550704956,-0.019330956041812897,-0.05932500958442688,-0.025885988026857376,-0.002481546951457858,0.013676244765520096,-0.010442398488521576,0.0019600428640842438,0.00594516284763813,-0.05502697080373764,-0.012453657574951649,0.036366887390613556,0.033140044659376144,0.028060808777809143,0.005408331286162138,-0.027927568182349205,-0.027029437944293022,0.03098202869296074,0.03623364493250847,-0.04836566001176834,0.0007566354470327497,0.04692249745130539,-0.03675737977027893,-0.03093898482620716,0.04540230706334114,-0.031854718923568726,-0.09672457724809647,-0.05646999552845955,0.020651357248425484,-0.027793128043413162,-0.0007982831448316574,-0.019118135794997215,0.004237998276948929,0.019014611840248108,-0.01891811192035675,0.07959124445915222,-0.05521664023399353,0.0008499306277371943,0.020867805927991867,-0.01320092286914587,-0.0021281298249959946,0.02314610965549946,-0.036674078553915024,-0.05226132646203041,0.029571082442998886,0.02965479902923107,0.031283847987651825,-0.07093239575624466,0.027682403102517128,-0.029133500531315804,-0.013711783103644848,0.034559138119220734,-0.04423127695918083,0.000647753244265914,-0.026779230684041977,-0.01217295229434967,0.04896436631679535,0.007700797636061907,-0.042460981756448746,-0.02501918561756611,0.04235338792204857,-0.02461078390479088,-0.0059410822577774525,0.0024839097168296576,0.0241058561950922,-0.010574420914053917,0.0043784985318779945,-0.06817682087421417,0.0043250261805951595,0.4817468225955963,-0.032108865678310394,0.06296153366565704,-0.04223038628697395,-0.009559216909110546,0.06292524188756943,0.018553070724010468,-0.028109680861234665,0.0052874269895255566,-0.035716913640499115,-0.00034209529985673726,-0.030593128874897957,0.00887283030897379,0.0003040296141989529,-0.01386750116944313,-0.04099038243293762,-0.06582267582416534,0.011359259486198425,0.03327056020498276,0.07242780178785324,-0.021486757323145866,-0.09529794752597809,-0.013593231327831745,0.0375569723546505,-0.041329409927129745,0.04374191164970398,-0.010001649148762226,0.004144552629441023,0.030703911557793617,0.0028385804034769535,0.007668676786124706,-0.005176033359020948,-0.04166444018483162,-0.008235271088778973,0.08488073199987411,0.05711091309785843,0.0019251825287938118,0.2032424956560135,0.08089535683393478,-0.020885871723294258,-0.006178053095936775,-0.02154567837715149,-0.055590830743312836,-0.04087794944643974,0.06780899316072464,0.08728887140750885,0.024546952918171883,0.03332352265715599,0.058534637093544006,-0.023268984630703926,0.05046205222606659,-0.02556026168167591,-0.0005117834080010653,0.03626742586493492,-0.03117223083972931,0.04964148998260498,0.02041664719581604,-0.0012965918285772204,0.0023036424536257982,-0.04105605185031891,-0.03009067289531231,0.0472702756524086,-0.05057352036237717,0.02228756994009018,0.005967405159026384,-0.06278017163276672,-0.0754082202911377,-0.029417218640446663,-0.053323134779930115,0.05137348920106888,0.005590768065303564,-0.014830376952886581,-0.032326921820640564,0.007353614550083876,-0.009639027528464794,0.0027342450339347124,0.054073743522167206,-0.03126148879528046,0.05511358380317688,-0.09282097220420837,-0.028406554833054543,0.009728052653372288,0.04792371019721031,-0.0073556117713451385,-0.003100272500887513,-0.0009375514346174896,-0.01953926309943199,-0.025839567184448242,-0.06409116089344025,-1.9453060303931125e-05,-0.03490051254630089,-0.013976427726447582,0.017471004277467728,0.12106510251760483,-0.058270007371902466,0.011964122764766216,0.003146525239571929,-0.017121626064181328,0.02996724098920822,-0.01813006028532982,0.01992093399167061,0.05432702228426933,-0.2763265073299408,-0.04585912823677063,-0.056583523750305176,0.02126692421734333,0.0039656697772443295,-0.073477603495121,0.00013544922694563866,-0.12755919992923737,-0.037628572434186935,-0.05514444410800934,-0.019381411373615265,0.030379338189959526,-0.0191962793469429,0.0426686629652977,0.017080232501029968,-0.030192356556653976,-0.023793227970600128,0.02382565103471279,-0.033697158098220825,0.04669718071818352,0.01754046604037285,0.021409112960100174,0.0242923554033041,-0.06327266991138458,0.01919597014784813,-0.03633565083146095,0.04936517775058746,-0.0277143195271492,0.00976170040667057,0.004720400553196669,0.0098115224391222,0.04664510115981102,-0.029250221326947212,-0.004120528697967529],"std":[0.17704500257968903,0.18225519359111786,0.1537151038646698,0.18995612859725952,0.17937688529491425,0.21349620819091797,0.17861239612102509,0.1883670836687088,0.16473478078842163,0.16815488040447235,0.18382570147514343,0.1942853182554245,0.20277023315429688,0.18207420408725739,0.19786852598190308,0.1713346242904663,0.20445217192173004,0.2268369346857071,0.16082437336444855,0.18207691609859467,0.17299744486808777,0.19857154786586761,0.16859854757785797,1.0510238409042358,0.18257413804531097,0.20053941011428833,0.2287781983613968,0.1677962839603424,0.15949971973896027,0.1613052636384964,0.17350704967975616,0.18499374389648438,0.1761884242296219,0.17580349743366241,0.20021964609622955,0.1982920914888382,0.18416902422904968,0.17730732262134552,0.18834611773490906,0.20023250579833984,0.18955017626285553,0.17450377345085144,0.17884358763694763,0.19014400243759155,0.1833191215991974,0.18192158639431,0.19221052527427673,0.18250665068626404,0.18700075149536133,0.20392532646656036,0.1636645644903183,0.18680231273174286,0.1978151500225067,0.15360866487026215,0.17472536861896515,0.20087608695030212,0.18133914470672607,0.19120930135250092,0.16974951326847076,0.22844262421131134,0.1886863112449646,0.1937789022922516,0.20514877140522003,0.17595793306827545,0.17241375148296356,0.1808483749628067,0.17565900087356567,0.17753373086452484,0.19382157921791077,0.2056826502084732,0.18924328684806824,0.32203078269958496,0.17905856668949127,0.20042800903320312,0.17567665874958038,0.16358403861522675,0.19052211940288544,0.1964312344789505,0.19200487434864044,0.17604400217533112,0.20169253647327423,0.16852985322475433,0.20285235345363617,0.18001998960971832,0.2273852527141571,0.19458980858325958,0.1831732541322708,0.22732162475585938,0.14898425340652466,0.16669370234012604,0.16701866686344147,0.17572875320911407,0.18502625823020935,0.17771850526332855,0.17822736501693726,0.16550765931606293,0.16180765628814697,0.1944061815738678,0.23678745329380035,0.18475741147994995,0.17112943530082703,0.15752285718917847,0.19685673713684082,0.17624112963676453,0.19689257442951202,0.202573761343956,0.18161740899085999,0.18856553733348846,0.16509346663951874,0.18289276957511902,0.17389316856861115,0.29215702414512634,0.16953706741333008,0.15366829931735992,0.17122140526771545,0.18738600611686707,0.18705415725708008,0.17670801281929016,0.16569945216178894,0.15259400010108948,0.18348796665668488,0.20145662128925323,0.2187446802854538,0.18055246770381927,0.1752774715423584,0.20427429676055908,0.19046582281589508,0.17511212825775146,0.1802341789007187,0.1771143227815628,0.1770484745502472,0.16323734819889069,0.18517419695854187,0.1958383321762085,0.17688797414302826,0.18416039645671844,0.17392443120479584,0.18262408673763275,0.18709540367126465,0.17210566997528076,0.1680552065372467,0.19373983144760132,0.1683102548122406,0.17897431552410126,0.17391899228096008,0.18687544763088226,0.16094925999641418,0.19124071300029755,0.23990190029144287,0.17171643674373627,0.1750800907611847,0.1925363391637802,0.17704646289348602,0.1786729097366333,0.18135233223438263,0.17651155591011047,1.2951561212539673,0.4977928400039673,0.16937021911144257,0.20662805438041687,0.17784057557582855,0.20105217397212982,0.1630997210741043,0.18382783234119415,0.1758565455675125,0.21958009898662567,0.17842015624046326,0.18129795789718628,0.17530407011508942,0.181766539812088,0.18458136916160583,0.1908320188522339,0.1578674018383026,0.1981038749217987,0.17341244220733643,0.16436268389225006,0.19575831294059753,0.17546682059764862,0.17286008596420288,0.20635303854942322,0.17913824319839478,0.15782950818538666,0.149438738822937,0.1897868514060974,0.16173261404037476,0.18433727324008942,0.2762281000614166,0.1748540699481964,0.21413098275661469,0.1977960169315338,0.18491153419017792,0.15784253180027008,0.18877747654914856,0.1842070072889328,0.1669398844242096,0.19761072099208832,0.16822023689746857,0.21316643059253693,0.191743403673172,0.16287870705127716,0.19623254239559174,0.18294577300548553,0.17218756675720215,0.18902714550495148,0.1981259286403656,0.17238549888134003,0.17753717303276062,0.18883244693279266,0.19012311100959778,0.18915888667106628,0.18318124115467072,0.1614285558462143,0.19500936567783356,0.1794068068265915,0.1584235280752182,0.18740594387054443,0.20229262113571167,0.1620541214942932,0.19431528449058533,0.16286985576152802,0.212575763463974,0.16740089654922485,0.1921526938676834,0.17225804924964905,0.18775036931037903,0.1689179241657257,0.19493736326694489,0.19352231919765472,0.18517956137657166,0.1888338029384613,0.1886870563030243,0.4056771397590637,0.16791373491287231,0.1881236433982849,0.17593076825141907,0.16645315289497375,0.21342909336090088,0.1962202787399292,0.1822635680437088,0.17087411880493164,0.19153501093387604,0.1770656853914261,0.19114507734775543,0.1814061403274536,0.15386080741882324,0.164637491106987,0.1660274863243103,0.1949741244316101,0.2680206894874573,0.1883121281862259,0.17039132118225098,0.18234121799468994,0.18403728306293488,0.1900060474872589,0.17389525473117828,0.22380301356315613,0.16119195520877838,0.17501728236675262,0.21592076122760773,0.18703362345695496,0.17950575053691864,0.2098838835954666,0.1900833398103714,0.1919572949409485,0.18916167318820953,0.17298848927021027,0.2021162509918213,0.17624753713607788,0.1822904795408249,0.1833748072385788,0.2000899314880371,0.20429779589176178,0.15803751349449158,0.16617058217525482,0.2024322897195816,0.22032473981380463,0.3054719567298889,0.18309152126312256,0.21615804731845856,0.17150796949863434,0.22075095772743225,0.18547813594341278,0.19023902714252472,0.1724989265203476,0.16769592463970184,0.20628486573696136,0.17528271675109863,0.17625658214092255,0.1834786832332611,0.19810037314891815,0.18463802337646484,0.2156219333410263,0.17859384417533875,0.18335840106010437,0.20674125850200653,0.16503335535526276,0.17349283397197723,0.17621047794818878,0.17565950751304626,0.19126124680042267,0.15977098047733307,0.1718270480632782,0.19845709204673767,0.17621886730194092,0.1861894726753235,0.1880473494529724,0.2339612990617752,0.17502164840698242,0.179008349776268,0.18128414452075958,0.16876643896102905,0.176747128367424,0.18748903274536133,0.20388932526111603,0.18249920010566711,0.17332050204277039,0.18773745000362396,0.1708640605211258,0.19871650636196136,0.18237435817718506,0.19278733432292938,0.1606394648551941,0.21280492842197418,0.20383869111537933,0.1788722276687622,0.1743672639131546,0.642346203327179,0.1938367336988449,0.1802556961774826,0.20313215255737305,0.19178956747055054,0.19908550381660461,0.16424520313739777,0.21251654624938965,0.18984313309192657,0.18712195754051208,0.16941164433956146,0.19810150563716888,0.1935083270072937,0.20236153900623322,0.16948701441287994,0.17817193269729614,0.1894865334033966,0.1571013629436493,0.189874529838562,0.19917050004005432,0.1879580020904541,0.1953970193862915,0.18209894001483917,0.19870205223560333,0.16710899770259857,0.18987113237380981,0.18079319596290588,0.24720627069473267,0.18246634304523468,0.1783977448940277,0.18086469173431396,0.18512991070747375,0.16203603148460388,0.17371010780334473,0.21745550632476807,0.16785724461078644,0.17391127347946167,0.1993149369955063,0.1740284264087677,0.2071572244167328,0.18864642083644867,0.17904222011566162,0.19137243926525116,0.17342446744441986,0.17145183682441711,0.16081967949867249,0.17650055885314941,0.1789073944091797,0.18414142727851868,0.1701524257659912,0.38816189765930176,0.1788211464881897,0.18843574821949005,0.18016317486763,0.16460543870925903,0.18250782787799835,0.17725007236003876,0.18527920544147491,0.18238677084445953,0.18947745859622955,0.2246234118938446,0.19307178258895874,0.208200141787529,0.16432060301303864,0.16517871618270874,0.18551230430603027,0.16471140086650848,0.166218563914299,0.14516513049602509,0.18570703268051147,0.1588030606508255,0.21061351895332336,0.18980848789215088,0.18863065540790558,0.16266541182994843,0.1983071118593216,0.19694793224334717,0.18026332557201385,0.1677124947309494,0.1802339255809784,0.17157110571861267,0.18546675145626068,0.1797514110803604,0.17810304462909698,0.1765221655368805,0.18241415917873383,0.16058135032653809,0.7569933533668518,0.22157233953475952,0.1808415800333023,0.1769281029701233,0.1782831996679306,0.2061048448085785,0.20739509165287018,0.18991219997406006,0.15973004698753357,0.20280106365680695,0.18071171641349792,0.1667417585849762,0.18323153257369995,0.18043674528598785,0.18495067954063416,0.19180786609649658,0.174737349152565,0.18047279119491577,0.17534077167510986,0.17332330346107483,0.20920434594154358,0.18140622973442078,0.17736324667930603,0.1836283951997757,0.17700345814228058,0.19967010617256165,0.17346228659152985,0.1875402331352234,0.19351741671562195,0.19352015852928162,0.17959196865558624,0.18019704520702362,0.17107024788856506,0.14191216230392456,0.19367051124572754,0.16550539433956146,0.18701596558094025,0.18709327280521393,0.16970305144786835,0.16288575530052185,0.17713749408721924,0.17108739912509918,0.1804233342409134,0.18634256720542908,0.1596498042345047,0.19640076160430908,0.16763398051261902,0.20500490069389343,0.18013978004455566,0.17287786304950714,0.195465087890625,0.18855668604373932,0.18113276362419128,0.17224903404712677,0.16478466987609863,0.17354625463485718,0.17878495156764984,0.17773626744747162,0.17250776290893555,0.19150958955287933,0.17022041976451874,0.17914433777332306,0.19682997465133667,0.1667299121618271,0.17577748000621796,0.37003543972969055,0.16850925981998444,0.18011729419231415,0.19648273289203644,0.1923263967037201,0.1812393069267273,0.18900951743125916,0.1722290813922882,0.17380128800868988,0.18368588387966156,0.19403621554374695,0.18613508343696594,0.16729316115379333,0.18930944800376892,0.17943812906742096,0.18281403183937073,0.16094501316547394,0.16919931769371033,0.18704602122306824,0.1816728413105011,0.2037990540266037,0.18429645895957947,0.16880887746810913,0.17736738920211792,0.17087167501449585,0.1653205007314682,0.17512448132038116,0.1669646054506302,0.18964533507823944,0.3063414394855499,0.18163882195949554,0.19496063888072968,0.19230246543884277,0.19400423765182495]}},"2":{"inputs":{"mean":[0.16195432841777802,0.15161718428134918,-0.06848487257957458,0.10156583786010742,-0.3544507324695587,-0.030371662229299545,0.09592044353485107,0.09984888136386871,-0.28268975019454956,-0.0997990071773529,0.11219022423028946,0.25178074836730957,-0.036906301975250244,0.286032110452652,0.10512542724609375,0.15657193958759308,0.4314647614955902,0.09117579460144043,0.06283149868249893,0.061633963137865067,0.09500709921121597,0.08677979558706284,-0.024143628776073456,2.8415935039520264,-0.12147661298513412,-0.021712636575102806,-0.002774385968223214,0.05077054351568222,-0.19168581068515778,-0.15293727815151215,-0.1324596107006073,0.11496855318546295,-0.361719012260437,-0.9484831690788269,-0.08789943903684616,-0.0356123223900795,-0.07652657479047775,0.1187579482793808,-0.027041267603635788,0.07302532345056534,-0.02252316102385521,-0.20118238031864166,-0.012069248594343662,-0.2695411443710327,0.138259619474411,-0.126081183552742,-0.2106911987066269,0.14760296046733856,-0.01326729916036129,-0.01595192961394787,0.13292795419692993,0.08943488448858261,-0.04678048565983772,0.10051356256008148,0.019255120307207108,0.300939679145813,-6.823133945465088,0.3121183514595032,-0.04414775222539902,0.39343059062957764,0.08043861389160156,0.16504159569740295,-0.09480372816324234,-0.22066102921962738,-0.3319161534309387,0.06906343251466751,0.33634406328201294,-0.11589372903108597,0.09373068809509277,0.20342735946178436,0.13958536088466644,0.05355892330408096,-0.09702356159687042,-0.1341436356306076,0.08830395340919495,0.18831273913383484,0.09018141031265259,0.22742155194282532,-0.08418859541416168,-0.005828356370329857,-0.20426194369792938,0.13990989327430725,0.4932364225387573,-0.03854207322001457,0.13480059802532196,0.1892644464969635,-0.23560483753681183,0.20612789690494537,-0.04605964198708534,-0.10328224301338196,-0.34993356466293335,0.15805178880691528,-0.203254833817482,-0.10179460793733597,-0.10149195790290833,0.14416539669036865,0.11460302770137787,0.23494917154312134,-0.020374763756990433,0.16141830384731293,-0.2442573457956314,0.05654880031943321,0.04861484467983246,0.3373247981071472,-0.1629869043827057,0.18303810060024261,0.09110716730356216,0.18227717280387878,0.04535777494311333,-0.058636777102947235,0.07326655834913254,0.5408217310905457,-0.12579035758972168,0.12015041708946228,-0.01574372686445713,-0.5473254919052124,0.33884984254837036,0.32418233156204224,0.12453974038362503,0.10544628649950027,-0.22598229348659515,0.27108681201934814,-0.00491497665643692,0.15876145660877228,0.030233558267354965,0.18325498700141907,0.062291305512189865,-0.09930036216974258,0.28954336047172546,0.124130018055439,-0.22041137516498566,-0.015601418912410736,0.08357471972703934,-0.06101891025900841,-0.01689833030104637,-0.1490946114063263,0.07425390928983688,0.0007425541407428682,-0.10571148991584778,0.02939510904252529,0.1546032428741455,-0.2190227210521698,-0.18015529215335846,0.037063777446746826,0.12032914161682129,0.16023458540439606,-0.21313689649105072,-0.10757970064878464,-0.7205064296722412,-0.01810898259282112,0.11086785048246384,0.26617205142974854,-0.12910373508930206,0.5128926038742065,-0.06247264891862869,0.055708494037389755,9.948301315307617,1.1399961709976196,-0.013962960802018642,-0.011178088374435902,0.1069803386926651,-0.15056827664375305,-0.11839144676923752,-0.12556977570056915,-0.001917563728056848,0.9257550835609436,-0.27803850173950195,0.05691516771912575,0.006167734041810036,-0.16690072417259216,0.0339643768966198,-0.06884732097387314,-0.057921018451452255,-0.0430292971432209,0.11115776747465134,0.005762777291238308,0.25699397921562195,0.07635775953531265,-0.20680458843708038,-0.2581535875797272,-0.05384347587823868,-0.10497958958148956,0.053218767046928406,-0.11414080858230591,0.22327715158462524,0.24767398834228516,-1.3776235580444336,0.13253265619277954,0.13872802257537842,-0.05947454646229744,-0.1860177218914032,-0.02431357651948929,0.12596383690834045,0.1776317059993744,0.005122686270624399,-0.05053669959306717,0.11072477698326111,0.08286009728908539,-0.020386409014463425,-0.19800101220607758,0.11075349897146225,-0.011146482080221176,0.16500242054462433,0.0818694680929184,-0.18063956499099731,0.001011894317343831,-0.06586851179599762,0.06498730182647705,-0.0004208517784718424,0.1717032492160797,0.02707098238170147,-0.2503643035888672,0.07413069158792496,-0.06664764136075974,0.06059309095144272,0.1047196164727211,0.17847803235054016,0.10590697824954987,0.22646576166152954,0.0952000766992569,0.1579284518957138,-0.16399602591991425,0.06958641856908798,0.07097239792346954,0.16900312900543213,0.055721964687108994,-0.1417800486087799,0.03105052001774311,0.1528339385986328,0.006660575047135353,0.16038094460964203,0.021884310990571976,0.015732325613498688,-0.0050352453254163265,-0.01584145799279213,0.04724448546767235,0.11676427721977234,-0.13137578964233398,0.018310008570551872,0.2966457009315491,-1.331871509552002,0.17786404490470886,-0.05987723171710968,0.048997167497873306,0.15503737330436707,-0.058197736740112305,-0.05194101110100746,-0.20295532047748566,-0.1051638200879097,-0.10216814279556274,-0.08101280778646469,0.42809247970581055,-0.06228281185030937,0.18366804718971252,0.0309797041118145,0.10304635018110275,-0.005535929463803768,-0.0969630628824234,0.0007190367323346436,0.1376108080148697,0.01955368183553219,0.07446162402629852,-0.14711353182792664,-0.03343001380562782,0.08775767683982849,-0.05491037666797638,-0.2683238387107849,-0.07568905502557755,0.030158722773194313,-0.22437964379787445,0.022044653072953224,0.14565490186214447,-0.014288797043263912,0.02173483557999134,0.24700820446014404,-0.3137224614620209,-2.224473714828491,-0.0949447825551033,0.16921456158161163,-0.1800818294286728,0.21233871579170227,-0.15468239784240723,-0.16682621836662292,0.3579384684562683,-0.3034641742706299,-0.0355323888361454,-0.019470104947686195,-0.0014438842190429568,-0.13626666367053986,0.37395259737968445,0.17370560765266418,0.1844407171010971,0.04538378119468689,0.07378233224153519,0.1844857633113861,0.1164882630109787,-0.10486526042222977,-0.1314646303653717,-0.09239626675844193,-0.1817007064819336,0.03340442478656769,0.34026315808296204,-0.10400181263685226,-0.041832227259874344,-0.0502789206802845,-0.22400860488414764,0.08703243732452393,0.1062505766749382,0.09926854819059372,0.20486679673194885,0.1747477501630783,-0.0361856073141098,0.05424833670258522,-0.09330740571022034,-0.16673441231250763,0.1474866420030594,-0.03307421877980232,-0.21773940324783325,-0.027186736464500427,0.06580672413110733,-0.16210490465164185,-0.0792807787656784,0.134563609957695,0.1361895352602005,0.21042756736278534,-0.08339881896972656,-0.25878649950027466,0.15299475193023682,-0.02121124044060707,0.02521698735654354,-0.2603297233581543,0.35377660393714905,-0.0469825342297554,-0.05814191326498985,-0.2866884768009186,0.09186189621686935,-0.11237563192844391,-0.027480293065309525,0.13110846281051636,0.04747623950242996,-0.18546448647975922,-0.08286350220441818,0.1673729568719864,0.12715907394886017,0.055828776210546494,0.1417178213596344,0.17774900794029236,-0.0182513315230608,0.09848139435052872,-0.00481411861255765,0.010433439165353775,0.033895041793584824,0.022179683670401573,-0.1622260957956314,-0.09587563574314117,0.0786016657948494,-0.04295044764876366,-0.3665044605731964,0.1709631234407425,0.03305268660187721,-0.16301533579826355,0.09308309108018875,-0.06919654458761215,0.10864148288965225,-0.21582037210464478,-0.461212158203125,-0.06058822572231293,0.060567740350961685,-0.2817649245262146,0.1048375591635704,-0.0600137896835804,-0.09640705585479736,-0.07874910533428192,0.0497257374227047,0.2564685344696045,0.057105645537376404,-3.351776599884033,-0.025059249252080917,0.18149548768997192,-7.263331644935533e-05,0.06843431293964386,-0.1677868813276291,0.024467652663588524,0.044636648148298264,-0.1106700450181961,0.2426677793264389,0.0872211679816246,0.07421183586120605,-0.16401231288909912,0.07417357712984085,-0.12898683547973633,0.020239844918251038,-0.041965194046497345,-0.10812224447727203,0.04970134049654007,-0.16099897027015686,-0.14183086156845093,0.20925390720367432,-0.05551004037261009,0.010333472862839699,-0.24469630420207977,0.043554194271564484,0.14733605086803436,-0.025018896907567978,-0.10289419442415237,-0.29688793420791626,0.12777179479599,-0.2180953174829483,0.056718673557043076,-0.11826703697443008,0.34335675835609436,0.09703268855810165,0.08152218163013458,-0.47564777731895447,-0.0673999935388565,0.1852438747882843,0.01537558063864708,0.07802446186542511,-0.2839529514312744,-0.027158688753843307,-0.1254289299249649,0.1597260981798172,0.15034130215644836,0.10324474424123764,-0.03137043118476868,0.053063392639160156,-0.07174497842788696,-0.07386130094528198,-0.2776018977165222,-0.07993592321872711,-0.157857745885849,-0.21232587099075317,0.10331202298402786,0.06702584028244019,-0.05178723484277725,0.1659710854291916,0.1674666404724121,0.6160920262336731,0.310693621635437,-0.0680105984210968,-0.06296021491289139,-0.03577225282788277,-0.054247431457042694,0.18712662160396576,0.16646230220794678,-0.003366058459505439,0.11381527036428452,0.3086307942867279,0.10288245975971222,0.059250686317682266,0.11789979785680771,0.0889776423573494,-0.2051478773355484,0.26968276500701904,0.03472306951880455,0.0409298837184906,0.06231454387307167,0.15371212363243103,0.19678457081317902,0.17660410702228546,0.13382872939109802,-0.07786700874567032,0.0226728655397892,-0.050298165529966354,0.29215767979621887,0.021377360448241234,-0.1253613829612732,-0.13613229990005493,-0.07448042184114456,0.13268496096134186,-0.18124347925186157,-0.015411022119224072,-0.05965085327625275,-0.14402830600738525,-0.07845007628202438,0.17285889387130737,-0.07768750190734863,0.08515678346157074,-2.73010516166687,-0.1625029444694519,-0.1669912338256836,0.04579643905162811,-0.053186845034360886,-0.10012653470039368,-0.12973935902118683,0.13379162549972534,-0.12814690172672272,-0.011192696169018745,0.22793176770210266,0.01027340441942215,-0.02970466949045658,-0.05007193610072136,-0.05751320719718933,-0.29353266954421997,0.09652204811573029,0.14770816266536713,0.051112741231918335,0.09032408893108368,0.1754627227783203,0.04963606595993042,-0.053899023681879044,-0.2269831746816635,-0.17265921831130981,0.11114361137151718,-0.1109442412853241,-0.12878523766994476,0.1613624393939972,-1.2439889907836914,0.07939758896827698,0.3109714984893799,0.12795470654964447,0.015095626935362816],"std":[0.762886106967926,0.7586259841918945,0.7288163304328918,0.8458088636398315,0.7902308106422424,0.794387698173523,0.8114457130432129,0.7479795813560486,0.7859703898429871,0.752244770526886,0.808029294013977,0.8387017846107483,0.7942100763320923,0.8012280464172363,0.7477383613586426,0.8175477981567383,0.806433379650116,1.7837036848068237,0.751274049282074,0.8165960311889648,0.8027978539466858,0.8657189011573792,0.7375351190567017,2.1354358196258545,0.7430452108383179,0.8138429522514343,0.8476933836936951,0.7085760235786438,0.8068868517875671,0.8494023084640503,0.7347292304039001,0.7812773585319519,0.8625341653823853,1.7902051210403442,0.7982791662216187,0.8239079117774963,0.7883667349815369,0.8011043667793274,0.8264469504356384,0.848626971244812,0.804145097732544,0.8278566598892212,0.7107744812965393,0.7893927097320557,0.8293769955635071,0.7775432467460632,0.9022671580314636,0.8262299299240112,0.7668774724006653,0.7138721942901611,0.748633086681366,0.9606441855430603,0.8587415814399719,0.8785197734832764,0.7678505182266235,0.8198012113571167,3.800151824951172,0.8126416206359863,0.7945034503936768,0.851938009262085,0.7278140783309937,0.8940249681472778,0.8373181223869324,0.8365866541862488,0.7435290813446045,0.7897562384605408,0.8634814023971558,0.7126337885856628,0.7601609230041504,0.8782387375831604,0.7913140654563904,1.8357442617416382,1.093144178390503,0.7677305936813354,0.8856194615364075,0.695096492767334,0.8133244514465332,0.7729622721672058,0.7477580904960632,0.8155576586723328,0.9443855881690979,0.9025281667709351,0.8836986422538757,0.7628859281539917,0.8171663880348206,0.78619384765625,0.7777161002159119,0.8969292044639587,0.8508763313293457,0.7315587401390076,0.8580789566040039,0.7272610664367676,0.7579392194747925,0.7526205778121948,0.7705578207969666,0.8477116227149963,0.7590200901031494,0.8205844163894653,1.4297634363174438,0.8055487871170044,0.7769389152526855,0.7189785242080688,0.7277997732162476,0.8328969478607178,0.87615567445755,0.8053064346313477,0.7691183090209961,0.8711997270584106,0.8938838839530945,0.80848628282547,0.8388383984565735,1.6168652772903442,0.7464167475700378,0.6901588439941406,0.7709111571311951,1.3958154916763306,0.9705056548118591,0.7721383571624756,0.7666816711425781,0.7889812588691711,0.7518950700759888,0.7219189405441284,0.6972913146018982,0.7541700005531311,0.7204329371452332,1.3394958972930908,0.8591939806938171,0.7433801889419556,0.7720792889595032,1.389176368713379,0.823712170124054,0.8360748291015625,0.74482262134552,0.7747446298599243,0.7748759984970093,0.7707309722900391,0.7819828391075134,0.998771607875824,0.7247922420501709,0.805462121963501,0.9286043643951416,0.7532691955566406,0.7743688225746155,0.7365929484367371,0.7471737265586853,0.8261134028434753,0.8069660067558289,0.7708117961883545,2.0390548706054688,0.7815559506416321,0.7865046858787537,0.9897247552871704,0.8055314421653748,1.076238751411438,0.7969231009483337,0.7305150032043457,3.363534450531006,1.4204351902008057,0.6981081366539001,0.8295427560806274,0.8700375556945801,0.713329553604126,0.723499059677124,0.7925539016723633,0.8305830955505371,1.5398246049880981,0.740199863910675,0.778773844242096,0.7873417139053345,0.8519496917724609,0.8046208620071411,0.739619791507721,0.7723663449287415,1.0249972343444824,0.8255642652511597,0.748746931552887,0.7998340725898743,0.7320628762245178,0.706524670124054,0.9029815196990967,0.8004031181335449,0.9719697833061218,0.7412625551223755,0.8981342911720276,0.79217129945755,0.8100403547286987,1.8130146265029907,0.7276090383529663,0.7541874051094055,0.8789350390434265,0.8379912376403809,0.8715030550956726,0.7789564728736877,0.8490862846374512,0.7801213264465332,0.7471680045127869,0.7862250208854675,0.860806405544281,0.6755332946777344,0.7622056007385254,0.7949624061584473,0.7213883996009827,0.8220363259315491,0.8294605612754822,0.7727454900741577,0.7885065078735352,0.7568371891975403,0.7252683043479919,0.7802272439002991,0.7391118407249451,0.8102232217788696,1.0405759811401367,0.7985420227050781,0.7158809304237366,0.7185478210449219,0.7794945240020752,0.7478563785552979,0.8309965133666992,0.8147510290145874,0.6954666376113892,0.7749486565589905,0.7462849020957947,0.7500796914100647,0.7166807651519775,0.8783073425292969,0.7504643797874451,0.7513470649719238,0.7656912207603455,0.7873714566230774,0.7645460367202759,0.8183152675628662,1.6878671646118164,0.7808133959770203,0.8021783232688904,0.7531988620758057,0.7994589805603027,1.3033316135406494,0.8077273964881897,0.8332232236862183,0.7639769911766052,1.5591182708740234,0.7971875667572021,0.7903249263763428,0.7848600745201111,0.7436190843582153,0.7383266091346741,0.7349655628204346,0.8592553734779358,1.6367307901382446,1.0374332666397095,0.7086343169212341,0.8201941251754761,0.8256599307060242,0.9038354754447937,0.7430468201637268,0.7929513454437256,0.7691465616226196,0.9066202044487,0.7522661089897156,0.7635980248451233,0.714918315410614,0.8280633091926575,0.7846425771713257,0.9674537777900696,0.7651267647743225,0.8266241550445557,0.8817125558853149,0.7409442067146301,0.8562484383583069,0.8925151824951172,0.8290289640426636,0.9214394092559814,0.7573560476303101,0.7418601512908936,0.9208385944366455,0.9050754904747009,3.618107557296753,0.7299734354019165,0.8807005286216736,0.8714470267295837,0.952406108379364,0.7974178791046143,0.8389489054679871,0.7721838355064392,0.8543998003005981,0.8985739350318909,0.7733951807022095,0.7577790021896362,0.7815892696380615,0.9311947822570801,0.7590927481651306,1.1315642595291138,0.7919096946716309,0.7614510655403137,0.7880322933197021,0.9369711875915527,0.784268856048584,0.6790584325790405,0.8210495710372925,0.8185320496559143,0.7348470687866211,0.8049759864807129,0.8475498557090759,0.7472975850105286,0.9072502851486206,0.80389404296875,0.8851216435432434,0.7511119246482849,0.8182570934295654,0.7866309881210327,0.7214205265045166,0.8440762758255005,0.804824948310852,0.8546833395957947,0.9221243262290955,0.7919814586639404,0.7978482246398926,0.7809995412826538,0.731407880783081,0.7434290051460266,0.8112732172012329,1.0639233589172363,0.7843793630599976,0.7734549641609192,0.8223598003387451,0.7174142599105835,1.404059648513794,0.8075687289237976,0.7530955672264099,0.7803537845611572,0.7647273540496826,0.7949619293212891,0.8104239106178284,0.869959831237793,0.779945433139801,0.774036169052124,0.8284652829170227,0.7539124488830566,0.8771035075187683,0.8475019931793213,0.7806717753410339,0.7797002196311951,0.7487971186637878,0.7920482158660889,0.7973244786262512,0.7375129461288452,0.7091947793960571,0.7819640636444092,0.7999548316001892,0.7958724498748779,0.7371469140052795,0.7212346792221069,0.8143019676208496,0.8115338087081909,0.8579317927360535,0.7588667869567871,0.7848263382911682,0.8840553760528564,0.8271242380142212,0.860345721244812,0.8023902773857117,0.7369619011878967,0.9537565112113953,0.7668749094009399,0.7286702394485474,1.054062843322754,0.722239077091217,0.7594685554504395,0.8583191633224487,1.321460485458374,0.7734456658363342,0.7478892207145691,0.7340279817581177,0.7539663910865784,0.8847493529319763,0.7414987087249756,5.356220722198486,0.8959921598434448,0.8069905638694763,0.9405511617660522,0.8685809969902039,0.703528881072998,0.8745167851448059,0.6572741270065308,0.7879287600517273,0.7482247352600098,0.842524528503418,0.7834345698356628,0.7388291358947754,0.758571445941925,0.9753873944282532,0.8071475625038147,0.714749813079834,0.7771514058113098,0.8347542881965637,0.7814317345619202,0.7487105131149292,0.8362826704978943,0.7973604798316956,0.7838307619094849,0.8899447917938232,0.837999165058136,0.8293520212173462,0.8336326479911804,0.8087937831878662,0.8316885232925415,0.7616366744041443,0.79234379529953,0.9161109328269958,0.811684250831604,1.4505908489227295,0.8346343636512756,0.9007092714309692,3.6648924350738525,0.7819784879684448,0.8083683252334595,0.8297783732414246,0.7656601071357727,0.8936927318572998,0.7376127243041992,0.702046811580658,0.8469238877296448,0.8258923888206482,0.8752315044403076,0.7858307957649231,0.8248700499534607,0.7347798943519592,0.7971380949020386,0.7093996405601501,0.8258927464485168,0.7997816801071167,0.789048433303833,0.72471684217453,0.750076413154602,0.8945879936218262,0.7596712708473206,0.8127009868621826,1.027669906616211,0.9586175084114075,0.7705675959587097,1.0293511152267456,0.82337486743927,0.7595701217651367,0.885800302028656,0.7510172724723816,0.778639554977417,0.7297387719154358,0.7662073373794556,0.9557909965515137,0.7404993176460266,0.8438634872436523,0.8132451176643372,0.8277848362922668,1.1306663751602173,0.7587993741035461,0.8155525922775269,0.785114049911499,0.7936831712722778,0.9357445240020752,0.758415937423706,0.7483689188957214,0.7717688679695129,0.7726063132286072,0.8041446805000305,0.8371825814247131,0.7855200171470642,0.7625306248664856,0.9037574529647827,0.7829698324203491,0.790412425994873,0.803817629814148,0.7124778628349304,0.811839759349823,0.7406261563301086,0.8711158037185669,0.8066736459732056,0.8745496869087219,0.9302436709403992,4.202030658721924,0.7516703009605408,0.7365808486938477,0.816944420337677,0.7828065752983093,0.7630429267883301,0.8046458959579468,0.709956169128418,0.7938889861106873,0.8181391358375549,0.8380960822105408,0.8256188631057739,0.7698171138763428,0.8054694533348083,0.8410089612007141,0.9320871233940125,0.7724783420562744,0.8229975700378418,0.8375017046928406,0.7610999941825867,0.7948927879333496,0.7624335289001465,0.7984170317649841,0.8128191232681274,0.8262227177619934,0.7602924108505249,0.7226263880729675,0.7585562467575073,0.8236991167068481,2.67978835105896,0.8011032938957214,0.7716647386550903,0.7667439579963684,0.74676913022995]},"targets":{"mean":[0.028515547513961792,-0.06832388043403625,0.0371050089597702,-0.023838991299271584,0.19268643856048584,-0.045196812599897385,-0.03757636994123459,-0.06674091517925262,0.0604848712682724,0.12703093886375427,0.060340702533721924,-0.048068419098854065,0.028877275064587593,-0.05327196419239044,0.021938655525445938,-0.14784462749958038,-0.19929471611976624,-0.13377860188484192,0.07326936721801758,-0.13297224044799805,0.13623613119125366,-0.009843943640589714,-0.03120272234082222,0.7217174768447876,0.0343533456325531,0.12319566309452057,0.02392788790166378,-0.06681432574987411,0.05151696130633354,-0.010208594612777233,0.034253764897584915,-0.11347950249910355,0.10161425918340683,0.10201015323400497,0.18657167255878448,-0.09932610392570496,-0.011381186544895172,-0.0978861078619957,-0.1337556093931198,-0.09521108865737915,0.03844155743718147,0.04082460328936577,0.020840555429458618,0.04308023303747177,-0.13620224595069885,0.07221752405166626,-0.010086230002343655,0.018620513379573822,-0.07271858304738998,7.623188139405102e-05,0.010784693993628025,0.06855045258998871,0.03591892868280411,0.009249581024050713,-0.024191154167056084,-0.07462535053491592,-0.061077386140823364,-0.10777884721755981,0.06453170627355576,-0.10527860373258591,-0.11438963562250137,-0.06455475091934204,-0.018166279420256615,0.09490035474300385,0.09774892777204514,0.08926961570978165,-0.03630881384015083,-0.022973744198679924,-0.08182026445865631,0.027432715520262718,0.027989143505692482,0.21781863272190094,0.019550541415810585,0.05094670131802559,-0.04841684177517891,0.017194120213389397,-0.011147513054311275,-0.010576550848782063,0.13602465391159058,0.029636025428771973,0.009363247081637383,-0.027453608810901642,-0.13064336776733398,-0.04094129428267479,-0.03343475982546806,-0.12388405948877335,0.09462276846170425,-0.04834046587347984,-0.03676105663180351,0.021014273166656494,0.15229205787181854,0.014517859555780888,0.15364712476730347,0.012484844774007797,-0.008473917841911316,-0.08430561423301697,-0.11918751895427704,-0.028329981490969658,-0.29654380679130554,0.03669613227248192,0.06783422082662582,-0.07671984285116196,-0.07684057950973511,-0.03226863592863083,-0.009958721697330475,-0.09218408912420273,-0.0261390320956707,-0.017548302188515663,-0.08683604001998901,0.024392206221818924,-0.031225746497511864,0.013237298466265202,-0.03095605969429016,-0.06769547611474991,-0.0029037166386842728,0.04951827600598335,-0.023318788036704063,-0.02062489092350006,-0.07125166803598404,-0.049861278384923935,0.11608393490314484,-0.05510785058140755,-0.06377135217189789,-0.11375635862350464,0.01187275443226099,-0.06320519000291824,-0.04325355216860771,-0.025435229763388634,-0.11158937215805054,-0.0016347253695130348,0.0992153063416481,0.0981489047408104,-0.008382041938602924,0.08662516623735428,-0.03252296522259712,0.0743754580616951,-0.08111914247274399,-0.025887280702590942,0.05744880810379982,0.013418306596577168,-0.10184184461832047,0.01811329275369644,-0.04621747136116028,-0.018653137609362602,0.014467336237430573,0.061829935759305954,0.08099552989006042,-0.059030622243881226,0.09541533887386322,-0.048305124044418335,-0.04460194706916809,-0.013790503144264221,0.0027056089602410793,0.047091662883758545,0.11380840092897415,-0.008768034167587757,1.443213701248169,0.09552937746047974,0.012655221857130527,0.03442152962088585,-0.05603690817952156,-0.06291958689689636,-0.016568131744861603,0.02790145017206669,0.06830623745918274,-0.1187482699751854,0.010316857136785984,0.09591958671808243,0.020481189712882042,0.037746675312519073,-0.11903258413076401,-0.07136348634958267,0.07854466885328293,0.03771308809518814,0.08372010290622711,-0.027823176234960556,-0.07000278681516647,-0.1028374508023262,0.009427360258996487,-0.007677763234823942,-0.15225286781787872,0.10883358120918274,0.0016447721282020211,0.000866738089825958,-0.09342817217111588,0.02949906326830387,-0.015344653278589249,-0.0763225182890892,-0.05114853382110596,0.12977184355258942,-0.004535782616585493,0.0698612704873085,-0.1337326318025589,-0.14249123632907867,-0.013406255282461643,0.03355224430561066,0.01073883194476366,-0.07609516382217407,0.030875368043780327,0.022666802629828453,-0.03912324830889702,-0.05969693139195442,0.022931525483727455,-0.05354268103837967,0.028462130576372147,0.03806128725409508,-0.02940250374376774,0.03316382318735123,-0.06473660469055176,-0.03575532138347626,0.02234792895615101,0.053721752017736435,-0.025813359767198563,0.05189495161175728,-0.07428844273090363,-0.03972297161817551,0.0028070672415196896,0.0059154462069272995,0.055321287363767624,-0.013059244491159916,-0.028750240802764893,0.045916859060525894,-0.08912508189678192,-0.07503333687782288,-0.08136412501335144,-0.09989406168460846,0.047969140112400055,0.030486077070236206,-0.06377895176410675,-0.05410822108387947,-0.0712558925151825,-0.15985576808452606,0.021986648440361023,-0.020945070311427116,-0.07416938245296478,0.05355268344283104,-0.05585072934627533,0.0471927747130394,-0.07026440650224686,-0.06181159242987633,-0.0235532708466053,-0.008281287737190723,-0.07469543069601059,0.05967994034290314,-0.003456076141446829,0.023052195087075233,0.02528117224574089,0.07196027040481567,-0.013612710870802402,-0.006653777323663235,-0.016625365242362022,0.056113291531801224,0.016380244866013527,-0.0584968738257885,0.0430515855550766,0.0979948490858078,-0.026512160897254944,-0.018485501408576965,0.10531464964151382,-0.023290686309337616,-0.008156661875545979,-0.010834320448338985,-0.022554835304617882,0.2847576141357422,0.03145755082368851,0.04029485955834389,-0.062453076243400574,-0.01955312304198742,-0.018430978059768677,0.1133899986743927,0.040245238691568375,-0.08042082190513611,0.018078410997986794,-0.09007761627435684,-0.14068244397640228,-0.023823605850338936,0.04774472489953041,-0.17127162218093872,-0.07862036675214767,0.03975081071257591,-0.02902473509311676,0.08499030023813248,-0.05265885964035988,-0.05910017713904381,0.035165056586265564,0.04662507027387619,0.009460913948714733,0.07229430973529816,0.07062786817550659,-0.17484666407108307,-0.08732502907514572,-0.04080908000469208,0.19427195191383362,0.008751346729695797,0.07332862168550491,0.003354091430082917,0.03098013810813427,0.12072711437940598,0.08990078419446945,0.06598324328660965,-0.008433699607849121,-0.059642694890499115,0.06467042118310928,0.11811527609825134,0.10285884886980057,-0.0008984754676930606,-0.025592738762497902,0.008478526026010513,0.010851697064936161,-0.1481473743915558,-0.13501228392124176,0.06020117178559303,0.027205947786569595,0.03699994087219238,-0.07656927406787872,0.00027417796081863344,-0.0006337372469715774,0.05997048318386078,-0.08018846064805984,-0.02106172777712345,0.062447287142276764,0.030168548226356506,-0.01781625673174858,-0.018803240731358528,-0.09285153448581696,-0.0032992446795105934,0.13372351229190826,-0.04858216270804405,0.009260760620236397,-0.06739512085914612,0.07257064431905746,-0.10526350885629654,0.06175985932350159,-0.09908044338226318,0.03987409174442291,-0.06713319569826126,0.0235176682472229,0.023518254980444908,-0.023872563615441322,0.019401004537940025,0.09610581398010254,-0.05487233027815819,-0.09788287431001663,-0.07277685403823853,0.05345431715250015,-0.01124985609203577,-0.02857760153710842,0.003550267079845071,-0.03604387119412422,0.04909483343362808,0.031578924506902695,-0.03819426894187927,0.009438689798116684,-0.02835315279662609,0.12848787009716034,-0.033005598932504654,0.06061158701777458,0.13824991881847382,-0.1812799870967865,0.041739046573638916,0.02395465224981308,-0.020189298316836357,-0.004686642438173294,0.012345734052360058,0.08874909579753876,0.03714825212955475,-0.05095953494310379,-0.037495050579309464,0.0011862348765134811,0.0018253152957186103,0.005984244402498007,0.04794887825846672,0.043189648538827896,0.0378403402864933,-0.05111975967884064,-0.05210946872830391,-0.002038706559687853,0.034304309636354446,0.04994390159845352,0.10548526793718338,0.037638694047927856,-0.0017437211936339736,-0.012270916253328323,0.00853809155523777,-0.04510209336876869,-0.04751306027173996,-0.033652909100055695,-0.028408445417881012,0.01953277736902237,-0.06690485775470734,0.02003485895693302,0.07145912945270538,-0.12706129252910614,0.03491748869419098,-0.051371581852436066,0.059628117829561234,0.0975123792886734,-0.040250711143016815,-0.02286725491285324,0.05772180110216141,-0.03162458911538124,-0.04381345212459564,-0.027892181649804115,-0.11536787450313568,-0.10874523967504501,0.12279229611158371,0.0009155030711553991,0.0041448757983744144,-0.018055029213428497,-0.02821207046508789,-0.03254462033510208,-1.121383138524834e-05,0.0483001247048378,-0.10574430227279663,0.03661419823765755,0.01703287474811077,0.009764728136360645,-0.02690267190337181,-0.02890114299952984,-0.013404954224824905,0.09598648548126221,-0.07837353646755219,-0.005493479780852795,-0.10806433856487274,-0.020419545471668243,0.028124142438173294,0.004701950121670961,0.09719026833772659,0.02168557606637478,-0.01876034587621689,-0.017162462696433067,-0.017179280519485474,-0.10549481958150864,-0.002573520876467228,0.035322852432727814,-0.0207932461053133,-0.04558068886399269,0.06510257720947266,-0.031809620559215546,-0.0786668136715889,-0.08411254733800888,-0.01702902652323246,0.07046286761760712,-0.03545545041561127,-0.004409308545291424,0.029883794486522675,-0.04685355722904205,-0.10163886845111847,-0.081655353307724,-0.10297136008739471,0.061529386788606644,-0.10842765867710114,0.07541540265083313,0.06215574964880943,-0.06771424412727356,-0.04391816630959511,-0.025818461552262306,-0.1430021971464157,-0.10189560055732727,-0.035569824278354645,0.014599824324250221,-0.03151742368936539,-0.0591839998960495,-0.000937041244469583,-0.044916652143001556,-0.055445682257413864,0.006252349354326725,0.04189472645521164,0.11933870613574982,-0.003432296449318528,0.11352292448282242,-0.08622819930315018,-0.007438907865434885,0.056378088891506195,-0.01032248418778181,-0.10435564070940018,0.05975302681326866,0.04462471976876259,0.019357455894351006,-0.009487609378993511,0.09757574647665024,-0.012728952802717686,0.06328374147415161,0.11769873648881912,0.004590158816426992,0.04290885850787163,-0.03681119903922081,0.14674200117588043,-0.07468477636575699,0.06865419447422028,-0.021378638222813606,-0.04523572325706482,-0.08821356296539307,0.10714881867170334,0.005335439462214708,0.12401182949542999,-0.053860899060964584,-0.06261307746171951,-0.02962922863662243,-0.029142213985323906,0.0038243753369897604,0.05187428370118141,0.019098607823252678,0.018027735874056816,0.007697491906583309,-0.03798159584403038,-0.09868527203798294,0.16503065824508667,-0.06772638112306595,-0.11888045817613602,-0.046959199011325836,0.11244217306375504],"std":[0.18452055752277374,0.26947715878486633,0.17696960270404816,0.2935599088668823,0.23126181960105896,0.23899126052856445,0.23173372447490692,0.24111342430114746,0.27333444356918335,0.24253569543361664,0.20140235126018524,0.23971882462501526,0.2547174394130707,0.21779179573059082,0.31243279576301575,0.2016148716211319,0.21695299446582794,0.3321225643157959,0.19496436417102814,0.19217094779014587,0.2839459776878357,0.2119881808757782,0.20002858340740204,9.075529098510742,0.21817722916603088,0.22017157077789307,0.21336832642555237,0.23139911890029907,0.2339414358139038,0.2543708384037018,0.22856396436691284,0.3168531656265259,0.1814989596605301,0.24428226053714752,0.22443565726280212,0.29224342107772827,0.2131371647119522,0.21845442056655884,0.23766767978668213,0.2483827769756317,0.20206589996814728,0.21961848437786102,0.2035757452249527,0.2554640769958496,0.31228941679000854,0.21421420574188232,0.19734500348567963,0.20038816332817078,0.3080374002456665,0.2209261655807495,0.18289922177791595,0.20420433580875397,0.18757079541683197,0.18798062205314636,0.19458448886871338,0.24238665401935577,1.4354268312454224,0.20678801834583282,0.2368042767047882,0.19907377660274506,0.21641813218593597,0.21339084208011627,0.2409801036119461,0.20393803715705872,0.19204983115196228,0.2459239959716797,0.19800923764705658,0.25297972559928894,0.22264569997787476,0.25707536935806274,0.21853141486644745,0.40460205078125,0.19291433691978455,0.20606133341789246,0.2194196879863739,0.22498145699501038,0.20337925851345062,0.19507770240306854,0.3042912185192108,0.18804925680160522,0.28144606947898865,0.239625483751297,0.18364617228507996,0.2924288809299469,0.22024191915988922,0.21334736049175262,0.20723438262939453,0.19914257526397705,0.19163089990615845,0.23941881954669952,0.2542867362499237,0.2365441918373108,0.20705735683441162,0.20785439014434814,0.22175991535186768,0.23153828084468842,0.22041258215904236,0.20014257729053497,0.7028789520263672,0.20189213752746582,0.16910232603549957,0.20911172032356262,0.2335474193096161,0.22363588213920593,0.19293877482414246,0.252828985452652,0.21340996026992798,0.31757453083992004,0.21673224866390228,0.23375654220581055,0.19301658868789673,0.6296444535255432,0.23770293593406677,0.2112986445426941,0.20542016625404358,0.21941761672496796,0.20323973894119263,0.23406246304512024,0.24705930054187775,0.17643925547599792,0.2652023732662201,0.21978534758090973,0.31499534845352173,0.23216718435287476,0.21492090821266174,0.2059594988822937,0.22431297600269318,0.21207895874977112,0.1906844526529312,0.25957414507865906,0.18567781150341034,0.21912533044815063,0.2310628443956375,0.23725134134292603,0.2018740177154541,0.20882785320281982,0.24745041131973267,0.2844841182231903,0.21197034418582916,0.2054908275604248,0.2002648264169693,0.2149198353290558,0.245056614279747,0.2070097029209137,0.20732423663139343,0.24187156558036804,0.21750107407569885,0.20931607484817505,0.40150153636932373,0.2291906327009201,0.19933603703975677,0.20812782645225525,0.26136311888694763,0.21272040903568268,0.29384201765060425,0.2532898187637329,10.52813720703125,2.5515363216400146,0.19611461460590363,0.21889281272888184,0.2297179400920868,0.3031320571899414,0.25424543023109436,0.25781556963920593,0.17936913669109344,0.25029852986335754,0.19476936757564545,0.28241217136383057,0.22062039375305176,0.2480250746011734,0.2674688398838043,0.27806735038757324,0.20646926760673523,0.33604156970977783,0.27955499291419983,0.23164625465869904,0.2813645601272583,0.19978387653827667,0.2020753175020218,0.24117448925971985,0.2603931427001953,0.21875636279582977,0.17696413397789001,0.24908794462680817,0.2081281691789627,0.21971367299556732,0.39708244800567627,0.24092191457748413,0.1899596005678177,0.3172948956489563,0.2022017538547516,0.21746741235256195,0.19291217625141144,0.315303236246109,0.23267747461795807,0.2377985268831253,0.1863061636686325,0.23940733075141907,0.25754716992378235,0.1909789741039276,0.19688278436660767,0.2671351730823517,0.1871344894170761,0.20831067860126495,0.20221464335918427,0.23638422787189484,0.19039307534694672,0.2001427263021469,0.23179991543293,0.18469180166721344,0.18566124141216278,0.1796850860118866,0.23359674215316772,0.22160860896110535,0.25577229261398315,0.24697133898735046,0.20097775757312775,0.19603413343429565,0.20821993052959442,0.20452779531478882,0.1995067000389099,0.2138700783252716,0.20682013034820557,0.25180643796920776,0.19914771616458893,0.23468121886253357,0.3719784617424011,0.25532427430152893,0.28846824169158936,0.2401706427335739,0.2090015858411789,0.8356143832206726,0.20314307510852814,0.23174433410167694,0.24913881719112396,0.20240771770477295,0.23991334438323975,0.21093301475048065,0.2555926442146301,0.19255058467388153,0.2914949357509613,0.235343798995018,0.29918524622917175,0.2006090134382248,0.2866419553756714,0.16768930852413177,0.2119992971420288,0.19418777525424957,0.7889791131019592,0.22623693943023682,0.1968315690755844,0.21964381635189056,0.24922612309455872,0.2151598185300827,0.20417878031730652,0.28372299671173096,0.2117777168750763,0.22028307616710663,0.27196168899536133,0.22070163488388062,0.20591729879379272,0.2157745510339737,0.22974388301372528,0.20935721695423126,0.22356197237968445,0.20387078821659088,0.28508955240249634,0.22598382830619812,0.2286682277917862,0.24516505002975464,0.21153351664543152,0.32741981744766235,0.1931663155555725,0.21157731115818024,0.21609200537204742,0.2354276329278946,0.4072260558605194,0.2882888913154602,0.2245723009109497,0.19253239035606384,0.2122560292482376,0.29023176431655884,0.24917244911193848,0.2126334011554718,0.2181096225976944,0.20323456823825836,0.2675485908985138,0.2313818335533142,0.2526787221431732,0.20427431166172028,0.2744743824005127,0.47956255078315735,0.21597127616405487,0.23446223139762878,0.23949141800403595,0.20018163323402405,0.2230815589427948,0.27349647879600525,0.20907191932201385,0.24521522223949432,0.18019132316112518,0.19825351238250732,0.20797641575336456,0.2860688269138336,0.24870328605175018,0.2889850437641144,0.21195603907108307,0.2543780207633972,0.18473948538303375,0.3543197214603424,0.2480720430612564,0.21693867444992065,0.2568419277667999,0.22252504527568817,0.24351419508457184,0.23522824048995972,0.2124810665845871,0.21529120206832886,0.20002403855323792,0.21814951300621033,0.24784409999847412,0.19363798201084137,0.22891347110271454,0.20815570652484894,0.20710860192775726,0.19882680475711823,0.7394145131111145,0.22888368368148804,0.23344282805919647,0.21313250064849854,0.20367789268493652,0.22783730924129486,0.2216266393661499,0.5137009620666504,0.18367847800254822,0.20139813423156738,0.20611554384231567,0.20070475339889526,0.20813636481761932,0.21056325733661652,0.2030452936887741,0.23241792619228363,0.279680073261261,0.2739783525466919,0.20959079265594482,0.20131279528141022,0.2193034142255783,0.2774178981781006,0.19210326671600342,0.2220914214849472,0.21637484431266785,0.22721485793590546,0.24366898834705353,0.21704551577568054,0.2066064476966858,0.2357364147901535,0.18044985830783844,0.18731138110160828,0.27795329689979553,0.19301238656044006,0.2315421998500824,0.18505370616912842,0.20791587233543396,0.19233064353466034,0.2665739059448242,0.21920587122440338,0.27854713797569275,0.21844419836997986,0.20468877255916595,0.21889226138591766,0.19244040548801422,0.2044973522424698,0.19750361144542694,0.2959946095943451,0.21600358188152313,0.1868363916873932,0.6317494511604309,0.2107512652873993,0.2239508330821991,0.21913325786590576,0.18051062524318695,0.19674421846866608,0.20416346192359924,0.22137148678302765,0.26666682958602905,0.22056463360786438,0.1985762119293213,0.20918230712413788,0.20550212264060974,0.18117758631706238,0.20496056973934174,0.22102612257003784,0.25853562355041504,0.2117844820022583,0.21109524369239807,0.26071444153785706,0.18340511620044708,0.22034373879432678,0.22956258058547974,0.23794478178024292,0.20106184482574463,0.2196388691663742,0.22630178928375244,0.22341185808181763,0.25305286049842834,0.2894621193408966,0.2412867695093155,0.20915555953979492,0.2217331826686859,0.21296778321266174,0.3636191189289093,0.1950499415397644,0.19102810323238373,0.7663929462432861,0.28174689412117004,0.2129690796136856,0.26339101791381836,0.22629621624946594,0.24120129644870758,0.22503365576267242,0.20086154341697693,0.23130306601524353,0.1845819652080536,0.22000253200531006,0.2023378312587738,0.272574782371521,0.2657201886177063,0.21861223876476288,0.19685165584087372,0.24811366200447083,0.22159749269485474,0.20335301756858826,0.23024152219295502,0.26363056898117065,0.21165607869625092,0.20863981544971466,0.2925303876399994,0.18918557465076447,0.20071150362491608,0.2794308662414551,0.21200811862945557,0.22159931063652039,0.22338710725307465,0.2029385268688202,0.1972126066684723,0.18848073482513428,0.16926461458206177,0.2361871600151062,0.19644516706466675,0.3145204782485962,0.21092568337917328,0.23214805126190186,0.23438474535942078,0.24591070413589478,0.19874981045722961,0.24312348663806915,0.20591603219509125,0.2030363827943802,0.18711525201797485,0.20368385314941406,0.23557817935943604,0.23624150454998016,0.19753749668598175,0.20253948867321014,0.21967114508152008,0.23402605950832367,0.1787538379430771,0.18477322161197662,0.23035846650600433,0.21343855559825897,0.20060883462429047,0.20835351943969727,0.23424075543880463,0.18910746276378632,0.24988363683223724,0.23969708383083344,0.19764232635498047,0.18897026777267456,0.31741446256637573,0.2056494802236557,0.23550274968147278,0.27263787388801575,0.2502981722354889,0.19818437099456787,0.2090001404285431,0.2922528088092804,0.1851174384355545,0.21823257207870483,0.19701248407363892,0.23364609479904175,0.20376454293727875,0.23101629316806793,0.23979009687900543,0.24372220039367676,0.21749155223369598,0.23234592378139496,0.29413077235221863,0.21995267271995544,0.2150447964668274,0.2446567714214325,0.17410314083099365,0.2543410062789917,0.18268847465515137,0.18606168031692505,0.22189749777317047,0.186330646276474,0.20999670028686523,0.5835283398628235,0.2017059028148651,0.2111358791589737,0.2295861691236496,0.2938460409641266]}},"3":{"inputs":{"mean":[0.03573684021830559,0.054998598992824554,-0.06573458760976791,0.2555110454559326,-0.0005656291032209992,-0.1171666830778122,-0.14442408084869385,0.08768609911203384,0.008348872885107994,-0.006596616469323635,0.24013729393482208,0.07301709800958633,-0.06217075139284134,0.10348450392484665,0.08127160370349884,-0.029351016506552696,0.16682371497154236,-1.0163384675979614,-0.05299648270010948,-0.3756678104400635,0.07460998743772507,0.007935351692140102,0.03164006397128105,0.865994930267334,0.08751223236322403,0.232021301984787,0.08753541111946106,0.06203757971525192,0.04817407950758934,-0.1151236966252327,-0.15541677176952362,0.031046956777572632,-0.21483708918094635,-0.3666970133781433,0.2722565531730652,-0.37051263451576233,-0.14496582746505737,0.061542656272649765,-0.1423237919807434,0.1579832285642624,0.08917998522520065,-0.0701909139752388,-0.047023508697748184,0.0183073952794075,0.11844909936189651,-0.007856449112296104,0.05902990698814392,0.3537117540836334,0.015327501110732555,-0.08533768355846405,-0.01481049694120884,-0.16726195812225342,0.29607686400413513,0.26120227575302124,0.13719169795513153,0.4728109538555145,-6.519637584686279,0.22988134622573853,0.07791328430175781,0.3514683246612549,-0.03766731172800064,0.0813743844628334,-0.25213712453842163,0.034739118069410324,-0.13968010246753693,0.26926231384277344,0.06960304081439972,0.25979092717170715,0.042113255709409714,-0.06299889832735062,0.06192615628242493,1.139083981513977,-0.3682553172111511,0.014214464463293552,-0.11485972255468369,0.05747770518064499,-0.15857969224452972,0.12181056290864944,0.014784976840019226,0.1290367841720581,-0.37087562680244446,-0.027135640382766724,0.10589867830276489,-0.14432178437709808,0.1795024871826172,0.17872436344623566,0.14166054129600525,0.4648146629333496,-0.02983533777296543,-0.4303697943687439,-0.10125336796045303,0.09297580271959305,-0.15592364966869354,-0.08385501056909561,-0.19306237995624542,-0.15124981105327606,0.06439189612865448,0.11987440288066864,-0.09838936477899551,0.3320901393890381,-0.015523066744208336,0.12042191624641418,0.01755955070257187,0.17881368100643158,-0.20496158301830292,-0.011731207370758057,-0.013527762144804,0.14198768138885498,-0.06518609076738358,-0.0062583331018686295,0.1696409434080124,0.18244053423404694,-0.39800530672073364,0.014880131930112839,-0.09180670976638794,-0.06945385038852692,0.024133354425430298,0.2133689820766449,-0.14256595075130463,0.09369783103466034,-0.05469490960240364,0.10673645883798599,-0.08047249913215637,0.13258576393127441,-0.02888770028948784,0.03557461127638817,0.016771428287029266,-0.04263441637158394,0.17022748291492462,0.24538761377334595,0.05376926437020302,0.433290958404541,-0.02510472573339939,0.038437619805336,-0.024778258055448532,-0.08742807060480118,-0.04449674114584923,0.05430319160223007,-0.06579110026359558,-0.11082769185304642,-0.03079853020608425,-0.13758328557014465,-0.24807694554328918,0.29254209995269775,0.2598912715911865,0.5894192457199097,0.09658762067556381,0.004133583512157202,-0.33323490619659424,-0.038867298513650894,0.29045242071151733,0.1709962785243988,0.011140424758195877,0.9688248038291931,0.10021355003118515,0.14275506138801575,5.5769782066345215,-0.059167079627513885,-0.029633330181241035,-0.12372927367687225,-0.09191293269395828,-0.1038915142416954,0.12476182729005814,0.06664811819791794,0.08200995624065399,0.24015037715435028,-0.2619188725948334,0.16969767212867737,-0.03403846174478531,0.1429983377456665,-0.14641781151294708,0.18783797323703766,0.030769042670726776,0.10547491163015366,0.21559664607048035,0.04661513864994049,-0.04846124351024628,0.15675346553325653,-0.07574712485074997,-0.1632705181837082,0.10532548278570175,-0.23411566019058228,0.1611461490392685,0.15735848248004913,0.18924713134765625,0.10136085748672485,-1.271185040473938,0.2648125886917114,0.032728612422943115,0.015863291919231415,-0.20627568662166595,0.11753228306770325,0.08824925124645233,-0.11656385660171509,0.14736731350421906,0.09415777027606964,0.21466860175132751,0.0900261327624321,-0.07007406651973724,-0.22003959119319916,0.039677754044532776,-0.12162057310342789,0.18853095173835754,-0.0551428347826004,-0.2136782556772232,0.18159763514995575,-0.002667952561751008,0.16995300352573395,-0.0630008652806282,0.02661779522895813,0.1034340113401413,-0.3327435851097107,0.04021172598004341,-0.0764261856675148,-0.04145652800798416,0.11700259149074554,0.10264448821544647,0.018231486901640892,0.32876765727996826,-0.008113224059343338,0.10409215092658997,-0.04460982233285904,0.0153152234852314,-0.10301139950752258,0.1811099797487259,0.12183677405118942,0.0049785440787673,-0.03490819036960602,0.4493227005004883,-0.12129721790552139,0.07137969136238098,-0.38056716322898865,0.0588248074054718,0.03814408555626869,0.02391509711742401,0.10070667415857315,0.12291758507490158,0.02701900154352188,0.12671761214733124,0.08083222806453705,-1.5685908794403076,0.06883040070533752,-0.12549711763858795,-0.06324443966150284,0.19722051918506622,0.026081379503011703,-0.10475140810012817,-0.2507408857345581,-0.7150213122367859,-0.19893625378608704,-0.26107966899871826,0.2770673632621765,0.10197404772043228,0.03249271959066391,-0.028881939128041267,0.10878731310367584,-0.041701868176460266,0.08982037007808685,0.0370040237903595,-0.12596601247787476,0.04139326140284538,0.15932467579841614,-0.11101867258548737,0.49412837624549866,-0.10422397404909134,0.06474142521619797,-0.06339358538389206,-0.13487879931926727,-0.143838033080101,0.19573065638542175,0.07657577097415924,-0.035239506512880325,0.08959890902042389,-0.07978052645921707,-0.10648472607135773,-0.0435209684073925,-1.4658676385879517,-0.03506356105208397,0.0325835645198822,0.002054926473647356,0.1950290948152542,-0.03559422865509987,0.2756461799144745,0.32469606399536133,-0.09299865365028381,0.110136017203331,0.2340879887342453,-0.2154621183872223,-0.09117963165044785,0.11950783431529999,0.18220339715480804,0.4254622459411621,0.33515289425849915,0.11600688099861145,0.5220767259597778,0.29769113659858704,0.058475371450185776,0.024063169956207275,0.17980153858661652,-0.20490692555904388,0.062020450830459595,0.1204925924539566,-0.0629204660654068,-0.10726092010736465,0.12853574752807617,-0.07995683699846268,0.06143694743514061,0.12723542749881744,0.15081240236759186,0.1222037523984909,0.0835985615849495,0.1880495548248291,0.002693889429792762,0.014688205905258656,-0.10482791811227798,0.3464255630970001,0.11342228949069977,-0.3717806935310364,-0.21393543481826782,-0.17683306336402893,-0.07498199492692947,0.045629121363162994,0.10667017847299576,-0.047295261174440384,0.13422946631908417,-0.11559245735406876,0.7885254621505737,-0.0305526964366436,-0.026167618110775948,0.12460485845804214,0.04059399664402008,0.00884273648262024,-0.006581668742001057,0.06915510445833206,-0.42360448837280273,-0.040526214987039566,-0.10129635781049728,0.026613906025886536,0.07426425814628601,0.032932162284851074,-0.13334061205387115,-0.1536223441362381,0.14510168135166168,-0.19034968316555023,0.0832216665148735,-0.03495020046830177,0.011017290875315666,-0.03797142207622528,0.010537887923419476,-0.10091731697320938,0.11363133788108826,0.1797078549861908,0.04443657025694847,0.024433501064777374,0.01686137355864048,-0.1338171660900116,0.053170278668403625,-0.1950896829366684,-0.057240962982177734,0.021292150020599365,0.12530609965324402,0.14409790933132172,-0.33297204971313477,0.2106352150440216,-0.07529012113809586,-0.3640159070491791,0.1320226639509201,-0.0803702250123024,0.05890783667564392,0.025086920708417892,-0.129534512758255,-0.046805836260318756,0.14950799942016602,0.4535326659679413,0.055314067751169205,-0.013955741189420223,-3.078688144683838,0.08688787370920181,0.06853066384792328,0.3265678286552429,0.12205257266759872,-0.2229367196559906,0.2071746438741684,0.1578833907842636,-0.35154950618743896,0.09336762130260468,0.02828557975590229,0.2305661141872406,-0.2386322319507599,0.01578599028289318,-0.3089076578617096,-0.05011182278394699,0.11925889551639557,0.023862242698669434,-0.027120908722281456,-0.12426937371492386,0.16752707958221436,-0.18000926077365875,-0.19838903844356537,0.09197013825178146,-0.1439211070537567,0.1128002181649208,-0.04019664600491524,-0.2844119369983673,-0.2707338035106659,-0.22665069997310638,0.15567965805530548,-0.2468242049217224,-0.024978676810860634,-0.06413445621728897,0.32259485125541687,-0.05643536522984505,0.15582506358623505,-0.20686306059360504,0.0748259499669075,0.03604865446686745,0.3323333263397217,-0.2141014188528061,-0.40580296516418457,0.1462731659412384,0.23360727727413177,0.13423781096935272,-0.12011764943599701,-0.03599542751908302,-0.080148845911026,0.23256602883338928,-0.018915414810180664,-0.027515336871147156,-0.1613297164440155,-0.041936010122299194,0.03842896595597267,-0.19992706179618835,0.1490650177001953,0.27146339416503906,-0.07217300683259964,0.2952374815940857,0.29617971181869507,1.2332651615142822,0.17669092118740082,-0.16007663309574127,-0.22831867635250092,0.013813866302371025,0.214023157954216,0.27684488892555237,0.15803921222686768,-0.27182066440582275,-0.02370765246450901,0.2157142162322998,-0.011330035515129566,0.10126044601202011,0.008313379250466824,-0.27684155106544495,-0.17127661406993866,0.1790924221277237,0.02853584662079811,-0.12328223139047623,0.0825861319899559,-0.06976964324712753,-0.25363972783088684,0.08871206641197205,-0.032981157302856445,0.21449854969978333,-0.05909040570259094,0.17625756561756134,0.17516940832138062,-0.09245848655700684,-0.01967806927859783,0.03625275194644928,0.058768924325704575,0.4228954315185547,-0.09703226387500763,-0.09972898662090302,-0.0514296256005764,-0.07063473016023636,-0.2923041880130768,0.14680853486061096,-0.128237783908844,0.08503708243370056,-4.151346206665039,-0.020468058064579964,-0.34554216265678406,0.21886605024337769,-0.11989018321037292,0.0020048210863023996,0.09006573259830475,0.36177965998649597,-0.004498750437051058,0.19903074204921722,0.13373412191867828,-0.2011965811252594,-0.04021347314119339,-0.1606001853942871,-0.11084673553705215,0.12025118619203568,0.035419173538684845,0.2448146641254425,-0.40048930048942566,0.02935931272804737,0.21118775010108948,0.2116999477148056,0.016932690516114235,-0.10956323146820068,-0.07616451382637024,0.15294800698757172,-0.006292202044278383,-0.23728974163532257,0.1147255226969719,-0.2190093696117401,-0.15414871275424957,0.10291469097137451,-0.058066848665475845,0.061875976622104645],"std":[0.8442214131355286,0.8451614379882812,0.8377705812454224,0.9785271286964417,0.9025105237960815,0.9690312743186951,1.0707457065582275,1.0548211336135864,1.0045260190963745,0.9039188027381897,0.898169755935669,1.0781869888305664,0.9853212237358093,0.9080445766448975,0.7847074866294861,0.9898229837417603,0.8617997169494629,3.045088768005371,0.8494731783866882,0.9929477572441101,0.8768638968467712,1.0331847667694092,0.9503677487373352,2.2216079235076904,1.038055419921875,0.9101426005363464,1.0503528118133545,0.9056141376495361,0.8860327005386353,1.1169934272766113,0.9815892577171326,0.9708013534545898,0.9507979154586792,1.480648398399353,0.9865104556083679,0.9412873387336731,1.0434386730194092,0.825886607170105,0.909549355506897,0.9704146385192871,0.9126665592193604,0.9406004548072815,0.7844940423965454,1.0340800285339355,0.991294801235199,0.9031311273574829,0.9570291042327881,1.183671474456787,1.0233594179153442,0.8991992473602295,0.9217062592506409,0.8972229361534119,1.0555522441864014,0.9587917327880859,0.9264613389968872,1.0643669366836548,2.7820160388946533,0.9185842871665955,0.8214908242225647,0.8872732520103455,0.848314106464386,0.9816380739212036,1.0975147485733032,0.9570120573043823,0.8951094150543213,0.9482239484786987,0.8754807710647583,0.8958288431167603,0.8445425629615784,0.8973540663719177,0.988558292388916,2.807441473007202,1.3929909467697144,0.9927304983139038,1.056077003479004,0.9471096396446228,0.9242474436759949,0.8491153120994568,1.0254426002502441,0.9109904170036316,1.2329554557800293,1.0070821046829224,0.8608478903770447,1.0275906324386597,0.8860971331596375,0.9052688479423523,0.9108537435531616,0.9304549694061279,0.8038559556007385,0.8245899081230164,0.8448077440261841,0.9071756601333618,0.8443848490715027,0.8990253210067749,0.872933030128479,1.0004312992095947,0.8329133987426758,0.983395516872406,2.3418350219726562,1.0143409967422485,0.8280033469200134,0.8719550967216492,1.013290286064148,0.9879148006439209,1.1443315744400024,0.9069852232933044,0.8419381380081177,0.9846515655517578,1.018065094947815,0.8949334621429443,0.842337429523468,1.800683617591858,1.0599713325500488,0.8881281614303589,0.8921756148338318,1.1777032613754272,0.984559178352356,0.876253604888916,1.0485059022903442,0.7359079122543335,0.8447932600975037,0.9979054927825928,1.0541338920593262,1.0512559413909912,0.8283689022064209,1.6990102529525757,1.0310245752334595,1.0057326555252075,0.8658533692359924,1.2402368783950806,0.806724488735199,0.9712974429130554,0.8586634397506714,1.161738634109497,0.8948178887367249,0.8599820137023926,0.8677073121070862,0.9944126605987549,0.964409589767456,0.9669989943504333,1.0981539487838745,0.9030829668045044,0.9365464448928833,0.9054233431816101,0.8519918918609619,1.2052091360092163,0.9258749485015869,0.9545373320579529,1.3885308504104614,0.9776242971420288,0.9750204682350159,0.9949969053268433,0.9926940202713013,1.8874728679656982,0.9048312902450562,0.9157795906066895,1.7150441408157349,1.3687095642089844,0.8339972496032715,0.9583812355995178,0.9917343258857727,1.1077924966812134,0.8278170824050903,0.9015402793884277,0.8177856206893921,1.1602214574813843,0.8284336924552917,0.9410032033920288,1.140742301940918,1.0328497886657715,1.0009422302246094,0.8728091716766357,0.8577582836151123,0.8608652353286743,0.8551994562149048,0.8760334849357605,1.0505762100219727,0.7642818093299866,0.8722259998321533,0.9717898368835449,0.8304470777511597,1.0343809127807617,0.8274164199829102,1.027885913848877,0.8409410119056702,0.9542762637138367,1.9332976341247559,0.8283385038375854,0.7946311831474304,1.0440701246261597,0.9614737033843994,0.9404608607292175,0.906187891960144,1.0937198400497437,1.0103708505630493,0.866126537322998,0.8829307556152344,1.1042927503585815,1.011950135231018,0.8661941289901733,0.9286736249923706,0.9030402302742004,0.9391880631446838,0.9199256896972656,1.0151379108428955,0.9508505463600159,0.9137775301933289,0.9475970268249512,0.9583606719970703,0.8006908893585205,0.9317656755447388,0.9050027132034302,1.0336204767227173,1.0298042297363281,0.8015474677085876,0.993703305721283,0.9402888417243958,0.8676975965499878,1.0189439058303833,0.7880055904388428,0.8771209120750427,0.8402633666992188,0.861484706401825,0.8873941898345947,0.9409782886505127,0.8318496942520142,1.061558485031128,0.9436801075935364,0.8852136731147766,1.1475857496261597,1.020114541053772,1.7716152667999268,0.9824998378753662,1.0466445684432983,0.9929006695747375,1.0730987787246704,0.9604072570800781,1.0303728580474854,0.9981140494346619,0.8122333288192749,2.8191137313842773,0.8633884787559509,1.0611718893051147,0.9486464858055115,0.8457894325256348,0.710966944694519,0.9526529312133789,0.8944557905197144,1.7918765544891357,1.0865225791931152,0.7617331743240356,1.065630555152893,0.9469073414802551,0.9463185667991638,0.8735187649726868,0.8789671063423157,0.9637361168861389,0.9322402477264404,0.9334011673927307,1.0483628511428833,0.8161255717277527,1.0946810245513916,0.9038478136062622,1.936727523803711,0.9752783179283142,0.9584788084030151,0.8976593613624573,0.9023935198783875,0.9945822358131409,1.0524104833602905,0.9440822005271912,2.1804306507110596,0.8949776291847229,0.7889434099197388,0.9433987140655518,1.1230822801589966,3.337664842605591,0.8625038266181946,0.9304686188697815,0.943744421005249,1.1485199928283691,0.9313879609107971,1.1831508874893188,0.9667524695396423,0.998258113861084,0.8674942255020142,1.475943922996521,0.9593634009361267,0.8961382508277893,0.99327152967453,0.8783560991287231,2.0896081924438477,0.940032958984375,0.8513943552970886,0.9666885733604431,0.9033336639404297,0.9880580902099609,0.8521226048469543,0.8818302154541016,1.015203833580017,0.8831265568733215,0.9789572954177856,0.9562327265739441,0.8796360492706299,0.8897767066955566,0.9712383151054382,0.9403125643730164,0.8660849332809448,0.9420942664146423,0.9276269674301147,0.8668395280838013,0.8348848819732666,1.0039764642715454,1.148683786392212,0.9626275897026062,0.9599934220314026,0.9197648167610168,0.8786379098892212,0.8290804028511047,0.9592369794845581,1.0512512922286987,1.063564419746399,0.8495090007781982,0.9148896336555481,0.9810458421707153,0.867030680179596,1.4532195329666138,1.0143290758132935,1.0455613136291504,0.8896602392196655,0.9797796010971069,0.9016855955123901,0.8778898119926453,0.9729127883911133,0.8479620218276978,0.8947709798812866,1.0082828998565674,0.7891546487808228,0.9418966770172119,0.9736369252204895,0.8222367763519287,0.8980550169944763,0.9541279673576355,0.9146493077278137,0.8890206217765808,0.8114356994628906,1.0010924339294434,0.8968501091003418,0.9216927289962769,0.8519335389137268,0.7616696953773499,0.9213059544563293,0.8413736820220947,1.0240230560302734,1.060339331626892,0.8070117831230164,0.8224892020225525,0.9265202879905701,1.0360594987869263,0.8731918931007385,0.9610698819160461,0.8733379244804382,1.0626124143600464,0.8789411783218384,0.8530135154724121,1.0538015365600586,0.9034944176673889,0.8150545954704285,1.081223487854004,1.6200484037399292,0.9195184707641602,0.8393932580947876,0.8202384114265442,1.1550445556640625,0.9397586584091187,0.7875452041625977,4.288115501403809,0.985155463218689,0.86375892162323,1.2103509902954102,0.8344260454177856,0.8995546698570251,0.8941749334335327,0.931842029094696,0.9300035238265991,0.8616327047348022,0.8453499674797058,0.9200338125228882,0.8468878269195557,0.8555176258087158,1.1161143779754639,0.980399489402771,0.867439329624176,0.8953157067298889,1.0090148448944092,0.8595530986785889,0.8540899753570557,0.7953268885612488,0.9315571784973145,0.8880949020385742,1.0897834300994873,0.9441341757774353,0.9649490714073181,1.031443476676941,0.9564733505249023,0.9313560724258423,0.8946598768234253,0.959801435470581,1.0506398677825928,1.0666109323501587,1.3958796262741089,0.9010088443756104,0.9238154292106628,2.4625701904296875,0.9827059507369995,0.9522054195404053,1.112531304359436,0.964773952960968,1.1914156675338745,1.2084122896194458,0.8803869485855103,0.9496801495552063,0.9225009083747864,1.1241360902786255,0.839162290096283,1.0294021368026733,1.1478277444839478,0.97989422082901,0.8454457521438599,0.8917062878608704,0.980089008808136,0.9292550683021545,0.8157872557640076,1.1214509010314941,1.048776626586914,0.8747810125350952,1.193474531173706,1.4958664178848267,0.9681840538978577,1.0451711416244507,1.1615865230560303,1.1549769639968872,1.134878158569336,0.8933972716331482,0.8466085195541382,0.8691505193710327,0.7501437664031982,0.9627565741539001,1.023957371711731,0.8196559548377991,0.9346698522567749,0.9765338897705078,0.8894429802894592,1.058086633682251,0.9112368822097778,1.062761902809143,1.0340203046798706,0.9130342602729797,1.0378481149673462,0.8795905709266663,1.0676299333572388,0.9087938070297241,0.8810932040214539,0.8742033839225769,0.8551775813102722,1.2015442848205566,0.7776622772216797,0.950516939163208,0.8869117498397827,1.121291160583496,0.9294102191925049,0.8726604580879211,0.8488003611564636,0.7984150052070618,0.9079813361167908,0.8323510885238647,0.9469248652458191,1.0243799686431885,4.741200923919678,0.8499985337257385,0.9796790480613708,0.9797505140304565,0.9510059356689453,0.8743205666542053,0.8925241231918335,0.953267514705658,0.8325154781341553,1.008860468864441,0.9239153265953064,0.9514459371566772,0.7710832953453064,0.8731249570846558,1.087904453277588,0.9617825746536255,0.8678908348083496,0.9209799766540527,1.2022221088409424,1.0877275466918945,0.8898099064826965,1.2085987329483032,0.9116409420967102,0.9410183429718018,0.8629544377326965,0.8620576858520508,0.8711367249488831,0.8284808397293091,1.0778285264968872,1.6315339803695679,0.929253339767456,0.9702932834625244,0.9595641493797302,1.0159964561462402]},"targets":{"mean":[0.1354474425315857,0.13852310180664062,0.006306031718850136,-0.15356943011283875,0.17228086292743683,-0.05824117362499237,0.012798216193914413,-0.11750564724206924,0.03542225435376167,-0.09728360921144485,-0.08221548050642014,0.1101718619465828,0.18797673285007477,-0.2259596288204193,0.20657336711883545,-0.042176369577646255,0.105360247194767,0.15138618648052216,0.07116137444972992,0.04328885301947594,0.101438008248806,0.04214667156338692,-0.1400890350341797,-0.46248239278793335,-0.04213548079133034,0.06160750240087509,0.05138473957777023,0.033514924347400665,-0.047861650586128235,0.12822897732257843,-0.1180434301495552,-0.21003295481204987,0.07828968018293381,-0.0023331502452492714,-0.13681693375110626,-0.14194931089878082,-0.04602024331688881,0.0716458410024643,0.1325952112674713,-0.11574877798557281,0.0027767943684011698,0.07387945801019669,0.019699955359101295,0.025565145537257195,-0.16229762136936188,-0.08343739062547684,-0.11356701701879501,-0.09306156635284424,-0.058359544724226,0.02203427627682686,-0.14745227992534637,0.1388181447982788,-0.09910065680742264,0.08145134896039963,-0.09669822454452515,-0.20537647604942322,0.7672778964042664,0.05261653661727905,0.1362549364566803,0.029536429792642593,-0.12148129940032959,-0.16821672022342682,-0.0832635909318924,-0.036788444966077805,0.10858239233493805,-0.08517804741859436,-0.018583856523036957,-0.09072211384773254,0.064865842461586,0.17829981446266174,-0.0017707636579871178,-0.3033759593963623,0.09347528964281082,0.37017616629600525,0.12290405482053757,-0.06219547241926193,0.20219704508781433,0.07940705120563507,-0.1595345139503479,-0.22011245787143707,-0.22449597716331482,-0.30889642238616943,0.10331574082374573,0.05228084698319435,-0.0828697457909584,-0.19906103610992432,0.0566755011677742,0.08270200341939926,0.048831548541784286,0.06454531103372574,-0.049953944981098175,-0.007891491055488586,0.06292212009429932,-0.06847089529037476,0.15511368215084076,0.10586752742528915,-0.19136467576026917,-0.000932711991481483,0.11993812769651413,-0.09642008692026138,0.14882805943489075,-0.0941864401102066,-0.0769762322306633,0.028750432655215263,-0.10414449125528336,0.037234049290418625,0.030827723443508148,-0.004799277056008577,0.1596480756998062,0.013713590800762177,-0.07866448163986206,-0.246403306722641,0.1725495308637619,-0.09507671743631363,0.009958738461136818,-0.08264638483524323,0.022062264382839203,-0.07290418446063995,0.13890451192855835,-0.09668966382741928,-0.07317043840885162,0.06455611437559128,-0.036619797348976135,0.12932410836219788,-0.07616478949785233,-4.5602897444041446e-05,-8.010742749320343e-05,0.002499503782019019,-0.25055670738220215,0.08225227147340775,-0.027029844000935555,-0.22223606705665588,0.041418273001909256,-0.09025725722312927,-0.09770969301462173,0.17033803462982178,0.0596998855471611,-0.29129695892333984,0.040301769971847534,0.1851804107427597,-0.10689950734376907,-0.04434502497315407,0.0006157449679449201,-0.12219631671905518,0.005247863940894604,-0.17133235931396484,0.008317392319440842,0.05102382227778435,0.06258811801671982,0.006745507940649986,-0.2337479591369629,0.0290993619710207,0.056301262229681015,-0.16835333406925201,-0.15202030539512634,-0.08624447882175446,-0.12968455255031586,-0.2384158819913864,0.1397477239370346,0.0011980715207755566,0.19969555735588074,0.050767913460731506,-0.12125576287508011,0.08977671712636948,-0.04889393970370293,-0.08324586600065231,0.08554555475711823,0.03178998827934265,-0.028943583369255066,0.01584337092936039,-0.13709542155265808,-0.23062817752361298,-0.23437464237213135,0.1020040512084961,-0.20408689975738525,-0.021659929305315018,-0.07799331098794937,-0.10260230302810669,-0.17213402688503265,0.15474186837673187,0.11247297376394272,0.021309319883584976,-0.05701730027794838,-0.06246346980333328,0.008882330730557442,-0.11474496871232986,0.08778902143239975,-0.12437108904123306,0.02092692255973816,0.44244301319122314,0.21504773199558258,0.03932148218154907,0.02633069083094597,0.08237089216709137,-0.01737375743687153,0.034215059131383896,-0.17892806231975555,-0.11639806628227234,0.17107468843460083,0.05066385492682457,0.10244036465883255,0.16407795250415802,-0.20873963832855225,-0.21579481661319733,-0.027876395732164383,0.06165250018239021,-0.10519883781671524,-0.047409217804670334,0.022424237802624702,-0.029594508931040764,-0.11109736561775208,0.18119341135025024,-0.07630109786987305,-0.015804314985871315,-0.2503705620765686,0.011012530885636806,0.03442970663309097,0.013337377458810806,-0.1191062182188034,0.197265625,0.045709915459156036,-0.05606400594115257,-0.1360105574131012,0.013173471204936504,-0.26399317383766174,-0.1067814826965332,-0.054702941328287125,0.2592597007751465,-0.11052191257476807,0.025287512689828873,-0.01907304860651493,0.12411709874868393,-0.13829074800014496,0.023813802748918533,-0.12490469962358475,0.0605383925139904,-0.22787992656230927,0.17279939353466034,0.12100467830896378,-0.27874475717544556,0.30408093333244324,-0.016071930527687073,-0.30553269386291504,0.02103251963853836,0.024325357750058174,0.018934978172183037,0.101106658577919,0.14660638570785522,0.09294738620519638,0.083139568567276,-0.08101479709148407,0.08333145081996918,-0.12371429800987244,-0.05980575829744339,0.1050645187497139,0.04284234717488289,-0.1656225621700287,-0.03204212337732315,-0.09593427181243896,-0.011690545827150345,-0.0516035333275795,-0.22027240693569183,0.15732331573963165,-0.1345135122537613,0.0755152478814125,-0.06689021736383438,-0.003339828457683325,0.032680485397577286,0.2190670371055603,0.1536971777677536,-0.07635490596294403,0.11720294505357742,-0.07309488952159882,0.49342796206474304,0.2148931473493576,-0.11785676330327988,0.28014761209487915,-0.17780590057373047,-0.009186794981360435,0.08067184686660767,0.03421636298298836,-0.23627221584320068,-0.23110495507717133,0.27409470081329346,0.10957249253988266,-0.09130975604057312,-0.13616235554218292,0.08554176986217499,-0.06734365969896317,0.16830924153327942,-0.056346721947193146,-0.11028590798377991,-0.041111480444669724,-0.17191942036151886,0.08562570065259933,0.1951807737350464,-0.08210037648677826,-0.07666901499032974,0.036091722548007965,-0.04423442482948303,0.0875987783074379,-0.12351542711257935,0.14081133902072906,0.1887565553188324,0.24413342773914337,-0.03212383762001991,-0.07753097265958786,-0.20779669284820557,-0.12426608800888062,-0.11064331233501434,0.023286079987883568,-0.03399958088994026,-0.001920133363455534,-0.1666669100522995,0.10108684748411179,0.02196134440600872,-0.092327781021595,0.1490636169910431,-0.015400208532810211,0.01821596920490265,-0.0966804251074791,-0.08587007969617844,0.02820407599210739,0.021376000717282295,-0.058139581233263016,0.13249877095222473,-0.07488133013248444,0.22275137901306152,0.009269780479371548,0.02945936843752861,0.18856669962406158,-0.2270219326019287,-0.052803508937358856,-0.0028295472729951143,-0.012614824809134007,0.02934648096561432,-0.004435829818248749,0.05728333815932274,-0.018031872808933258,0.006565020885318518,0.08466490358114243,-0.04286007210612297,-0.00393356429412961,0.01750868931412697,-0.3519185483455658,0.08540632575750351,-0.07043755054473877,0.022355567663908005,0.0025365666951984167,-0.06402900069952011,0.1738990843296051,0.15062952041625977,-0.19065791368484497,-0.10309332609176636,0.059898741543293,-0.1845177710056305,0.11401360481977463,-0.0013450143160298467,0.12878304719924927,-0.03466695174574852,0.05740348622202873,-0.023683680221438408,0.11774994432926178,-0.05261802300810814,0.05378449708223343,0.2491377741098404,-0.18282781541347504,-0.1056145578622818,-0.02837345562875271,-0.02372913435101509,-0.01461399719119072,0.0970112755894661,-0.15010975301265717,-0.21094679832458496,0.11138688772916794,0.18682962656021118,0.13688811659812927,0.03705192357301712,-0.06000945717096329,-0.031313855201005936,0.06770186126232147,0.07298590242862701,-0.394724577665329,-0.08823637664318085,-0.20682372152805328,-0.1118636354804039,-0.3236972391605377,0.08465342968702316,0.053084101527929306,0.1696254014968872,0.04181821271777153,-0.049026377499103546,-0.06385549157857895,-0.08620606362819672,-0.03843848779797554,-0.03373197838664055,0.07323189824819565,-0.1544136106967926,0.15339915454387665,0.1318913698196411,-0.05223875492811203,-0.0875680148601532,0.19633586704730988,0.12361679971218109,0.18014486134052277,0.05327689275145531,-0.041253283619880676,0.20509974658489227,0.1834656000137329,0.08152318745851517,-0.2133697122335434,-0.006326436996459961,-0.09817387163639069,0.07695945352315903,0.16897419095039368,-0.054080668836832047,0.020051509141921997,0.043987542390823364,0.09060091525316238,0.005397854372859001,-0.06512975692749023,0.10241059213876724,0.0664445012807846,-0.03605617955327034,0.08198042958974838,-0.1053701713681221,0.03906998038291931,-0.034661803394556046,0.1178644672036171,-0.01793823577463627,-0.09830517321825027,-0.13145817816257477,-0.16707287728786469,-0.051308147609233856,0.1144559383392334,-0.013524012640118599,-0.15281927585601807,-0.3270733952522278,-0.004634228069335222,0.09267190843820572,0.08917902410030365,0.040682267397642136,-0.14716991782188416,-0.02829652465879917,0.05176467075943947,0.09775663912296295,0.12953773140907288,-0.28217223286628723,0.07212215662002563,0.13355335593223572,-0.0727965459227562,0.16967540979385376,-0.14044030010700226,0.04485883563756943,0.08973906934261322,0.12037015706300735,-0.037629831582307816,0.07942033559083939,0.01670265942811966,0.030796732753515244,0.04535085707902908,-0.07568837702274323,0.20758214592933655,-0.25559359788894653,-0.0714561939239502,-0.173729807138443,0.18243944644927979,0.14853034913539886,-0.05816897004842758,-0.12465360015630722,0.08666332066059113,-0.12214649468660355,0.02343231812119484,0.08871805667877197,-0.09735536575317383,-0.06459912657737732,0.11168088763952255,0.015691176056861877,0.3089295029640198,0.03754251450300217,0.14211659133434296,0.1502729058265686,0.0837692990899086,0.00832881685346365,-0.26891985535621643,0.02323790453374386,-0.05321655422449112,0.005278836470097303,0.1621152013540268,0.06104494631290436,0.15946882963180542,0.1401568055152893,0.030490456148982048,0.04486043378710747,0.0002632250252645463,-1.4722927517141216e-05,0.15008802711963654,-0.07298534363508224,0.022791612893342972,0.018978619948029518,0.3978983461856842,-0.0012909239158034325,-0.09099887311458588,-0.015555966645479202,0.03430972993373871,0.031289562582969666,-0.018639756366610527,0.1902633011341095,0.023654283955693245,0.2112269103527069,0.052250828593969345,-0.0131595553830266],"std":[0.2695806324481964,0.43643617630004883,0.26831284165382385,0.3483780026435852,0.41825997829437256,0.40847066044807434,0.42086929082870483,0.3798539936542511,0.3198370933532715,0.39549094438552856,0.29720091819763184,0.2715364694595337,0.31302914023399353,0.41121169924736023,0.31276601552963257,0.31997284293174744,0.4442138969898224,0.7433618307113647,0.2675473093986511,0.2919885218143463,0.30441755056381226,0.3091874122619629,0.2819403409957886,0.9051080942153931,0.3680773079395294,0.4515374004840851,0.3797571063041687,0.3535038232803345,0.38421258330345154,0.3373183310031891,0.40548625588417053,0.3741329610347748,0.2707899212837219,0.20840533077716827,0.2774010896682739,0.34052374958992004,0.37189120054244995,0.27500343322753906,0.29135435819625854,0.36450818181037903,0.3100172281265259,0.3725738823413849,0.31855344772338867,0.34769144654273987,0.3402951955795288,0.3534243702888489,0.3477194905281067,0.34881123900413513,0.3412509262561798,0.34662145376205444,0.34792783856391907,0.29142051935195923,0.2786393463611603,0.2656008005142212,0.39738699793815613,0.3615056872367859,0.5538997650146484,0.2724302113056183,0.2729562819004059,0.4053536355495453,0.27006903290748596,0.3369871973991394,0.33586403727531433,0.3320969343185425,0.27178525924682617,0.3239951431751251,0.3164101243019104,0.30992308259010315,0.4350569546222687,0.37964820861816406,0.41586586833000183,0.628943920135498,0.3519461452960968,0.45766788721084595,0.32705020904541016,0.35115543007850647,0.377109557390213,0.2723610997200012,0.397543728351593,0.33704444766044617,0.34384915232658386,0.5011255741119385,0.3402656316757202,0.3308679461479187,0.307070791721344,0.3272843062877655,0.3125242590904236,0.3210245370864868,0.23627600073814392,0.35067713260650635,0.3342796564102173,0.46956056356430054,0.2961256504058838,0.4834979772567749,0.2968760132789612,0.3475743532180786,0.3054453432559967,0.31133466958999634,0.6333972215652466,0.34962573647499084,0.28891173005104065,0.33316928148269653,0.4104956090450287,0.4110940098762512,0.24286359548568726,0.3258449137210846,0.26436847448349,0.3812772035598755,0.3106805682182312,0.28105059266090393,0.35842442512512207,0.5492690205574036,0.34262967109680176,0.30186590552330017,0.2995624244213104,0.27341917157173157,0.2684783637523651,0.2630613446235657,0.3030185103416443,0.3905332088470459,0.3086850941181183,0.3754705786705017,0.5154637098312378,0.34567350149154663,0.28677040338516235,0.4132918119430542,0.33740270137786865,0.430867463350296,0.351592481136322,0.3518258035182953,0.29583901166915894,0.32417812943458557,0.39355263113975525,0.5251556038856506,0.3167128264904022,0.4523419439792633,0.31599757075309753,0.4347039759159088,0.353341668844223,0.2930452525615692,0.3265983462333679,0.30140313506126404,0.3774701952934265,0.3513983190059662,0.2826922833919525,0.2848193943500519,0.36451247334480286,0.30290552973747253,0.24577470123767853,0.34585073590278625,0.4190666973590851,0.2871789336204529,0.32172855734825134,0.4007214605808258,0.2711851894855499,0.349052757024765,1.4398142099380493,0.36986875534057617,0.31153976917266846,0.3447670638561249,0.3312033712863922,0.4960070848464966,0.31836384534835815,0.35333195328712463,0.28551462292671204,0.24800091981887817,0.33601048588752747,0.2545158267021179,0.33200734853744507,0.35117143392562866,0.3989322781562805,0.31766477227211,0.3648163974285126,0.32686927914619446,0.29165369272232056,0.3097801208496094,0.4244612753391266,0.3089231848716736,0.347781777381897,0.3611428737640381,0.2936580777168274,0.24938362836837769,0.2628953754901886,0.4467354118824005,0.31588736176490784,0.3781697452068329,0.27665820717811584,0.3249315023422241,0.30401185154914856,0.4483751952648163,0.3070961534976959,0.27182328701019287,0.45667287707328796,0.3745853006839752,0.29803571105003357,0.3625381588935852,0.2623998820781708,0.36779776215553284,0.3829346299171448,0.27580955624580383,0.3379755914211273,0.2913379967212677,0.3712446689605713,0.31461551785469055,0.42793455719947815,0.2966982126235962,0.3099791705608368,0.35840487480163574,0.3076377511024475,0.28857487440109253,0.31069689989089966,0.23376837372779846,0.41049718856811523,0.35906124114990234,0.2727509140968323,0.34014686942100525,0.3397478759288788,0.3899034261703491,0.36512866616249084,0.3092590570449829,0.3305559456348419,0.27512285113334656,0.3051237165927887,0.36318865418434143,0.2970297336578369,0.28418847918510437,0.5391978621482849,0.36966100335121155,0.3082437217235565,0.47274601459503174,0.33024126291275024,0.5056676864624023,0.4679678678512573,0.40630391240119934,0.38468945026397705,0.2924244701862335,0.2886599898338318,0.49210602045059204,0.3316786587238312,0.29386669397354126,0.5114253163337708,0.3114274740219116,0.5080795884132385,0.3346725106239319,0.29407212138175964,0.3101722002029419,0.2894066870212555,0.3067801594734192,0.33837687969207764,0.38689279556274414,0.34572941064834595,0.3050418198108673,0.3295406699180603,0.2996525168418884,0.3314613103866577,0.30381500720977783,0.33137956261634827,0.334503173828125,0.2867412567138672,0.4619627892971039,0.2510458827018738,0.47540464997291565,0.292140930891037,0.4602612853050232,0.32336780428886414,0.27183496952056885,0.2952454090118408,0.29609718918800354,0.3175317943096161,0.40238165855407715,0.35137656331062317,0.737694263458252,0.31367191672325134,0.40428781509399414,0.4338849186897278,0.30339503288269043,0.5525482296943665,0.31227144598960876,0.34187155961990356,0.261324942111969,0.32664257287979126,0.48724883794784546,0.49059951305389404,0.32719364762306213,0.33171790838241577,0.2801404595375061,0.5754698514938354,0.34547609090805054,0.34706565737724304,0.3203883171081543,0.2901437282562256,0.8047022819519043,0.31392714381217957,0.29414740204811096,0.4496247470378876,0.38719508051872253,0.4659475088119507,0.28768011927604675,0.28930899500846863,0.29684382677078247,0.29118168354034424,0.37960049510002136,0.37998688220977783,0.28937914967536926,0.2712395191192627,0.2761373817920685,0.4029388129711151,0.3013482391834259,0.3419002294540405,0.325819194316864,0.273525595664978,0.27355659008026123,0.31931421160697937,0.49017661809921265,0.2946917712688446,0.3846568763256073,0.2802255153656006,0.2763162851333618,0.4636582136154175,0.3438083231449127,0.46893465518951416,0.2605888247489929,0.3326363265514374,0.3031327724456787,0.3243454098701477,0.26785212755203247,0.33179599046707153,0.3642348051071167,0.4122418463230133,0.28031766414642334,0.3899768590927124,0.339398056268692,0.29231974482536316,0.2635372281074524,0.3961416780948639,0.3662927746772766,0.32914504408836365,0.3061232566833496,0.3624149262905121,0.3681738078594208,0.2604302763938904,0.2724292278289795,0.4205847978591919,0.2688887119293213,0.3833988904953003,0.2394479513168335,0.446083664894104,0.2867599129676819,0.4251647889614105,0.40462547540664673,0.4210285246372223,0.3773198425769806,0.38255488872528076,0.4214616119861603,0.3758898675441742,0.31428027153015137,0.25977590680122375,0.30433863401412964,0.37939536571502686,0.28833043575286865,0.30454221367836,0.35577309131622314,0.28960058093070984,0.3269469738006592,0.41722288727760315,0.34342965483665466,0.43304643034935,0.2721102237701416,0.38163456320762634,0.47490251064300537,0.26914387941360474,0.345650315284729,0.310892254114151,0.31299343705177307,0.3143710792064667,0.4411376416683197,0.45597022771835327,0.2870878279209137,0.2539640963077545,0.3439653813838959,0.2617490291595459,0.32819753885269165,0.45161348581314087,0.3436413109302521,0.38407933712005615,0.429666668176651,0.34716862440109253,0.31915906071662903,0.33879798650741577,0.3627054989337921,0.2323436588048935,0.27685052156448364,0.2763851284980774,0.2915117144584656,0.2939239740371704,0.33778685331344604,0.32465285062789917,0.41304001212120056,0.3154921531677246,0.3414577841758728,0.3076166808605194,0.2662022113800049,0.32301944494247437,0.31715747714042664,0.2733216881752014,0.3369980454444885,0.29363295435905457,0.29844677448272705,0.4178193211555481,0.31073009967803955,0.2672080397605896,0.26213985681533813,0.293670654296875,0.7489050030708313,0.4287463426589966,0.372967004776001,0.3247275948524475,0.33057376742362976,0.24061155319213867,0.6222230792045593,0.3241179585456848,0.2773574888706207,0.249301016330719,0.33174625039100647,0.289382666349411,0.316491037607193,0.39805343747138977,0.33802688121795654,0.45387473702430725,0.368535578250885,0.3106325566768646,0.3152889311313629,0.2824312150478363,0.4575115740299225,0.34872037172317505,0.37151992321014404,0.4174692928791046,0.32598528265953064,0.4340564012527466,0.33824312686920166,0.27593958377838135,0.34253180027008057,0.47422876954078674,0.39460068941116333,0.3376432955265045,0.2970139682292938,0.280712753534317,0.4178052842617035,0.246645987033844,0.35096603631973267,0.3483314514160156,0.3590562343597412,0.44481170177459717,0.38536620140075684,0.2531293034553528,0.3966931700706482,0.3396272361278534,0.2702360153198242,0.3133605420589447,0.2757762670516968,0.34454235434532166,0.2720717191696167,0.28505751490592957,0.30445629358291626,0.34964558482170105,0.3593073785305023,0.30398228764533997,0.2488660365343094,0.33034762740135193,0.26298606395721436,0.335808128118515,0.28561416268348694,0.28056806325912476,0.2531808614730835,0.4003917872905731,0.32465696334838867,0.3041667938232422,0.24612459540367126,0.4586673974990845,0.3044731616973877,0.31949132680892944,0.46753358840942383,0.2867411971092224,0.39963147044181824,0.31297656893730164,0.3836905360221863,0.2753647267818451,0.40336641669273376,0.39097556471824646,0.33758416771888733,0.28255993127822876,0.30730774998664856,0.3926507830619812,0.29068639874458313,0.31573060154914856,0.2662818133831024,0.3975723385810852,0.3855951130390167,0.31451717019081116,0.5272418856620789,0.4164947271347046,0.3364604413509369,0.3555232584476471,0.2967448830604553,0.36234331130981445,0.2299959510564804,0.3850599527359009,0.3869437873363495,0.33008134365081787,0.42118799686431885,0.3484780192375183,0.3601015508174896]}},"4":{"inputs":{"mean":[0.2551973760128021,0.13982856273651123,-0.15561093389987946,-0.166558176279068,0.46224209666252136,-0.07368680089712143,-0.17354677617549896,0.13263952732086182,0.034526873379945755,-0.09686369448900223,0.043703094124794006,0.18846645951271057,0.09943018108606339,-0.42977574467658997,-0.011990128085017204,-0.04879074916243553,0.5781168937683105,-0.9490346312522888,0.013527214527130127,-0.22389988601207733,0.0697733536362648,0.06331617385149002,0.12852375209331512,-0.26684924960136414,-0.09748217463493347,0.9352509379386902,0.21657948195934296,0.21662044525146484,0.07884354144334793,0.2056475579738617,-0.5703871250152588,-0.6848471164703369,-0.10094209015369415,-0.7322160601615906,0.4490170478820801,-0.3240799903869629,0.00820065662264824,0.2642062306404114,0.02652853913605213,0.053763292729854584,0.029508747160434723,0.03307962417602539,0.021096669137477875,0.0014829400461167097,0.294750452041626,-0.21803808212280273,0.06280665844678879,0.22934181988239288,0.06884627044200897,-0.1557054966688156,-0.4104584753513336,-0.10346601158380508,0.26072776317596436,0.10920260846614838,-0.06650041043758392,0.13994847238063812,-3.207385778427124,0.23035968840122223,0.08029332756996155,-0.07048886269330978,-0.054326388984918594,-0.23135505616664886,-0.25082579255104065,0.1621326059103012,0.06146542355418205,0.313856303691864,0.02501661516726017,0.032569922506809235,0.006562379188835621,0.6402265429496765,0.08685822784900665,0.8145895600318909,-0.2963387668132782,0.7893805503845215,-0.06994098424911499,-0.1201976016163826,0.14597946405410767,-0.12432222813367844,-0.1442548781633377,-0.5671284794807434,-1.3994886875152588,-0.9405880570411682,0.3144381642341614,-0.027490541338920593,-0.2646300494670868,-0.032701101154088974,-0.041779886931180954,0.6365413069725037,0.07832912355661392,-0.3033622205257416,-0.10968300700187683,0.03335493057966232,0.021735839545726776,-0.010228022933006287,-0.08466674387454987,0.041463952511548996,-0.11475072056055069,-0.23251771926879883,-0.14863355457782745,0.07832008600234985,0.03477257490158081,0.004710536915808916,0.11925546824932098,0.09860876202583313,-0.28936660289764404,0.10372734069824219,-0.14039000868797302,0.43279406428337097,-0.005717276129871607,-0.01585989259183407,-0.04400784522294998,-0.10417269170284271,-0.28242993354797363,0.3131667673587799,-0.29557034373283386,0.022359440103173256,-0.06746619939804077,0.11789949238300323,0.02453605830669403,0.03707980737090111,-0.238145649433136,0.1562589406967163,-0.08576001226902008,0.3154780864715576,0.17809179425239563,0.1937992423772812,-0.06401287764310837,0.19196905195713043,0.017796603962779045,0.17064428329467773,0.12445950508117676,-0.33194032311439514,0.055620383471250534,0.01304858922958374,-0.03359806537628174,0.5958356261253357,-0.21532423794269562,-0.6677411794662476,-0.0023411004804074764,0.05195452645421028,0.09340272843837738,-0.25852784514427185,-0.08056216686964035,0.1488008052110672,0.4008999764919281,0.8090556263923645,0.1771610528230667,0.03783011808991432,-0.17287349700927734,0.14189495146274567,-0.28425946831703186,0.06195269525051117,0.2570059895515442,0.5216267108917236,0.031857527792453766,-0.07016056030988693,6.848662376403809,-1.165407419204712,-0.15072517096996307,-0.1514204442501068,-0.08757823705673218,0.007849366404116154,0.17198173701763153,0.5950494408607483,-0.0685335099697113,0.1463412195444107,-0.17598825693130493,0.242584228515625,-0.18418826162815094,0.0161406509578228,-0.6219485998153687,0.1437615305185318,-1.0745739936828613,0.3695778548717499,0.28338563442230225,0.01584748364984989,-0.23114673793315887,-0.030082356184720993,-0.542910099029541,0.03051144629716873,0.42429035902023315,-0.33588507771492004,0.15913870930671692,0.2249285727739334,0.320612370967865,-0.013125773519277573,-2.3049747943878174,0.13924816250801086,0.1341072916984558,0.7505612969398499,-0.21364209055900574,0.18584637343883514,0.5928970575332642,0.2903655171394348,0.09650874882936478,0.40337854623794556,0.2595288157463074,-0.16248835623264313,-0.04381805658340454,-0.08975370228290558,-0.1069277822971344,-0.13226202130317688,-0.2539675533771515,-0.40367960929870605,-0.2089792788028717,0.23149041831493378,-0.13110806047916412,-0.06875184178352356,0.17357750236988068,0.1483980268239975,-0.10587266087532043,-0.20737606287002563,-0.35700973868370056,0.03771425783634186,-0.5845438241958618,0.1419198215007782,0.00533785717561841,0.4570910334587097,0.40131038427352905,0.43470942974090576,-0.03479070961475372,-0.08947527408599854,-0.010437585413455963,-0.27996960282325745,-0.07949980348348618,0.04093865305185318,0.22274388372898102,0.4020087420940399,0.30627748370170593,-0.09011013805866241,-0.0921371579170227,-0.3803951144218445,-0.3970087766647339,-0.2133478969335556,-0.25183892250061035,0.2354452759027481,-0.13542185723781586,0.24002574384212494,0.27945250272750854,-0.19150952994823456,-1.4068043231964111,0.025371983647346497,-0.7035839557647705,0.03142277151346207,0.33157268166542053,0.23970912396907806,-0.10096527636051178,0.1020531877875328,-0.989168107509613,-0.2379305362701416,-0.3107427656650543,0.12653324007987976,-0.2682465612888336,-0.022198595106601715,0.017026178538799286,0.029335150495171547,0.16268806159496307,-0.42916810512542725,0.08991377800703049,-0.2560666799545288,0.18476317822933197,0.07002253830432892,-0.36903300881385803,0.5925756692886353,0.1006690189242363,0.11090309172868729,-0.2200527787208557,0.03229333832859993,0.04315776750445366,0.34584665298461914,0.08720092475414276,-0.3909519612789154,-0.30193910002708435,0.8500490188598633,0.09508813172578812,0.017557218670845032,-1.626968264579773,-0.0977298766374588,-0.17650318145751953,0.18220354616641998,0.23317882418632507,-0.7001587152481079,-0.03091820888221264,0.045863062143325806,0.34110161662101746,-0.07288123667240143,0.12721911072731018,-0.22413145005702972,-0.19844117760658264,0.22411414980888367,0.08657906949520111,0.46194708347320557,0.003958226181566715,0.057276755571365356,1.3070683479309082,0.9130173921585083,0.09214464575052261,0.14212869107723236,0.2504320442676544,-0.12428445369005203,0.23877809941768646,0.19943645596504211,0.05785796791315079,-0.16423991322517395,0.18482379615306854,0.010269364342093468,0.06061943992972374,-0.06819086521863937,0.5412561893463135,0.11090759187936783,0.05567481741309166,0.050790052860975266,0.002337522804737091,-0.18506790697574615,0.4104560613632202,0.6735721826553345,0.1013106107711792,-0.293159157037735,0.20160657167434692,0.03102528676390648,-0.02765868790447712,-0.0021657277829945087,0.0022869431413710117,-0.1366417109966278,0.0385691337287426,0.33986029028892517,1.1536625623703003,0.3021041750907898,0.0907544195652008,-0.151487797498703,0.538733720779419,-0.5274903178215027,-0.14946256577968597,-0.1405050903558731,-0.39508354663848877,-0.30076733231544495,-0.28035810589790344,0.0724567174911499,0.0006194604211486876,0.1038079708814621,-0.06477347016334534,-0.2550564706325531,0.11175797134637833,-0.02631128579378128,-0.8168550133705139,0.14331263303756714,0.021407950669527054,0.04419892281293869,-0.21842893958091736,-0.6152669191360474,0.4229769706726074,0.5048588514328003,-0.3403370976448059,-0.21992748975753784,-0.16287745535373688,-0.11211363226175308,-0.0702410414814949,-0.15724335610866547,0.10661109536886215,-0.186703160405159,-0.27417710423469543,0.02215893194079399,-0.14910529553890228,0.057057395577430725,0.09377002716064453,0.06693214923143387,-0.20309029519557953,0.17651520669460297,-0.03299245238304138,-0.009782467968761921,-0.021494999527931213,0.128927543759346,0.08326587826013565,0.44675904512405396,-0.05932942032814026,0.37856847047805786,-3.6981594562530518,-0.008749188855290413,0.23683114349842072,0.2136404663324356,0.3059115707874298,-0.04128566384315491,-0.26049381494522095,-0.05243586003780365,-0.31400614976882935,-0.31305375695228577,-1.0837254524230957,0.01241468545049429,-0.5031916499137878,0.48430487513542175,-0.5389645099639893,-0.037604909390211105,0.053639866411685944,-0.07326976954936981,0.08893736451864243,-0.1410001814365387,0.3440309762954712,-0.9523500204086304,-0.03592643886804581,0.5265704989433289,-0.20658139884471893,-0.07088072597980499,-0.1293954998254776,-0.22263285517692566,-0.23536087572574615,-0.3439551293849945,-0.016497578471899033,-0.261259526014328,0.3835345506668091,-0.30085912346839905,-0.1749386042356491,0.03213392198085785,0.1639409363269806,0.08112569153308868,0.9573893547058105,-0.20041044056415558,0.29302284121513367,-0.22265709936618805,-0.5234261155128479,-0.03517570346593857,0.5746062397956848,0.21629415452480316,0.09679507464170456,0.02516135945916176,-0.10763508826494217,0.25722822546958923,0.06276679784059525,-0.0762375146150589,0.022040028125047684,-0.05191958323121071,0.047488439828157425,-0.0865795835852623,0.14468316733837128,0.33487048745155334,-0.22028318047523499,0.16806113719940186,0.3324618637561798,0.6656785011291504,0.04025205597281456,0.2255469262599945,-0.24353700876235962,0.0461447611451149,0.05346176028251648,-0.03525190055370331,-0.16808611154556274,-0.29059261083602905,0.34313851594924927,-0.33113688230514526,0.11017970740795135,0.4051837623119354,-0.27038440108299255,-0.04623524099588394,-0.3233727812767029,0.08882863819599152,-0.024550296366214752,-0.052231766283512115,0.11891545355319977,-0.21285580098628998,-0.03852058947086334,0.30069389939308167,0.0906451866030693,0.17159661650657654,-0.02152150310575962,-0.1270919293165207,0.41120773553848267,0.03270445391535759,0.254741907119751,0.12452065944671631,0.12122520059347153,0.5609778165817261,0.04293167218565941,-0.12417159974575043,-0.29286426305770874,-0.08950037509202957,-0.5885184407234192,0.04415295273065567,0.0904630795121193,-0.05103003978729248,-3.302928924560547,-0.2609906494617462,-0.13995908200740814,0.5909655094146729,-0.1199989765882492,0.16534610092639923,0.10348114371299744,0.6239076852798462,-0.10166757553815842,0.7438380122184753,0.6957718133926392,-0.15494082868099213,-0.04343806952238083,0.1859017163515091,0.22977368533611298,0.2977968156337738,-0.10718600451946259,0.42548611760139465,0.18712075054645538,0.17076264321804047,0.196945458650589,0.03895839676260948,0.3879956007003784,-0.1301143914461136,-0.46089255809783936,0.16740183532238007,0.02742224931716919,-0.367110013961792,-0.07451467216014862,0.5610930323600769,0.004808198194950819,0.26053598523139954,0.03968510404229164,0.32840484380722046],"std":[0.8962441086769104,1.4159574508666992,1.0237390995025635,1.139562964439392,1.3578929901123047,0.9329168200492859,1.07382071018219,0.8903496861457825,0.988594114780426,1.2775487899780273,1.2842671871185303,1.1122697591781616,1.012203574180603,1.2438901662826538,0.9885759353637695,1.0730942487716675,1.3597196340560913,2.6892588138580322,0.9262822270393372,1.036469578742981,0.9295099377632141,1.134114146232605,1.003392219543457,3.0788722038269043,1.1465665102005005,1.5194206237792969,1.096166729927063,0.9953557848930359,1.3448472023010254,1.3331583738327026,1.3209285736083984,1.303645133972168,1.0695936679840088,1.822115182876587,1.1221952438354492,1.141370177268982,1.1055728197097778,0.9643417000770569,1.098885416984558,1.2024027109146118,0.8877071738243103,1.0733088254928589,0.978722870349884,0.867394208908081,1.1236265897750854,1.2550948858261108,0.89676833152771,0.9316319227218628,0.991698682308197,0.8468524217605591,1.2515861988067627,1.0238220691680908,1.2004002332687378,1.0780723094940186,1.2778644561767578,0.9225835800170898,2.494060754776001,0.9623299241065979,1.035753607749939,1.29703950881958,0.9037483930587769,1.113166093826294,1.0701828002929688,0.9377313256263733,1.0455105304718018,1.1514497995376587,0.9980730414390564,1.0266932249069214,0.8474858999252319,1.240415334701538,1.365280270576477,2.330289125442505,1.3536968231201172,1.5237764120101929,0.9042384028434753,1.2787257432937622,1.1587581634521484,0.8918098211288452,0.9232689738273621,1.4150536060333252,1.8342348337173462,1.7522848844528198,1.1925561428070068,0.892418384552002,0.9511536955833435,0.893307626247406,0.9488526582717896,1.3268390893936157,0.9705334901809692,1.3654272556304932,0.9556989669799805,1.2846266031265259,0.8891863822937012,1.3488433361053467,0.9372085332870483,1.2571980953216553,1.1063337326049805,1.0608844757080078,1.7818354368209839,0.9927809238433838,1.2476916313171387,1.1666064262390137,0.849899172782898,1.2382316589355469,1.2662750482559204,1.038339614868164,0.8532094359397888,1.256544828414917,0.9550014138221741,0.9038345217704773,1.4335588216781616,1.995405912399292,0.9424486756324768,1.0032445192337036,0.9861593246459961,1.2006081342697144,0.9325366020202637,0.9319384098052979,1.0574983358383179,1.3538415431976318,1.174046277999878,0.8910510540008545,0.8874578475952148,1.0358479022979736,1.0786914825439453,1.2448756694793701,1.090572714805603,1.0439019203186035,1.106575846672058,1.1609516143798828,1.1822725534439087,1.2806434631347656,1.2340505123138428,1.0192066431045532,0.8837707042694092,1.4586198329925537,0.9371455311775208,2.0310823917388916,0.7752307057380676,0.9460790753364563,1.4319225549697876,0.922618567943573,1.2042698860168457,0.9977602958679199,1.0091382265090942,1.3434827327728271,1.2986156940460205,0.9242488145828247,1.3932631015777588,0.9815224409103394,1.3748465776443481,1.0802515745162964,0.822565495967865,1.5886214971542358,0.9480879306793213,0.9575465321540833,3.0406999588012695,2.125663995742798,1.017472505569458,0.8310858607292175,1.1597875356674194,1.057131052017212,1.0090219974517822,1.3920855522155762,1.2724010944366455,1.1394916772842407,1.117182731628418,0.9857953190803528,1.0712288618087769,1.2499206066131592,1.2957037687301636,1.0803132057189941,1.6427780389785767,1.3105082511901855,1.1415657997131348,1.1413416862487793,1.2949978113174438,0.9787713289260864,1.1308144330978394,1.1239432096481323,1.1749907732009888,1.4398207664489746,0.9074661135673523,1.016385793685913,1.1816303730010986,1.0583548545837402,2.664984941482544,1.0265525579452515,1.0235035419464111,1.5633485317230225,0.9230477809906006,0.9043686985969543,1.7113393545150757,1.509162187576294,0.9926848411560059,1.2553330659866333,0.9552955627441406,1.094118356704712,0.9054762125015259,0.9116275906562805,1.0229942798614502,0.830038845539093,1.3133469820022583,1.301123023033142,1.0847599506378174,0.9644861817359924,0.9407281279563904,1.1301671266555786,0.8697737455368042,0.9035186767578125,1.020967721939087,1.064528226852417,1.196646809577942,1.1087394952774048,1.2351477146148682,1.1261403560638428,1.0034013986587524,1.3933745622634888,1.1455405950546265,1.132204532623291,0.8700546026229858,0.9928242564201355,0.8986261487007141,1.0265395641326904,1.0375407934188843,0.9314344525337219,1.114071249961853,1.271823763847351,0.997974157333374,0.9744811654090881,0.9147598147392273,1.9689418077468872,1.6601126194000244,0.9627488851547241,0.890670120716095,1.0283710956573486,1.1897478103637695,0.9621394276618958,1.162422776222229,1.0842628479003906,3.0096776485443115,1.3394112586975098,1.8849443197250366,0.9706424474716187,1.077060341835022,1.089341163635254,0.9038287401199341,0.9512966871261597,2.0307087898254395,1.3618898391723633,1.331855297088623,0.9255478978157043,0.9220997095108032,0.9649621248245239,1.283288598060608,0.9267531037330627,1.0363281965255737,1.277075171470642,0.9377767443656921,0.981772243976593,0.873978853225708,0.916092574596405,1.1421455144882202,1.9385539293289185,0.8929336071014404,0.920616626739502,1.0498323440551758,0.9337923526763916,1.1436792612075806,1.5330063104629517,1.036666989326477,2.205035448074341,1.3154479265213013,1.5117374658584595,1.5349634885787964,1.1774400472640991,2.7584073543548584,0.9674714803695679,0.9360420107841492,1.0615195035934448,1.2371906042099,1.6840752363204956,1.6533530950546265,0.942290186882019,1.229939341545105,1.0667181015014648,1.0330709218978882,0.9382162690162659,1.1562254428863525,1.2226479053497314,0.9515202045440674,1.9468775987625122,0.9986429810523987,0.8838167190551758,1.4523974657058716,1.3282909393310547,1.5004467964172363,0.904242753982544,1.052108883857727,1.1766927242279053,0.8797640800476074,0.9595362544059753,0.837081253528595,0.8830759525299072,0.9806246757507324,0.8751857876777649,1.3434550762176514,0.9935843348503113,1.1455947160720825,0.9653688669204712,0.9292324185371399,0.9309324622154236,0.9820274114608765,1.3660340309143066,1.138966679573059,1.4005043506622314,0.9414416551589966,0.8779333829879761,1.4260252714157104,0.8584648370742798,0.8524304032325745,1.0514228343963623,0.9464001655578613,0.8714264631271362,1.138131856918335,1.083301067352295,1.625262975692749,1.101649284362793,1.1445209980010986,1.120278000831604,1.1392050981521606,1.3739184141159058,0.9537304639816284,1.192139983177185,1.2159621715545654,1.352831482887268,1.0554161071777344,1.1259702444076538,0.9291209578514099,1.102620005607605,0.9083225727081299,0.9183185696601868,0.9398446679115295,0.9413217306137085,1.7876056432724,0.9145470261573792,0.8078475594520569,0.9218719601631165,1.430652379989624,1.5671018362045288,1.5548694133758545,1.334378719329834,1.6045904159545898,0.9600249528884888,1.046677827835083,1.2376604080200195,0.9276917576789856,1.0113403797149658,0.9875592589378357,0.989407479763031,1.074622631072998,1.3353632688522339,1.1033284664154053,0.9480336904525757,1.3973256349563599,1.5491117238998413,1.3992196321487427,0.9533253908157349,1.042014241218567,1.272591471672058,0.9538207054138184,1.2279884815216064,1.0919018983840942,1.0479696989059448,1.0215915441513062,1.6872626543045044,4.152089595794678,1.0806869268417358,1.0262502431869507,1.1125714778900146,0.9489316344261169,0.999049186706543,1.3916412591934204,0.8879417181015015,1.1666561365127563,1.3304558992385864,1.5691295862197876,0.8055963516235352,0.9738685488700867,1.3690286874771118,1.296852469444275,1.0543473958969116,0.9452621340751648,1.160622239112854,0.8350589275360107,1.1556438207626343,1.2803746461868286,1.615817666053772,0.9139925241470337,1.3613523244857788,1.2065519094467163,1.0688989162445068,1.10590660572052,1.1877559423446655,0.9882596135139465,1.2296158075332642,0.9176514148712158,0.9709566831588745,1.45980966091156,1.0761594772338867,1.394712209701538,0.9745535254478455,1.156211495399475,1.559259295463562,1.5376278162002563,1.3329877853393555,0.8913387656211853,0.9259396195411682,1.3352477550506592,0.9027385711669922,1.1559182405471802,0.9265924096107483,0.9331107139587402,1.034796118736267,0.9517326951026917,1.1982752084732056,0.9902791976928711,1.0155576467514038,1.5112148523330688,1.207955241203308,0.9640612006187439,0.8732272982597351,0.9265583157539368,1.165842890739441,0.9816838502883911,1.2822843790054321,1.1276558637619019,1.3415472507476807,1.5205743312835693,0.9421762228012085,1.2389096021652222,1.1890164613723755,0.9338593482971191,1.1537450551986694,0.9528006315231323,1.0030086040496826,1.0000141859054565,1.3418177366256714,1.1750684976577759,1.105342149734497,1.14814293384552,1.1882320642471313,1.7145215272903442,1.503661870956421,0.9791936278343201,1.0186127424240112,1.0440412759780884,0.8929283618927002,1.2473913431167603,1.029238224029541,0.9883561134338379,0.9653071761131287,0.9229952096939087,1.1258686780929565,1.2782682180404663,0.8823931813240051,1.1615790128707886,1.0511789321899414,0.9940997362136841,1.2688854932785034,0.8751219511032104,0.8550192713737488,0.8514509201049805,0.8504303097724915,1.385695457458496,1.0328441858291626,1.1703933477401733,1.0403295755386353,4.485245227813721,0.9675867557525635,0.9074082970619202,1.635124921798706,0.9016739130020142,1.3051923513412476,0.9568267464637756,1.3013359308242798,1.0915770530700684,1.4990406036376953,1.396503210067749,0.8957762718200684,0.9355117082595825,1.0956299304962158,1.2148185968399048,0.987937867641449,1.0152255296707153,1.0083073377609253,1.250917673110962,0.9451555013656616,1.035412073135376,1.041396141052246,1.4121813774108887,1.2715107202529907,1.372660756111145,1.0500364303588867,1.0522171258926392,0.8967688083648682,0.9347976446151733,1.6972733736038208,0.9573320746421814,1.422251582145691,0.9054698944091797,1.3008720874786377]},"targets":{"mean":[-0.14065101742744446,-0.404281347990036,-0.11836359649896622,-0.011352595873177052,-0.13983675837516785,-0.05716639384627342,-0.03194306418299675,0.022410904988646507,0.11955894529819489,-0.06894826889038086,0.08400008082389832,-0.0880264937877655,-0.07795116305351257,0.05223549157381058,0.055204953998327255,0.05339175462722778,-0.057202357798814774,0.15188023447990417,-0.12910082936286926,-0.18235419690608978,0.5670202374458313,-0.5400547385215759,-0.018567821010947227,-0.4273890256881714,0.047394685447216034,0.15248648822307587,-0.0559447817504406,-0.11304906010627747,-0.36019742488861084,-0.14300040900707245,0.2635028064250946,0.41615718603134155,-0.08332394808530807,0.1973830610513687,-0.10908305644989014,0.10987760126590729,0.08176575601100922,-0.16113615036010742,0.06071192026138306,-0.08028263598680496,-0.11124888807535172,-0.0039758156053721905,0.0006729487795382738,0.06404007971286774,0.12200850993394852,0.06074794381856918,0.28052064776420593,-0.09642884135246277,-0.05706789344549179,-0.17191004753112793,0.20696161687374115,-0.07149641215801239,0.06997030228376389,-0.15945808589458466,0.1473758965730667,-0.02007371373474598,0.3939094841480255,-0.032843202352523804,-0.1583864688873291,0.19332298636436462,-0.14915762841701508,0.20718151330947876,-0.5733452439308167,-0.011838987469673157,0.015008360147476196,0.04133189469575882,-0.09377217292785645,0.0014951485209167004,0.12072945386171341,-0.4178028106689453,0.19962598383426666,-0.17182013392448425,0.16076399385929108,-0.43404340744018555,-0.11260496824979782,-0.02219409868121147,0.08037970215082169,0.02951735444366932,-0.07706266641616821,0.3984260857105255,0.25184088945388794,0.646672785282135,0.21338911354541779,0.041240591555833817,0.1302512288093567,0.09010136127471924,-0.08902670443058014,0.032679785043001175,0.025596916675567627,0.097307488322258,-0.10071597993373871,0.008565345779061317,-0.23900540173053741,0.0015947397332638502,-0.1268792301416397,0.08624932914972305,0.0990348681807518,-0.0533536858856678,-0.039087727665901184,0.034161265939474106,-0.06898791342973709,-0.18371455371379852,0.054153263568878174,-0.3386107385158539,-0.10095541179180145,0.07983742654323578,-0.09586162120103836,-0.587040901184082,0.10616237670183182,-0.06475672870874405,0.0576651468873024,-0.05163443461060524,0.06975933909416199,-0.003049221821129322,0.11898120492696762,-0.16333290934562683,0.07268062978982925,-0.01117927860468626,-0.10456740111112595,-0.042531974613666534,-0.08027345687150955,-0.05507422983646393,0.001853459165431559,-0.22430720925331116,-0.24987924098968506,-0.06615468114614487,-0.47251537442207336,-0.13250955939292908,-0.49998265504837036,0.038192082196474075,-8.68023416842334e-05,0.12434455752372742,-0.01627749390900135,-0.018044745549559593,-0.05018527805805206,-0.09399698674678802,-0.021150628104805946,0.22107698023319244,-0.19428005814552307,0.025931254029273987,0.03224378451704979,-0.0826990157365799,0.1168210357427597,-0.40001893043518066,-0.28537821769714355,-0.3210213780403137,0.1658988893032074,0.07678001374006271,0.06139044091105461,0.08535159379243851,0.20686231553554535,-0.24446170032024384,-0.024217886850237846,-0.24722544848918915,-0.03730864077806473,0.010475965216755867,-2.646933078765869,-0.006174259819090366,-0.04900330677628517,0.05117814242839813,-0.4151935875415802,-0.12025796622037888,0.06643863022327423,-0.22307147085666656,0.05777990072965622,0.097317174077034,-0.06377434730529785,-0.03261967748403549,-0.05830352380871773,0.11914725601673126,0.15981730818748474,0.168402761220932,0.3614220917224884,0.6511176824569702,0.889994740486145,0.22848130762577057,0.4960588216781616,0.10804860293865204,0.14781111478805542,0.009641009382903576,-0.2342977374792099,0.1474226415157318,0.008312015794217587,0.02513798698782921,0.04235060513019562,-0.26290076971054077,0.2841784954071045,0.13229045271873474,-0.09510307759046555,-0.30263495445251465,0.04163624718785286,-0.05837357044219971,-0.1470971256494522,0.07939676940441132,-0.07690242677927017,-0.17963144183158875,0.04190860688686371,-0.42964261770248413,-0.09927377104759216,-0.05152006447315216,0.024178870022296906,-0.30676743388175964,-0.5459522008895874,0.1605820655822754,0.05952572077512741,-0.14402206242084503,0.05679906904697418,-0.11391589790582657,-0.07100723683834076,0.042330145835876465,-0.10751494765281677,-0.06737634539604187,0.23178905248641968,-0.01754838414490223,0.19429466128349304,-0.020594030618667603,-0.001345931552350521,-0.03439723327755928,-0.07650508731603622,-0.21117755770683289,0.13811057806015015,-0.17811597883701324,-0.04052947834134102,0.00929112546145916,0.0804249569773674,0.17499203979969025,-1.254870891571045,-0.10021884739398956,-0.01409220602363348,0.02390103042125702,0.04267076030373573,0.0470115952193737,0.114801786839962,0.00862991064786911,0.2404257357120514,0.0670531690120697,-0.00446765311062336,1.8367092609405518,-0.2752055823802948,0.27403712272644043,0.03275620937347412,-0.02407156676054001,0.2703281044960022,-0.02647414058446884,0.10575344413518906,-0.14436952769756317,-0.036058202385902405,-0.07010702043771744,-0.021014293655753136,-0.028543073683977127,-0.0027957127895206213,-0.12544402480125427,0.03346606716513634,0.1279117912054062,-0.17263548076152802,-0.06776702404022217,0.02514379471540451,0.15881770849227905,0.07478275895118713,0.32081836462020874,-0.14463050663471222,-0.08265932649374008,-0.08586015552282333,-0.009312025271356106,-0.11319375038146973,-0.15487346053123474,-0.15343961119651794,0.12033246457576752,0.13959664106369019,-0.09341269731521606,0.07152910530567169,0.06132766604423523,0.15163196623325348,-0.3111746311187744,0.463848352432251,0.07200514525175095,0.41446518898010254,0.08712722361087799,0.05777643993496895,0.07106588780879974,-0.08381283283233643,0.2984069287776947,0.1883532553911209,0.02337542735040188,-0.013163781724870205,0.049773961305618286,-0.15498945116996765,0.10312220454216003,0.08456425368785858,0.0013015875592827797,-0.08889058232307434,-0.21592192351818085,-0.12955519556999207,-0.08490297198295593,-0.5821910500526428,-0.326975554227829,-0.02518344484269619,-0.05802963301539421,-0.2661997377872467,0.30681654810905457,-0.23164117336273193,-0.09016160666942596,-0.0968533381819725,-0.20707836747169495,-0.21092532575130463,0.003976487088948488,0.2853410243988037,0.18202899396419525,-0.1160428449511528,0.09421303868293762,-0.09814544022083282,-0.18386968970298767,0.003545817220583558,0.1557529866695404,0.04932533577084541,-0.24887646734714508,0.02169376239180565,-0.12846383452415466,0.08824838697910309,-0.1302974373102188,-0.020951183512806892,-0.09580116719007492,0.01755705289542675,0.31842029094696045,-0.1751769334077835,-0.13867604732513428,-0.05472816899418831,-0.09212304651737213,-0.2527361810207367,-0.001070961239747703,0.5927103161811829,0.2905508279800415,-0.11813903599977493,0.3640146255493164,0.09359274059534073,0.09879446774721146,0.025331733748316765,-0.2518960237503052,-0.11793726682662964,-0.09184301644563675,-0.04709021374583244,-0.06224215403199196,-0.010943770408630371,0.14201785624027252,0.24999769032001495,0.11281661689281464,0.11292318254709244,0.030744481831789017,-0.003061097813770175,0.1696278154850006,-0.2104799896478653,-0.03811733424663544,0.16538001596927643,0.14428791403770447,0.01526610367000103,0.20532605051994324,0.014816895127296448,-0.0432228222489357,-0.07303985208272934,0.0869816467165947,0.18242450058460236,-0.03550337255001068,0.21455447375774384,0.044588517397642136,-0.08977115154266357,-0.2100595086812973,0.13310687243938446,0.08599832653999329,-0.06057826802134514,0.006355698220431805,0.03840235248208046,-0.31724193692207336,0.09579068422317505,-0.12097711116075516,-0.03602777048945427,-0.2028316855430603,0.35080376267433167,-0.062453798949718475,-0.03534986451268196,-0.049759600311517715,-0.012782391160726547,0.011533583514392376,0.34754714369773865,0.05722852051258087,0.23712410032749176,0.04057212173938751,0.25301679968833923,-0.07857063412666321,0.01866254210472107,-0.14245577156543732,0.1622931808233261,0.11154620349407196,-0.13974665105342865,-0.07551391422748566,-0.07507465779781342,0.15693449974060059,-0.12478123605251312,0.16090501844882965,-0.13257639110088348,-0.31392332911491394,-0.07513830065727234,-0.016666429117321968,0.09874530881643295,0.12305259704589844,-0.1354021579027176,-0.0246967151761055,-0.005192729644477367,-0.10904081165790558,-0.24292795360088348,-0.014851064421236515,0.015690946951508522,-0.0777273029088974,0.0793951004743576,0.18105335533618927,-0.5817776918411255,0.06143829971551895,-0.09048817306756973,0.18961699306964874,0.10840485990047455,0.008392834104597569,-0.1722283661365509,-0.0554703064262867,-0.03934316337108612,0.16566714644432068,0.005854108836501837,0.05719900131225586,0.16499526798725128,-0.0965319499373436,-0.2589607834815979,-0.07334907352924347,0.16569413244724274,-0.00110805022995919,0.10666297376155853,0.026786871254444122,0.08484509587287903,0.5155134201049805,-0.06793995946645737,-0.13297708332538605,-0.14111699163913727,-0.07500738650560379,0.13034163415431976,-0.08039030432701111,0.12993906438350677,0.09724541753530502,0.006637278012931347,0.027266845107078552,-0.22463026642799377,0.24104635417461395,-0.07993665337562561,-0.12423238158226013,0.04132435843348503,0.9342520236968994,-0.06913349032402039,-0.2301378697156906,-0.19297052919864655,-0.020953666418790817,0.2054261863231659,0.020858649164438248,0.12826679646968842,0.008663312532007694,-0.05457600951194763,0.019030258059501648,-0.08731230348348618,0.046713557094335556,-0.17810237407684326,0.13334353268146515,-0.17096580564975739,-0.17398270964622498,-0.06383346021175385,-0.09462494403123856,-0.06543770432472229,0.08840606361627579,-0.039109956473112106,-0.09163279831409454,0.11483731865882874,0.09083359688520432,-0.1146736592054367,-0.07669398933649063,0.33799344301223755,-0.0997750535607338,0.00929002370685339,-0.08463018387556076,-0.14030486345291138,0.031521935015916824,-0.008634937927126884,-0.1780707985162735,-0.09499731659889221,-0.2144411951303482,-0.2320762425661087,0.008748355321586132,-0.11711642146110535,-0.0465887188911438,0.12657147645950317,-0.27886906266212463,-0.017727959901094437,-0.004208890255540609,-0.1966513693332672,-0.016449272632598877,0.07176634669303894,-0.02266695350408554,0.7766644358634949,-0.08395468443632126,0.1542598456144333,-0.3841724991798401,-0.1010836586356163,-0.13254544138908386,-0.023784328252077103,-0.03759893402457237,-0.09337494522333145,-0.17476606369018555,0.07080447673797607,-0.2316516935825348],"std":[0.28717249631881714,0.6821798086166382,0.31856414675712585,0.35578837990760803,0.5773621201515198,0.336181104183197,0.3421241343021393,0.31672927737236023,0.3585669696331024,0.5195315480232239,0.3943691551685333,0.31324514746665955,0.35905301570892334,0.5230845808982849,0.40953949093818665,0.31329265236854553,0.568777322769165,0.4864718019962311,0.3318498134613037,0.3056468665599823,0.9266337156295776,1.3520601987838745,0.35148191452026367,3.4391608238220215,0.40018215775489807,0.647861123085022,0.37236908078193665,0.3662690818309784,0.5576484799385071,0.46012067794799805,0.4035812020301819,0.4407692551612854,0.27606692910194397,0.30672189593315125,0.3166235685348511,0.3697825074195862,0.4181806743144989,0.3100992441177368,0.3650973439216614,0.41388797760009766,0.32472774386405945,0.38746586441993713,0.36056649684906006,0.3141021430492401,0.43524929881095886,0.4678678810596466,0.32467952370643616,0.33503180742263794,0.30730485916137695,0.33074820041656494,0.6857799291610718,0.3233172595500946,0.36498528718948364,0.31261879205703735,0.44948244094848633,0.3403841257095337,0.8251305818557739,0.30100712180137634,0.3460523188114166,0.5126072764396667,0.30851632356643677,0.36765578389167786,1.335989236831665,0.3286663293838501,0.3102675974369049,0.3776897192001343,0.3997932970523834,0.3421756625175476,0.3380512297153473,0.44405612349510193,0.5078933238983154,0.44942277669906616,0.32598087191581726,0.5435938239097595,0.33054670691490173,0.39008262753486633,0.47419095039367676,0.3037573993206024,0.31594833731651306,0.5122787356376648,0.34120064973831177,1.0645076036453247,0.4510195255279541,0.30228108167648315,0.3174760937690735,0.33847200870513916,0.329533189535141,0.3531704246997833,0.2944250702857971,0.49354761838912964,0.3814361095428467,0.4990490674972534,0.3573136031627655,0.5047202110290527,0.3429985046386719,0.3891076147556305,0.37360233068466187,0.3299591541290283,0.5049199461936951,0.4060472548007965,0.3608378767967224,0.4947209358215332,0.3440755605697632,0.580815315246582,0.3196563422679901,0.34448766708374023,0.299581378698349,0.5521260499954224,0.33206140995025635,0.3090273141860962,0.46886369585990906,0.6078273057937622,0.3255079686641693,0.32894688844680786,0.31302955746650696,0.3439193367958069,0.309787392616272,0.31726837158203125,0.3749363422393799,0.5541000366210938,0.43852007389068604,0.3378269076347351,0.3648984134197235,0.34234005212783813,0.3508310914039612,0.3626445531845093,0.4133974313735962,0.4250619113445282,0.7250616550445557,0.3249507546424866,0.36136093735694885,0.4276316463947296,0.5392776131629944,0.3471262753009796,0.3345535099506378,0.5670962929725647,0.33179861307144165,0.5986327528953552,0.30275776982307434,0.3223362863063812,0.47159072756767273,0.3186347484588623,0.4198106825351715,0.5914931297302246,0.33296114206314087,0.29692596197128296,0.41083666682243347,0.3191588521003723,0.30368202924728394,0.3523172438144684,0.5722445845603943,0.4186594784259796,0.3102909326553345,0.3554037809371948,0.3048241138458252,0.3228002190589905,7.141279220581055,0.8638433218002319,0.3381165564060211,0.2922069728374481,0.5057299733161926,0.40526628494262695,0.368448942899704,0.456683486700058,0.37093085050582886,0.2954535186290741,0.42687636613845825,0.2882964313030243,0.3153238892555237,0.35523754358291626,0.4576704502105713,0.34757113456726074,0.4694531261920929,0.9217352867126465,1.3754277229309082,1.013498067855835,0.5886054635047913,0.41475746035575867,0.3535855710506439,0.31979987025260925,0.44043469429016113,0.2903446853160858,0.3704829514026642,0.38751935958862305,0.4039747714996338,0.4421747922897339,0.32064634561538696,0.4144364893436432,0.3825794458389282,0.6461566686630249,0.3116582930088043,0.30620908737182617,0.7115174531936646,0.581433117389679,0.3448032736778259,0.48344486951828003,0.3298710882663727,0.7368583083152771,0.3547089695930481,0.3374501168727875,0.377620130777359,0.3935171365737915,0.6419349908828735,0.3834226131439209,0.39885881543159485,0.3138071894645691,0.3301060199737549,0.38571402430534363,0.3218933641910553,0.335872083902359,0.34860947728157043,0.34421852231025696,0.4225703775882721,0.3865639865398407,0.3822280168533325,0.4232613444328308,0.3328428566455841,0.46731671690940857,0.3440728187561035,0.43121665716171265,0.3188275396823883,0.2996349632740021,0.3465815782546997,0.4015524685382843,0.3320164680480957,0.4128703474998474,0.9512244462966919,0.43415459990501404,0.35843437910079956,0.3453028202056885,0.31768783926963806,0.4200018644332886,0.6402829885482788,0.3553040325641632,0.3581397533416748,0.2780519723892212,0.3128225803375244,1.7066526412963867,0.40435007214546204,0.39719513058662415,0.42860862612724304,0.4403644800186157,0.6786095499992371,0.30112189054489136,0.597664475440979,0.4335218369960785,0.3595702350139618,0.34560707211494446,0.5960462093353271,0.4957413375377655,0.4816776514053345,0.3306809067726135,0.30146482586860657,0.34164199233055115,0.4428623616695404,0.35027188062667847,0.32482966780662537,0.43650922179222107,0.2988502085208893,0.4383315443992615,0.29393553733825684,0.33834558725357056,0.33415570855140686,0.33901655673980713,0.34026315808296204,0.3053895831108093,0.36554673314094543,0.31024810671806335,0.3405132591724396,0.5802409052848816,0.3561515510082245,0.8993860483169556,0.43051451444625854,0.5283994078636169,0.8384876251220703,0.3279220163822174,0.43346545100212097,0.35503387451171875,0.32996007800102234,0.32185599207878113,0.2965396046638489,0.7269458174705505,0.6446871757507324,0.3479877710342407,0.40954700112342834,0.3437117040157318,0.37504225969314575,0.3178398311138153,0.39821189641952515,0.4549514651298523,0.32943710684776306,0.5011593103408813,0.3288235366344452,0.3046071231365204,0.5805875062942505,0.44971057772636414,0.5839014053344727,0.32920992374420166,0.3949248492717743,0.4952576756477356,0.30127856135368347,0.36004215478897095,0.32810357213020325,0.3625519871711731,0.3871109187602997,0.3117245137691498,2.97727632522583,0.3557993173599243,0.39780187606811523,0.4043860137462616,0.3052249252796173,0.3444882035255432,0.3144886791706085,0.46097710728645325,0.38827961683273315,0.47745847702026367,0.31600314378738403,0.32124412059783936,0.5900911092758179,0.2956567406654358,0.35011762380599976,0.3122052550315857,0.3568808138370514,0.32831960916519165,0.39798274636268616,0.3529743254184723,0.47024601697921753,0.3918403387069702,0.4420390725135803,0.33374249935150146,0.9038617014884949,0.46238455176353455,0.30932852625846863,0.3338210880756378,0.42774686217308044,0.49927636981010437,0.30275338888168335,0.36740899085998535,0.34480738639831543,0.35781335830688477,0.30109962821006775,0.30884164571762085,0.33648616075515747,0.3056011497974396,0.506272554397583,0.3265744149684906,0.35469916462898254,0.3398391604423523,0.5671020746231079,0.5331270098686218,0.6007223129272461,0.38526174426078796,0.7245450615882874,0.4123769998550415,0.32898572087287903,0.4375420808792114,0.29702073335647583,0.31834229826927185,0.3290260136127472,0.3272455930709839,0.3791687786579132,0.46849268674850464,0.31349503993988037,0.3101266920566559,0.5410990118980408,0.47269168496131897,0.5186715722084045,0.364725798368454,0.33751964569091797,0.4001389443874359,0.3110259473323822,0.42632344365119934,0.3744611144065857,0.29117584228515625,0.32775720953941345,0.6824948787689209,0.4140361249446869,0.3107379376888275,0.3091367483139038,0.33990389108657837,0.29416999220848083,0.33308565616607666,0.5613095760345459,0.35739409923553467,0.4601828455924988,0.5533940196037292,0.4659297466278076,0.2974969744682312,0.38759851455688477,0.4736388325691223,0.28006380796432495,0.30762946605682373,0.31515052914619446,0.33467620611190796,0.3297257721424103,0.39940187335014343,0.4479462802410126,0.621249794960022,0.34889164566993713,0.5177088379859924,0.29107630252838135,0.32169243693351746,0.37627601623535156,0.4519991874694824,0.3185913562774658,0.48220059275627136,0.3115691542625427,0.3983812928199768,0.5643302798271179,0.3504781424999237,0.3161984384059906,0.28819894790649414,0.3473670482635498,0.36843371391296387,0.5834531784057617,0.5012344717979431,0.3274596333503723,0.3221474587917328,0.28553637862205505,0.38517430424690247,0.41097134351730347,0.2785581350326538,0.31334105134010315,0.37830260396003723,0.30488821864128113,0.32586976885795593,0.3689238727092743,0.3560635447502136,0.5920165777206421,0.46100711822509766,0.3435707092285156,0.3295890688896179,0.34152624011039734,0.4069323241710663,0.33526238799095154,0.6026612520217896,0.35232120752334595,0.3030814528465271,0.5407330989837646,0.3028048872947693,0.3150348663330078,0.317954421043396,0.36601534485816956,0.4596583843231201,0.35931044816970825,0.3647293150424957,0.3572039306163788,0.8164305686950684,0.30889829993247986,0.38710668683052063,0.4183841347694397,1.9224295616149902,0.6358291506767273,0.5484833121299744,0.32221347093582153,0.37837886810302734,0.4534616470336914,0.30462056398391724,0.2916753590106964,0.34499284625053406,0.3545397222042084,0.3040088415145874,0.3197207450866699,0.4121134877204895,0.4352244436740875,0.3263706862926483,0.4062426686286926,0.33790504932403564,0.34053122997283936,0.3063034415245056,0.302876353263855,0.32796764373779297,0.31928300857543945,0.30585038661956787,0.47227713465690613,0.3645077049732208,0.3494335412979126,0.29793205857276917,0.5397739410400391,0.37286290526390076,0.31357577443122864,0.609643816947937,0.31596291065216064,0.46806639432907104,0.34495463967323303,0.4732632339000702,0.3678026795387268,0.5565999746322632,0.44512850046157837,0.3199790418148041,0.3413833677768707,0.36147016286849976,0.5998517274856567,0.3573093116283417,0.3693939447402954,0.3240072727203369,0.4140840172767639,0.3530345559120178,0.3429459035396576,0.37059450149536133,0.8983098268508911,0.42888426780700684,0.4606264531612396,0.440778911113739,0.4244993031024933,0.29979726672172546,0.33063793182373047,0.3807574212551117,0.33242541551589966,0.501208484172821,0.3552962839603424,0.40449395775794983]}},"5":{"inputs":{"mean":[0.5667797923088074,-0.4425159692764282,-0.1232665404677391,-0.4805834889411926,0.10470238327980042,-0.1625356376171112,0.18144430220127106,0.17006072402000427,0.510013997554779,-0.1434447467327118,-0.034966882318258286,0.10707132518291473,0.013561461120843887,-0.007508033886551857,0.047198306769132614,0.08408746123313904,0.24387122690677643,-0.761760950088501,0.0798880085349083,-0.12375637888908386,1.6277751922607422,-0.49457091093063354,-0.004494925029575825,-0.022769825533032417,-0.02513645589351654,0.9867644309997559,0.34678730368614197,0.3044351637363434,-0.2644336521625519,0.024015527218580246,-0.01515234261751175,0.33554354310035706,-0.4338136613368988,0.12100833654403687,0.5454981327056885,-0.33242374658584595,0.2889866530895233,0.19355438649654388,-0.06140557676553726,-0.147211492061615,0.23500078916549683,0.03782473877072334,0.036928482353687286,0.21440233290195465,0.49215757846832275,-0.18998464941978455,0.37081634998321533,0.09337060153484344,0.3321336805820465,0.25658297538757324,-0.4870881736278534,0.1375667154788971,0.3354395627975464,-0.25396963953971863,0.11393021792173386,0.17268957197666168,-1.2365728616714478,0.37675392627716064,0.043199360370635986,0.11934693157672882,-0.12312943488359451,0.08465129137039185,-1.546708583831787,0.08309967070817947,0.21570716798305511,0.3668675124645233,0.209586039185524,0.08197544515132904,0.3807392716407776,0.12079507112503052,0.38388684391975403,0.4877951145172119,0.01840813085436821,0.23361895978450775,-0.1629452407360077,-0.23020298779010773,0.38624876737594604,0.12356976419687271,-0.03526456654071808,0.05217507854104042,-1.0528117418289185,0.3514059782028198,0.6874650716781616,0.2361573576927185,0.11454342305660248,0.1380791962146759,-0.04924939200282097,0.7943342328071594,-0.09703564643859863,0.1577836275100708,-0.11838231980800629,0.11224526911973953,-0.1804112195968628,-0.118077851831913,0.010194836184382439,0.1380450427532196,0.13202162086963654,0.1900312453508377,-0.9072489142417908,0.22364290058612823,0.32463401556015015,-0.1966812163591385,0.22454841434955597,-0.5121370553970337,-0.9022671580314636,0.14162620902061462,0.14243578910827637,-0.6824671626091003,0.11264270544052124,0.14749868214130402,0.11645998060703278,0.08737428486347198,-0.29450124502182007,0.039038654416799545,0.20028021931648254,-0.33875587582588196,0.3191632032394409,0.19987212121486664,0.15666159987449646,-0.04408257454633713,-0.279549241065979,0.3668660819530487,-0.004058930091559887,0.14516404271125793,-0.6769968867301941,-0.02501644566655159,-0.16689243912696838,0.12472930550575256,-1.3089743852615356,0.1839342713356018,-0.07835645228624344,0.045368026942014694,0.3169133961200714,-0.07889898121356964,0.005490223877131939,0.1775931715965271,-0.19520777463912964,0.20122641324996948,0.041091207414865494,0.9168247580528259,0.20373643934726715,-0.14907681941986084,0.07970588654279709,-0.16846859455108643,0.20408883690834045,0.06597944349050522,0.3243555426597595,0.13228537142276764,0.07407233864068985,0.13866601884365082,0.15431605279445648,0.1585644781589508,-0.14650015532970428,0.06460943073034286,-0.017667416483163834,0.04655374959111214,8.108222007751465,-0.8803310394287109,0.03779732063412666,0.11050494015216827,-0.4489752948284149,-0.2173420637845993,0.12657713890075684,0.01590319164097309,-0.010066824033856392,0.16731694340705872,-0.019536375999450684,0.4578861594200134,0.022527864202857018,0.07912828028202057,-0.052978552877902985,-0.05416843295097351,-0.05086088180541992,2.0874085426330566,1.9083985090255737,0.8251051902770996,0.5672033429145813,0.04737994074821472,-0.3174251616001129,0.04604219272732735,0.1833251416683197,-0.052292127162218094,0.17228244245052338,0.42390263080596924,0.48917335271835327,-0.3570418953895569,-0.5336315631866455,0.050722118467092514,-0.2729276418685913,0.43683505058288574,-0.03956574201583862,0.49901342391967773,0.03208424150943756,0.5480448007583618,-0.06121378019452095,-0.2795991599559784,0.31929531693458557,-0.8488425612449646,-0.04940584674477577,0.19312606751918793,0.37056809663772583,-0.3119897246360779,-1.4402546882629395,-0.31255048513412476,-0.032989878207445145,0.6572951674461365,0.2247939556837082,-0.1365707814693451,0.03148205950856209,0.20059721171855927,-0.8829682469367981,-0.06756372004747391,0.12191178649663925,0.21335206925868988,-0.21985271573066711,0.3842742443084717,0.5610519051551819,0.4894837737083435,0.401792049407959,0.25798627734184265,0.5726619958877563,-0.07373271137475967,-0.21553370356559753,0.016153233125805855,0.07128793746232986,0.02189738117158413,-2.3353142738342285,0.5149086117744446,0.10968939960002899,0.32696110010147095,0.1720075011253357,0.09637963771820068,-0.023528365418314934,0.1396147608757019,-0.1427365094423294,1.2485973834991455,-0.24782492220401764,4.279930591583252,-0.4827822744846344,-0.11814022809267044,-1.1443681716918945,0.16659298539161682,0.08127632737159729,0.0757395550608635,0.682033360004425,-0.06425164639949799,0.26422128081321716,0.2601242661476135,-0.3753238022327423,-0.09489874541759491,-0.38505738973617554,0.5489510297775269,-0.19833247363567352,0.328265517950058,-0.057112328708171844,0.3335653245449066,-0.1107746958732605,0.015733594074845314,0.46887022256851196,0.030142391100525856,0.14905330538749695,-0.09138790518045425,-0.21212457120418549,1.1236850023269653,0.25412845611572266,-0.13059766590595245,-0.23986269533634186,0.0064758420921862125,0.47020789980888367,0.3852561414241791,0.07934044301509857,-2.348506450653076,-0.20145224034786224,0.27821218967437744,0.8742591142654419,-0.48927634954452515,-0.26061007380485535,-0.20701342821121216,-0.02167193591594696,0.37091758847236633,0.40553683042526245,-0.22276940941810608,-0.24890005588531494,0.2386905699968338,-0.14451716840267181,0.2472819834947586,0.04129932448267937,0.05766872689127922,0.036674268543720245,0.4036668837070465,0.08784446120262146,-0.0784197673201561,-0.11507715284824371,-0.05831821262836456,0.13057103753089905,0.13751809298992157,-0.05922814831137657,0.04690565541386604,0.0719989463686943,0.48849987983703613,0.1506025195121765,-0.02632948011159897,0.2803812325000763,0.035343237221241,-0.0032734572887420654,0.19737330079078674,0.23008424043655396,0.13343457877635956,0.0630955919623375,0.2608585059642792,-0.24274441599845886,-0.022519541904330254,0.4145578145980835,0.05777382850646973,0.5096544027328491,0.38485538959503174,0.05251427739858627,-0.0944088026881218,0.4008512496948242,-0.11895690113306046,0.0319085493683815,-0.12399066984653473,-0.19753509759902954,0.5582188963890076,-0.07655355334281921,0.2863089144229889,0.8495234847068787,0.06047074869275093,0.006302437279373407,0.022736607119441032,1.710054636001587,-0.20416352152824402,-0.0943600982427597,1.0180635452270508,-0.07078403234481812,0.1864231377840042,-0.05932790786027908,0.056065596640110016,-0.12547184526920319,0.13294312357902527,-0.0161244198679924,-0.16513575613498688,0.13822443783283234,0.17187604308128357,-0.528517484664917,0.4147014915943146,0.2612123191356659,0.4103931188583374,-0.11739669740200043,-0.2491498738527298,0.009499351494014263,0.2687077522277832,0.35060375928878784,-0.11030060052871704,-0.06609804183244705,0.19754666090011597,0.015264630317687988,0.07459825277328491,-0.004484089091420174,0.08946133404970169,0.16722387075424194,0.06358997523784637,0.2862761616706848,0.12143462896347046,0.13582919538021088,0.06656476110219955,0.06667765229940414,0.3309115469455719,-0.3324999213218689,-0.2967555820941925,0.059075977653265,-0.12866927683353424,0.30493640899658203,0.12417147308588028,-0.13821057975292206,0.19672463834285736,-0.9930293560028076,0.16272732615470886,0.29126274585723877,0.06508158892393112,0.40965020656585693,0.07391954213380814,0.19850991666316986,-0.06938115507364273,-0.03201458603143692,-0.22390876710414886,-0.09134566783905029,0.23379307985305786,-0.2115473449230194,0.13826069235801697,0.07050589472055435,0.06406902521848679,-0.45805129408836365,-0.10666767507791519,0.2106505036354065,0.1946014165878296,0.24587266147136688,-0.36327531933784485,0.2377757728099823,0.19565735757350922,-1.0234155654907227,-0.44873374700546265,0.03416094183921814,-0.11783771216869354,-0.4774205982685089,-0.30093443393707275,0.0060171824879944324,-0.0868697389960289,-0.013654833659529686,-0.33585044741630554,0.012863226234912872,0.21422424912452698,0.0011150045320391655,0.39701762795448303,-0.14076244831085205,-0.033593788743019104,0.272451251745224,0.3841094672679901,-0.32316794991493225,0.5008153319358826,0.10862307995557785,0.2831766605377197,0.5469761490821838,0.14145144820213318,-0.16626527905464172,0.3847962021827698,0.27582648396492004,0.04319576546549797,-0.4621790945529938,-0.0644289180636406,0.07403096556663513,-0.042561084032058716,0.28153470158576965,0.4738052189350128,-0.23607631027698517,1.6758537292480469,0.18620361387729645,0.30954501032829285,-0.13857340812683105,0.10948055237531662,-0.1613065004348755,-0.7066699266433716,-0.09910057485103607,-0.11466729640960693,-0.0650140568614006,-0.09173562377691269,0.07063364237546921,-0.36613383889198303,0.21576955914497375,0.454520046710968,0.2277347892522812,1.8531025648117065,-0.11904087662696838,0.04082472622394562,0.14087988436222076,-0.11809003353118896,0.558983564376831,0.050983406603336334,0.1058814749121666,0.294534832239151,0.37240034341812134,-0.08407356590032578,-0.018386531621217728,-0.17469942569732666,0.33337506651878357,0.041830845177173615,0.1920471042394638,0.2875622808933258,-0.03286110982298851,0.37147125601768494,0.28411975502967834,-0.08663815259933472,0.1766895353794098,-0.1918422430753708,-0.39390823245048523,0.20379939675331116,0.12852807343006134,0.04548458755016327,-1.4179928302764893,-0.22457334399223328,-0.02335899882018566,0.5299072265625,0.11150234937667847,0.3766653537750244,-0.19964055716991425,0.36797887086868286,-0.1516086459159851,0.4128393530845642,0.14510589838027954,0.12392453849315643,0.11545529216527939,-0.011233340948820114,0.11180652678012848,0.039559684693813324,-0.013614017516374588,0.6677011847496033,-0.12066982686519623,-0.02242947928607464,0.6891175508499146,0.3545283377170563,1.8062663078308105,-0.2353033870458603,-0.24531514942646027,-0.7202675342559814,0.0826590433716774,-0.5048542022705078,-0.12400298565626144,0.6752520799636841,-0.14569862186908722,0.3275317847728729,0.30177828669548035,0.31209009885787964],"std":[1.1088049411773682,1.1885653734207153,0.9314184188842773,1.1612892150878906,0.9322824478149414,0.9151812791824341,1.1056575775146484,0.895958423614502,1.1969342231750488,0.98281329870224,1.0219179391860962,1.4989956617355347,1.0961241722106934,1.1404987573623657,0.9215648174285889,1.1190693378448486,0.9991716146469116,2.2341461181640625,1.0625925064086914,0.9309999942779541,2.280987024307251,3.5358376502990723,1.150573968887329,3.6399428844451904,1.0281836986541748,1.4271396398544312,1.0815027952194214,0.923650324344635,1.0435515642166138,1.0754857063293457,0.9607465863227844,1.1413273811340332,1.3630887269973755,1.4663983583450317,1.2285521030426025,1.1164501905441284,1.1782294511795044,1.030530571937561,0.9689896702766418,1.018526315689087,0.9611892104148865,0.9162178039550781,0.9449597597122192,0.9676430821418762,1.0920379161834717,0.9883325099945068,1.2105120420455933,0.9838648438453674,1.0251855850219727,0.8825869560241699,1.993930459022522,1.0895639657974243,1.132359504699707,1.151934027671814,1.0986601114273071,1.0111732482910156,1.6656293869018555,1.1601483821868896,1.1347416639328003,1.0286647081375122,0.9877068400382996,1.0806756019592285,3.132484197616577,0.9447292685508728,1.3037528991699219,1.0745600461959839,1.0060818195343018,1.2447776794433594,1.0041519403457642,1.0644218921661377,1.1057829856872559,1.7369619607925415,1.1998732089996338,1.2225275039672852,0.9423138499259949,1.0419142246246338,1.0484949350357056,1.0246825218200684,1.0980336666107178,1.0320366621017456,1.9118772745132446,2.327252149581909,1.0544474124908447,1.1273396015167236,1.131940484046936,1.00576913356781,0.9760457873344421,1.259194016456604,1.0775574445724487,1.1124604940414429,1.080349087715149,1.0536024570465088,0.9889411330223083,0.9957222938537598,1.185703992843628,1.2063401937484741,1.0334899425506592,1.0977760553359985,1.985843539237976,1.2840691804885864,1.0319772958755493,1.0819107294082642,0.949432373046875,1.5803545713424683,1.3908674716949463,1.1069105863571167,0.9635332226753235,1.6627053022384644,0.9221398234367371,1.1030384302139282,1.0956130027770996,1.983554482460022,1.0062005519866943,0.9104597568511963,0.9658888578414917,1.083844780921936,1.0511066913604736,1.0661731958389282,0.9423402547836304,0.9762290120124817,1.0202126502990723,1.0232446193695068,0.9055675864219666,0.9371993541717529,1.1130551099777222,1.1310560703277588,1.0106570720672607,1.055890440940857,1.7807456254959106,1.101866364479065,1.1162408590316772,1.0667293071746826,1.2166037559509277,1.1119023561477661,1.0543805360794067,1.0235275030136108,1.025382161140442,1.2565420866012573,0.9089049100875854,1.2114578485488892,1.0840927362442017,0.8453413844108582,1.0020170211791992,1.6348198652267456,1.006750226020813,1.2063809633255005,1.0947012901306152,0.9993298053741455,1.3450636863708496,0.9313523173332214,1.292409896850586,1.1167445182800293,0.8687532544136047,1.3919734954833984,1.1261837482452393,0.8745918273925781,3.8678581714630127,2.338635206222534,1.2176960706710815,0.9662548899650574,1.03290593624115,1.0679590702056885,1.002197504043579,1.0941566228866577,1.030668020248413,1.0682251453399658,0.9969128966331482,1.3305214643478394,1.3688411712646484,1.6292356252670288,0.9781359434127808,1.0462123155593872,1.1236629486083984,2.8186194896698,3.563520908355713,3.0145490169525146,1.2659908533096313,0.9350396394729614,1.0176920890808105,1.0454849004745483,0.8703893423080444,1.5469309091567993,0.9075558185577393,1.0726386308670044,1.1440008878707886,0.9748832583427429,1.8860275745391846,0.8973411321640015,1.0000239610671997,1.4500422477722168,1.0278260707855225,1.0554977655410767,1.0043234825134277,1.2693346738815308,0.9511781930923462,0.9537456035614014,1.175412893295288,2.2288622856140137,0.902548611164093,1.0339564085006714,1.0763709545135498,1.250740647315979,1.9250752925872803,1.1764357089996338,0.9281800985336304,1.296829342842102,0.9976781010627747,0.9508371949195862,0.924402117729187,1.0107035636901855,1.2937086820602417,1.028080940246582,1.070704698562622,1.078612208366394,1.115484595298767,1.0999698638916016,1.40972900390625,1.181905746459961,0.9273030757904053,0.9800136685371399,1.216255784034729,0.970812201499939,0.9620644450187683,1.196471929550171,1.0523343086242676,1.0784711837768555,2.016594648361206,1.253912091255188,0.9038140773773193,1.027294635772705,1.0726913213729858,1.7210627794265747,1.0583049058914185,0.9550451040267944,1.0550076961517334,1.5144425630569458,0.9594460725784302,3.9879419803619385,1.123984456062317,1.0647194385528564,2.390364170074463,1.2048414945602417,1.1887917518615723,1.0107550621032715,2.0241031646728516,0.9691190719604492,0.9608607292175293,1.013178825378418,1.8074830770492554,1.129040002822876,0.9969409108161926,1.152255892753601,0.9646332263946533,1.1103686094284058,1.0278767347335815,1.1385244131088257,1.0976508855819702,1.00686514377594,1.199960708618164,1.0338070392608643,1.100930094718933,1.0615854263305664,1.1266196966171265,2.3453187942504883,1.0363073348999023,0.9517128467559814,1.0839331150054932,0.9815146923065186,1.4886932373046875,1.260289192199707,1.016878604888916,3.139418363571167,1.139973759651184,1.0367965698242188,1.9179625511169434,1.6420724391937256,2.008201837539673,0.9987010955810547,0.9819496870040894,1.3150136470794678,1.5935633182525635,0.9994975328445435,1.1051080226898193,0.9658746719360352,1.22832190990448,1.1535807847976685,1.1164346933364868,1.075191855430603,1.0962367057800293,1.342747688293457,1.0730863809585571,1.8695770502090454,0.9485960602760315,0.9285420179367065,0.9745503067970276,1.0088417530059814,1.0363123416900635,1.0251563787460327,1.0347362756729126,1.4634549617767334,0.9957645535469055,1.0958672761917114,1.1170830726623535,0.9842130541801453,1.0321476459503174,1.0195833444595337,5.820615768432617,0.8937151432037354,1.0954909324645996,1.076097846031189,1.0026952028274536,0.908961832523346,1.235076665878296,1.348839282989502,1.1878231763839722,1.0612248182296753,0.9448922276496887,1.0142614841461182,1.0226587057113647,0.9032106399536133,0.968553364276886,1.1723188161849976,1.0019630193710327,1.084743618965149,0.9231547713279724,1.0611523389816284,1.6747105121612549,0.9478343725204468,0.8863413333892822,1.0459367036819458,2.471264123916626,1.0607870817184448,1.0484249591827393,1.730549931526184,0.9277442693710327,0.982876181602478,1.3413585424423218,1.017042875289917,1.0719172954559326,1.2561407089233398,1.0394083261489868,1.0176119804382324,0.9611338376998901,1.1392205953598022,1.593040108680725,1.0046489238739014,0.9824184775352478,1.023564338684082,1.0454670190811157,1.1016596555709839,1.1150152683258057,0.9595906734466553,2.0859687328338623,0.969886302947998,1.1449108123779297,1.072435975074768,1.063983678817749,1.3669233322143555,0.9341345429420471,1.035976767539978,1.0451223850250244,1.0355560779571533,1.1106537580490112,0.9613049030303955,0.9014309048652649,1.0862038135528564,1.0043941736221313,1.0729039907455444,1.2144060134887695,1.2049347162246704,1.0539931058883667,1.090492606163025,0.968864917755127,0.9449458718299866,1.1516876220703125,1.1178165674209595,2.8134875297546387,1.058035135269165,1.0950881242752075,1.0694549083709717,1.091100811958313,0.9953089952468872,0.980286717414856,0.8691515922546387,0.9256632924079895,1.01731538772583,1.1280101537704468,0.8988597393035889,0.9320658445358276,1.016994833946228,1.6473222970962524,1.025731086730957,1.0590239763259888,1.2081457376480103,0.9656961560249329,1.0902624130249023,1.1175919771194458,1.4481487274169922,0.9362125992774963,1.1829824447631836,1.5193090438842773,1.7440024614334106,1.0195642709732056,1.1934709548950195,1.118513822555542,1.0854411125183105,1.0968616008758545,0.983398973941803,1.0973789691925049,1.0901433229446411,1.4064264297485352,1.2188268899917603,1.1518250703811646,1.5521663427352905,1.0139881372451782,0.9702075123786926,0.9729220271110535,1.0325018167495728,1.4773085117340088,1.0112297534942627,1.0401062965393066,1.1182081699371338,1.120177984237671,1.0524275302886963,0.9525608420372009,1.1762794256210327,1.066487193107605,1.164300560951233,0.9946976900100708,1.0122195482254028,0.9477225542068481,1.0589011907577515,1.086647868156433,1.0884109735488892,1.0178604125976562,1.6061298847198486,1.1623930931091309,1.1643164157867432,0.9676904082298279,1.0939044952392578,1.4137767553329468,1.2768677473068237,0.974616527557373,1.0865099430084229,0.8994085788726807,1.0526673793792725,0.9166043996810913,1.8968364000320435,1.5578162670135498,1.0651687383651733,1.120597004890442,4.231472015380859,1.029419183731079,1.0866652727127075,0.8571023941040039,0.9495735764503479,1.1840946674346924,1.0264501571655273,1.5090566873550415,1.1136735677719116,1.0038169622421265,1.1977572441101074,0.9970454573631287,1.1805486679077148,0.9799599051475525,1.1183760166168213,1.0464869737625122,1.0751876831054688,1.1223517656326294,1.1831333637237549,1.0645520687103271,0.921992838382721,1.000641107559204,0.9072855710983276,1.069800853729248,1.0074341297149658,1.256532073020935,1.1165649890899658,3.272353410720825,1.058629035949707,0.9812114834785461,1.245706558227539,1.0601578950881958,1.1244378089904785,0.9720378518104553,1.0093203783035278,0.9630333185195923,1.0564799308776855,1.0302263498306274,1.1049610376358032,1.0256726741790771,1.0225906372070312,1.6496078968048096,0.9727838039398193,0.9682955145835876,1.2548086643218994,1.1292885541915894,0.9962273240089417,1.4153295755386353,1.0801489353179932,2.493082046508789,1.2574894428253174,1.0599644184112549,1.2889626026153564,0.951856255531311,1.1693928241729736,1.1654683351516724,1.5674810409545898,0.939032793045044,1.1090086698532104,1.071168303489685,1.1959599256515503]},"targets":{"mean":[1.1216014623641968,-1.461837887763977,-1.3209768533706665,-2.060528039932251,-1.9708969593048096,-1.6769986152648926,-1.7147334814071655,1.768643856048584,1.293479323387146,-1.7666473388671875,-2.5505459308624268,-1.7461624145507812,-1.7579765319824219,1.3085566759109497,2.450470447540283,1.9619379043579102,-2.865201950073242,-1.3135005235671997,1.5220626592636108,-1.491450309753418,1.6341909170150757,1.9634037017822266,0.5590910911560059,1.051126480102539,2.1695966720581055,0.9451316595077515,-2.850964307785034,2.0382204055786133,2.1900200843811035,-1.6032923460006714,-1.7609847784042358,0.5394458770751953,2.829035997390747,1.4080026149749756,-2.0767128467559814,-1.8705329895019531,-1.647161602973938,-0.5467559099197388,2.513197898864746,-1.4305024147033691,-1.8053531646728516,2.090672016143799,-2.024144172668457,1.9949012994766235,-0.8891842365264893,-1.9826751947402954,2.107764720916748,1.640048623085022,1.9416048526763916,-3.127026319503784,-0.9141002297401428,1.5232361555099487,-2.5739290714263916,-1.3608061075210571,1.6849747896194458,1.6585389375686646,2.148258924484253,-3.138793706893921,0.7031439542770386,1.427210807800293,-1.2658151388168335,-2.4645254611968994,-2.2211713790893555,-2.2217400074005127,-1.699634313583374,2.584017753601074,1.8012700080871582,2.209411144256592,1.6682770252227783,-1.6099627017974854,1.2412090301513672,1.130327820777893,1.2721072435379028,-0.09234550595283508,2.8439087867736816,-2.1193530559539795,1.6406556367874146,-0.44513365626335144,1.4940614700317383,2.008073568344116,2.6368913650512695,1.1163303852081299,1.0085175037384033,0.9423640966415405,1.891387939453125,-1.4106793403625488,2.7441928386688232,1.4100732803344727,-1.2579851150512695,-2.1084179878234863,2.332484006881714,2.3386402130126953,-1.9298231601715088,-1.5233391523361206,-2.2074592113494873,1.768441081047058,-2.3888795375823975,-1.597048044204712,-2.2658801078796387,-1.7330350875854492,1.4545081853866577,1.4540824890136719,1.5748264789581299,-0.9972296357154846,-2.182338237762451,1.4787464141845703,-1.9848270416259766,-1.5801513195037842,1.5197054147720337,-2.1151680946350098,1.55901300907135,1.3024659156799316,-2.0280659198760986,-1.1195403337478638,-1.9883469343185425,-1.1435725688934326,2.295607805252075,1.6837505102157593,-1.8919652700424194,-1.5167264938354492,2.2304275035858154,2.1492106914520264,2.489305257797241,-2.000783681869507,-0.3338882029056549,1.949423909187317,2.501568555831909,1.6919437646865845,3.145209550857544,-2.4795126914978027,2.535464286804199,1.4207878112792969,2.2211341857910156,2.713447093963623,1.6422919034957886,1.3231713771820068,-1.2343696355819702,-1.130472183227539,-2.37216854095459,0.8834884166717529,-1.4870878458023071,-1.8711497783660889,-2.1559979915618896,-1.641414761543274,-1.9761394262313843,-2.365182399749756,2.4472036361694336,-2.5696072578430176,-1.2761024236679077,2.2373878955841064,0.17711254954338074,1.8817681074142456,-1.4153773784637451,-1.9745447635650635,-0.7608063817024231,-1.8643442392349243,-3.4046688079833984,-2.000197410583496,-1.4665039777755737,1.1996175050735474,-0.9590976238250732,-1.3860441446304321,1.2892508506774902,2.363046884536743,1.3169474601745605,1.9598031044006348,-1.483457326889038,1.6679660081863403,-2.362917184829712,3.719223737716675,-1.7252086400985718,-1.4842379093170166,-2.0016191005706787,1.70046865940094,-2.6897895336151123,-2.7190001010894775,1.4799036979675293,2.2468602657318115,-1.137483835220337,0.9238899946212769,-1.4907780885696411,3.60343861579895,2.3526246547698975,2.2742340564727783,1.6864290237426758,-1.3766950368881226,2.145547866821289,-2.4696717262268066,-1.5738390684127808,-1.8284425735473633,2.642178773880005,1.3482577800750732,-2.0511441230773926,2.059826374053955,2.2983267307281494,-2.0175068378448486,0.5675615072250366,2.1980490684509277,-1.5872474908828735,1.5691839456558228,-1.9732991456985474,-1.2565027475357056,-0.7061893939971924,-1.9317349195480347,-1.7018016576766968,1.6855005025863647,1.5731114149093628,-1.3789395093917847,2.39631724357605,2.02378249168396,-0.8691877722740173,-2.2496626377105713,1.5383857488632202,-0.5675469040870667,-1.155966877937317,1.6809241771697998,-1.8605608940124512,2.4562594890594482,-2.048666000366211,-2.134040355682373,1.8012166023254395,-1.9833377599716187,-1.381710171699524,-2.1025145053863525,-1.8754196166992188,2.9393134117126465,2.4377753734588623,2.4008986949920654,-1.7901355028152466,1.0750445127487183,0.5866016745567322,-2.6108462810516357,2.1171774864196777,1.4025741815567017,2.5115480422973633,0.2570948302745819,1.9356091022491455,0.6379868388175964,-1.5004465579986572,-0.3515719175338745,-1.358394742012024,-2.563643217086792,1.645427942276001,-1.7053428888320923,-1.266418695449829,1.6756346225738525,-2.5657002925872803,1.5501192808151245,3.0340185165405273,-2.39198899269104,2.8058085441589355,2.368558406829834,-2.2913689613342285,2.0586092472076416,1.9187283515930176,1.7154021263122559,0.7010351419448853,-2.4639148712158203,1.0845353603363037,1.4643222093582153,-0.8636987805366516,-1.1164674758911133,1.997086763381958,-1.616334319114685,1.2150765657424927,-0.8104525208473206,-1.788057804107666,2.186830759048462,1.1224879026412964,1.7277941703796387,2.0316109657287598,-1.7170461416244507,-0.6403553485870361,0.8952392935752869,-2.324934482574463,-2.2874624729156494,3.456616163253784,-2.0227043628692627,2.7483606338500977,1.6195085048675537,-3.2568440437316895,2.5050644874572754,2.264115810394287,1.3395203351974487,-1.83922278881073,1.810677409172058,-1.3555980920791626,-2.318849563598633,0.969586193561554,1.764126181602478,-1.5192292928695679,-1.3812201023101807,-1.495947241783142,-1.6333742141723633,-2.403005838394165,1.6619967222213745,-1.6916741132736206,1.3325425386428833,-1.7755242586135864,2.0030758380889893,1.7835850715637207,-2.289194107055664,1.9143880605697632,2.619858741760254,-1.7258776426315308,-2.543825149536133,-0.8430247902870178,-2.553929328918457,0.5019799470901489,1.5147185325622559,-1.927667498588562,-1.641927719116211,-3.3642358779907227,-1.8982529640197754,2.1309659481048584,-1.884688138961792,1.723720908164978,-1.9592701196670532,1.8193559646606445,0.01897224225103855,-1.9485669136047363,-2.5369203090667725,-1.5033042430877686,1.4896010160446167,-1.3576397895812988,1.3461990356445312,-2.5823209285736084,-1.8183356523513794,-2.091539144515991,-1.5191701650619507,1.0740234851837158,2.1657514572143555,-2.3056037425994873,1.6892486810684204,2.4562644958496094,-1.6838419437408447,3.4144222736358643,2.0804319381713867,-1.672151803970337,-1.3913973569869995,1.781989336013794,-1.935463309288025,-1.385297417640686,2.6323776245117188,-0.9805365204811096,1.2357165813446045,1.4706331491470337,1.5147355794906616,-2.2236135005950928,-2.4647061824798584,-1.7272251844406128,1.6851625442504883,-2.1318581104278564,2.2196712493896484,1.8145123720169067,1.5970762968063354,1.7235828638076782,-3.4302492141723633,1.9901132583618164,-2.2523739337921143,2.859745979309082,1.8376092910766602,2.2609732151031494,1.7210400104522705,-1.7784788608551025,-1.800513505935669,-1.686579704284668,1.279515027999878,-1.704689860343933,-2.363776922225952,1.3923252820968628,-2.1638829708099365,1.6332929134368896,-1.7688959836959839,2.5358901023864746,-2.4729058742523193,2.920792579650879,-1.3835275173187256,2.3202767372131348,2.097085952758789,0.21234914660453796,-1.711317539215088,-2.1741437911987305,1.542661428451538,-1.154098629951477,-1.6426022052764893,-1.976514458656311,1.7214521169662476,-1.911193609237671,-2.555912971496582,2.9998507499694824,1.601225733757019,-1.4924300909042358,-1.2728774547576904,-2.875699996948242,-1.9939616918563843,1.9836817979812622,1.9751144647598267,1.3194366693496704,-2.246950387954712,-1.1247999668121338,-1.3127527236938477,2.818990468978882,1.597246527671814,-1.3298779726028442,-2.2046220302581787,1.0080403089523315,1.9437789916992188,-2.565155029296875,-1.508670687675476,0.7454959154129028,-2.486504077911377,-1.1746623516082764,0.17879898846149445,-1.605216383934021,-1.5189087390899658,-2.3045401573181152,1.9084806442260742,3.162837266921997,1.8888996839523315,2.21328067779541,-1.179821252822876,1.4199000597000122,-2.330434799194336,1.4230904579162598,-0.503574550151825,-1.9781774282455444,1.6319890022277832,-1.1934270858764648,-1.9305065870285034,2.0913093090057373,2.121459484100342,2.3869338035583496,1.7291104793548584,-2.0275843143463135,1.4007208347320557,-2.1182258129119873,-2.153635025024414,-2.347247362136841,-2.6961686611175537,1.8978939056396484,-2.1159963607788086,2.5216851234436035,-2.174409866333008,-2.110276460647583,-2.2214560508728027,-2.1676084995269775,-1.3034868240356445,-1.661950707435608,1.4919089078903198,2.6730470657348633,-2.9585251808166504,-1.8034615516662598,2.022796392440796,1.2316582202911377,-1.897068977355957,-2.832772731781006,-2.715162515640259,1.6063884496688843,0.9628423452377319,1.2085795402526855,-1.4943764209747314,1.9842908382415771,2.01776123046875,-1.5249097347259521,1.9840829372406006,2.903189182281494,-2.5769011974334717,1.2940024137496948,-2.5887484550476074,1.3325523138046265,1.9322060346603394,1.8967434167861938,2.669830322265625,-1.3215281963348389,-2.125185966491699,2.003948211669922,-1.7493358850479126,-0.6338385939598083,1.4008735418319702,2.116454839706421,1.434266448020935,2.4382174015045166,1.4919021129608154,-1.6771905422210693,1.7640657424926758,-1.9254530668258667,1.0584803819656372,-1.2618672847747803,1.7777589559555054,-2.117722988128662,2.774895191192627,2.063840389251709,2.0065934658050537,2.1379268169403076,1.1713550090789795,-1.8958728313446045,-1.5481903553009033,1.9058489799499512,0.9297998547554016,-2.8356361389160156,-2.473862648010254,-1.5193296670913696,1.5962530374526978,-2.167602300643921,-1.4395002126693726,-2.0031423568725586,-2.474191665649414,-2.541952610015869,1.5199693441390991,1.7769643068313599,1.8679355382919312],"std":[0.5564829111099243,0.7433574795722961,0.6020784378051758,0.7992135882377625,0.8118776082992554,0.716907262802124,0.6573719382286072,0.7580227851867676,0.6230155229568481,0.7756410241127014,0.9185754656791687,0.7496447563171387,0.6734939813613892,0.6295859813690186,0.958511233329773,0.7406591176986694,1.0128005743026733,0.6881116628646851,0.7289594411849976,0.6628257632255554,1.2272685766220093,1.7785316705703735,0.479551762342453,6.136740207672119,0.8214017748832703,0.6707850098609924,0.9918012619018555,0.8418967127799988,0.840177595615387,0.7498527765274048,0.7426143288612366,0.528468132019043,0.9726012349128723,0.5513532161712646,0.7034465670585632,0.764310896396637,1.7725462913513184,0.6947558522224426,0.9261590838432312,0.6956281661987305,0.7337014079093933,0.7568246126174927,0.7850385308265686,0.7732824087142944,0.6250987648963928,0.8090337514877319,0.8040491342544556,0.6687778830528259,0.7472856044769287,1.0764679908752441,0.9514319896697998,0.6445825695991516,0.9282929301261902,0.660576581954956,0.7377737760543823,0.7236718535423279,0.8582971096038818,1.021952748298645,0.5598652958869934,0.7735064625740051,0.5992396473884583,0.9032360315322876,1.5653393268585205,0.8831077218055725,0.6807410717010498,1.027841329574585,0.7540544271469116,0.7125946879386902,0.7532965540885925,0.6842825412750244,0.639091432094574,0.7652320861816406,0.6260491609573364,0.5678463578224182,0.9973434805870056,0.8672549724578857,0.703866720199585,0.5817335247993469,0.706230103969574,0.8512921333312988,0.9083283543586731,0.8119340538978577,0.613407552242279,0.7019847631454468,0.7398185133934021,0.6058284044265747,1.0587457418441772,0.6029648780822754,0.6227182745933533,0.8455660939216614,0.8424551486968994,0.8421577215194702,0.7741352319717407,0.6571418046951294,0.8713333010673523,0.7696969509124756,0.9984081387519836,0.6696410179138184,0.982507050037384,0.7462032437324524,0.5830297470092773,0.7068425416946411,0.6693837642669678,0.7480073571205139,0.8946192264556885,0.7057533264160156,0.8244970440864563,0.8484830856323242,0.7629632353782654,0.7912955284118652,0.6610518097877502,0.7618929743766785,0.7939929962158203,0.5555296540260315,0.7734676003456116,0.6285424828529358,0.8723398447036743,0.7287377715110779,0.7450594305992126,0.6492385864257812,0.8972576856613159,0.859626829624176,0.9174363613128662,0.7868229150772095,0.613420307636261,0.7921123504638672,0.9781654477119446,0.6569313406944275,1.301011085510254,0.9286540150642395,0.87407386302948,0.6157049536705017,0.9360408782958984,0.9828128218650818,0.6729259490966797,0.5641726851463318,0.5794644355773926,0.5657780766487122,0.8592585325241089,0.5666124224662781,0.6267505288124084,0.8217502236366272,0.8026059865951538,0.8292081356048584,0.8068791031837463,0.9568866491317749,0.9626200199127197,0.9216792583465576,0.6878443360328674,0.8516280651092529,0.944233238697052,0.786735475063324,0.5906838178634644,0.7486742734909058,0.5347417593002319,0.7726650238037109,6.090473175048828,2.0397512912750244,0.6845788955688477,0.6484391093254089,0.6173756122589111,0.6665924787521362,0.5877417922019958,0.8482936024665833,0.6460336446762085,0.8441663384437561,0.6138083934783936,0.7252650260925293,0.8574649691581726,1.3351019620895386,0.7905512452125549,0.6982964277267456,0.8042474985122681,1.1007466316223145,1.7934768199920654,1.3785593509674072,0.6437765955924988,0.8355434536933899,0.6529392600059509,0.47222745418548584,0.6860584020614624,1.315005898475647,0.9014983773231506,0.8841133713722229,0.6873121857643127,0.7065418362617493,0.7269154191017151,0.9979069828987122,0.709822416305542,0.6898667216300964,0.9702228903770447,0.5956725478172302,0.854516327381134,0.8723084330558777,0.8779978156089783,0.7758681178092957,0.41424110531806946,1.2549906969070435,0.7008237242698669,0.7160458564758301,0.7469795346260071,0.5929726958274841,0.879030704498291,0.7902127504348755,0.6825283169746399,0.6376955509185791,0.635125458240509,0.6882973909378052,0.8339636325836182,0.797481894493103,0.49034392833709717,0.9344314932823181,0.6385691165924072,0.7079562544822693,0.5807346701622009,0.7613940238952637,0.7302417159080505,0.9904671907424927,0.8244754672050476,0.7693682312965393,0.818959653377533,0.8307980298995972,0.6187407374382019,0.8435673713684082,0.7999456524848938,1.0540021657943726,1.3113361597061157,0.8970590233802795,0.7366994023323059,0.6026667356491089,0.5382426381111145,1.11697256565094,0.8510141968727112,0.6532395482063293,0.9133919477462769,0.45452365279197693,0.8039146661758423,1.855228304862976,0.7202281951904297,0.6524007320404053,0.9625086188316345,0.841979444026947,0.6667935252189636,0.6480714082717896,0.8676932454109192,0.690345287322998,0.9264253377914429,0.6856637597084045,1.094726324081421,0.8978069424629211,1.0362766981124878,0.9548830986022949,0.9321008920669556,0.8006269335746765,0.7387309670448303,0.7246018052101135,0.5381988883018494,0.8808568120002747,0.5036309361457825,0.6521108150482178,0.609636127948761,0.5607083439826965,0.8042992353439331,0.6159325838088989,0.5858755707740784,0.6588428020477295,0.7548004388809204,0.8762534856796265,0.5812650322914124,0.6902965903282166,0.8451219797134399,1.4542129039764404,0.4459446370601654,0.5404672622680664,1.1004678010940552,0.8850557208061218,1.2836090326309204,0.8438512086868286,0.9554013609886169,0.7086166739463806,1.1440107822418213,0.9359563589096069,0.7777902483940125,0.6663875579833984,0.7625077366828918,0.7793753743171692,0.5830160975456238,0.8108965754508972,0.4987805485725403,0.7069060206413269,0.6252568364143372,0.729069173336029,0.6297782063484192,0.6607720255851746,0.8840755820274353,0.7342306971549988,0.7474290728569031,0.6443934440612793,0.7576307654380798,0.7836106419563293,0.6785404086112976,0.8793600797653198,0.7730464935302734,0.9861844778060913,0.6741628646850586,0.9040364623069763,3.3180346488952637,0.9940059185028076,0.5104385018348694,0.6931859254837036,0.8276370167732239,0.7710236310958862,1.1474570035934448,0.7685844302177429,0.8692535758018494,0.7518417239189148,0.7019345760345459,0.7378891706466675,0.8094130754470825,0.46630293130874634,0.8073338270187378,0.902798056602478,0.6754810214042664,0.7043752074241638,0.6447250843048096,0.6131227016448975,1.3253421783447266,0.7656845450401306,0.8334600329399109,0.6378273367881775,1.0406372547149658,0.9150850176811218,0.8997966647148132,0.7379967570304871,0.9543817043304443,0.7487276792526245,1.1975185871124268,0.8156201839447021,0.7353942394256592,0.6399677991867065,0.7086482048034668,0.775818407535553,0.5567075610160828,0.9404956102371216,0.7743450403213501,0.6986601948738098,0.6387740969657898,0.6256686449050903,0.8009212017059326,0.9025200009346008,0.7162173390388489,0.7952141761779785,0.9786732196807861,0.8423128724098206,0.7167902588844299,0.677738606929779,0.7178803086280823,1.2170456647872925,0.7853929400444031,0.8486148118972778,1.037298560142517,0.6438712477684021,0.9099158644676208,0.7191031575202942,0.6761648058891296,0.7613091468811035,0.7548006772994995,0.627755343914032,0.6932373642921448,0.8983660936355591,0.6733412146568298,0.7517543435096741,0.734383761882782,0.7947863340377808,0.9262974262237549,0.9559845328330994,1.1149451732635498,0.6032542586326599,0.8607161641120911,0.9022477865219116,0.44722723960876465,0.6941090226173401,0.861355185508728,0.7313514351844788,0.612460196018219,0.7799213528633118,0.8021910786628723,0.6755238175392151,0.7360377907752991,0.9290183186531067,1.025633692741394,0.7624297738075256,0.7014487981796265,0.5943097472190857,1.0632259845733643,0.78773432970047,0.7366231083869934,0.963168740272522,0.5651944279670715,0.8477705717086792,0.6533799767494202,0.6697475910186768,1.110675573348999,0.6393288373947144,0.6941466331481934,0.8933335542678833,0.5375111699104309,0.7621954679489136,1.010022759437561,0.6561768651008606,0.5859791040420532,0.8920879364013672,0.6327280402183533,0.9126942157745361,0.7303264737129211,0.741256833076477,0.8647463321685791,0.7510743737220764,1.0813597440719604,0.7596217393875122,0.8341452479362488,0.5676863193511963,0.660084068775177,0.9165506958961487,0.6071789860725403,0.7343584895133972,0.7900566458702087,0.7136982679367065,0.7254959940910339,0.7609877586364746,0.7662927508354187,0.7921196222305298,0.9049140810966492,0.6940629482269287,0.7607915997505188,0.7879973649978638,0.8539950251579285,0.7459123730659485,0.9531207084655762,0.9672923684120178,0.7055345177650452,0.8844491839408875,0.8943928480148315,0.7757021188735962,0.7972539067268372,0.8228138089179993,0.8115091919898987,1.0257744789123535,0.6325130462646484,0.657230794429779,1.0451289415359497,2.1619367599487305,0.7506955862045288,0.8242056965827942,0.6306281685829163,0.7394904494285583,1.003075122833252,0.9503892064094543,0.751217782497406,0.534829318523407,0.7025290727615356,0.7525923848152161,0.759630560874939,0.7191784977912903,0.7667137384414673,0.7128772139549255,0.9911338686943054,0.978018581867218,0.5943021774291992,0.8587459921836853,0.6282225251197815,0.7628401517868042,0.7673917412757874,1.012059211730957,0.5768417119979858,0.788499116897583,0.7738405466079712,0.6435175538063049,0.8407078385353088,0.6468503475189209,0.8560301065444946,0.6076666712760925,0.909589409828186,0.657075047492981,0.7440749406814575,0.7094142436981201,0.7930682301521301,0.5552033185958862,0.633401095867157,0.7791839241981506,0.8174335360527039,1.0376802682876587,0.8332479596138,0.764937698841095,0.8375549912452698,0.5925666689872742,0.7749258279800415,0.690742552280426,0.7549168467521667,0.5305911898612976,1.498509407043457,0.9473984241485596,0.6791673898696899,0.7407816052436829,0.8585972785949707,0.660700261592865,0.8243281841278076,1.1560777425765991,0.9396423101425171,0.6951852440834045,0.691362738609314,0.718109667301178]}}} \ No newline at end of file diff --git a/scripts/analysis/check_model.py b/scripts/analysis/check_model.py new file mode 100644 index 0000000..15963da --- /dev/null +++ b/scripts/analysis/check_model.py @@ -0,0 +1,21 @@ +# %% +import torch +import os +import json +from safetensors.torch import load_file + +# Load model from safetensors file +model_path = "/Users/curttigges/Projects/crosslayer-coding/conversion_test/gpt2_32k/full_model.safetensors" +device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + +if os.path.exists(model_path): + state_dict = load_file(model_path, device=device.type) + print(f"Loaded model from {model_path}") +else: + print(f"Model file not found at {model_path}") + +# %% +state_dict.keys() +# %% +state_dict["decoder_module.decoders.0->1.weight"].shape +# %% diff --git a/scripts/compare_norm_stats.py b/scripts/compare_norm_stats.py new file mode 100644 index 0000000..6567fca --- /dev/null +++ b/scripts/compare_norm_stats.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Compare two norm_stats.json files layer-by-layer. + +Usage: + python scripts/compare_norm_stats.py path/to/a/norm_stats.json path/to/b/norm_stats.json +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Dict, Any, Tuple +import numpy as np + + +def _load_norm(path: Path) -> Dict[int, Dict[str, Any]]: + with open(path) as f: + raw = json.load(f) + # cast layer keys to int for convenient lookup + norm: Dict[int, Dict[str, Any]] = {int(k): v for k, v in raw.items()} + return norm + + +def _diff_stats( + a: Dict[int, Dict[str, Any]], b: Dict[int, Dict[str, Any]] +) -> Dict[int, Dict[str, Tuple[float, float]]]: + """Return {layer: {"inputs_mean": (abs_diff, rel_diff%), ...}}""" + out: Dict[int, Dict[str, Tuple[float, float]]] = {} + for layer in sorted(set(a) | set(b)): + layer_res: Dict[str, Tuple[float, float]] = {} + for section in ("inputs", "targets"): + for field in ("mean", "std"): + key = f"{section}_{field}" + if layer in a and layer in b and section in a[layer] and section in b[layer]: + vec_a = np.asarray(a[layer][section][field], dtype=np.float64) + vec_b = np.asarray(b[layer][section][field], dtype=np.float64) + if vec_a.shape != vec_b.shape: + layer_res[key] = (float("nan"), float("nan")) + continue + abs_diff = float(np.mean(np.abs(vec_a - vec_b))) + denom = np.mean(np.abs(vec_a)) + 1e-12 + rel_diff = float((abs_diff / denom) * 100.0) + layer_res[key] = (abs_diff, rel_diff) + else: + layer_res[key] = (float("nan"), float("nan")) + out[layer] = layer_res + return out + + +def main(): + parser = argparse.ArgumentParser(description="Compare two norm_stats.json files") + parser.add_argument("file_a", type=Path) + parser.add_argument("file_b", type=Path) + parser.add_argument( + "--top-n", type=int, default=5, help="Show detailed stats for top-N layers with biggest mean differences" + ) + args = parser.parse_args() + + norm_a = _load_norm(args.file_a) + norm_b = _load_norm(args.file_b) + + diffs = _diff_stats(norm_a, norm_b) + + print(f"Compared {len(diffs)} layers\n") + worst_layers = sorted(diffs.items(), key=lambda kv: np.nan_to_num(kv[1]["inputs_mean"][0], nan=0.0), reverse=True) + + print("Layer | inputs_mean | targets_mean | inputs_std | targets_std (abs diff / % rel diff)") + print("------- | ------------- | -------------- | ------------ | ------------") + for layer, stats in worst_layers[: args.top_n]: + im = stats["inputs_mean"] + tm = stats["targets_mean"] + isd = stats["inputs_std"] + tsd = stats["targets_std"] + print( + f"{layer:5d} | {im[0]:10.4g} / {im[1]:6.2f}% | {tm[0]:10.4g} / {tm[1]:6.2f}% | " + f"{isd[0]:10.4g} / {isd[1]:6.2f}% | {tsd[0]:10.4g} / {tsd[1]:6.2f}%" + ) + + print( + "\nTip: large relative differences (>5-10 %) mean you should use the training norm_stats.json during evaluation." + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/convert_batchtopk_to_jumprelu.py b/scripts/convert_batchtopk_to_jumprelu.py index dfdbf91..2822e30 100644 --- a/scripts/convert_batchtopk_to_jumprelu.py +++ b/scripts/convert_batchtopk_to_jumprelu.py @@ -4,7 +4,7 @@ import argparse import sys import logging -from typing import Dict, List +from typing import Dict, List, Optional import math # Ensure the project root is in the Python path @@ -17,7 +17,7 @@ from clt.models.clt import CrossLayerTranscoder from clt.training.data.local_activation_store import LocalActivationStore # Default store for this script from clt.training.evaluator import CLTEvaluator # Added for NMSE check - from safetensors.torch import load_file as load_safetensors_file # Added for safetensors support + from safetensors.torch import load_file, save_file # Added for safetensors support # Add other store types if needed, e.g., RemoteActivationStore except ImportError as e: @@ -124,14 +124,32 @@ def main(args): else: # Standard single-file checkpoint if args.batchtopk_checkpoint_path.endswith(".safetensors"): logger.info(f"Loading BatchTopK model state from safetensors file: {args.batchtopk_checkpoint_path}") - state_dict = load_safetensors_file(args.batchtopk_checkpoint_path, device=device.type) - state_dict = _remap_checkpoint_keys(state_dict) # Remap keys - model.load_state_dict(state_dict) + state_dict = load_file(args.batchtopk_checkpoint_path, device=device.type) else: logger.info(f"Loading BatchTopK model state from .pt file: {args.batchtopk_checkpoint_path}") state_dict = torch.load(args.batchtopk_checkpoint_path, map_location=device) - state_dict = _remap_checkpoint_keys(state_dict) # Remap keys - model.load_state_dict(state_dict) + + state_dict = _remap_checkpoint_keys(state_dict) # Remap keys + + # --- Add config validation against checkpoint --- + first_encoder_weight_key = "encoder_module.encoders.0.weight" + if first_encoder_weight_key in state_dict: + checkpoint_num_features = state_dict[first_encoder_weight_key].shape[0] + config_num_features = clt_config_batchtopk.num_features + if checkpoint_num_features != config_num_features: + logger.error("--- CONFIGURATION MISMATCH ---") + logger.error( + f"The 'num_features' in your config file ({config_num_features}) " + f"does not match the number of features in the checkpoint ({checkpoint_num_features})." + ) + logger.error(f" Config path: {args.config_path}") + logger.error(f" Checkpoint path: {args.batchtopk_checkpoint_path}") + logger.error( + "Please ensure you are using the correct config file that was used to train this model." + ) + return # Exit the script + + model.load_state_dict(state_dict) model.eval() logger.info("BatchTopK model loaded and set to eval mode.") @@ -174,7 +192,7 @@ def main(args): try: estimated_thetas = model.estimate_theta_posthoc( - data_iter=activation_store_theta, + data_iter=iter(activation_store_theta), num_batches=args.num_batches_for_theta_estimation, default_theta_value=args.default_theta_value, device=device, # Pass device to ensure buffers are on correct device @@ -193,7 +211,12 @@ def main(args): # 5. Save the Converted JumpReLU Model and its Config logger.info(f"Saving converted JumpReLU model state to: {args.output_model_path}") os.makedirs(os.path.dirname(args.output_model_path), exist_ok=True) - torch.save(model.state_dict(), args.output_model_path) + if args.output_model_path.endswith(".safetensors"): + save_file(model.state_dict(), args.output_model_path) + logger.info(f"Saved converted JumpReLU model state as safetensors to: {args.output_model_path}") + else: + torch.save(model.state_dict(), args.output_model_path) + logger.info(f"Saved converted JumpReLU model state as .pt to: {args.output_model_path}") logger.info(f"Saving converted JumpReLU model config to: {args.output_config_path}") os.makedirs(os.path.dirname(args.output_config_path), exist_ok=True) @@ -204,10 +227,6 @@ def main(args): # 6. Perform a quick L0 and NMSE check on the converted model logger.info("Performing L0 and NMSE check on the converted JumpReLU model...") - all_sample_inputs_for_l0: Dict[int, List[torch.Tensor]] = {i: [] for i in range(model.config.num_layers)} - all_sample_targets_for_nmse: Dict[int, List[torch.Tensor]] = {i: [] for i in range(model.config.num_layers)} - # total_tokens_collected_per_layer: Dict[int, int] = {i: 0 for i in range(model.config.num_layers)} # No longer needed with direct cat - l0_check_fetch_batch_size = ( args.l0_check_batch_size_tokens if hasattr(args, "l0_check_batch_size_tokens") and args.l0_check_batch_size_tokens is not None @@ -218,7 +237,12 @@ def main(args): f"Collecting data for L0/NMSE check using {args.num_batches_for_l0_check} batches with fetch batch size {l0_check_fetch_batch_size} tokens." ) - mean_tg_for_eval = None # To store normalization stats for Evaluator + # --- Initialise accumulators for batch-wise processing to conserve memory --- + total_l0_per_layer: Dict[int, float] = {i: 0.0 for i in range(model.config.num_layers)} + total_tokens_for_l0_per_layer: Dict[int, int] = {i: 0 for i in range(model.config.num_layers)} + all_reconstructions_for_nmse: Dict[int, List[torch.Tensor]] = {i: [] for i in range(model.config.num_layers)} + all_targets_for_nmse_check: Dict[int, List[torch.Tensor]] = {i: [] for i in range(model.config.num_layers)} + mean_tg_for_eval = None std_tg_for_eval = None try: @@ -235,7 +259,6 @@ def main(args): ) data_iterator_for_l0_check = iter(activation_store_for_l0_check) - # Retrieve mean_tg and std_tg if store has them (after iter is created, stats should be available if auto) if hasattr(activation_store_for_l0_check, "mean_tg") and hasattr(activation_store_for_l0_check, "std_tg"): mean_tg_for_eval = activation_store_for_l0_check.mean_tg std_tg_for_eval = activation_store_for_l0_check.std_tg @@ -246,123 +269,102 @@ def main(args): "mean_tg or std_tg not available from L0 check store. NMSE will be on potentially normalized values." ) + # --- Process data batch by batch --- for batch_idx in range(args.num_batches_for_l0_check): try: sample_inputs_batch, sample_targets_batch = next(data_iterator_for_l0_check) - for layer_idx in sample_inputs_batch.keys(): # Iterate over layers present in the input batch - if layer_idx in all_sample_inputs_for_l0: - input_tensor_data = sample_inputs_batch[layer_idx] - if input_tensor_data.dim() == 3: - num_tokens = input_tensor_data.shape[0] * input_tensor_data.shape[1] - all_sample_inputs_for_l0[layer_idx].append( - input_tensor_data.reshape(num_tokens, model.config.d_model) - ) - elif input_tensor_data.dim() == 2: - all_sample_inputs_for_l0[layer_idx].append(input_tensor_data) - - if ( - layer_idx in sample_targets_batch and layer_idx in all_sample_targets_for_nmse - ): # Check if target exists for the layer - target_tensor_data = sample_targets_batch[layer_idx] - if target_tensor_data.dim() == 3: - num_tokens = target_tensor_data.shape[0] * target_tensor_data.shape[1] - all_sample_targets_for_nmse[layer_idx].append( - target_tensor_data.reshape(num_tokens, model.config.d_model) - ) - elif target_tensor_data.dim() == 2: - all_sample_targets_for_nmse[layer_idx].append(target_tensor_data) + # Flatten inputs and targets from (B, S, D) to (B*S, D) + flat_inputs_batch = { + layer_idx: (t.reshape(-1, t.shape[-1]) if t.dim() == 3 else t) + for layer_idx, t in sample_inputs_batch.items() + } + flat_targets_batch = { + layer_idx: (t.reshape(-1, t.shape[-1]) if t.dim() == 3 else t) + for layer_idx, t in sample_targets_batch.items() + } + if not flat_inputs_batch: + continue + + # 1. Compute and accumulate L0 statistics + with torch.no_grad(): + l0s_batch = run_quick_l0_checks_script( + model, flat_inputs_batch, args.num_tokens_for_l0_check_script + ) + for layer_idx, l0_val in l0s_batch.items(): + if layer_idx in flat_inputs_batch and not math.isnan(l0_val): + tokens_in_batch = flat_inputs_batch[layer_idx].shape[0] + total_l0_per_layer[layer_idx] += l0_val * tokens_in_batch + total_tokens_for_l0_per_layer[layer_idx] += tokens_in_batch + + # 2. Get reconstructions for NMSE, moving to CPU to free up VRAM + with torch.no_grad(): + reconstructions_batch = model(flat_inputs_batch) + for layer_idx, recon_tensor in reconstructions_batch.items(): + if layer_idx in flat_targets_batch and recon_tensor.numel() > 0: + all_reconstructions_for_nmse[layer_idx].append(recon_tensor.cpu()) + all_targets_for_nmse_check[layer_idx].append(flat_targets_batch[layer_idx].cpu()) except StopIteration: logger.warning( - f"Activation store exhausted after {batch_idx + 1} batches during L0/NMSE check data collection. Proceeding with collected data." + f"Activation store exhausted after {batch_idx + 1} batches. Proceeding with collected data." ) break - if hasattr(activation_store_for_l0_check, "close") and callable( - getattr(activation_store_for_l0_check, "close") - ): + if hasattr(activation_store_for_l0_check, "close"): activation_store_for_l0_check.close() except Exception as e_l0_fetch: - logger.warning( - f"Error initializing or fetching batches for L0/NMSE check: {e_l0_fetch}. Check might use zero or incomplete input." - ) - - final_sample_batch_for_l0_inputs: Dict[int, torch.Tensor] = {} - for layer_idx, tensor_list in all_sample_inputs_for_l0.items(): - if tensor_list: - final_sample_batch_for_l0_inputs[layer_idx] = torch.cat(tensor_list, dim=0) - logger.info( - f"Layer {layer_idx}: Collected {final_sample_batch_for_l0_inputs[layer_idx].shape[0]} total input tokens for L0/NMSE check." - ) - else: - logger.warning(f"Layer {layer_idx}: No input tokens collected for L0/NMSE check.") - final_sample_batch_for_l0_inputs[layer_idx] = torch.empty( - (0, model.config.d_model), device=device, dtype=model.dtype - ) + logger.error(f"Error during L0/NMSE check data processing: {e_l0_fetch}. Check may be incomplete.") + # Allow to proceed with any data collected so far - final_sample_targets_for_nmse_check: Dict[int, torch.Tensor] = {} - for layer_idx, tensor_list in all_sample_targets_for_nmse.items(): - if tensor_list: - final_sample_targets_for_nmse_check[layer_idx] = torch.cat(tensor_list, dim=0) - logger.info( - f"Layer {layer_idx}: Collected {final_sample_targets_for_nmse_check[layer_idx].shape[0]} total target tokens for NMSE check." - ) - else: - logger.warning(f"Layer {layer_idx}: No target tokens collected for NMSE check.") - final_sample_targets_for_nmse_check[layer_idx] = torch.empty( - (0, model.config.d_model), device=device, dtype=model.dtype - ) - - model_for_l0_check = model - - empirical_l0s_per_layer = run_quick_l0_checks_script( - model_for_l0_check, final_sample_batch_for_l0_inputs, args.num_tokens_for_l0_check_script - ) - logger.info(f"Empirical L0 per layer (out of {args.num_tokens_for_l0_check_script} sampled tokens):") + # --- Finalize and Log Metrics --- + logger.info("Empirical L0 per layer (averaged over collected data):") total_empirical_l0 = 0.0 - for l_idx, l0_val in empirical_l0s_per_layer.items(): - logger.info(f" Layer {l_idx}: {l0_val:.2f}") - if not (isinstance(l0_val, float) and math.isnan(l0_val)): - total_empirical_l0 += l0_val + empirical_l0s_per_layer = {} + for layer_idx in range(model.config.num_layers): + if total_tokens_for_l0_per_layer.get(layer_idx, 0) > 0: + avg_l0 = total_l0_per_layer[layer_idx] / total_tokens_for_l0_per_layer[layer_idx] + empirical_l0s_per_layer[layer_idx] = avg_l0 + logger.info(f" Layer {layer_idx}: {avg_l0:.2f}") + if not math.isnan(avg_l0): + total_empirical_l0 += avg_l0 + else: + empirical_l0s_per_layer[layer_idx] = float("nan") + logger.info(f" Layer {layer_idx}: nan (no tokens processed)") logger.info(f"Total Empirical L0 across all layers: {total_empirical_l0:.2f}") # Compute and log NMSE - logger.info("Computing NMSE...") - evaluator = CLTEvaluator(model=model_for_l0_check, device=device, mean_tg=mean_tg_for_eval, std_tg=std_tg_for_eval) - - # Ensure inputs and targets for NMSE have corresponding layers and non-empty tensors before calling - valid_layers_for_nmse = set(final_sample_batch_for_l0_inputs.keys()) & set( - final_sample_targets_for_nmse_check.keys() - ) - inputs_for_nmse_metric = { - k: v for k, v in final_sample_batch_for_l0_inputs.items() if k in valid_layers_for_nmse and v.numel() > 0 + logger.info("Computing NMSE on collected data...") + final_reconstructions = { + layer_idx: torch.cat(tensors).to(device) + for layer_idx, tensors in all_reconstructions_for_nmse.items() + if tensors } - targets_for_nmse_metric = { - k: v for k, v in final_sample_targets_for_nmse_check.items() if k in valid_layers_for_nmse and v.numel() > 0 + final_targets = { + layer_idx: torch.cat(tensors).to(device) for layer_idx, tensors in all_targets_for_nmse_check.items() if tensors } - if ( - not inputs_for_nmse_metric - or not targets_for_nmse_metric - or not any(v.numel() > 0 for v in inputs_for_nmse_metric.values()) - or not any(v.numel() > 0 for v in targets_for_nmse_metric.values()) - ): + if not final_reconstructions or not final_targets: logger.warning( - "Insufficient data for NMSE calculation after filtering (empty inputs or targets for common layers). Skipping NMSE." + "Insufficient data for NMSE calculation after processing (empty reconstructions or targets). Skipping NMSE." ) else: - # Get reconstructions from the model using the collected inputs - with torch.no_grad(): # Ensure no gradients are computed during this forward pass - reconstructions_for_nmse = model_for_l0_check(inputs_for_nmse_metric) - - reconstruction_metrics = evaluator._compute_reconstruction_metrics( - targets=targets_for_nmse_metric, reconstructions=reconstructions_for_nmse - ) - nmse_value = reconstruction_metrics.get("reconstruction/normalized_mean_reconstruction_error", float("nan")) - explained_variance = reconstruction_metrics.get("reconstruction/explained_variance", float("nan")) - logger.info(f"Normalized Mean Squared Error (NMSE) on collected data: {nmse_value:.4f}") - logger.info(f"Explained Variance (EV) on collected data: {explained_variance:.4f}") + evaluator = CLTEvaluator(model=model, device=device, mean_tg=mean_tg_for_eval, std_tg=std_tg_for_eval) + # Filter to common layers with data + valid_layers_for_nmse = set(final_reconstructions.keys()) & set(final_targets.keys()) + reconstructions_for_metric = {k: v for k, v in final_reconstructions.items() if k in valid_layers_for_nmse} + targets_for_metric = {k: v for k, v in final_targets.items() if k in valid_layers_for_nmse} + + if not reconstructions_for_metric or not targets_for_metric: + logger.warning("No common layers with data between reconstructions and targets. Skipping NMSE.") + else: + reconstruction_metrics = evaluator._compute_reconstruction_metrics( + targets=targets_for_metric, reconstructions=reconstructions_for_metric + ) + nmse_value = reconstruction_metrics.get("reconstruction/normalized_mean_reconstruction_error", float("nan")) + explained_variance = reconstruction_metrics.get("reconstruction/explained_variance", float("nan")) + logger.info(f"Normalized Mean Squared Error (NMSE) on collected data: {nmse_value:.4f}") + logger.info(f"Explained Variance (EV) on collected data: {explained_variance:.4f}") # --- Optional Layer-wise L0 Calibration --- # if args.l0_layerwise_calibrate: @@ -404,7 +406,7 @@ def main(args): state_dict_to_populate_orig = _remap_checkpoint_keys(state_dict_to_populate_orig) original_model.load_state_dict(state_dict_to_populate_orig) elif original_checkpoint_path.endswith(".safetensors"): - state_dict_orig = load_safetensors_file(original_checkpoint_path, device=device.type) + state_dict_orig = load_file(original_checkpoint_path, device=device.type) state_dict_orig = _remap_checkpoint_keys(state_dict_orig) original_model.load_state_dict(state_dict_orig) else: @@ -462,7 +464,18 @@ def main(args): final_calibration_inputs: Dict[int, torch.Tensor] = {} for layer_idx_cal, tensor_list_cal in calibration_inputs_collected.items(): if tensor_list_cal: - final_calibration_inputs[layer_idx_cal] = torch.cat(tensor_list_cal, dim=0) + concatenated = torch.cat(tensor_list_cal, dim=0) + # --- NEW: Down-sample to avoid huge tensors that crash MPS --- + num_tokens_available = concatenated.shape[0] + max_tokens = ( + args.num_tokens_for_l0_check_script + if args.num_tokens_for_l0_check_script > 0 + else num_tokens_available + ) + if num_tokens_available > max_tokens: + sel_indices = torch.randperm(num_tokens_available, device=concatenated.device)[:max_tokens] + concatenated = concatenated[sel_indices] + final_calibration_inputs[layer_idx_cal] = concatenated else: logger.warning(f"Layer {layer_idx_cal}: No calibration input tokens collected.") final_calibration_inputs[layer_idx_cal] = torch.empty( @@ -488,7 +501,7 @@ def main(args): # 4. Calibrate the converted JumpReLU model (model_for_l0_check is the one already converted) calibrate_layerwise_theta_for_l0_matching( - model_to_calibrate=model_for_l0_check, + model_to_calibrate=model, calibration_inputs=final_calibration_inputs, target_l0s_per_layer=target_l0s, num_tokens_for_l0_check=args.num_tokens_for_l0_check_script, @@ -502,54 +515,107 @@ def main(args): # 5. Log final L0s of the calibrated model logger.info("--- L0s after Layer-wise Calibration ---") calibrated_l0s_per_layer = run_quick_l0_checks_script( - model_for_l0_check, final_calibration_inputs, args.num_tokens_for_l0_check_script + model, final_calibration_inputs, args.num_tokens_for_l0_check_script ) total_calibrated_l0 = 0.0 - for l_idx, l0_val in calibrated_l0s_per_layer.items(): - logger.info(f" Layer {l_idx}: {l0_val:.2f} (Target: {target_l0s.get(l_idx, float('nan')):.2f})") + for layer_idx, l0_val in calibrated_l0s_per_layer.items(): + logger.info(f" Layer {layer_idx}: {l0_val:.2f} (Target: {target_l0s.get(layer_idx, float('nan')):.2f})") if not (isinstance(l0_val, float) and math.isnan(l0_val)): total_calibrated_l0 += l0_val logger.info(f"Total Empirical L0 across all layers (Calibrated): {total_calibrated_l0:.2f}") # 6. Re-save the calibrated model logger.info(f"Re-saving calibrated JumpReLU model state to: {args.output_model_path}") - torch.save(model_for_l0_check.state_dict(), args.output_model_path) + if args.output_model_path.endswith(".safetensors"): + save_file(model.state_dict(), args.output_model_path) + logger.info(f"Re-saved calibrated JumpReLU model state as safetensors to: {args.output_model_path}") + else: + torch.save(model.state_dict(), args.output_model_path) + logger.info(f"Re-saved calibrated JumpReLU model state as .pt to: {args.output_model_path}") # Config remains the same (JumpReLU), only log_thresholds changed. logger.info("--- Layer-wise L0 Calibration Step Finished ---") # --- Re-evaluate NMSE/EV after L0 calibration --- logger.info("--- NMSE/EV after Layer-wise Calibration ---") - if ( - not inputs_for_nmse_metric # This was defined before the calibration block - or not targets_for_nmse_metric - or not any(v.numel() > 0 for v in inputs_for_nmse_metric.values()) - or not any(v.numel() > 0 for v in targets_for_nmse_metric.values()) - ): - logger.warning( - "Insufficient data for NMSE re-evaluation after calibration (empty inputs or targets for common layers). Skipping." + logger.info("Re-fetching data for post-calibration NMSE/EV evaluation...") + + post_calib_recons: Dict[int, List[torch.Tensor]] = {i: [] for i in range(model.config.num_layers)} + post_calib_targets: Dict[int, List[torch.Tensor]] = {i: [] for i in range(model.config.num_layers)} + + try: + store_for_post_calib_eval = LocalActivationStore( + dataset_path=args.activation_data_path, + train_batch_size_tokens=l0_check_fetch_batch_size, + device=device, + dtype=args.activation_dtype or clt_config_batchtopk.expected_input_dtype or "float32", + rank=0, + world=1, + seed=args.seed + 1, # Use same seed as pre-calibration check + sampling_strategy="sequential", + normalization_method="auto", ) + iterator_post_calib = iter(store_for_post_calib_eval) + + for _ in range(args.num_batches_for_l0_check): + try: + inputs, targets = next(iterator_post_calib) + flat_inputs = { + layer_idx: (t.reshape(-1, t.shape[-1]) if t.dim() == 3 else t) + for layer_idx, t in inputs.items() + } + flat_targets = { + layer_idx: (t.reshape(-1, t.shape[-1]) if t.dim() == 3 else t) + for layer_idx, t in targets.items() + } + + if not flat_inputs: + continue + + with torch.no_grad(): + recons = model(flat_inputs) # Use calibrated model + + for layer_idx, recon_tensor in recons.items(): + if layer_idx in flat_targets and recon_tensor.numel() > 0: + post_calib_recons[layer_idx].append(recon_tensor.cpu()) + post_calib_targets[layer_idx].append(flat_targets[layer_idx].cpu()) + except StopIteration: + logger.info("Store exhausted during post-calibration evaluation.") + break + + if hasattr(store_for_post_calib_eval, "close"): + store_for_post_calib_eval.close() + except Exception as e_post_calib: + logger.error(f"Error during post-calibration NMSE data processing: {e_post_calib}") + + final_post_calib_recons = { + layer_idx: torch.cat(tensors).to(device) for layer_idx, tensors in post_calib_recons.items() if tensors + } + final_post_calib_targets = { + layer_idx: torch.cat(tensors).to(device) for layer_idx, tensors in post_calib_targets.items() if tensors + } + + if not final_post_calib_recons or not final_post_calib_targets: + logger.warning("Insufficient data for post-calibration NMSE calculation. Skipping.") else: - # Ensure the evaluator uses the potentially updated model_for_l0_check (calibrated model) - # If evaluator was initialized with model, and model_for_l0_check is the same instance that was modified, this is fine. - # If not, evaluator might need to be updated or re-initialized with the calibrated model. - # Assuming model_for_l0_check is the same instance that evaluator holds or that CLTEvaluator uses the model passed at evaluation time. - # CLTEvaluator constructor takes a model, but its _compute_reconstruction_metrics does not, it uses self.model. - # So, we need to ensure the evaluator has the *calibrated* model. evaluator_after_calib = CLTEvaluator( - model=model_for_l0_check, device=device, mean_tg=mean_tg_for_eval, std_tg=std_tg_for_eval + model=model, device=device, mean_tg=mean_tg_for_eval, std_tg=std_tg_for_eval ) - with torch.no_grad(): - reconstructions_after_calib = model_for_l0_check(inputs_for_nmse_metric) + valid_layers = set(final_post_calib_recons.keys()) & set(final_post_calib_targets.keys()) + recons_metric = {k: v for k, v in final_post_calib_recons.items() if k in valid_layers} + targets_metric = {k: v for k, v in final_post_calib_targets.items() if k in valid_layers} - metrics_after_calib = evaluator_after_calib._compute_reconstruction_metrics( - targets=targets_for_nmse_metric, reconstructions=reconstructions_after_calib - ) - nmse_after_calib = metrics_after_calib.get( - "reconstruction/normalized_mean_reconstruction_error", float("nan") - ) - ev_after_calib = metrics_after_calib.get("reconstruction/explained_variance", float("nan")) - logger.info(f"NMSE (post-L0-calibration): {nmse_after_calib:.4f}") - logger.info(f"EV (post-L0-calibration): {ev_after_calib:.4f}") + if recons_metric and targets_metric: + metrics_after_calib = evaluator_after_calib._compute_reconstruction_metrics( + targets=targets_metric, reconstructions=recons_metric + ) + nmse_after_calib = metrics_after_calib.get( + "reconstruction/normalized_mean_reconstruction_error", float("nan") + ) + ev_after_calib = metrics_after_calib.get("reconstruction/explained_variance", float("nan")) + logger.info(f"NMSE (post-L0-calibration): {nmse_after_calib:.4f}") + logger.info(f"EV (post-L0-calibration): {ev_after_calib:.4f}") + else: + logger.warning("No common layers with data for post-calibration NMSE calculation. Skipping.") logger.info("Conversion script finished successfully.") @@ -569,6 +635,16 @@ def calibrate_layerwise_theta_for_l0_matching( logger.info("Starting layer-wise L0 calibration...") model_to_calibrate.eval() # Ensure model is in eval mode + if model_to_calibrate.config.activation_fn != "jumprelu": + logger.error( + f"Model activation function is '{model_to_calibrate.config.activation_fn}', not 'jumprelu'. " + "Cannot perform theta calibration. Exiting calibration." + ) + return + if model_to_calibrate.log_threshold is None: + logger.error("model_to_calibrate.log_threshold is None. Cannot perform calibration. Exiting.") + return + # Detach original log_thresholds to use as base for scaling, to avoid them changing with each layer's calibration original_log_thetas_exp = model_to_calibrate.log_threshold.exp().detach().clone() @@ -605,7 +681,8 @@ def calibrate_layerwise_theta_for_l0_matching( # We are calibrating one layer at a time. When checking layer_idx, # we modify only model.log_threshold.data[layer_idx]. current_scaled_theta_layer = base_theta_layer * mid_s - model_to_calibrate.log_threshold.data[layer_idx] = torch.log(current_scaled_theta_layer.clamp_min(1e-9)) + if model_to_calibrate.log_threshold is not None: + model_to_calibrate.log_threshold.data[layer_idx] = torch.log(current_scaled_theta_layer.clamp_min(1e-9)) # Check L0 with this new theta for the current layer # run_quick_l0_checks_script returns a dict {layer_idx: l0_val} @@ -653,7 +730,8 @@ def calibrate_layerwise_theta_for_l0_matching( # Set the layer's log_threshold to the one corresponding to the best scale found final_scaled_theta_layer = base_theta_layer * current_best_scale - model_to_calibrate.log_threshold.data[layer_idx] = torch.log(final_scaled_theta_layer.clamp_min(1e-9)) + if model_to_calibrate.log_threshold is not None: + model_to_calibrate.log_threshold.data[layer_idx] = torch.log(final_scaled_theta_layer.clamp_min(1e-9)) logger.info(f"Layer {layer_idx}: Set final scale {current_best_scale:.3f}. Log_threshold updated.") logger.info("Layer-wise L0 calibration finished.") @@ -766,6 +844,38 @@ def run_quick_l0_checks_script( return empirical_l0s_per_layer +def _override_store_norm_stats(store, stats_path: Optional[str]): + """If stats_path is provided, load that JSON and inject into the store for normalisation & evaluator.""" + if not stats_path: + return None, None + try: + with open(stats_path) as f: + stats_json = json.load(f) + except Exception as e: + logger.error(f"Failed to load norm_stats from {stats_path}: {e}") + return None, None + # Populate the tensors the same way ManifestActivationStore._prep_norm does + mean_tg: Dict[int, torch.Tensor] = {} + std_tg: Dict[int, torch.Tensor] = {} + mean_in: Dict[int, torch.Tensor] = {} + std_in: Dict[int, torch.Tensor] = {} + device = store.device + for layer_idx_str, stats in stats_json.items(): + li = int(layer_idx_str) + if "inputs" in stats and "mean" in stats["inputs"] and "std" in stats["inputs"]: + mean_in[li] = torch.tensor(stats["inputs"]["mean"], device=device, dtype=torch.float32).unsqueeze(0) + std_in[li] = (torch.tensor(stats["inputs"]["std"], device=device, dtype=torch.float32) + 1e-6).unsqueeze(0) + if "targets" in stats and "mean" in stats["targets"] and "std" in stats["targets"]: + mean_tg[li] = torch.tensor(stats["targets"]["mean"], device=device, dtype=torch.float32).unsqueeze(0) + std_tg[li] = (torch.tensor(stats["targets"]["std"], device=device, dtype=torch.float32) + 1e-6).unsqueeze(0) + if mean_in and std_in: + store.mean_in, store.std_in = mean_in, std_in + store.mean_tg, store.std_tg = mean_tg, std_tg + store.apply_normalization = True + logger.info(f"Overrode normalisation stats with {stats_path}") + return mean_tg, std_tg + + if __name__ == "__main__": parser = argparse.ArgumentParser( description="Convert a BatchTopK CLT model to JumpReLU with post-hoc theta estimation." @@ -904,6 +1014,13 @@ def run_quick_l0_checks_script( help="Maximum iterations for binary search per layer during L0 calibration.", ) + parser.add_argument( + "--norm-stats-path", + type=str, + default=None, + help="Optional path to a norm_stats.json file to override the store's statistics (use the training stats for consistent NMSE/EV).", + ) + # Note: clt_dtype for the model will be taken from the loaded config_dict_batchtopk initially. args = parser.parse_args() diff --git a/scripts/download_norm_stats.py b/scripts/download_norm_stats.py deleted file mode 100644 index bc78f1e..0000000 --- a/scripts/download_norm_stats.py +++ /dev/null @@ -1,30 +0,0 @@ -import requests -import json - -url = "http://34.41.125.189:8000/datasets/EleutherAI%2Fpythia-70m%2Fpile-uncopyrighted_train/norm_stats" -output_filename = "norm_stats_downloaded.json" - -try: - print(f"Attempting to download from {url}...") - response = requests.get(url, timeout=60) - response.raise_for_status() # Raises an HTTPError for bad responses (4XX or 5XX) - data = response.json() - with open(output_filename, "w") as f: - json.dump(data, f, indent=4) - print(f"Successfully downloaded and saved to {output_filename}") -except requests.exceptions.HTTPError as http_err: - print(f"HTTP error occurred: {http_err}") -except requests.exceptions.ConnectionError as conn_err: - print(f"Connection error occurred: {conn_err}") -except requests.exceptions.Timeout as timeout_err: - print(f"Timeout error occurred: {timeout_err}") -except requests.exceptions.RequestException as req_err: - print(f"An error occurred during the request: {req_err}") -except json.JSONDecodeError: - print("Failed to decode JSON from the response. The content might not be valid JSON.") - # Optionally, save the raw content for inspection - with open("norm_stats_raw_content.txt", "w") as f: - f.write(response.text) - print("Raw response content saved to norm_stats_raw_content.txt") -except Exception as e: - print(f"An unexpected error occurred: {e}") diff --git a/scripts/train_clt.py b/scripts/train_clt.py index 6c2c163..bfca521 100644 --- a/scripts/train_clt.py +++ b/scripts/train_clt.py @@ -389,6 +389,11 @@ def parse_args(): action="store_true", help="Enable computation of detailed sparsity diagnostics during evaluation.", ) + train_group.add_argument( + "--enable-profiling", + action="store_true", + help="Enable detailed performance profiling to identify bottlenecks.", + ) # --- Sampling Strategy --- sampling_group = parser.add_argument_group("Sampling Strategy (TrainingConfig)") @@ -706,6 +711,7 @@ def main(): # Dead Features & Diagnostics dead_feature_window=args.dead_feature_window, compute_sparsity_diagnostics=args.compute_sparsity_diagnostics, + enable_profiling=args.enable_profiling, # WandB enable_wandb=args.enable_wandb, wandb_project=args.wandb_project, diff --git a/tutorials/1A-end-to-end-training-gpt2-relu.py b/tutorials/1A-end-to-end-training-gpt2-relu.py index f4ec2af..ebea21c 100644 --- a/tutorials/1A-end-to-end-training-gpt2-relu.py +++ b/tutorials/1A-end-to-end-training-gpt2-relu.py @@ -135,9 +135,8 @@ # Storage Parameters activation_dir=activation_dir, output_format="hdf5", - compression="gzip", - chunk_token_threshold=8_000, - activation_dtype="float32", # Explicitly set desired storage precision + chunk_token_threshold=32_000, + activation_dtype="float16", # Explicitly set desired storage precision # Normalization compute_norm_stats=True, # NNsight args (defaults are usually fine)