From eb1506f365a9146242c1e3084fd652b68683acc3 Mon Sep 17 00:00:00 2001 From: wangwl Date: Thu, 11 Dec 2025 06:41:16 +0000 Subject: [PATCH 1/4] add VovNet --- .../VovNet/Test/VovNetV1/coverage.txt | 3 + .../VovNet/Test/VovNetV1/vovnetv1.py | 227 ++++++ .../VovNet/Test/VovNetV1/weloTrainStep.py | 647 ++++++++++++++++++ .../VovNet/Test/VovNetV2/coverage.txt | 3 + .../VovNet/Test/VovNetV2/vovnetv2.py | 217 ++++++ .../VovNet/Test/VovNetV2/weloTrainStep.py | 647 ++++++++++++++++++ .../VovNet/Test/vovnetv1_loss.jpg | Bin 0 -> 34590 bytes .../VovNet/Test/vovnetv1_loss.txt | 29 + .../VovNet/Test/vovnetv2_loss.jpg | Bin 0 -> 35744 bytes .../VovNet/Test/vovnetv2_loss.txt | 29 + .../VovNet/Test/weloTrainStep.py | 647 ++++++++++++++++++ 11 files changed, 2449 insertions(+) create mode 100644 PyTorch/build-in/Classification/VovNet/Test/VovNetV1/coverage.txt create mode 100644 PyTorch/build-in/Classification/VovNet/Test/VovNetV1/vovnetv1.py create mode 100644 PyTorch/build-in/Classification/VovNet/Test/VovNetV1/weloTrainStep.py create mode 100644 PyTorch/build-in/Classification/VovNet/Test/VovNetV2/coverage.txt create mode 100644 PyTorch/build-in/Classification/VovNet/Test/VovNetV2/vovnetv2.py create mode 100644 PyTorch/build-in/Classification/VovNet/Test/VovNetV2/weloTrainStep.py create mode 100644 PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.jpg create mode 100644 PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.txt create mode 100644 PyTorch/build-in/Classification/VovNet/Test/vovnetv2_loss.jpg create mode 100644 PyTorch/build-in/Classification/VovNet/Test/vovnetv2_loss.txt create mode 100644 PyTorch/build-in/Classification/VovNet/Test/weloTrainStep.py diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/coverage.txt b/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/coverage.txt new file mode 100644 index 000000000..18fc47ec9 --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/coverage.txt @@ -0,0 +1,3 @@ +all api: ['_amp_foreach_non_finite_check_and_unscale_', '_amp_update_scale_', '_copy_from', '_has_compatible_shallow_copy_type', '_local_scalar_dense', '_log_softmax', '_log_softmax_backward_data', '_pin_memory', '_reshape_alias', 'add', 'add_', 'addmm', 'as_strided', 'as_strided_', 'cat', 'convolution', 'convolution_backward', 'copy_stride', 'div', 'eq', 'fill_', 'fused_sgd', 'is_pinned', 'linear', 'max_pool2d', 'maxpool2d_backward', 'maxpool2d_forward', 'mean', 'mm', 'mul', 'mul_', 'native_batch_norm', 'native_batch_norm_backward', 'nll_loss_backward', 'nll_loss_forward', 'reciprocal', 'relu_', 'sum', 'threshold_backward', 'topk_out', 'view', 'zero_'], total: 42 +fallback op: [], total: 0 +coverage rate: 100.00% diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/vovnetv1.py b/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/vovnetv1.py new file mode 100644 index 000000000..abfa11f5a --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/vovnetv1.py @@ -0,0 +1,227 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F +from collections import OrderedDict + + +__all__ = ['VoVNet', 'vovnet27_slim', 'vovnet39', 'vovnet57'] + + +model_urls = { + 'vovnet39': 'https://dl.dropbox.com/s/1lnzsgnixd8gjra/vovnet39_torchvision.pth?dl=1', + 'vovnet57': 'https://dl.dropbox.com/s/6bfu9gstbwfw31m/vovnet57_torchvision.pth?dl=1' +} + + +def conv3x3(in_channels, out_channels, module_name, postfix, + stride=1, groups=1, kernel_size=3, padding=1): + """3x3 convolution with padding""" + return [ + ('{}_{}/conv'.format(module_name, postfix), + nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups, + bias=False)), + ('{}_{}/norm'.format(module_name, postfix), + nn.BatchNorm2d(out_channels)), + ('{}_{}/relu'.format(module_name, postfix), + nn.ReLU(inplace=True)), + ] + + +def conv1x1(in_channels, out_channels, module_name, postfix, + stride=1, groups=1, kernel_size=1, padding=0): + """1x1 convolution""" + return [ + ('{}_{}/conv'.format(module_name, postfix), + nn.Conv2d(in_channels, out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + groups=groups, + bias=False)), + ('{}_{}/norm'.format(module_name, postfix), + nn.BatchNorm2d(out_channels)), + ('{}_{}/relu'.format(module_name, postfix), + nn.ReLU(inplace=True)), + ] + + +class _OSA_module(nn.Module): + def __init__(self, + in_ch, + stage_ch, + concat_ch, + layer_per_block, + module_name, + identity=False): + super(_OSA_module, self).__init__() + + self.identity = identity + self.layers = nn.ModuleList() + in_channel = in_ch + for i in range(layer_per_block): + self.layers.append(nn.Sequential( + OrderedDict(conv3x3(in_channel, stage_ch, module_name, i)))) + in_channel = stage_ch + + # feature aggregation + in_channel = in_ch + layer_per_block * stage_ch + self.concat = nn.Sequential( + OrderedDict(conv1x1(in_channel, concat_ch, module_name, 'concat'))) + + def forward(self, x): + identity_feat = x + output = [] + output.append(x) + for layer in self.layers: + x = layer(x) + output.append(x) + + x = torch.cat(output, dim=1) + xt = self.concat(x) + + if self.identity: + xt = xt + identity_feat + + return xt + + +class _OSA_stage(nn.Sequential): + def __init__(self, + in_ch, + stage_ch, + concat_ch, + block_per_stage, + layer_per_block, + stage_num): + super(_OSA_stage, self).__init__() + + if not stage_num == 2: + self.add_module('Pooling', + nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True)) + + module_name = f'OSA{stage_num}_1' + self.add_module(module_name, + _OSA_module(in_ch, + stage_ch, + concat_ch, + layer_per_block, + module_name)) + for i in range(block_per_stage-1): + module_name = f'OSA{stage_num}_{i+2}' + self.add_module(module_name, + _OSA_module(concat_ch, + stage_ch, + concat_ch, + layer_per_block, + module_name, + identity=True)) + + +class VoVNet(nn.Module): + def __init__(self, + config_stage_ch, + config_concat_ch, + block_per_stage, + layer_per_block, + num_classes=1000): + super(VoVNet, self).__init__() + + # Stem module + stem = conv3x3(3, 64, 'stem', '1', 2) + stem += conv3x3(64, 64, 'stem', '2', 1) + stem += conv3x3(64, 128, 'stem', '3', 2) + self.add_module('stem', nn.Sequential(OrderedDict(stem))) + + stem_out_ch = [128] + in_ch_list = stem_out_ch + config_concat_ch[:-1] + self.stage_names = [] + for i in range(4): #num_stages + name = 'stage%d' % (i+2) + self.stage_names.append(name) + self.add_module(name, + _OSA_stage(in_ch_list[i], + config_stage_ch[i], + config_concat_ch[i], + block_per_stage[i], + layer_per_block, + i+2)) + + self.classifier = nn.Linear(config_concat_ch[-1], num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight) + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.Linear): + nn.init.constant_(m.bias, 0) + + def forward(self, x): + x = self.stem(x) + for name in self.stage_names: + x = getattr(self, name)(x) + x = F.adaptive_avg_pool2d(x, (1, 1)).view(x.size(0), -1) + x = self.classifier(x) + return x + + +def _vovnet(arch, + config_stage_ch, + config_concat_ch, + block_per_stage, + layer_per_block, + pretrained, + progress, + **kwargs): + model = VoVNet(config_stage_ch, config_concat_ch, + block_per_stage, layer_per_block, + **kwargs) + if pretrained: + state_dict = load_state_dict_from_url(model_urls[arch], + progress=progress) + model.load_state_dict(state_dict) + return model + + +def vovnet57(pretrained=False, progress=True, **kwargs): + r"""Constructs a VoVNet-57 model as described in + `"An Energy and GPU-Computation Efficient Backbone Networks" + `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _vovnet('vovnet57', [128, 160, 192, 224], [256, 512, 768, 1024], + [1,1,4,3], 5, pretrained, progress, **kwargs) + + +def vovnet39(pretrained=False, progress=True, **kwargs): + r"""Constructs a VoVNet-39 model as described in + `"An Energy and GPU-Computation Efficient Backbone Networks" + `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _vovnet('vovnet39', [128, 160, 192, 224], [256, 512, 768, 1024], + [1,1,2,2], 5, pretrained, progress, **kwargs) + + +def vovnet27_slim(pretrained=False, progress=True, **kwargs): + r"""Constructs a VoVNet-39 model as described in + `"An Energy and GPU-Computation Efficient Backbone Networks" + `_. + Args: + pretrained (bool): If True, returns a model pre-trained on ImageNet + progress (bool): If True, displays a progress bar of the download to stderr + """ + return _vovnet('vovnet27_slim', [64, 80, 96, 112], [128, 256, 384, 512], + [1,1,1,1], 5, pretrained, progress, **kwargs) + +def Model(num_classes=100): + return vovnet39(num_classes=num_classes) \ No newline at end of file diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/weloTrainStep.py b/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/weloTrainStep.py new file mode 100644 index 000000000..2c191729c --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/weloTrainStep.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import random +import sys +import time +import json +import argparse +from collections import OrderedDict +from pathlib import Path +import numpy as np +import pandas as pd +from tqdm import tqdm +import importlib + +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" # 强烈推荐在 shell/最顶端设置 +os.environ["PYTHONHASHSEED"] = "12345" +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +def ensure_cublas_workspace(config=":4096:8"): + """ + 尝试为 cuBLAS 设置可复现 workspace。强烈建议在主脚本入口处（import torch 之前） + 通过 export 设置该 env。此函数会在运行时设置，但如果 torch 已经被 import， + 则可能为时已晚——函数会打印提醒。 + """ + already = os.environ.get("CUBLAS_WORKSPACE_CONFIG") + if already: + print(f"[seed_utils] CUBLAS_WORKSPACE_CONFIG 已存在：{already}") + else: + os.environ["CUBLAS_WORKSPACE_CONFIG"] = config + print(f"[seed_utils] 已设置 CUBLAS_WORKSPACE_CONFIG={config} （注意：请在 import torch 前设置以保证生效）") + +def set_global_seed(seed: int = 42, set_threads: bool = True): + """ + 统一随机性设置。注意：若希望完全发挥效果，请在主脚本入口（import torch 之前） + 先调用 ensure_cublas_workspace(...) 或在 shell 中 export CUBLAS_WORKSPACE_CONFIG。 + """ + ensure_cublas_workspace() # 会设置 env 并提醒 + os.environ["PYTHONHASHSEED"] = str(seed) + + if set_threads: + os.environ["OMP_NUM_THREADS"] = "1" + os.environ["MKL_NUM_THREADS"] = "1" + + random.seed(seed) + np.random.seed(seed) + + # 现在导入 torch（晚导入以便前面 env 生效） + import torch + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + # 强制确定性（如果存在不确定性算子，PyTorch 会报错并提示） + try: + torch.use_deterministic_algorithms(True) + except Exception as e: + print("[seed_utils] 设置 deterministic 模式时出错：", e) + print("[seed_utils] 请确认 CUBLAS_WORKSPACE_CONFIG 已在 import torch 之前设置。") + + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + if set_threads: + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + + print(f"[seed_utils] 全局 seed 已设置为 {seed}") + +set_global_seed(2025) + +""" +通用训练模版（优先从本地导入 Model -> 支持 DDP / 单卡，AMP，resume，日志，checkpoint） +保存为 train_template_localmodel.py +""" +import torch +import torch.nn as nn +import torch.optim as optim +import torch.backends.cudnn as cudnn +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as tv_models + +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +from torch.sdaa import amp +# from torch.cuda import amp + + +# ---------------------------- +# Helper utilities (self-contained) +# ---------------------------- +class AverageMeter(object): + def __init__(self, name='Meter', fmt=':.4f'): + self.name = name + self.fmt = fmt + self.reset() + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / max(1, self.count) + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} (avg {avg' + self.fmt + '})' + return fmtstr.format(name=self.name, val=self.val, avg=self.avg) + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k + 返回一个 list，每个元素是 tensor（百分比形式） + """ + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + # output: (N, C) -> pred: (maxk, N) + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() # (maxk, N) + correct = pred.eq(target.view(1, -1).expand_as(pred)) # (maxk, N) bool + + res = [] + for k in topk: + # 把前 k 行展平后求和（返回 0-dim tensor），随后换算为百分比 + correct_k = correct[:k].reshape(-1).float().sum() # 注意：不传 keepdim + # 乘以 100.0 / batch_size，保持返回 tensor（和之前代码兼容） + res.append(correct_k.mul_(100.0 / batch_size)) + return res + +def save_checkpoint(state, is_best, save_dir, filename='checkpoint.pth'): + save_path = os.path.join(save_dir, filename) + torch.save(state, save_path) + if is_best: + best_path = os.path.join(save_dir, 'model_best.pth') + torch.save(state, best_path) + +def set_seed(seed, deterministic=False): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + cudnn.deterministic = True + cudnn.benchmark = False + else: + cudnn.deterministic = False + cudnn.benchmark = True + +# ---------------------------- +# Argument parser +# ---------------------------- +def parse_args(): + parser = argparse.ArgumentParser(description='Generic PyTorch training template (DDP/AMP) with LocalModel priority') + parser.add_argument('--name', default='run', type=str, help='experiment name (log/checkpoints dir)') + parser.add_argument('--seed', default=42, type=int, help='random seed') + parser.add_argument('--arch', default='None', type=str, help='model name') + parser.add_argument('--deterministic', action='store_true', help='set cudnn deterministic (may be slower)') + parser.add_argument('--dataset', default='cifar10', choices=['cifar10','cifar100','imagenet','custom'], help='which dataset') + parser.add_argument('--datapath', default='./data', type=str, help='dataset root / imagenet root / custom root') + parser.add_argument('--imagenet_dir', default='./imagenet', type=str, help='if dataset=imagenet, path to imagenet root') + parser.add_argument('--custom_eval_dir', default=None, help='if dataset=custom, provide val dir') + parser.add_argument('--num_workers', default=4, type=int, help='dataloader workers per process') + parser.add_argument('--epochs', default=200, type=int) + parser.add_argument('--steps', default=0, type=int, help='max steps to run (if >0, training will stop when global_step reaches this).') + parser.add_argument('--batch_size', default=128, type=int) + parser.add_argument('--model_name', default='resnet18', help='torchvision model name or python path e.g. mypkg.mymodule.Model (used if no local Model)') + parser.add_argument('--num_classes', default=None, type=int, help='override num classes (auto-detect for common sets)') + parser.add_argument('--pretrained', action='store_true', help='use torchvision pretrained weights when available') + parser.add_argument('--optimizer', default='sgd', choices=['sgd','adam','adamw'], help='optimizer') + parser.add_argument('--lr', '--learning_rate', default=0.1, type=float) + parser.add_argument('--momentum', default=0.9, type=float) + parser.add_argument('--weight_decay', default=5e-4, type=float) + parser.add_argument('--nesterov', action='store_true') + parser.add_argument('--scheduler', default='multistep', choices=['multistep','step','cosine','none'], help='lr scheduler') + parser.add_argument('--milestones', default='100,150', type=str, help='milestones for multistep (comma sep)') + parser.add_argument('--step_size', default=30, type=int, help='step size for StepLR or cosine max epochs') + parser.add_argument('--gamma', default=0.1, type=float) + parser.add_argument('--scheduler_step_per_batch', action='store_true', help='call scheduler.step() per batch (for some schedulers)') + parser.add_argument('--resume', default='', type=str, help='path to checkpoint to resume from') + parser.add_argument('--start_epoch', default=0, type=int) + parser.add_argument('--print_freq', default=100, type=int) + parser.add_argument('--save_freq', default=10, type=int, help='save checkpoint every N epochs (rank0 only)') + parser.add_argument('--amp', action='store_true', default = True,help='use automatic mixed precision (AMP)') + parser.add_argument('--grad_accum_steps', default=1, type=int, help='gradient accumulation steps') + parser.add_argument('--local_rank', default=None, type=int, help='local rank passed by torchrun (if any). Use -1 or None for non-distributed') + parser.add_argument('--cutmix_prob', default=0.0, type=float) + parser.add_argument('--beta', default=1.0, type=float) + parser.add_argument('--seed_sampler', default=False, action='store_true', help='set sampler epoch seeds to make deterministic distributed shuffling') + args = parser.parse_args() + args.milestones = [int(x) for x in args.milestones.split(',')] if args.milestones else [] + return args + +# ---------------------------- +# build model (优先 LocalModel) +# ---------------------------- +def build_model_with_local_priority(args, device=None): + """ + 用参数 args.arch 作为模块名导入 Model() + 如果模块不存在或没有 Model 类，则报错停止。 + """ + try: + # 动态导入模块，比如 args.arch = "rexnet" + mod = importlib.import_module(args.arch) + Model = getattr(mod, "Model") # 从模块中获取 Model 类 + except Exception as e: + raise RuntimeError( + f"无法导入模型模块 '{args.arch}' 或未找到类 Model。" + f"\n错误信息：{e}" + ) + + # 解析数据集类别数 + if args.dataset == 'cifar10': + num_classes = 10 + elif args.dataset == 'cifar100': + num_classes = 100 + else: + print(f"[ERROR] 不支持的数据集类型：{args.dataset}，无法确定类别数。程序终止。") + sys.exit(1) + + + # 实例化 + try: + model = Model(num_classes) + except Exception as e: + raise RuntimeError( + f"Model() 实例化失败，请检查模型构造函数。\n错误信息：{e}" + ) + + return model + +# ---------------------------- +# Data loader factory +# ---------------------------- +def build_dataloaders(args, rank, world_size): + if args.dataset == 'cifar10' or args.dataset == 'cifar100': + mean = (0.4914, 0.4822, 0.4465) + std = (0.2470, 0.2435, 0.2616) if args.dataset == 'cifar10' else (0.2023, 0.1994, 0.2010) + # train_transform = transforms.Compose([ + # transforms.RandomCrop(32, padding=4), + # transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + # test_transform = transforms.Compose([ + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + + train_transform = transforms.Compose([ # 2025/12/3 从visformer模型开始 + transforms.Resize(256), # 先放大到 256 + transforms.RandomCrop(224), # 再随机裁剪为 224（更符合 ImageNet 风格增强） + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + root = args.datapath + if args.dataset == 'cifar10': + train_set = datasets.CIFAR10(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR10(root=root, train=False, download=False, transform=test_transform) + num_classes = 10 + else: + train_set = datasets.CIFAR100(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR100(root=root, train=False, download=False, transform=test_transform) + num_classes = 100 + + elif args.dataset == 'imagenet': + train_dir = os.path.join(args.imagenet_dir, 'train') + val_dir = os.path.join(args.imagenet_dir, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = args.num_classes or 1000 + + elif args.dataset == 'custom': + train_dir = os.path.join(args.datapath, 'train') + val_dir = args.custom_eval_dir or os.path.join(args.datapath, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = len(train_set.classes) + else: + raise ValueError("Unknown dataset") + + if dist.is_initialized() and world_size > 1: + train_sampler = DistributedSampler(train_set, num_replicas=world_size, rank=rank, shuffle=True) + else: + train_sampler = None + + train_loader = DataLoader(train_set, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + num_workers=args.num_workers, + pin_memory=True, + sampler=train_sampler, + drop_last=False) + val_loader = DataLoader(val_set, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + return train_loader, val_loader, num_classes, train_sampler + +# ---------------------------- +# Train & validate +# ---------------------------- +def train_one_epoch(args, epoch, model, criterion, optimizer, train_loader, device, scaler, scheduler=None, train_sampler=None, global_step_start=0, max_global_steps=None): + """ + 现在支持：若 max_global_steps 非 None，则当 global_step 达到该值时提前退出 + 返回: epoch_summary_dict, step_logs_list, global_step_end + step_logs_list: list of dicts with per-step info (for logging to CSV if需要) + """ + batch_time = AverageMeter('Time') + data_time = AverageMeter('Data') + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.train() + end = time.time() + optimizer.zero_grad() + + iters = len(train_loader) + step_logs = [] + global_step = global_step_start + + for i, (images, targets) in enumerate(train_loader): + # check global steps limit + if (max_global_steps is not None) and (global_step >= max_global_steps): + break + + data_time.update(time.time() - end) + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + + if args.amp: + with amp.autocast(): + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + else: + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + + if args.amp: + scaler.scale(loss).backward() + else: + loss.backward() + + if (i + 1) % args.grad_accum_steps == 0: + if args.amp: + scaler.step(optimizer) + scaler.update() + else: + optimizer.step() + optimizer.zero_grad() + if scheduler is not None and args.scheduler_step_per_batch: + scheduler.step() + + with torch.no_grad(): + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + losses.update(loss.item() * args.grad_accum_steps, images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + + batch_time.update(time.time() - end) + end = time.time() + + # increment global step AFTER processing this batch + global_step += 1 + + # per-step print (controlled by print_freq) + # 输出格式调整为：Epoch[23]:step[1/32] step_train_loss 3.0075 acc1 25.95 acc5 54.46 + # 使用 i+1 / iters 更贴近人类可读的“第几步 / 总步数（该 epoch 内）” + if ((global_step % args.print_freq == 0) or (i == iters - 1)) and ((dist.get_rank() if dist.is_initialized() else 0) == 0): + lr = optimizer.param_groups[0]['lr'] + # note: losses.val is 当前 batch 的 loss（经过 grad_accum 处理后还原），losses.avg 是到目前为止的 epoch 平均 + print(f"Epoch[{epoch}]:step[{i+1}/{iters}] step_train_loss {losses.val:.4f} acc1 {top1.val:.2f} acc5 {top5.val:.2f}") + + # collect per-step log + step_logs.append({ + 'epoch': epoch, + 'batch_idx': i, + 'global_step': global_step, + 'lr': optimizer.param_groups[0]['lr'], + 'loss': losses.val, + 'loss_avg': losses.avg, + 'acc1': top1.val, + 'acc1_avg': top1.avg, + 'acc5': top5.val, + 'acc5_avg': top5.avg, + 'time': batch_time.val + }) + + # if reached max_global_steps inside epoch, break (handled at loop start next iter) + if (max_global_steps is not None) and (global_step >= max_global_steps): + # optional message + if (dist.get_rank() if dist.is_initialized() else 0) == 0: + print(f"[Info] 达到 max_global_steps={max_global_steps}，将在 epoch 内提前停止。") + break + + if scheduler is not None and not args.scheduler_step_per_batch: + scheduler.step() + + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]), step_logs, global_step + +def validate(args, model, val_loader, criterion, device): + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.eval() + with torch.no_grad(): + for i, (images, targets) in enumerate(tqdm(val_loader)): + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + outputs = model(images) + loss = criterion(outputs, targets) + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]) + +# ---------------------------- +# Main +# ---------------------------- +def main(): + args = parse_args() + + # handle local_rank from env if not provided + local_rank_env = os.environ.get('LOCAL_RANK', None) + if args.local_rank is None and local_rank_env is not None: + args.local_rank = int(local_rank_env) + + distributed = (args.local_rank is not None and args.local_rank != -1) + if distributed: + dist.init_process_group(backend='nccl', init_method='env://') + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + + if distributed: + torch.cuda.set_device(args.local_rank) + device = torch.device('cuda', args.local_rank) + else: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + set_seed(args.seed + (rank if distributed else 0), deterministic=args.deterministic) + + save_dir = os.path.join('models', args.name) + if rank == 0: + os.makedirs(save_dir, exist_ok=True) + with open(os.path.join(save_dir, 'args.json'), 'w') as f: + json.dump(vars(args), f, indent=2) + if distributed: + dist.barrier() + + train_loader, val_loader, auto_num_classes, train_sampler = build_dataloaders(args, rank, world_size) + if args.num_classes is None: + args.num_classes = auto_num_classes + + # 使用本地 Model 优先（LocalModel 已在文件顶部尝试导入） + model = build_model_with_local_priority(args, device) + model.to(device) + + if distributed: + model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) + + criterion = nn.CrossEntropyLoss().to(device) + params = [p for p in model.parameters() if p.requires_grad] + if args.optimizer == 'sgd': + optimizer = optim.SGD(params, lr=args.lr, momentum=args.momentum, + weight_decay=args.weight_decay, nesterov=args.nesterov) + elif args.optimizer == 'adam': + optimizer = optim.Adam(params, lr=args.lr, weight_decay=args.weight_decay) + elif args.optimizer == 'adamw': + optimizer = optim.AdamW(params, lr=args.lr, weight_decay=args.weight_decay) + else: + raise ValueError('Unknown optimizer') + + scheduler = None + if args.scheduler == 'multistep': + scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.milestones, gamma=args.gamma) + elif args.scheduler == 'step': + scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma) + elif args.scheduler == 'cosine': + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) + elif args.scheduler == 'none': + scheduler = None + + scaler = amp.GradScaler() if args.amp else None + + start_epoch = args.start_epoch + best_acc = 0.0 + if args.resume: + if os.path.isfile(args.resume): + ckpt = torch.load(args.resume, map_location='cpu') + model_state = ckpt.get('state_dict', ckpt) + if isinstance(model, DDP): + model.module.load_state_dict(model_state) + else: + model.load_state_dict(model_state) + if 'optimizer' in ckpt: + optimizer.load_state_dict(ckpt['optimizer']) + start_epoch = ckpt.get('epoch', start_epoch) + best_acc = ckpt.get('best_acc', best_acc) + print(f"=> resumed from {args.resume}, start_epoch={start_epoch}") + else: + print(f"=> resume path {args.resume} not found") + + log_columns = ['epoch', 'lr', 'loss', 'acc1', 'acc5', 'val_loss', 'val_acc1', 'val_acc5'] + log_df = pd.DataFrame(columns=log_columns) + # step-level log + step_log_columns = ['epoch', 'batch_idx', 'global_step', 'lr', 'loss', 'loss_avg', 'acc1', 'acc1_avg', 'acc5', 'acc5_avg', 'time'] + step_log_df = pd.DataFrame(columns=step_log_columns) + + total_epochs = args.epochs + # global_step计数器（训练过程中跨epoch持续） + global_step = 0 + + epoch = start_epoch + # loop until either epoch criteria or step criteria met + while True: + if train_sampler is not None: + if args.seed_sampler: + train_sampler.set_epoch(epoch + args.seed) + else: + train_sampler.set_epoch(epoch) + + if rank == 0: + print(f"==== Epoch {epoch}/{total_epochs - 1} ====") + + # 如果传入了 args.steps (>0)，则把剩余允许的 step 数传给 train_one_epoch， + # 否则 max_global_steps=None（按整 epoch 执行完） + if args.steps and args.steps > 0: + max_global_steps = args.steps + else: + max_global_steps = None + + train_log, step_logs, global_step = train_one_epoch( + args, epoch, model, criterion, optimizer, train_loader, device, scaler, + scheduler, train_sampler, global_step_start=global_step, max_global_steps=max_global_steps + ) + + # 如果启用了按 steps 的模式且已经达到上限，直接退出 main（跳过 validate） + if max_global_steps is not None and global_step >= max_global_steps: + if rank == 0: + print(f"[Main] 达到 max_global_steps={max_global_steps}（global_step={global_step}），提前退出训练（跳过验证）。") + # 直接返回 main()，不再执行后续 validate / 保存逻辑 + return + + # 验证并记录 epoch 级别日志（如果在 step 模式下很可能在中间某个 epoch 提前结束，但我们仍做一次 validate） + val_log = validate(args, model, val_loader, criterion, device) + current_lr = optimizer.param_groups[0]['lr'] + + if rank == 0: + # epoch summary print, 格式与示例对齐 + print(f"Epoch[{epoch}]: epoch_train_loss {train_log['loss']:.4f} acc1 {train_log['acc1']:.2f} acc5 {train_log['acc5']:.2f} | " + f"val_loss {val_log['loss']:.4f} acc1 {val_log['acc1']:.2f} acc5 {val_log['acc5']:.2f} lr {current_lr:.6f}") + row = { + 'epoch': epoch, + 'lr': current_lr, + 'loss': train_log['loss'], + 'acc1': train_log['acc1'], + 'acc5': train_log['acc5'], + 'val_loss': val_log['loss'], + 'val_acc1': val_log['acc1'], + 'val_acc5': val_log['acc5'], + } + new_row_df = pd.DataFrame([row]) + log_df = pd.concat([log_df, new_row_df], ignore_index=True) + log_df.to_csv(os.path.join(save_dir, 'log.csv'), index=False) + + is_best = val_log['acc1'] > best_acc + if is_best: + best_acc = val_log['acc1'] + if (epoch % args.save_freq == 0) or is_best or ( (max_global_steps is None) and (epoch == total_epochs - 1) ) : + state = { + 'epoch': epoch, + 'state_dict': model.module.state_dict() if isinstance(model, DDP) else model.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + 'args': vars(args) + } + save_checkpoint(state, is_best, save_dir, filename=f'checkpoint_epoch_{epoch}.pth') + + # increment epoch + epoch += 1 + + # stopping conditions: + # 1) if steps mode enabled and reached steps -> stop + if args.steps and args.steps > 0: + if global_step >= args.steps: + if rank == 0: + print(f"[Main] 已达到指定 steps={args.steps}（global_step={global_step}），训练结束。") + break + + # 2) if steps not used, stop when epoch >= epochs + else: + if epoch >= total_epochs: + if rank == 0: + print(f"[Main] 已达到指定 epochs={total_epochs}（epoch={epoch}），训练结束。") + break + + if dist.is_initialized(): + dist.barrier() + if rank == 0: + print("Training finished. Best val acc1: {:.2f}".format(best_acc)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/coverage.txt b/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/coverage.txt new file mode 100644 index 000000000..7b95886fe --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/coverage.txt @@ -0,0 +1,3 @@ +all api: ['_amp_foreach_non_finite_check_and_unscale_', '_amp_update_scale_', '_copy_from', '_has_compatible_shallow_copy_type', '_local_scalar_dense', '_log_softmax', '_log_softmax_backward_data', '_pin_memory', '_reshape_alias', 'add', 'add_', 'addmm', 'as_strided', 'as_strided_', 'bernoulli_', 'cat', 'convolution', 'convolution_backward', 'copy_stride', 'div', 'div_', 'eq', 'fill_', 'fused_sgd', 'hardtanh_', 'hardtanh_backward', 'is_pinned', 'linear', 'lt', 'max_pool2d', 'maxpool2d_backward', 'maxpool2d_forward', 'mean', 'mm', 'mul', 'mul_', 'native_batch_norm', 'native_batch_norm_backward', 'nll_loss_backward', 'nll_loss_forward', 'reciprocal', 'relu_', 'resize_', 'sum', 'threshold_backward', 'topk_out', 'view', 'zero_'], total: 48 +fallback op: ['hardtanh_', 'hardtanh_backward'], total: 2 +coverage rate: 95.83% diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/vovnetv2.py b/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/vovnetv2.py new file mode 100644 index 000000000..e003f8120 --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/vovnetv2.py @@ -0,0 +1,217 @@ +""" VoVNet as per https://arxiv.org/pdf/1904.09730.pdf (v1) and +https://arxiv.org/pdf/1911.06667.pdf (v2). """ + +import collections + +import torch +from torch import nn + +# The paper is unclear as to where to downsample, so the downsampling was +# derived from the pretrained model graph as visualized by Netron. V2 simply +# enables ESE and identity connections here, nothing else changes. +CONFIG = { + # Introduced in V2. Difference is 3 repeats instead of 5 within each block. + "vovnet19": [ + # kernel size, inner channels, layer repeats, output channels, downsample + [3, 64, 3, 128, True], + [3, 80, 3, 256, True], + [3, 96, 3, 348, True], + [3, 112, 3, 512, True], + ], + "vovnet27_slim": [ + [3, 64, 5, 128, True], + [3, 80, 5, 256, True], + [3, 96, 5, 348, True], + [3, 112, 5, 512, True], + ], + "vovnet39": [ + [3, 128, 5, 256, True], + [3, 160, 5, 512, True], + [3, 192, 5, 768, True], # x2 + [3, 192, 5, 768, False], + [3, 224, 5, 1024, True], # x2 + [3, 224, 5, 1024, False], + ], + "vovnet57": [ + [3, 128, 5, 256, True], + [3, 160, 5, 512, True], + [3, 192, 5, 768, True], # x4 + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 224, 5, 1024, True], # x3 + [3, 224, 5, 1024, False], + [3, 224, 5, 1024, False], + ], + "vovnet99": [ + [3, 128, 5, 256, True], + [3, 160, 5, 512, True], # x3 + [3, 160, 5, 512, False], + [3, 160, 5, 512, False], + [3, 192, 5, 768, True], # x9 + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 192, 5, 768, False], + [3, 224, 5, 1024, True], # x3 + [3, 224, 5, 1024, False], + [3, 224, 5, 1024, False], + ], +} + + +class _ESE(nn.Module): + def __init__(self, channels: int) -> None: + # TODO: Might want to experiment with bias=False. At least for + # MobileNetV3 it leads to better accuracy on detection. + super().__init__() + self.conv = nn.Conv2d(channels, channels, 1, padding=0) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = x.mean([2, 3], keepdim=True) + y = self.conv(y) + # Hard sigmoid multiplied by input. + return x * (nn.functional.relu6(y + 3, inplace=True) / 6.0) + + +class _ConvBnRelu(nn.Sequential): + def __init__(self, in_ch: int, out_ch: int, kernel_size: int = 3, stride: int = 1): + super().__init__( + nn.Conv2d( + in_ch, + out_ch, + kernel_size, + stride=stride, + padding=kernel_size // 2, + bias=False, + ), + nn.BatchNorm2d(out_ch), + nn.ReLU(inplace=True), + ) + + +class _OSA(nn.Module): + def __init__( + self, + in_ch: int, + inner_ch: int, + out_ch: int, + repeats: int = 5, + kernel_size: int = 3, + stride: int = 1, + downsample: bool = False, + ) -> None: + super().__init__() + self.downsample = downsample + self.layers = nn.ModuleList( + [ + _ConvBnRelu( + in_ch if r == 0 else inner_ch, + inner_ch, + kernel_size=kernel_size, + stride=stride, + ) + for r in range(repeats) + ] + ) + self.exit_conv = _ConvBnRelu(in_ch + repeats * inner_ch, out_ch, kernel_size=1) + self.ese = _ESE(out_ch) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + # Pass through all modules, but retain outputs. + input = x + if self.downsample: + x = nn.functional.max_pool2d(x, 3, stride=2, padding=1) + features = [x] + for l in self.layers: + features.append(l(x)) + x = features[-1] + x = torch.cat(features, dim=1) + x = self.exit_conv(x) + x = self.ese(x) + # All non-downsampling V2 layers have a residual. They also happen to + # not change the number of channels. + if not self.downsample: + x += input + return x + + +class VoVNet(nn.Module): + def __init__( + self, + in_ch: int = 3, + num_classes: int = 1000, + model_type: str = "vovnet39", + has_classifier: bool = True, + dropout: float = 0.2, + ): + """ Usage: + >>> net = VoVNet(3, 1000) + >>> net = net.eval() + >>> with torch.no_grad(): + ... y = net(torch.rand(2, 3, 64, 64)) + >>> print(list(y.shape)) + [2, 1000] + """ + super().__init__() + + # Input stage. + self.stem = nn.Sequential( + _ConvBnRelu(in_ch, 64, kernel_size=3, stride=2), + _ConvBnRelu(64, 64, kernel_size=3, stride=1), + _ConvBnRelu(64, 128, kernel_size=3, stride=1), + ) + + body_layers = collections.OrderedDict() + conf = CONFIG[model_type] + in_ch = 128 + for idx, block in enumerate(conf): + kernel_size, inner_ch, repeats, out_ch, downsample = block + body_layers[f"osa{idx}"] = _OSA( + in_ch, + inner_ch, + out_ch, + repeats=repeats, + kernel_size=kernel_size, + downsample=downsample, + ) + in_ch = out_ch + self.body = nn.Sequential(body_layers) + self.has_classifier = has_classifier + + self.classifier = nn.Sequential( + nn.AdaptiveAvgPool2d(1), + nn.Flatten(), + nn.Dropout(p=dropout, inplace=True), + nn.Linear(in_ch, num_classes, bias=True), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + y = self.stem(x) + y = self.body(y) + if self.has_classifier: + y = self.classifier(y) + return y + +def Model(num_classes=1000, model_type="vovnet39", in_ch=3, dropout=0.2): + """ + 简化工厂函数，用于快速实例化 VoVNet 模型 + Args: + num_classes (int): 分类数 + model_type (str): vovnet19, vovnet27_slim, vovnet39, vovnet57, vovnet99 + in_ch (int): 输入通道数，默认 3 (RGB) + dropout (float): 分类器 dropout，默认 0.2 + Returns: + VoVNet 实例 + """ + return VoVNet( + in_ch=in_ch, + num_classes=num_classes, + model_type=model_type, + has_classifier=True, + dropout=dropout + ) \ No newline at end of file diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/weloTrainStep.py b/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/weloTrainStep.py new file mode 100644 index 000000000..2c191729c --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/weloTrainStep.py @@ -0,0 +1,647 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import random +import sys +import time +import json +import argparse +from collections import OrderedDict +from pathlib import Path +import numpy as np +import pandas as pd +from tqdm import tqdm +import importlib + +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" # 强烈推荐在 shell/最顶端设置 +os.environ["PYTHONHASHSEED"] = "12345" +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +def ensure_cublas_workspace(config=":4096:8"): + """ + 尝试为 cuBLAS 设置可复现 workspace。强烈建议在主脚本入口处（import torch 之前） + 通过 export 设置该 env。此函数会在运行时设置，但如果 torch 已经被 import， + 则可能为时已晚——函数会打印提醒。 + """ + already = os.environ.get("CUBLAS_WORKSPACE_CONFIG") + if already: + print(f"[seed_utils] CUBLAS_WORKSPACE_CONFIG 已存在：{already}") + else: + os.environ["CUBLAS_WORKSPACE_CONFIG"] = config + print(f"[seed_utils] 已设置 CUBLAS_WORKSPACE_CONFIG={config} （注意：请在 import torch 前设置以保证生效）") + +def set_global_seed(seed: int = 42, set_threads: bool = True): + """ + 统一随机性设置。注意：若希望完全发挥效果，请在主脚本入口（import torch 之前） + 先调用 ensure_cublas_workspace(...) 或在 shell 中 export CUBLAS_WORKSPACE_CONFIG。 + """ + ensure_cublas_workspace() # 会设置 env 并提醒 + os.environ["PYTHONHASHSEED"] = str(seed) + + if set_threads: + os.environ["OMP_NUM_THREADS"] = "1" + os.environ["MKL_NUM_THREADS"] = "1" + + random.seed(seed) + np.random.seed(seed) + + # 现在导入 torch（晚导入以便前面 env 生效） + import torch + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + # 强制确定性（如果存在不确定性算子，PyTorch 会报错并提示） + try: + torch.use_deterministic_algorithms(True) + except Exception as e: + print("[seed_utils] 设置 deterministic 模式时出错：", e) + print("[seed_utils] 请确认 CUBLAS_WORKSPACE_CONFIG 已在 import torch 之前设置。") + + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + if set_threads: + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + + print(f"[seed_utils] 全局 seed 已设置为 {seed}") + +set_global_seed(2025) + +""" +通用训练模版（优先从本地导入 Model -> 支持 DDP / 单卡，AMP，resume，日志，checkpoint） +保存为 train_template_localmodel.py +""" +import torch +import torch.nn as nn +import torch.optim as optim +import torch.backends.cudnn as cudnn +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as tv_models + +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +from torch.sdaa import amp +# from torch.cuda import amp + + +# ---------------------------- +# Helper utilities (self-contained) +# ---------------------------- +class AverageMeter(object): + def __init__(self, name='Meter', fmt=':.4f'): + self.name = name + self.fmt = fmt + self.reset() + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / max(1, self.count) + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} (avg {avg' + self.fmt + '})' + return fmtstr.format(name=self.name, val=self.val, avg=self.avg) + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k + 返回一个 list，每个元素是 tensor（百分比形式） + """ + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + # output: (N, C) -> pred: (maxk, N) + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() # (maxk, N) + correct = pred.eq(target.view(1, -1).expand_as(pred)) # (maxk, N) bool + + res = [] + for k in topk: + # 把前 k 行展平后求和（返回 0-dim tensor），随后换算为百分比 + correct_k = correct[:k].reshape(-1).float().sum() # 注意：不传 keepdim + # 乘以 100.0 / batch_size，保持返回 tensor（和之前代码兼容） + res.append(correct_k.mul_(100.0 / batch_size)) + return res + +def save_checkpoint(state, is_best, save_dir, filename='checkpoint.pth'): + save_path = os.path.join(save_dir, filename) + torch.save(state, save_path) + if is_best: + best_path = os.path.join(save_dir, 'model_best.pth') + torch.save(state, best_path) + +def set_seed(seed, deterministic=False): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + cudnn.deterministic = True + cudnn.benchmark = False + else: + cudnn.deterministic = False + cudnn.benchmark = True + +# ---------------------------- +# Argument parser +# ---------------------------- +def parse_args(): + parser = argparse.ArgumentParser(description='Generic PyTorch training template (DDP/AMP) with LocalModel priority') + parser.add_argument('--name', default='run', type=str, help='experiment name (log/checkpoints dir)') + parser.add_argument('--seed', default=42, type=int, help='random seed') + parser.add_argument('--arch', default='None', type=str, help='model name') + parser.add_argument('--deterministic', action='store_true', help='set cudnn deterministic (may be slower)') + parser.add_argument('--dataset', default='cifar10', choices=['cifar10','cifar100','imagenet','custom'], help='which dataset') + parser.add_argument('--datapath', default='./data', type=str, help='dataset root / imagenet root / custom root') + parser.add_argument('--imagenet_dir', default='./imagenet', type=str, help='if dataset=imagenet, path to imagenet root') + parser.add_argument('--custom_eval_dir', default=None, help='if dataset=custom, provide val dir') + parser.add_argument('--num_workers', default=4, type=int, help='dataloader workers per process') + parser.add_argument('--epochs', default=200, type=int) + parser.add_argument('--steps', default=0, type=int, help='max steps to run (if >0, training will stop when global_step reaches this).') + parser.add_argument('--batch_size', default=128, type=int) + parser.add_argument('--model_name', default='resnet18', help='torchvision model name or python path e.g. mypkg.mymodule.Model (used if no local Model)') + parser.add_argument('--num_classes', default=None, type=int, help='override num classes (auto-detect for common sets)') + parser.add_argument('--pretrained', action='store_true', help='use torchvision pretrained weights when available') + parser.add_argument('--optimizer', default='sgd', choices=['sgd','adam','adamw'], help='optimizer') + parser.add_argument('--lr', '--learning_rate', default=0.1, type=float) + parser.add_argument('--momentum', default=0.9, type=float) + parser.add_argument('--weight_decay', default=5e-4, type=float) + parser.add_argument('--nesterov', action='store_true') + parser.add_argument('--scheduler', default='multistep', choices=['multistep','step','cosine','none'], help='lr scheduler') + parser.add_argument('--milestones', default='100,150', type=str, help='milestones for multistep (comma sep)') + parser.add_argument('--step_size', default=30, type=int, help='step size for StepLR or cosine max epochs') + parser.add_argument('--gamma', default=0.1, type=float) + parser.add_argument('--scheduler_step_per_batch', action='store_true', help='call scheduler.step() per batch (for some schedulers)') + parser.add_argument('--resume', default='', type=str, help='path to checkpoint to resume from') + parser.add_argument('--start_epoch', default=0, type=int) + parser.add_argument('--print_freq', default=100, type=int) + parser.add_argument('--save_freq', default=10, type=int, help='save checkpoint every N epochs (rank0 only)') + parser.add_argument('--amp', action='store_true', default = True,help='use automatic mixed precision (AMP)') + parser.add_argument('--grad_accum_steps', default=1, type=int, help='gradient accumulation steps') + parser.add_argument('--local_rank', default=None, type=int, help='local rank passed by torchrun (if any). Use -1 or None for non-distributed') + parser.add_argument('--cutmix_prob', default=0.0, type=float) + parser.add_argument('--beta', default=1.0, type=float) + parser.add_argument('--seed_sampler', default=False, action='store_true', help='set sampler epoch seeds to make deterministic distributed shuffling') + args = parser.parse_args() + args.milestones = [int(x) for x in args.milestones.split(',')] if args.milestones else [] + return args + +# ---------------------------- +# build model (优先 LocalModel) +# ---------------------------- +def build_model_with_local_priority(args, device=None): + """ + 用参数 args.arch 作为模块名导入 Model() + 如果模块不存在或没有 Model 类，则报错停止。 + """ + try: + # 动态导入模块，比如 args.arch = "rexnet" + mod = importlib.import_module(args.arch) + Model = getattr(mod, "Model") # 从模块中获取 Model 类 + except Exception as e: + raise RuntimeError( + f"无法导入模型模块 '{args.arch}' 或未找到类 Model。" + f"\n错误信息：{e}" + ) + + # 解析数据集类别数 + if args.dataset == 'cifar10': + num_classes = 10 + elif args.dataset == 'cifar100': + num_classes = 100 + else: + print(f"[ERROR] 不支持的数据集类型：{args.dataset}，无法确定类别数。程序终止。") + sys.exit(1) + + + # 实例化 + try: + model = Model(num_classes) + except Exception as e: + raise RuntimeError( + f"Model() 实例化失败，请检查模型构造函数。\n错误信息：{e}" + ) + + return model + +# ---------------------------- +# Data loader factory +# ---------------------------- +def build_dataloaders(args, rank, world_size): + if args.dataset == 'cifar10' or args.dataset == 'cifar100': + mean = (0.4914, 0.4822, 0.4465) + std = (0.2470, 0.2435, 0.2616) if args.dataset == 'cifar10' else (0.2023, 0.1994, 0.2010) + # train_transform = transforms.Compose([ + # transforms.RandomCrop(32, padding=4), + # transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + # test_transform = transforms.Compose([ + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + + train_transform = transforms.Compose([ # 2025/12/3 从visformer模型开始 + transforms.Resize(256), # 先放大到 256 + transforms.RandomCrop(224), # 再随机裁剪为 224（更符合 ImageNet 风格增强） + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + root = args.datapath + if args.dataset == 'cifar10': + train_set = datasets.CIFAR10(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR10(root=root, train=False, download=False, transform=test_transform) + num_classes = 10 + else: + train_set = datasets.CIFAR100(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR100(root=root, train=False, download=False, transform=test_transform) + num_classes = 100 + + elif args.dataset == 'imagenet': + train_dir = os.path.join(args.imagenet_dir, 'train') + val_dir = os.path.join(args.imagenet_dir, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = args.num_classes or 1000 + + elif args.dataset == 'custom': + train_dir = os.path.join(args.datapath, 'train') + val_dir = args.custom_eval_dir or os.path.join(args.datapath, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = len(train_set.classes) + else: + raise ValueError("Unknown dataset") + + if dist.is_initialized() and world_size > 1: + train_sampler = DistributedSampler(train_set, num_replicas=world_size, rank=rank, shuffle=True) + else: + train_sampler = None + + train_loader = DataLoader(train_set, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + num_workers=args.num_workers, + pin_memory=True, + sampler=train_sampler, + drop_last=False) + val_loader = DataLoader(val_set, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + return train_loader, val_loader, num_classes, train_sampler + +# ---------------------------- +# Train & validate +# ---------------------------- +def train_one_epoch(args, epoch, model, criterion, optimizer, train_loader, device, scaler, scheduler=None, train_sampler=None, global_step_start=0, max_global_steps=None): + """ + 现在支持：若 max_global_steps 非 None，则当 global_step 达到该值时提前退出 + 返回: epoch_summary_dict, step_logs_list, global_step_end + step_logs_list: list of dicts with per-step info (for logging to CSV if需要) + """ + batch_time = AverageMeter('Time') + data_time = AverageMeter('Data') + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.train() + end = time.time() + optimizer.zero_grad() + + iters = len(train_loader) + step_logs = [] + global_step = global_step_start + + for i, (images, targets) in enumerate(train_loader): + # check global steps limit + if (max_global_steps is not None) and (global_step >= max_global_steps): + break + + data_time.update(time.time() - end) + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + + if args.amp: + with amp.autocast(): + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + else: + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + + if args.amp: + scaler.scale(loss).backward() + else: + loss.backward() + + if (i + 1) % args.grad_accum_steps == 0: + if args.amp: + scaler.step(optimizer) + scaler.update() + else: + optimizer.step() + optimizer.zero_grad() + if scheduler is not None and args.scheduler_step_per_batch: + scheduler.step() + + with torch.no_grad(): + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + losses.update(loss.item() * args.grad_accum_steps, images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + + batch_time.update(time.time() - end) + end = time.time() + + # increment global step AFTER processing this batch + global_step += 1 + + # per-step print (controlled by print_freq) + # 输出格式调整为：Epoch[23]:step[1/32] step_train_loss 3.0075 acc1 25.95 acc5 54.46 + # 使用 i+1 / iters 更贴近人类可读的“第几步 / 总步数（该 epoch 内）” + if ((global_step % args.print_freq == 0) or (i == iters - 1)) and ((dist.get_rank() if dist.is_initialized() else 0) == 0): + lr = optimizer.param_groups[0]['lr'] + # note: losses.val is 当前 batch 的 loss（经过 grad_accum 处理后还原），losses.avg 是到目前为止的 epoch 平均 + print(f"Epoch[{epoch}]:step[{i+1}/{iters}] step_train_loss {losses.val:.4f} acc1 {top1.val:.2f} acc5 {top5.val:.2f}") + + # collect per-step log + step_logs.append({ + 'epoch': epoch, + 'batch_idx': i, + 'global_step': global_step, + 'lr': optimizer.param_groups[0]['lr'], + 'loss': losses.val, + 'loss_avg': losses.avg, + 'acc1': top1.val, + 'acc1_avg': top1.avg, + 'acc5': top5.val, + 'acc5_avg': top5.avg, + 'time': batch_time.val + }) + + # if reached max_global_steps inside epoch, break (handled at loop start next iter) + if (max_global_steps is not None) and (global_step >= max_global_steps): + # optional message + if (dist.get_rank() if dist.is_initialized() else 0) == 0: + print(f"[Info] 达到 max_global_steps={max_global_steps}，将在 epoch 内提前停止。") + break + + if scheduler is not None and not args.scheduler_step_per_batch: + scheduler.step() + + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]), step_logs, global_step + +def validate(args, model, val_loader, criterion, device): + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.eval() + with torch.no_grad(): + for i, (images, targets) in enumerate(tqdm(val_loader)): + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + outputs = model(images) + loss = criterion(outputs, targets) + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]) + +# ---------------------------- +# Main +# ---------------------------- +def main(): + args = parse_args() + + # handle local_rank from env if not provided + local_rank_env = os.environ.get('LOCAL_RANK', None) + if args.local_rank is None and local_rank_env is not None: + args.local_rank = int(local_rank_env) + + distributed = (args.local_rank is not None and args.local_rank != -1) + if distributed: + dist.init_process_group(backend='nccl', init_method='env://') + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + + if distributed: + torch.cuda.set_device(args.local_rank) + device = torch.device('cuda', args.local_rank) + else: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + set_seed(args.seed + (rank if distributed else 0), deterministic=args.deterministic) + + save_dir = os.path.join('models', args.name) + if rank == 0: + os.makedirs(save_dir, exist_ok=True) + with open(os.path.join(save_dir, 'args.json'), 'w') as f: + json.dump(vars(args), f, indent=2) + if distributed: + dist.barrier() + + train_loader, val_loader, auto_num_classes, train_sampler = build_dataloaders(args, rank, world_size) + if args.num_classes is None: + args.num_classes = auto_num_classes + + # 使用本地 Model 优先（LocalModel 已在文件顶部尝试导入） + model = build_model_with_local_priority(args, device) + model.to(device) + + if distributed: + model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) + + criterion = nn.CrossEntropyLoss().to(device) + params = [p for p in model.parameters() if p.requires_grad] + if args.optimizer == 'sgd': + optimizer = optim.SGD(params, lr=args.lr, momentum=args.momentum, + weight_decay=args.weight_decay, nesterov=args.nesterov) + elif args.optimizer == 'adam': + optimizer = optim.Adam(params, lr=args.lr, weight_decay=args.weight_decay) + elif args.optimizer == 'adamw': + optimizer = optim.AdamW(params, lr=args.lr, weight_decay=args.weight_decay) + else: + raise ValueError('Unknown optimizer') + + scheduler = None + if args.scheduler == 'multistep': + scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.milestones, gamma=args.gamma) + elif args.scheduler == 'step': + scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma) + elif args.scheduler == 'cosine': + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) + elif args.scheduler == 'none': + scheduler = None + + scaler = amp.GradScaler() if args.amp else None + + start_epoch = args.start_epoch + best_acc = 0.0 + if args.resume: + if os.path.isfile(args.resume): + ckpt = torch.load(args.resume, map_location='cpu') + model_state = ckpt.get('state_dict', ckpt) + if isinstance(model, DDP): + model.module.load_state_dict(model_state) + else: + model.load_state_dict(model_state) + if 'optimizer' in ckpt: + optimizer.load_state_dict(ckpt['optimizer']) + start_epoch = ckpt.get('epoch', start_epoch) + best_acc = ckpt.get('best_acc', best_acc) + print(f"=> resumed from {args.resume}, start_epoch={start_epoch}") + else: + print(f"=> resume path {args.resume} not found") + + log_columns = ['epoch', 'lr', 'loss', 'acc1', 'acc5', 'val_loss', 'val_acc1', 'val_acc5'] + log_df = pd.DataFrame(columns=log_columns) + # step-level log + step_log_columns = ['epoch', 'batch_idx', 'global_step', 'lr', 'loss', 'loss_avg', 'acc1', 'acc1_avg', 'acc5', 'acc5_avg', 'time'] + step_log_df = pd.DataFrame(columns=step_log_columns) + + total_epochs = args.epochs + # global_step计数器（训练过程中跨epoch持续） + global_step = 0 + + epoch = start_epoch + # loop until either epoch criteria or step criteria met + while True: + if train_sampler is not None: + if args.seed_sampler: + train_sampler.set_epoch(epoch + args.seed) + else: + train_sampler.set_epoch(epoch) + + if rank == 0: + print(f"==== Epoch {epoch}/{total_epochs - 1} ====") + + # 如果传入了 args.steps (>0)，则把剩余允许的 step 数传给 train_one_epoch， + # 否则 max_global_steps=None（按整 epoch 执行完） + if args.steps and args.steps > 0: + max_global_steps = args.steps + else: + max_global_steps = None + + train_log, step_logs, global_step = train_one_epoch( + args, epoch, model, criterion, optimizer, train_loader, device, scaler, + scheduler, train_sampler, global_step_start=global_step, max_global_steps=max_global_steps + ) + + # 如果启用了按 steps 的模式且已经达到上限，直接退出 main（跳过 validate） + if max_global_steps is not None and global_step >= max_global_steps: + if rank == 0: + print(f"[Main] 达到 max_global_steps={max_global_steps}（global_step={global_step}），提前退出训练（跳过验证）。") + # 直接返回 main()，不再执行后续 validate / 保存逻辑 + return + + # 验证并记录 epoch 级别日志（如果在 step 模式下很可能在中间某个 epoch 提前结束，但我们仍做一次 validate） + val_log = validate(args, model, val_loader, criterion, device) + current_lr = optimizer.param_groups[0]['lr'] + + if rank == 0: + # epoch summary print, 格式与示例对齐 + print(f"Epoch[{epoch}]: epoch_train_loss {train_log['loss']:.4f} acc1 {train_log['acc1']:.2f} acc5 {train_log['acc5']:.2f} | " + f"val_loss {val_log['loss']:.4f} acc1 {val_log['acc1']:.2f} acc5 {val_log['acc5']:.2f} lr {current_lr:.6f}") + row = { + 'epoch': epoch, + 'lr': current_lr, + 'loss': train_log['loss'], + 'acc1': train_log['acc1'], + 'acc5': train_log['acc5'], + 'val_loss': val_log['loss'], + 'val_acc1': val_log['acc1'], + 'val_acc5': val_log['acc5'], + } + new_row_df = pd.DataFrame([row]) + log_df = pd.concat([log_df, new_row_df], ignore_index=True) + log_df.to_csv(os.path.join(save_dir, 'log.csv'), index=False) + + is_best = val_log['acc1'] > best_acc + if is_best: + best_acc = val_log['acc1'] + if (epoch % args.save_freq == 0) or is_best or ( (max_global_steps is None) and (epoch == total_epochs - 1) ) : + state = { + 'epoch': epoch, + 'state_dict': model.module.state_dict() if isinstance(model, DDP) else model.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + 'args': vars(args) + } + save_checkpoint(state, is_best, save_dir, filename=f'checkpoint_epoch_{epoch}.pth') + + # increment epoch + epoch += 1 + + # stopping conditions: + # 1) if steps mode enabled and reached steps -> stop + if args.steps and args.steps > 0: + if global_step >= args.steps: + if rank == 0: + print(f"[Main] 已达到指定 steps={args.steps}（global_step={global_step}），训练结束。") + break + + # 2) if steps not used, stop when epoch >= epochs + else: + if epoch >= total_epochs: + if rank == 0: + print(f"[Main] 已达到指定 epochs={total_epochs}（epoch={epoch}），训练结束。") + break + + if dist.is_initialized(): + dist.barrier() + if rank == 0: + print("Training finished. Best val acc1: {:.2f}".format(best_acc)) + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.jpg b/PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.jpg new file mode 100644 index 0000000000000000000000000000000000000000..a849a202a2e789e3aa3d5f276dc75ceb91e4d703 GIT binary patch literal 34590 zcmeEu1z22LmUZC{!GcSWU;z@`B?J%d9z0M;a0?WH013ef5FogFkis>%y99T4m!kM9 zZ@PQlOTRbM(>?S0%|BiBov&`mz4x43`>eh8+WVfnsk=qMJvk{^DF7TC9Ka0r1l-L4 zBmk($$SBB2s3<5XXlSVDm;_ju7#Nr&c=$L36r@y?6r|+j)O2hN)HIK1$;lacnI1iU z!o|f!#lSDj$05YV$;I*gAaH1CXqXt7#8_Cw91qAJaQv4acg+A?RCsN~F9>in0C-$D z1YEegHUJd>0EYzY?e`A<=>Z3ifQW>Qf{KQY0s8^?9snK=0RbKn0SO5a5%z0u*!KWL zTqL{)PsEY&Rg6(+90)kxM5dwAJ}qq_R2@E~<1}&fLqoq$L`*_T|B!)^iJ6O=hnJ6E zK;oICl(dYjoSM3Zrk1vjuBn;1h2={tYbR$HS2uSLPye?8fkE%y2S-JJjERl=6rYfu zk(rg9lbe@cR$c+Ftg5c5{o2~r-qG3B-7_*eHa;;q^=*1-d1ZBNePeTL`{?-O^z8iN z^6L8gxZnT?|2C|@jO>ST;ljoRkBEqXi1K}0aPV%hhJcHR^xz3Hp12B%u>(F0#~W0F zr;%x;Eoiiys)vLoj>G8p>A05YkG>D>r;+__1M~a4M)sG1{rkA40hkDIus;t07a$6_ zxMocEMgLd(*ERT;4g8<60lt)XjqKw>w8$}R@mBUu=kI6r>GIpIWXWW19!vLw6**I~ z#E(!q<9x_QPro_y71?Fg(YRUpbFfyb$cJW<$HqPu$u7Jc;fibbK@XF}49|UDm}z#; zHnXdkLGLg-C(Rw-C{Z(tU$VMlv|^+viWP55NA}dTM*I%&N*N_*Cz)-io?l4WI_@je zngmT-q)^Eh7n58xjD7b!wtBU+Z<|kcqOFy&c$6FUx8>jSI-?`Y<8LAZqf@lM9LBXZ zINkx;@NQ#gRLU#wYdO*q&wWPHc&l^@?;r18e zd-xHVLGNX~IiC}w$o{ard3HcKr)zbwY|AYCkO@+Lzaz|)D|p5qu)XzOgYYO zUoXF$C?8+dV(LMlxmAG`JVYBA!Qo@U@cFFpfZTfi(cykf-j1&GHH+1G#8m-Odtnh% zGYL|xn7~GrT-;aCG2i6^^y@v)7MJ=#u+i~zA&EPHp)I5xAWrnbV#prayS!&Ahp%AA z#=7{WG|3>&b#y$%p___vBX5&M>JHG^NIFV^ccSe6;aOD5sg<@=2nR+)CIiZ4sSXX} z>ScG1V5&uUYZM2b$kxe_nGa%kM=3;pS+MemvMS5)(r-g1j6kyV3%FF)tVuHjPeOoa zx0EHI<4Hfj_2N1NG$TT#?Gg~k()lr~wq+e-=z#@5^bjue>LA1|pO*@^o&o5Stb zj<{#fyKhFrpgHB1RIO5tTfIpBO2J{C%)$%jZNPhm-p@{(taR`>qc1){2`Af}){4!> zxcwmB%e-(YUCJnKtouSnRuGCAz(7_v#*RujRynTCBd@~EKUYagBZ6Gwmejs7v zmoU4}()b=6Nb`#Du`B_mQ%?|AJ%igvh?~NM82cPMTk-IeK*{JEtm$Fw-->D8<@@Mx zpd5J@lHWRsQN|Qrwu%q2o0N}%kpkDBogf0qx->XL_bMm~_NNHFWG5bR&k5U~_Qq6$ zEuD1;We^JpZOpJ2PZq!qv^t{@rX$+6a^MJenRTRYLr6dV<9Jn$BGL$KZ63w>92fmd z=_)AUT5%U9+Ic}k>Eo6V?{uX&$~i^^OT|exdm-E6mu1qZm0zmBu@9BqqS@o?90r~o zW4^$?wf1&u7vK%T#K;8= zoFc>c`#sJI1=KEe?*QuK_I9W8EKPW_xm!f?Xvnlx7!Oo4k!)mMHN%sQ1y^C0U~1WZm_Z*CLw#!iUb#^(v0Lg?Ol_KeIj`z61{vs$$92IojZK|eLar{ zFUa`#ToC>JyH4|c7>_6>VqCJf1T-?;JUg=YE4Mgfg1N-wBD=b$;mEON`3NHyLAXZS zwH;|DF4|{v9P`AhA2h;dx^GwRBh!Y*bOFe7i3hn3h|#$0iQPf%N{FLY6QzSzoZ|ogFG(jvp#LVo=Hs2s$;gpa;+T|yGs|41Ane3?f=k-rY!$cnQiabVR z$aXSxVR&D$)kx-UDHfU~kd-x`Wba|S_84dpW`?zHX6!*sSwozP^Stq>DyU*l)0S-= z^5IerrAmg*jjTibYhG6ehcp5$U%$S8o2NpDKTj3K6m;N?xs`>P`4lakhIJ=nYK2^Vf5o>1{86t~!XZER`fCrXL!(}<*)q%+&Ek_49g zBsk!qrsTf)srOl17BLUKS>7H|-V!ZKo~Pxy2cC$R)|dJT>U9;ZMA0ZWZpJ5(8TTS1 zgsh&mGfdU9TQ0G9WglrkWTxxmCTi>S0=~Uc54?Kakch393s_woLP_78-cKr-?smxT zJMJXAvTr1MAM*tpy&CbAwB1=!{&tmR2Ps>CDSO;HT~$>f0siVk0(RQaft{`mbn6sf z@ScQT>~jZJ;}^@}gnaqkW)bZpZ_Pe=GQ zx*!_mcL2Cl)bz=Er4sAudI9*&^YdEsZ{_w??7HX-+cVaOU-#=?7$@@5dYHeA$079^`wVh|FJ#OW0~El7LnB<6wL)($ioEp27%g7 z^0kk(={X)ex0`?CMomKU2tYy!&dGC6ksRTw^On0%4rOI?eMbQo(O+%u%Iv2{n+#WM z**Y-<`s#Lsc_kYcJFFL$v77I36@4|ylsYG!xw(J^zEyHNS13*q!ke{uR5>D$D`n%1 z$in7jIqxehtpvl4d@KCz%7hSBbNgx~mNlWMj7Q{;kz~jh-)K08lU=$QB*-}LN%tr4 z)vu6A(Af;%0X!ra*@P##Hw236=jlgZdlhU}mg{fDg}*RHBcZZACgxg$TYF;^C)!>` z*m%7jzk);Qu4#lLV;85ui*KkHSyqT!A&F=sLu_FpV%WsyrHoMMLoZo;DXL#naZ&`9 zmj`|`z5_gqGkI5qXgiR|`86tYBbU}+7fNz8|Dix4OIsv2YJ2j2SD#R}MwFOuwdVA_ zFZ?}uD=1+s(Ia3zViC&A*4B~POwW3Z{BHV;BkHFk_ojr^15rfI(sCuqDX}jdOKDOS zBor~;gjmqvY$^a;1ggS>_|p~|V$<>-x#8woX_$B*ik)&0s^U+s(-dw)nTDNNfWXkF zx43L!tZgqxDrPNL)e*mR@*|K>ehO3|(mUb~t6ARJL!Gzg)R)Hj_P~8VEdnb`XT_w` zzDG;Qc1!m$1#e639UuTqR+kufc)e+B3iNiu$60 zw|Sla*iT)7%{&w}!;LnwgKz6KcaiimJC@aSJ@AV`a~TK7q4b1((NPqA zRR}ImM?T+uMeRN5eAhOH$kx4Vt*)m=C=q0_8>L0zftn-J5dPs9Z;1RS-A)o&0fQ~iSz zq4%gzKtl!TphZL@hyuWwZ1kr0YPdnPzA4G+4shsuDF_WPz`p|kXzl>ZFYf?vQCb#G znKx4|;fU@4_+k{ur_f`lv%S&*`-c7<075N-eFsR>dUgkBMs2!G5M6>hbh|~bWq<-C zVKj9JMf|5D|8$Z`(K`Stj(^jp#<1uiJUjGZ_@&ruiaWs5;`es|UtG`;7u?WK$Cdhi z3YWnI;!JYwxn8e|L@sNec0JdlDPh0lNl?fvJkrafGz*P&r7M3TlJ#CRROLs!lKv5(0GmI7 zWQHfm10=9<2lypk!F=>D5Xx5oYwLCmvN@kJZ!}PI2lx@NK)*z&RBZpBj{9#o4EiNr zQT>Rm2&q3EHuU=$c2fQrJ@(f32$ZNnY%?>GNpwEu_k znDja{Cb=j_U)wM@YGMh~F*TxiO;}1C1-XzZ^d1d?hKesT0RGDl&hY=dasBU|7XKYR zR{+6ZveEZm8cC>Z+n7+@umYU*e#X5-4+GuFpaBgiFzS{5C+fx7yl~38DSCKXUvNehHI8@nl|-@K!@l*F)s-r zEEw~e_lGermG8`J^k>Yg55~Mm7VWR^pHe~K|HQlo5kNy#sds=yf}dF9%s;*7Z+_}| z0KoaDu8h7xgvS;?(4TR~Xqnfc{W{I?-0#tM25U7Bx3aVl*tAPzdk095^%Y?L2WdP1 zAlV(D3wZ|mwbl(9D`7z4l+1evh`_444RAiVeH2(L0ANfH{@Yr9ZD59dwr>1B8A5vQ zcIOa!EJi>Qfv-kS;LDkVhn{Axx#yAVtBqNX@^J~%c*Bx}W&s2$Vn~0qrWJwUb&r-$ zO0=g?9p?@}ReA@QMeC_~sxx52J zhF8QLi?%`rQoipJTCgd~ye|3$qHzb12!g6*+yO|-bF4(DE1QCW3wYjTb54yorF34G z)UTUYUo1ii1RUg-jl!?6)T2QN5YfK*znna_zC_=fQ|V}S^6i#xKcuR1=}|T&dy!IC zQ;pCX$^tDqzYW;T+LN-NK+g)kYgU@pyLewv1_DOGzG~s84HLZ{PXl|StQkJAy#26x zDbEk{)Fn3-ID7UD8y<;?pd{$PuI*suJg3s+nM_y{PnunVMpz!OSQ6)B>@oL=6dNH% zeWI`SoYK=QnHO-~#DMeu??dA$Nre=&L8-SRF%Iq-binp0%8^O2WJe4KJ=T!I>uB+n zxp6lC`uVKg%NAg1t%#Yfb?@A$&C(~wi8f>fa+xp-PQIHz5UG2B+Qa%fggXG;L95X@M&%t~SYC{R@b5Kn^PS`eBM@H8^9IeO zCI6IoV~@vxZay(Zzy^1oeokAcVyEIIsJLivm1oZ-XI2~#^Ytof9ErDd$u*P~O)@g8*rEjDlI>S#53Y?Dp^NnFn(Qh^!7m+5^`j3t1ei`1Bg-5^JP>cF zcJwry^Sxx|Cmt4wTCx}}NfKxV8cPx=!~Kz(vM2;330(m_Q5*XXfWx5bg+67(Qw`qn zip^CsuNw^)I*y8bRNB(@%!KQaIa-h6v`{mWr^f(dFUt}Yl|AF)f~U8iZ#dcs4Ownv zSE6L>(jsX|TxH?Vg$J!!&oBh@(Dmk8INwB-b`h^k1{2B%0;|3F3${v4f?41X+Lc=i z2D7KF62daLD;L1rnJ-o0TdAK|=zqTVs)mntqR`(bxS5iDbRdYbXm$1q2Xqdo({hVH z52xju3$3f!`RqDK=)a;w$&-`@?3Ar)Iw>S*b0J=o_9Hn&Rf~?25@WDZR-#iz@zUlM zxL?FQR@S~er#jE;*o*0Uo~4X`A^uXf`GRf~+=N#gXy#For06AMdSY2u6-vq%S3~{9 z)R)6GZ@DrflKwqis6VZ5Piq=VS}GhtK65z{H9&iXVIkPX5uu$yas0LF^v4ldOJgTn ztimT!7_m`w4+!95EnHVb8UnOou)CrZJgA&--UIDfZhmH7p3HsnE~c_W3rP5)|JaYr z58FYELYQ!1aJju=bY%TRYMeDYcFi`m!ueek3_hfHfG-AB&|uwfRd;}JE75t-I*_bc z^jHzXeFu0^XMamj0XM87NdWi<8r*>U0#uA=;hb=^>9mY=bLaC>6U5iHX_qAjv}MLU zJoFzLx=uo6!ga>YIKe2)~$#%N1weWg}vm) z5}J5d!FPZ+C%hT%^XaZt>#(GT|1tMcAd9;uqp9|aN`MA2e2OGttt<=e{TVmdJX5;| z8ylR2PQw0-Le%o0RTK@y$B=!MF&_9ewrciy1 zfa5U#!CPct;_3N$#M9Lm6%GM>r^|7J(NOO!$Ry-#R>$oHc~=97OO8_vUNK( z!j22y1omS}w%t7WMsq60)@0Y@YYce<#XgX*fc4xT5G?L(2u(`D4t0-N_^ZR=54$8!C8 zKkfKK1D*hCzviX#2poyv1EDL?jcKEPP)E3k($#s~iP2FiHb#3R;+V^=l4gwY2s>?K zrFK!)7)JTjp4pr>9?z^o#?o}N%*W`HcIMaP0 zEWcfeSqVjCoaykq+>56uZ`*a6>!K=l=v(KDfnz-n#@+sg9(Z{o`=#dQ9Rua~W`J%N zU=z;hDC=ie7_iLx`ayn6#)59IX~t}elLL9=mKKqktgT&ewIH3@BjQT3w*b>{-ObnZ z84E8Hp2bd1V$jwfJaV8bVb)9Qpw$A!`*x~@H2{k2|9m|%zWG8_I#KD5%NLB)y1)oeHXDkB(1ccsz z8X->n(f{Fa{hJoy?U3Vd^VY*#$6rU};Xgzye0@ufc`{J}BsSEbXW6Q{N>m(vYijFu z2cU*cc+FhfvZc=@ZSqC1pB>xj^* z6fmyI+rJ0Tf0auPs~Qr@kB%b zXoiZL+f-^Q&oIorhre~0&&iS_kC26q|~2&OP~TPtoVCli+{p?SikoF*TMv%zqXZ^2b zY7;OD+=Wi&&k8+uI(fwY5J19!iq=fN4rcjy8JRb=wSaRr-cTPnS6p%PjvBAph(iM3 zSW8&;G!@HRzD>X|5@_$>@v5ha4i%3tS(ct}u;(2HB|jgzMvW*5Or>3*)q+Q`OVqi};n70+rG=`BZG z$l9gkfqSak+hZIt4{48LSalejB<+{0M)W!3^-Rw>ihKCBbnZBsY; z+P51C()v_2d!4#_o4?643S;Y%LHP8U9E5)JcqhInef<~GXGO9UCZU;50m|AbV$8A8 zLT(fO%6b}a_WAANU!Vb^wKXoXh#Bmf=msJ+?+U{V*-A#wVDusPAboV=Da5 zTY0J%3#-PiN$3dBcajcPdP2Uqk_^t!nGSuEeB5(HubnX7ZDF@5O;cK!8TC0iEeF;%((%QKLV7>T-tNZiB%w@OLJn!(b++rpi%xO@PWRjHSx(?%Tu*wj8 z>Ml z5@eLTKE8ZavzG}TwcoOVS%uw;f0@Rwn9e_98o}#JthVOW(fd*XgNw|%17j{Im9?ed zK<+5{c-M%z2%Jw7E*5F`7tXvxnOX*9A0$%|p~!>*^C;Fn$A_&XrG<%6`V;1f!R`ua zrU0ZTF*=G%c%f%rt*_0(RN}H6RbR*6*e#oMzF5J)5FZ&R40cQ-)lG4;i?%0gME@fi)1QRjZ!7vL?use z3A(ammt;-(rImZU#d9XM7Sv6~EB{6M&P$0?;YhoA4F;Mo2+v3I5k#cn@- zFCim+YcG2%vV|7;Q`GSn=tG8t4yzuA(29lqsc9@JyqL2YNI#7BR6hAR_xZ)PO=;n< zkRnG|mbRzAVtQanDub;Ysj9e@{cRFqcl@3hPF?;+WI~gUlDUu}ZP+;yGD+H#S*H-( z3#Fzk>j^Hl^yTrQ^wX_NpNHdxtC>j$C4(g^moP4*NBoD(DEvUSejg$P!)f@~yaMgb zN|5y!!@V|rsU{{1S7Aa8@kdIjh@;L$7Oi@f9=i-2kL*k%4S7OW9Q?tNzUF+?i*H&8 zV^(BX^%Tn*tH;wuGQZTzzhf)Le=YxT;yGVePmC*tv`W9(C7eCmSqUDH(~xx!y!+)5 z?~`WZU^Q}gi%?kxOXeE7)x33IZpaT7*ycWf5WM5ss-4*iEdMQz+RvwmcsI#5bhM?^}q zrE68q*=sb{N?HH$gPx@p;;SH(vx4#X+YCB^JHQkJ>X+Vo?!>QTDa0>|J*Ei>r7lJx zO&TY>0UHV}tT16pbf;?+<~viH|K=3hZ_MQ1V}IZspuj$@81xH45IiJ$U3pEq0r_uB zYW!&~kpDHAsrfM#`B%)4P))VzgF=~Cyg!E@^4PX4Uvq&=QmzhLhTF@)CHNVchf$PS zO?9X1WrgdhSG-%d9(GoelKoD5V_+*FQc{_?JyG1fZj9u3{=CifiV$^;*fyTk{jb4{ z*|1>#w}rc0hVrV~D3XWo4VG+=O1E=nj`KD%mop28#y%Sif5@|%q*bnJER9|4vJ~es z(SH!)vT|6I-*~0WoaFPo<@WEbLK2Wf*FhbVS(1V%K||sHrE`f%>LQ?ncgtIfhFAiK2J=Sc6I9Y-FKf? zkX+e{mgUFB?*L_|f{0^m#zq{QoO{}nWgFQmQ12w;6&8#SS!GSZ-W*q9h`jO@PwoKb zwN#_rUai;Uy7!Y-gE(fp=FQvoF0!1)Qnu89q>{x|*}dkjz6Iei%~z3ik1q%doJ)Fb z1yRh;v+)my&NN}=q)8DSCwBnW6(|v9L%9LHIUk=Or?!HE0n96^ zYX6FO1V`8<0c4LSj^QCsBf;vF3wmA`cBA08%-H5o3&|A?A~!uPu#EKAx#rGvC3T#Z zepUW1>5ciW>;Mi!WJ31|a{)MIA*)GEqjxvOtE5g38VK{ey792H{qYj|KkL))=6b&e z=LkRKJ|_8BKv#en(3F>`K z(_?N!5vByd(RWH1E_OKc#t|bTiX)8@oEJ?Xcsq{a_h)syrurpV1{tvV0!DnxGZ1Z(_ zjN=2v4L9dug*(Y$@p|!PEPB}qXM8SZj!6Nh1LCtO52{cXXU)>QoJTRkAM=3yL>3yO zXJiVRJ%YVVvD*5tTB}Ma!oj-5O3FheT)>TS2u}u^lS@wY+s~dQ7kZEzGFM8j5xeXg z?%wVkgA2biq4Jcxm9y+E11AZM%QUH?6jlYQ#aryaU@-}RI3nd+)rp5KKT>@iQX#w< zqwTJv+KvlK>nFrm)lT8MlZ6%apWPGW89=j%#>{LeD6Tl%0d^XEvKNq1-mJbAzT$;w zsjUAEir)YH3@VmaJ90iWT(GLttUSogg52lH1k;m z#f%4J6~__1CtWAg!-!{<7>Mn^7bP1`T^2si*n&WfFySk(fW2J?)TQGs|N3RHQQQ2P zZ4V_sDbMsPj+&>^l`NWHUXwW5)w9~{Vbwp6Qo#3)=aE{YcV;UXK(L*=1Hcz;0|v&y z{eluqj^6#dMH~ZRTmiI#Z^WAz;t*edb`&oV#r{+o;Mo##lqBUEcA2N< ziH0&jebi0o5OC!Vc%h8+BOij5Sd1Ltk$Fu_x}a1EG$#9e?M8{>bi3E_%Q8{e6sD+^%Cwhq$gPC9Xhfs!yEmyBLirmoV&ej)DjMrO)WB` zboKs1T9y&&>~;wK(Zd4n1OA&wC_TPD2(i~kdR}Zsp))7YhUeZ@yoWIV#@+;Ze;{O) zBh3#Ujk47|fqLi^FNzcm;SA&&=w3Ey2k|1$cYK>-stEAsR~xIl@x%s?f1A>^8bW=v zm+tof@+22V6iRf4`dK%E8PI4y0Qvj~Q372zQuNU`+QxFaccU&VPEJmT7H||ZQYxY^ zU5<}WJU?W)8_5jM^i@`M`_Es|LscfqF(74$e z_f6Ju7nXdcErij1gw&tYbrz9zc|PtBoR6N{vPs*4M=J-YI{0|oV{ZAYVR#>n-vk6Q& z#DPRNVS%RDX6^uztj;*lqQ*d2nK|GN&@Fg4dF!{=bgDXpBKad?{w?qI@8{(!xGXoQ z4y?47Ub+e*h3_7{^%dIgBJR;ucC34^&nB|e?*?w4caQ2+X0(5eU#$OWKy6@T`23b* zRvEd;A53y)g#OJ?-@`(BK3yhbDk{H@xvOAGz!z4kXI%wMI(%wS60icC@%rGASyva= zQ%xMHvQKA*=z1kLDfyFO{p}&kJyhxt|7loA0ecW5fMJGtZ%&^+D{~UhyDOp;!c4*D zVOKH5W*JHwtB&2`>E-h&7pzn|`|h+Z=apSKSS@eHe-iwXIY2od0C)j6U8>MiP(IAw z2_;?dD3{}kO=#+cQo?S&uTYV+G5Ix#l(6q|dj8oc_4r;KuYms7C5Jfyf!inD*IEkC zW9gEK0yVJ6gEc;0p!hLw^~TCQQx6i-mzvR+b^~JdvDPvv3s5ObNy_V2`K@`$*gInc zKD6a9d8efLzD5Fa2awx-nMbL^MX)hQ0E<&{AN|?ZnUnL(xUo2v#}3VuK3?;xTrU@^ zsPW~scj$4XF`zNA4ogc{Cv3=8MJ@BnR_?u8L5%DGQGz|o8fSzk>2An(u^QS0Qx`Eg zZkG;rbiqkz43v616jZ&C3$7wE`GjdrDi%8Wu`y9!pC zt^qG6bcO33&3an5ud8Zx*}rQlrSE=C1k08M%gDYL2;M7)1z}qmw zF*@N1dbHAYu?3ZiAtVI1xw#I{KAp*klwisOnvH5nDRAw=jQu~B31Ec(JL&&tG`Nyq zL)uBPUxj6Wv-&@nbo;9k@gId|=f6}9Xd0ZIb{A@E>k;+^WG2$`i9;WPSp=102bHT6 zl>_hNF#$o`P4C6`2IV@yb7@2t3yW1eS>_&K=eXr_u9xY-Q*D37mtT8jLC`wPDo%4c zzmYXk8tY_nv2PJQzz$*Y3CP%6B-0? zVaBr7bS=-}NqgBS|876cfsTV}$@rFeRtvXl4=0gKftb=H?0*OZc9Kh5v&IL4WSg?Nx%G{aD*})m_oNZQHHvqQ)K=o@*O4|9&_?rsy2-xHI=FnuEYkpy~>&CuO~8~ zZz|D{0@@Z*m-e7QlY}eRZ2|v=v&GX`wdxWGqOB23i&gsTBIm!2fitF}vUS~X<|Kcq z_e~dcHQ~D+^E40Mns&Y)3|s94ty66*WgySmb<}W&)LBH&JL*1X*X;+KS{9QpK@FO! zKwJEqM4ps71xevE$OsJOs(w3_@@KDC0LszXjtvER+eLq!(+E;}~ zsH@~Y8H`(;fpZJD)k0;CyJ}tN74?Tn!-{J(H7U^4A`@IpH^0c6d*x1Z*|E0G+pvd! z3?!kZvYtv0nO$-%gx$j8@*<6e!37G-1pWaFs$ow1kC^(){HgvaFDsLpBLo=Z(np9wSGe$d)8{B~{uM{H`8J#h2+UiIGoSN^y6jaBO{z@OuTv zON(D4p}+Q-*^VLQmq{9@P-Ni&uJir1VyfMf-M^ zbqP{|HoA+;@~p278q2c=jh$=4G&-a;gx_5tg9DVxg|*f;O$9tpB&4sd<6aiqf1S>1 zeSfN;(6nr6d|Xa-WlMF~CG6e-x;`6GSH#Bdt5#6t6_+GXkNw^;bL*<;sRgSMl=i}? zPucp|g(7MxMb_h9Q)#iokWIvGfA??-nN?M*i_&eLdvk4%%h()}gQ^Qn_s65`y;UJp zo3`wp30Aflpx|41uHT~Jzjg(F&(Hoh6|DT6%Kg?hzY+0t_shz8+H$q?2L3k@?2roI zlz63$6gyycRdw+8G|M|}F4a_L<5}ED`A4!s?3nqaysCvcX()aypAMZ%OeUce)h6bn zS-0JGB^Zu5kn7-?m7Mh^_DfJ-=VIO(4A9UaVfn#~g$jP(J#AUQXHptN!x?GL4Bz#6 z5i#MO@RJen%iJVxR?ngolOMT)E|FwNfk>jACXbq&hpgq(|MPj-p--lgct zu2rv`{%rBY^m;piiUBg*&j|k?Vf&AWV}CF0F4k&`(6_3cNLgkPw8}YtQ&k^~G&_j9 zO77OOaa^1qT&)x{`@S_vdt(;@Of?jZJ3sTYS?qMoroqV(@&OOQqau>~&*vLi#R`0s z5}c8-RDE*Oh8UiSV3WeTeCj4F#-hx6pzj;k_z*0P3M5oh14o@GCZJ}$ z%rd{C&?Baeyws>=HYa4PZp@NW8t!8`tmK#r;>{|=S3J^?~037l8JE0{T(A`HPv%0cQbF-F0T?a0wybhhoO|uKEg_` zepOfaJ;Cee=lb1%471--ziK#YR*08~mzT}73S22RBG-)f3Jtc#EnO7gtU$nfHEseZ zRH?k zp^$6Fy|HDp%ADDY#07HTTXIHEvJXDg}m^M+#om0_%tl4REAC#|`u|3c; z7^9rszV^7mpre%OpOqlNJyjX>GOhI}?g03g^^an$1*6+t&k4LRv$VU$-i-w7=81-I}&W?h~40JnKdigbF%t#O^ zgNCkgxAfF6qVg6%Mwbt{8YZ^RXGWoL)_+D-gd%f&BYES(>6xzFN?x?J4h@G&@u^!c z%_0dWAkwTc!+SH07mK{PTRIvcnGKy5O$DT?A5Cu0TyCuuBvnSa>BrI{>&ppBWlWmG`<$nn|FVZVYv%_Nbxad7F2tMllGx8+#raGo^{`N* ze!Vk+e3me;B;Z$7685Y(&Ec826#Ijs+phhljh%6w`5Ei`c`jBQ3J4(uXT%xb?daFtqXsJToV zh_WydwV@*LjAzF=jtEdpR~;tgEA5E#e8j~Zc}~)N0T$L2V5zFfs=0YL^e^S^-D>4uu{!0?edH3pN3yUx5lyC{D%pyaHZz1UxuY~QS}Qw$qo?W}AkQ;i z)NsnpSU4xDTh$m@G^abiCvsL_&%3nEB3GOwGZxN8G0d66Y#%Gv2h&M@wGQ*ovHJh? z>+h!p{G>4cP}2BcyWQ}=NUZ>3w1{S;Yy0&ywswXWtyQ7f`lYkOI0uF!LS2O?1DhIecD1>AjEWs*AH0^F$-( zC~WP$`x)l>xjUFc@3e2xk5Y<6a7gte1ZIG)S_04ZnmLP2Om|oO&$i^Jx*-DBpxC9o z^hSDnqm!wv&Fs!cPEuvhwvWE_lzsLS4izZlp#{1d@`k2`WPC1*AvQhOQTi!wkxEdZ zS|(9vhWAZRNl2|1ovV~QBeG}0rE8LvjV|HaQhk+)%$tr*l}#K_MyC`Q%?W7n>M)LM*Jd{hz26>(7*l^@@O@n@@=@${ z!q%`P)~>B{&PIaFA+ghx^WK%}M&A9=&`Eh02pEFWjYcTF_saR#Zo(uFmRa7k$Mp zB+thAr5Y5NWF=^v7Y$I2jRQ44j zq%fzg8RMio@x}?ZsNVU!-wt+W2435&v~y$V!NDD1r)VrjfuRX>^7xhWncryD6)19K zSo@mjoh`U}j)%M9R7hV}zO@6Cdr=hpU4gPFjf$#;QGm?UKSc}hG3(LBJ~PEdrjezg zZN{Z%Nk1kj+~v7-wSMEnGV3a%n1t^2}706b~NEj%(&dX$?EidC!nkvgbcL zewbOghsU;-Oe5tTJXWa-4ogj)(a+$$Kx@TmKGCxNJgxEeg2{=#(8BD+SFy*A0+tjY z(qy>^S@AiK1K9mx%`C~EXPh*Bj9~H^VvVG^q1E+=IpOx?{Gq_keZ$9Fqg7cV29gz1 zg_FvyT>Yi%A;yGtmZPF8(v@rWNnCLTthb&YrcUU)_7@cFtYBG~n%~O8{6&oajV#T7 z`TO7P_C5~qtt&guHwg#&to~6Y2Z{>i8N~U44RjG#mC3R3B#vv~jT+ar?R@f5@D;BV z$Ew5N{#>9a`O!%3Wg1rvSA~M9T#9N1C!2BLaItFy8m5Xk9i}IxOyMAct^)A3*rdFs zba<^6D@S)WPb-v%(ed7Ku>KQthcI`NR18DjUoL~QEM%EAPsejnRJ@I2=i)7N8xVy< zXR-ev?;iGdEA81o%1#{Tx;1=oJGv?dI@gU_=(u+O zq__QLc1w%ll5ORI(r~Tm>~gL3Pt)}|?_+5$>3o^AHxMkXijYRe+i6fUQTb;d5^q={ zq6Ke$fBJ}bMqg3@N{Pl+*U1nVRPeP z7mZ+-O(~>w<8XPgKv+dvPjGDQgIRqZ*j%JE1XIs!2)bb>d6Jg$WLP3T>1XYeC&)ovYt)2zs?J%R zyA_7I*pg?7Y`F|D^;=q>@7PToaj`Uqq%w=5w@Ve;#T1Q#+P?LCESx)-v~P}LGL7eL zr;*9IjCn(Fon@P;qjR-Bfr}RJTJzNV+S4UJ@+C4_e_TdE>OSVbR+NPT$%)v8Qi&nO8y zoaXE;C#sIjo!^|}7(a{-tJ-Utt4Wj81dlyBa5@Km%9`eQ`k2mJQV}L;6%u=qPegt+ zD(PJjA8l#}}Zm`DC+G9HrNoItN?P)`Fma7ma>%;EI3@TCyiQk}h<;Io7>O-~wI^a3bqAUs9GbP*2Y&8t(+zZPy5d`yFZHhFj8FSu=)0ifeA;A`1e3&!U!|S}BbS+r$dbj8NC?-aR+b)0L4#1L8 zonHOvweWIoA!U!?9vd-5)x8f7_*ZK9DUf7smj(&+$aCJANX%T=Ec#$NNNC{8&I}qf zVTsEnDimBdcqQN+_Eo+q=bqb`JKk35noC#ON(a7 z<$@g*^m;B%!9A}$87mGpfS&W6rjoDDr;!_k!EII39VM84rMi$6B zklW^7MzQw$9dN`LIce>^wDPdO{RWCn;Sf&pw#8~{pgIw}X7Q^lJK{-n*KAcED-Oo> z=R!Yfq7Yawt_tC%5a3qO6Hn$>k~Qgmf9r*pGX-^aYN<>;ussE>mGrEvCCK*c-dHyc zDkX(|4PJe48I%=Evrn=&Nlp=RB`U_0vCI?wnaeS}9n_IFCdaHGD9QkpWHeM{Dk$NG zTAE5askl#kg)KyqK5aD6SO7tjVPiZWqqG};o^ z=*8*n96$c}(Y|lb7CSk310`IDOG+vobVz9geHhD5;qGYq?hJ2!8STl{UiK1Q@eosc z-9u+}O_s|l@lXt&ENq7)fBL~;hv+L!v7@$&Nkgg7bOl#wBM~#-s++VLuBaxXhM5x- z7ErSSRN7S1A#~-uGA6xoXg@ttk;F8Hjv8uG)VWe5+{~f9tqjkBIfR7hMSgOr-rd1e zMQ`3%pwyO|C!3kRe?Gi(Ng+wtTNQRXoD}OBQoU{J?eQ8`uol z&86gu`B>%-)T4IeSp@0&ba?yENHndxsB19_8GCNvUYe>i%ra()Fpt0gdZ~FKx+j+& zZpzNyfKnNhG7HO4Z|4?kLGct`hm^d{Y1yhhLOIQXNrb^v`|i>wW2Scibm5Tt#xtn# zUdFfEK}cA3@EWM{weFk{Cp-+ze1fjj&*Pv>vBw0p!$l9O*bu|m{5eV-Q9OmC_Zv^q z%e<*(2f|GJ zme_m+HuX~u>A;FtGmT&4cPMk%nhHphOJ6PMm%?F3@UCI zBDyAG*QQoWL-3R)33+4RXgp;jc!BAhqoN?iV#dHEJk>(*;xTF}FXoepR8n)_NVEHz zOJjkWxNPEQ1F))SKFR$zit-jJwZj!V-$JT*-qQv7b@FG~(zNX2wHJ-eJeVzAT_lg* zNnS4=R@t>OXw)daw$rN2Gpk8Xv{o(~uDBI3<<2T=e_)X1kKeiMosWDt5VEUN_%X^+ z@f^0)(?oi7s|qltMQ@m(lrVZ-&F##bY>u*3#o4RQftGqYvy_jXbmS5=@fdr!=FlAG3KRv)fM zyS+LYd`YRrZ!zD>aLAn}sWhH@e09k^V~J|iepvT^{^ z1b6&6b*=M?sP)pX`nLd@l5q_mde`LZ!j)WIO>eGhvXdbCS|^X$ zf9pIC8#Tve+~~&+r9A91^@(8k2OY<;ZeWd`(r^h_qz_b{WnV-ZYUiYcSxInnPILeH zf{9-o8b6mD5f)x7j5NPJcbL%9^YwMFP|>87Lp>K7^u14M#=%eANOAAI>Du!!KMn+H zA&CT*hvjANxh^Q}c-L%!VfC!2ovT1f159(1K%Q%siF^PBO&hLWT)zLF^$(^?-*t6= z*Xgw(uK_eEtHcc=Z|~-UXWnbM-}#3oWwd=Ee{^;K#{-$@k8TK*SPx$7yD*8J+e>p( zlP^7}CuOE{$cndKFvnZuH0a1d`Yg)^ETM$Ts#G2g7b3?9j*fHsa`9YQms>MmUz@qN z3Z`xWMol>1Ku&N+NzE1@RI_7K1qs@}{Av~8cj!X;#1n^Vg4r0pJhYc)b3;w zg$+f^g~dz+qVpjDO}zY-?c2wqE7771*RKau(N9r=8l|V?=mFNx?^x|! z159X-sC7M@pd&a=8xW^40Fb!3FGyiC$h%`*=ymWy!Q>>zHT)@QJ4Xo)`+1mO#P#oF zGd8@i8G)9-@2p6LYaVo5wrbpQC&{J(3FQ)c8D`A!aDB2nEYrTdlXmJ(GOf8wfzjc~ z0q$&7Xod|V;+}lgcH6DXpQe95YrB7{>;FRC`7(MR{&u(+zi(GpTCn4Paszu8a zko4-y;6M#YJ0EdYTh*p`931!fZLi8s*6zKnqFMWN%lAIw;a7E$8MZ?;>VU;FcWUkP z^F|W~tzIKewIz$ORZqV$d?*h)$Y;SuRdcarYtLs9r$lR|QbFMn(m)Kfa5tWlvn*No9mT9dnyu~y`hQ$-2b=^=( z9~E43!vllu8lOh)nk{~3Q>T|*m2w<)7JkmbWJe0lu+85Vl$24?aeLi{=rmI>G-Nex zmbj5=jX3eJF4{&|>E^8OG5KRuoPz7blYZoQK03%_&l38Y%c1W>z>ZOnc^*$+q4ecE za`3JJC$TAi_FDKx13V~g@V>qIqAWzXumXlG#CtY5j)$mC{tko~g_Z->JxbmnXJn5e z3rw1_B=(C{_Vjb#R%aZs;am7>Q%)%$o>yd=)i$^)Z9t{gB$Uh%%PASIO61isBd$gM zz&6!ciT-cC1b8%Sb@K2OxUnt=+UCjK4|b4?45Ww;r;{9?yC#ZxN$S&7<7zRjgWRs-inH7krlq~*`OziDie z4-P{DWgD@PS*E&Q+zR0aDE>*F!%@0X+-&eavqK104WH-`1gSt=sGicvdWkjs5V zA>0`1mgT3a8HZR3lNVjF zt@vs?Q_AYfkG@4(o{WGG^B}${i+5L-Sr*aX+*_>n0sKzev2LlJzA{(kM`TKKTFCuC zD2*8%YgEQ5V*$}m${1G8)-p5*rMkB2L7He4e*6psi`Ny;LbqNG3-xq|sr2+&M`c+} zCLsLK36Z9yrMba&rv>0DK>7M5iq8{ALi=TzY|TS2k^=mxG`F!KrwbDP;V5 zj0h7%3s(+T4=$EUF0}iIguQX&KN`g(Ruas5Try$ij!b?XO@+dIOg40nOyh<`AYOrGSB30Pk z6e^*v)jg{P%B?6#75ScRX_scFyZiFH!80T~K(lWl5ATh{cs~FulV??KWqR&(nnG(t zOzoonw(>XXX}^>A(O^PUGT8{geKPlR+uLa;KWSVrw!|lCRqiMhok)RQ!Jx z1M#oU_xs(?8(qIV+Ptpd_qDLoB7%#oA5Tk$%%;Gq_+oLSi2=|vACX6^-A^qmOAB6i z*t(Q~o%HtGlT3%?8ATKg4T;_K+&X~QB@6{tRbE+>W>sc|=;iUm!S|rF@_sm@%5dcq(PJEEd;CCkcg~=nh`nUe_f8ACkx z0o;Gl29;l2+yJ{d7DQ_i>KT6PwmCJEaA~R@m_|39f7Spr=}TUT4N`n!=yMo<_^$Y& zR^ey3H|_@C_+<*T#)&&7CpHPRH_WGg0a~=YQ2OIMUxX(Go{kd_Ylrg2pzIf7*%4BG zO#4cU^KF&gCpF@gW#S2N0phq~SR(Frjr0MG?=xT*2HU53@$?c?Ayy-+m%EPa@!t6k z=4oh0wf1Xn1DFioG>WOBsM&OvO*BXzZ#Ay0h-93FT-BjT__~>0Z2)mvlft=Zn5XBf zI+q7#+Rq+VYq_B4?4ab(WE>yz0sum%`{w5^;BL9u+hEqPWZ>@Z4{pIB!JPz$AOV60cTaHW5L^O*4iF%CfB*^Z?$$UVcyM=jY20bL z|2}8t%$+%R?*EP4nLFp+X`Xua)7^XTuBu(N*89F|RRN!dF9YxuWff!rNJvNk3&bA) zJ_nEnV4$I)qoHD;qoZSDV%*0j#=*wI!X_sq!Xu`pprxUvprU%nz{UKKj)R_xibar> zTIMBS#FpX3qYY_aBgwky9`-F|)9;@$m}? z3JHryKbMh}lUGpG)Y8_`)zddHx3ILbwz0KyadmU|@O@OioSD%&x4it#52@ZSU+JpPZhZ zUtC^Y-~7}rBmna7n)Qd4{iBljOL@U1W&|J1afTlSYV%>Qp{*&iDAAMKh2U?U?T9v(6Q zKmu@i!;<;_{=dt=`{3VW;Q!1R5K8;l!aX5Mj~2)E)z-n~B4pl(p|Imxfl~hVvD`q7 z3U6ADar0F`MMa#s*~FMwp;ZsMBF*_i=K=t`?t#mHzb?LJkYsbO^OuaR#NBWs7d|rG=Ww*$2bP!jtXW)9nZhP_Rx(Kzi zxP1n7HP5i;F==PEda~g>Sf8?)(XT4> z=~&jUdh?#9TSc|U@DPgcoDN&~qC|C7K$TXX)*jPVgsPy_`&*ouk=!$-bRP?S z^?A^beTu2>T%OUy6<@VW4s)ATjn|+CZ9YFm0aEs8^F~0bsf!M#NxT&%IrtaE9M1aU z>T0ZA^-1JWib!5t;4Yso)i}}Xk3m_F={u@`qx_Y2(FV=o14NI%YVed$e8SZeP+81# zGrE!k!$>wtd$2Jtikd4Pw}ty;swU7ZupnEjOmR7g+Lzi?%3;)&gKOfM)I4IjzcxAz zJ~_dDiF;?~tcoXmZV?n#RTuu@QYd{^6M+a(x$jggEOkl=oP-quXG7xsvpO zySQ$mQ>#sa(2a$!>(sG$nDsG|sJe?+u=F@zw7qo!DidRja9V$qBQ2{ajThu19ruUo zgKdHseqg{vS$61@x} zFx_qF$}n@&J73^gBwG*Dj-2bgTYG>;9~IXFpvos3;yWb6A%&5ji3`Ig2R{Z|yf81Fa%0a5gPoy(OcmCrihB(Q*t9u7>E? zb8SMyt`yP1@(do7U6S7mdb)Vzkm-d6i~>5ml)C~1z|_;QK|f1dD+|kMdIoW=h;ROR z;qIj#?a^{@0G4Xk*lQZ=gqIl=Jv5qBpftq|w;s#jP7fH;b5P|~m|C!!20GRo=P7Bi z%HcCOAh#N6+bb%(nwkkq*nvCLr}7COK{26d=@)d5_(DBgIkhLMNu9GO6xbAVyN)R$ zEB?|v_h9B!?+eo(aJDUDANjC zDYyTmoATPBg)}6t9QS@5ilf|{^OV9}uyq#&SD-m}!X^V4Tue-~&P2>jA2GPsvvuDt z?R^bI+OT#!aG~GhViJ6hU0H?Q<^DqMH-%YHLr19`I+Z4u*6h2N<#h=k53BXQaVQFRFi6%-E5P{z3q#O2FsZ$2XF|Df`v9}B>wxb)0Z?Z{y?kSP7zQr zHmhxM^%!{`y?C?e=kbu7oC82kQ<^GWx>C-@+r8}FyA$=-R zsdfA0FzCC-G4{1WLi~tfWY&J6({=3i6l?l|T=w=7PR*UF$AxlfnkeD?YmVAck$looYT^wPL+K_G$xCK>M^Q3)v}+qYZT>`hjF63D@1M} zQK)H+QkvoQ2psTMnuSYjihoO_q-l|H?45Vfc5Ri>PD0enSD56q_9tX~8%P^|rU??A zV3L-b&97^CG@d%9c=B%&lm&@QRH7@3394mKUdxkNnTeaUa(SyE7yB~GlwL^~)mNXE z)F>$dXI{Yp&lAi(f>G=TlX$$=BmAYbUxdw39GP|G3)rD)F}S6m{t(F_<2UY3>6LTm2(AM zy0WwimS4CP9p1Jwz)b`kDJqw$g7UkI)4~YiR)Fxf=E{t`=l^qC- zkh&w_isbCD8LghTUe`h??-oX;n)(u?Olo+{A6dV;2f6Rc;*H*(PAQ@wZAIkWu3mkCt{^}_#(3l7g0MNk!t2S`J2lTe3 zGq&xtD?g8<=4L z83arnMwk5g&OiUilmr}rffvxatvw=fbdMXx6m|9N9W@*vRT=^Zye9x1^C1oYd|%n0 ze!^`C4)~6|e0W3RstN^!!2uyz|E~JKd;Q-7=70M%#hc=TxZkXcN3ZIg^}H~o5v)*v zy-rI}copD}E=+~XwiJBkt%_?Skf@wpd~8@iV-XSW&QSG4JSRjVLjBkFO77PN3b6gN zk<9V}y#`32-(1BkUz{6y!+>3v5P5K(EZ-LoeQMOJ|(h5?83d zgI)rE1igl!k5bVP$jbpCDt=@68F^tJ3Ep9O;lYr8hrE>k2zl`$?3%S->$c%A=;;2> z$V*xj2Z6j60}#kd{U_u#_D9I8AA!8cmmO{%oY6w>{T_J@A%lj&>2Sa@@z1Dn?ti@I zem~bd5a9ZARmS{K?%_%v8qIlPwJqw?f1hQ3;m`3Cg0)>lT3cBPYTYNbhXXR>-;1#Q zt+1V~HjC=iHo?j8iD7WyBoj_`5SydRT;ggI9XOGu%sL17X{j{F&EkazA&}Rk{iz(d z|Jp(*;u_sAoDeA;_a=0$g-(*s>;A6$P2gSeRZT@V}xEVmK zE`uoQgC`7xqUzU`Y%v9ml#xQ@kY|MIdN=4FA5L-3zqNPpMw)Q~GA~B7=3kbC7RYqY z#FDeJr)P$;DgGvdUu6%`E*{D{hJ)ht@@Zib=5RpiPm-^4jSu>U?8F&uoD1!=FB@Wh zw8H_c2*VrktGB@YHR#?k96(QN%e@W?zD^PU=^{$qVPEZj(5tH=STG(mPY%X!tup#f z(jG)S)xvOV*|7DF)=d8L=-IVjPg@(5#dS5Iqm}lAqJRm8eg*p4{D&Xsd}~s7W0kg6 zUxqnalXUH1|6TtDnW;fqG|2hw54;vxV&@sVp7Ap=U7gxd)abfM*<0gz_vmXoTrP()GrIqbQQ~TOW zE$|glYfKr1(z0@{12@Bs*ht?B0xB9(0~J4gLymcj1W7`)Ov~O>@>(|pDmg(%y7;g$ z0&$I@O0@W(2__BfWnnmDL;xWAV!JO;crWx~7BDz^4z$-l9K~4IG0IbzOchPYZ~(($yXgg1EgUeSWQgdyV+ix@5Ol!Y6CI}T-(eT%>VqKy zRg#;%q+h~QEtG>_gj=ebJ=MXc!4KiGXl$4`H=G-@XRH?F{ox0RJFe>@Adz+yH^W@k zm6HLuG+cg|G94|c-l-|kS;UY+KcV3qi(~sSXd=*yD!E3j;$o;HFseTzW46y2;>d|pjt0wmwg#za4t#rb_AS%PIF#m1Z!Qn z=C+E`8r^s>+gW!v-`dUsTe}G`lvPxU0WsgXKKX1!D=H4W*}G0bB?()B1MuaFyBUa{ z+t;wW)NW4PWSr9!`T*^!J{uW-&9=xc^$%q!U}4D&1^ku@8^RYyZMV)lY_fSF+|Akz z_a)_ZC?GX7_og<5Dm)5gSDGE96DNt{E;=4-YG9b{z(4+^IDp0E88Y7M^IbNvV*Uvm z)2cKpz09Ickgce*0?s%OgE?34!$)%U5Sf+?Yqv*A-gQilJUR$H%H{c%Z|QM@KTIgIEbciy}mUYlr2 zN_%t&`$lSaa4J5sZcSMM_7HQ4oLMb@XS>!lR?X?TBX9|G^Zyc=MTK!BEsk{G*;&2^ z1;U289tt=FqHRe{j& zed&IjB}aJ4@BrHpO=Ute2a<>Iae05Dxf+1HKD% zJ-p)xY7qI8sqpO>Jvb32f6ghO4lJ6BT6iHM^wWIP@C0?R7Zt}(J)tz#nA@nUd4qsp zZ+*1jfHa?mHpDfN*&|ESN*Bf^b6>d7k3r>&g`lAvdvaDT1!$VX6&g=i*jDg>2f}1J zsO$OC9%D?rn|s>R(pNa=96UAMnT?$<=zf61Tid zJB4s>wyw#gF+*>auFI~*y;*>n8EzGc^5clhCV^S{h@gIPYRhcMM>89$670 z-pVRksalP3Yn~Pu6)}z4V1EJ?+(nlGSCju#hlx*)Ri4rvQX{S(nZ_h4S0{Bf6pYtT zzlx7-Ra?3*<14gVYynuEJJ6FpEZgFps>BG zzOS{OGSl-g^{UfVV3FlZ69*JHu>ka`!c47PM)^~U!diihSLIBGZrw9V;=?#o47?G|gDWFNUeG}7>Z zf>n02sd@HUs@`wyagyD!4;jHRk^OV?=?oR_Fe+@^nhDHg0BZ4j-1j7iDx3b&!|Awm zQ{O3yDa6i;YemHRNd*adY2uvR_~IwGivlWTrP{(?)jdNKIxh|WZXk9MlC5Cx<5>B0 z@T8ki@g%2#%8cLR`U|$3(BAO<$v^55(Vw2AD&k4Lr*wb=XhuOza6td0;Cs+|$`ugS z=Du{p3Vg$01NkC^6uJ43<}1S_^B;d_gPfTe$^!U1%3to&fAQ_u-1wf}x0-#;RSEkI zHDG~Db~qp%T$pw`Pt*>31RJyEaLBynDzWdl*lN91+RcMrFuRvSC08~k;@OMzeftUx z-%Vb!kJBFfd<}ILBV(^bECj(pyVRWC9bK0qke%`mc4Wl8?4=a=AhO(*yBAPpr0uoa zon0Jd*c+6dcgG{Tctz)t?~51_T(;S&~M?JpQL2`A?4? z+Z5l^hyUYRDyDL2j>D;;F|)RK@PulPbh@YCVNUFe@znCiwglMUjCdpb<~OKegd=LH|Qt+3)W8o2kg#m1JjQ1ps}quplPl-I6#Ke6%SU@ z5`@ru0dPR?)1#?7e@N?@#xS~!46z#0-&AzQ4OU$HYO9B4pL3agLphvvYP;-og5gZ? zbAu(#SFs?rZVJWFrOMq07rjK~bDMn1_fM;nUnL2>UTX$;@aeJ^8QaY!Aheir*9GP3 zp5=1Vm?0`w`o^b~bSrI0vR%yKslp@~E7T@yMn}ylce9xb0#~^wUBtbV)vurJV`_jX z0A;_T2c`5~UU$i#=5~O%>0D@Kc@dDj!AGkH^swsOA*gq13K0AU0`ZHn`Q?c118P6o zbEM}8TD(DSncWdTv%iz8v6#IY6cfROh|0EFj+Z>^(eA0EMwZ)4Q-!sN?3RfI<_vTQ z63xjl3{!O+y#FlTT?3>Dov+fB)yzE{K(}(5QeK7VEYyTKyIz$XbJnJXH!ZuJif0L# z!U4u^UgAX#CAMm&bF?wq8>(7wc7584Cc2g20OH^fC>+q3WmjpYdeA0OR_<^*TDn`f zp*UaN3QDzo8(J{7~X79%&Pd<)!CvhAdQkTq*ZjJ`LR>YB#8!{WN~g=`MU& z0?G03O|K-7LzYLgU^5N_|7apaHxVvHFuct}s9=5{2v}%)GitCKVtes4B)gh0dnaXP zrry2WzQe_@&L79l#z2$;Ed-b|*Vs$lVBY@;I@l4@@6*cwDSS>0BCC%rbD4$xRy+|}`TnmN`v zk&E`i;r5&N@zPN#2*|>;+qIYoOMv3seYP0QW53zCnI6k5Y|EGA%+|$KZusF|ICLWL zc8#Wqq6Wcv6?eWWa&h$|xND}=?>Y6o!d(%c+M>JxOAneWOXFfOvF^G(o&4M+m2G9F z;z7f9ufV}HFBrvy#e&P~Y>t0?igzuHp6mThIJqH2H0&dzNs+&x zFWC^yM60!PLao51`dx39KIyhB&>%iW)7ayjC(jeCd^R@*LlPra*Gu{>``CYB-+KSj zU1#d(T30Q*Ao+cJ#>H9Tq?g(lp_4UVHV^E8HM#Q9RvuV!B(usjePhVp`U>G~txdKKd~UB}zCC!; z^4c0`Jw;!(rhhW=!Dnh7ao z@m!3Yj6g%DAh$Gp)^;Y4Bn4EeXIojUOPKcfM4Pxmeb;GNdpa-FCM!k|)zY|k&T(%v zlm8W~P;P0cYaUqve-@?Nn*?_kLxoGGh%AQ;GX8*N!{CJP<&xYtdve~k=ovc8zIvrG z^`>|T(0SF^A!5725Y0_b)7U)-64iadpT~H5C!v|>r4@E=TDI+`r7mOf`gd~mPm=X- zuON);1bBE#Vwv`u*%tvF%8kLWP=jsSJ3}8F7Z?&85aY2@c8fX(+RjPG4gJMZ{EbIk zmo7Nq1PLKrmY1f$0Xd(hF`X37UrYkxk{4wB4M ze@1?B!x)%zISa`+v1mh8|EPK8T3E^aQD?cEs@5J${CSSNh0$3zNO3l^+|$!wfL+uJ1Z=ddl^* zz}C5wBEYd#lsN-ic==*TvFkWfrG>I)ToIJLbBVt|V)7OpTRa&Sr>D!PJ0n(-bv|aw z^F)y=p~i3#;swl$i!2RY$65LaRlz8i9dX^0f64eYEZGp?rVOwMb=u4yR5%`?H`H z;EI@kf7i78E8Oybg*$Wv0{Aix_pPJXV`i6ElGi`K(Y=`*ur@NKG`1QY@9=+`zSMMB zFQ@vWyvXJU@0&^Y+cW6CgW^eedfZ{zLfM9kY`p|g8&`C`;Z^1U78!b+4`Lej$t&!g zo1Ts~{IIBC7;e&Fu~|^8O#MKU*-$VPuPbaPohm?xxmpj7^fVIHU~zpl9MHqh=)8D8 z8e)G~&HCl5YhM7xRzXOMg~ZTVX>MrrdaR_%yJHQ6l$T!q8yp$`tWlvR4~TI@_gX1| zDiqp@3y#b#I??#OOJGMaF^F}fS9=HPuwZ50K+Y%Ma{PwwBI9hrGx>(EZ-H)lwu_q~2)gN}95tRfo zPJGaqGY%k%sGGf$ws7V}SkR#T!P;2Lm${A~y2vbuBH2X>>K3^MfO!em`?ktOvx+-A zWohTmxv?3xiBzgIDR$+-!sapkOB!#BfrnS|Q0cMK2ATXhv8o#f(=W7Tl71vP zir-R|;bGOV(a;-IALmI^^U~boRr`&Zoae;R1UgKiYF_1{r4ke)#rZC>J4O0C zJHwM;j^eTA$(Cjn?>15gw-u%52_Yp__7BBEsgSPC8YtE|*B!7uYrSpj#@s?)l=sdG z4=rk*L$181OpUb@n)_?Ll0Ta^{zWe3FRoZJ>BO_lEQGF6@&|G~`rYjEbn`e7qepYHKGY*A-Fa9vhZ2b|cZhknQJJVOdC|pa^+oxQ zo}NL!(DF77ImP(Ov#^~~rskSPddPz@f^S`FC%N@Y9PG@{A4~47n{Vp#WXd+C4#%J& z{LOSYE{cg=Ywy=s)(VBT+jsO^NGEgZ6DdqnJYzw~I)PkJc%A zB0blE#1hztW?P7nyvm)$hR z(RQ@JvfzMm5s{F#wwZ+Ano|CcSHsb0!_J?k(0-o+PX<2Fnr^Z9Xl$nArt_)HF;s~F zHsI1=lEYaR>KaA~BI#TuD|=unz)OW%5~o%k_kvBJ;AKHT@&Vb7u}Vz|$O1#|LMO+U zX(6pba&x~+{F0u0h*sJtr)lTyY6OyNE0&#s))V7C{`pG1VuH%MR7=XxRFzF5tM_8} zeox5%9)kT}gcI%$j5F7aZYo`Z`}+G(Vxm?|I&O91bX%$-VO>Cd-~F)g8=CGT{uG=hxa#yB-$3^6m% z6xoTUIfdY(x_zGriT1?RJuj!XM7_;^O-1>5dh666WJK_y#E_)-D(J_W+_+pwbzNU& z>D;Xw91uT(w@M4aJ3SmhA8by#21O6>_j22`fVNIVey}x1Kw7daaW;zmqF9p7&gy$_$6uRe3ia5} z&?mFw3wJw7_?-ir&86a357v!7G{&=7+jKZ9P$VPM;nm$a*P-Vj3#Fs~DbU*sInMi7 zt?&{&YCg$9t3lC{nmh6a{|83n$LK2I<$GSb_J4I0mR!MH4E!`TIkl!St`8?tl{?vht|zA*KC1bMp=Ioafr2k93DF-9_-v1eLaW&E zG{p_~9qR0o5L%TvHcC6!?4ZA5{v~t(1 zM|OE^G*&SmMSIB6eSWoz(N^m4ZA{cl^hRso>seoLt?;g{u;y9Y9eZA>I%rk zy=eA;=A`@QG*^+`yhB;_>${~RT!Ym8HSUBZH~?jLJ2SLn`u~M!{uRFe=TPHcU^>-* zmaFT3o#6gA36h%Q{Z9JLV{@GChRAHJyBG&zY(T>gjn$#667_6ov>yxfrCD_V{*NUO z^>1F_>ph!&tt;;G%~|XGS;r@K(gBL1D$^*a;QH%&W99{4(<3ph@T=kCTU-Xl8bCKP}E)rJVw2WV%0&K|iD}>~=A-1(A-II?=3` zI25!zQJUTyx#`V5Eva1x7Bb7bQq3=9{+^f^-o4uV;_ zkaR(2-Cp0&15f8srN+F9iPtNX*)MZh-EWrP)l9aDdI%h=61|Dk!VMlLh*lSQ!*UV>7)sx8JLORB9s5)ZTzJ) zc{JL-`T-Szdy?k?#H#i);xV(VjS~bIUHTm`Q1CTa zwp*>eNP(EnBcG(@3PIU(OVtposk2reO1LlSfxE#ev~Ym=P2y}?Gr8Q>CFt9ef`cEt z*FK13EtS%Lb7*QPU58HkKupz1=Oax0OUVPW9>1$>ZLKdww%;v0jN=q7$`$i`AL?4D+H^Kdm5eO(VF?n^eVP`7nu@wVgjI+S#>#u!5Z) zRu@9an+jmgaUcIfR?Hu%cz@Any=Dq4p4nuUi00S(e28GFH?8%=$?|W6R9R7s`L%;U&kMvbN`6D)Tz;s$K0}XB+>KT)1cPvW$`4!3WNF=vxG~f zmBve}eI%h`MJAX?-)9!%DZfyy7)o?J^_t9y|H4O_xO2Er^1?fhFnj|JXoj2>7iu}B zWEFcgLhju5ZB?j_nyM3I5W)V^lz|4;ik_IRO4+vPAZ@?pP&d1j{tJW$`Sa2L|Dn(S zIrH$I2*<~sU2?sb0mGr|s_hOqovYOB+Lfc#d#dHU^`Y!CH0H^#Ux&XeAg7{fE-K4> z|Cpci-mT72cmv~Rsfl;s$pP2o{G%A28Mphmq^t@vuAISZoy5OuYjGW$sh`LMnbx*eI z!;4PcqX9^N5Mo*Igt2vF`6KLbiK*x98|J%CQ%9xo^%D392$Iw>tJBmZ{pglt1`e>S z*O8FEV^_ov4Xy|s%PWHtv7IJJ)H;~7q#fZHK7Ey*(;E*#OUAH?JN?ot8h&F{6Boc@ z`Ou#S<4`$~Wh-zs4!LU$JKIsla^@Q!nU!JK#4U#7LqfL~7tEvegcqDL+%6|@fZsKB z#I42pQtLGw&;}jwVnL^;3*8p}{x3xS|Cd0TKMRHQZ#KUq4u*K-wMh7o-&eKT%{zLi z6G%K}yKrAYLH~;{;rNo)(HWk&%s4r?4n03wNcd8ewyWlVbF~5N$^7iNuATKzPcRj0 z>`j?6Evey;t2ed+ys3xjMCo$(bba-Z-i_ln*nXU0N7G`&sc^v7(Rguntje)?C=Dyh*g- zgIxiSjEI_k4G6h?$=IHz9G^ybHyr|FS^AY^3H6$AZEAeT+@A;@ks)~=_;b8iOWEn3 z!bbH-5gg!Jj((CoYZ$?F!I=48k{q2b)a-xCGXHZd@Grvz{nralOaF7m+Fz|i{v~{t zDm5$1#KvP3C{7dQ=+bbgO2yIM$eaaAGq1_9tEG7~Hn}-n9Fi>hM%-qup?n{xSl_JK z&U4p%ZhwDoUnCZ9k(E3YWKGio2f!NU68T9vE)QW=qLsPV8DNJK!K|v@W%re-1yRWaqn7Bhhe*tyJcgvru|cL z4sH$}WHGazx#5W9^fzR_X7#%(=6RwIl5DRjM>`dkVpNAr`v+o;48eA}v@s>~_71utW3mZ6-yMvKF4}55LkkDn-vjtE{ofv&xyh z@fCqq3pBIxyRI$H!YJOIb1Cqt%Z@70(Qz+Yy#(AOX4#kOEiQ%`>Qh*?F-GH#obxl* z#+n1IE}{*~X66qp%=C!5%Z9b{S0jlr3E3WQ7d}RnSQ4mJDl=_#=rL_jI9O6i+q(XW ztk~4jbb9_YZ9(=n6Z}5T1P?Vz!){|SBF|LOvc6h@fgUwd@{TRFaaD*9#I-0id#IlF zk(2JW-$yn5vKS7i@*G0Qy~aMTa%4e482y!a1#cwSM#)&5Z!fA2OZvAW()csHOK0_r zJv4*cFXCt4eBS5&dWw^+u-O=pyv6J{q`C9L^ufYT%fB*x`M?6=%5OB>0KqK)jk$|x~M(pU|AOk zmOg8n_l5(KZ84bYb252SWYajbXQoGt@{a;1Q(ZEEQ}K!-(gMuk-hc%O1JkP#h)`Fw zZ)z)_!_`IJ{wj=|yYJ?<6JpWA(mHf=8<`ur(KJNW_q{FS8=i+PS=5hM7=f{p`j+7_ z&ioIrHXD>DceuiIQC10^jMu?64l7~M2p_o)5!tx3+TF*`+Y8_H!2wJ3X`=;mS99$i zLNi6r5Q}J1zeXW2#l2$N#RnsN#gxhds7`PICzXT4K&2g|>+36zwCd?O<sZ<+|CsYbcAVt@UroPvOS;6^933K?7U+VP8U*|`!%6f4I^Azqg<*CbIDyx zhmV${-msAzKPX6-e{{^54|H~Hh%1bxh2nT2lB|UC_CL3vgSavvMxc)+TZkn3$q{u! zW68H~V&`7OO0BVN&!8U{0I4Hl>;78W^p~7J_2-t20n`8760^T!bCQ2d5dN=`tb;a3 zXHU4fb{ARKVEg%GzOwbr(rp4tNdwp*t6lY!5d}*84{(56LN4Ce>5j2BiDLoF>z))L z)m2%)pBjTU!$2KZI$}%Z5qE7ruTNXg`J<`r;g8e2czPS2LUr!feBX8WOO``#J<6X^DvH7&O z&B5sxB@Haa8_dk^wq*H5&k=nX6G~B2GtrqYpjoX;@HqOhuP*(S;q>UV2`3~*Rg~uW zhke&ih)CK)>k^Z~%UXfih}LK~{20+Ka(eWs&@Q!@*kqd8xjZPc%^(*Wk%6gi>(Pq= zV7owKOYcbHD2(P+V?eO<;jUPb0JTGhmtB&voMhsS-XQ8k5>6$akvo4G>X+e+`Y%Qc zs&)l&el5;*d!eniZ&A-H=MCig4HrA4dgpL|mC&SVhAlwm$n68M}<$PhI5y zGo1H3v^Pt6=>0KXS4fLPkm#)7f!@Lbp^UY3keajZ-i}7jgZ-~6JO)#z2mf+(3=!2Rf;RGQ@;WktTR@pFCWE&-BFokfm)?aq1wx5 zZ-)ApGBf=*u|6+3@Hv-VW;wQ>6#(fGIc&0)G93FP&w3e_k)C16xCpk0j@UCj={Pk( zLAXlBV((E#*s0(PA6`V@zn?UIWZqD*vi;$FWbnh#dd<(;>mMWJ6=u2CtJP&BU;S09 z5!J`}Qv?;1FE7j6x}LdTM-9741in;S)91>v8gUH^%kPFZ8xf!&FS)qgIO!Dh7R{;L zVQ*-#o-COx_^y={QbdAtZCM*&H)KLRIu2j%h@FA4o}$X7)BL>?V}0CJ`wi9yu=?ow zwUeSn+9TtZ6yEv8c6+|kX7YF6^QK%}(sJagv$JE~=t97YXSb!MjJgl@v)?a5)+g@{`=J~hP}guo}qkDH?cpqD2!jX9T0^=bMW`9tbJl0L*9`k!auu)OMX0RcI;h(Z(`7Qp;v*Z`5d|ymjLEM;K)lNQ}u| z7GVeI-I7ajY9=g|ZI!J8Az4f*@^`bort1Vfv9gu2yW8!LKCe@l&r4G|H~z>lqs-oV zD$6^dSwE@)lrQ1jiWyzN)_f}4!Oyb)-os-`ILcA=Xp(ovNwNY{@qTEedC~* z534=j7s$g@?&^gXOvc4l2~Jc5-FXJXnPt<`Nhu`?7h@Y6s?y>^(qpgnL?47~;(CDl zXU==`%Umge1q0*Uj3rCZix10b`G*G88lGM%qREBrdZC3z(7{c62xKwwli8qQkN>K^ zdDoGT`{kj|Yi`InyMy>l7?XRlKZbZv@r2T?FjvV7_e*Xd6J%As8{{;vwx)s8`?Pdv z9kF_rxS*G%#(D`lb^Tzwcn_0bMnP9Ch|t_LUf%HT)s1_83IDC_flTr7fVK0I`_sU= z#%=FWy^pp7Ik(%mAI>)>7i#zma~2cw5V7*^jfk4=rV4i2aqR~741pS19Of)xr9<>? ztG#?rpnXsl=_5&Tt@hh+F^QqN`mC$MQJCl{UpRZRY%v>Qq+hXhvDf z-NJ9tS~WQ6iW8Rl4#f{sWhkW^PNQB-h88APpP%kp>khN+|uUrNE>hQ`@ zNGqhhG4&h3Ymz}kAY-*&X6qyJobBDqF@m0+{ZWEJpVl~IW5MsU;stLiW(?)L^f10> zSZR2o&+QoqHpH!yGEv&I0h0k9hZuR=%GMm(6bdI|z85Wun-$+D3#Y0Le0re^rG~ZT z)s;23{593;lHqS{-Xot=?|n&-AAk46Ys#1Q!-DOpfc2P?`779&#n}xgx|tIK^)fI9 zSra)mUdFFJlQ&BJ;3m{oP<^L3r5P|#`)(~ry8Y1^U2ZCy=sbAAWbg(KF!f@~C8;}g zAKje=nYug%t?h+3fKctiUODd1 z;viDXGW)wFtX!bjpaySRSo3Sd=H6p`At-qCMkZiP;G4#E^D}A*3TkF=Ijswf$%ryB ziuu{hdi6rjmLsb<$?7W2kuok1-&b#mLAIWa%A1$+)#GLd7ld(^h#eKc%tu+%EqjNO zkOXT7SvM_R4pHx%t|uPA{MpC1ef&k(zWOD~%pG1Tbw;+A1?+i?QuoU*+W>2ep9@sj zAS4vy>-(v>CB#}1uWN4=YHy#5dy+qKpsYCC)1Mw)AJ=$VY-2plmfvgU2q?7z(QG@; zR|6SNrGg2Kr7=Ct#RG%S&`KYFhLJckYdpS6iA)VG02WM}8YCqd_AtK3PS8c+o*FwY z_AaB^)*1Vvv2l>>b6DABabHY<%2$icCALzM?Nex@wVaGgx#wZK2RgE*{9Iie`IGC= zhIh&t*LaU7_P-J+3CxTq`ydSnDgr$#*s2#tP2c8lj2L!|4$#R_gjY2J0GCV^Z$zzE z`i!mSD?&TxIF3RZP7jxE4#5oA`V9HzhBp|%t zlv~l^e4BS^2)!kZ!uO2aHXHh0nJ&@o=2)=?L!Jy)ylLCnvfwwERR*pni4^AK+uiua z(cJF!*mtW@e`DSjN&8-|61SS7n|AJ=k=?6(81$|G3InTTs?+XuM@p{OW4oE+?DhrG25L91w!go`xJz-mW%h`hXKMp!B?l8*G*^}pE zu39`rugcDuPO)eYStR;(m0hxCP5ZFH(br{!%Utp=ZdR)iSYCPXpvbw^r)5okTx-98 zkd&z(Z=58UK)*mgfj#Ily>;7^!#HfClV4`=A_D1p>R)GnzlxdT_!K zs{UC3i?wSknMRVVtppEy`_e#*;=|O5)Kp^N+Tx3H>%Q|H|a}k)b)K#;=#Ht zms6(#H=W8H5c`r0S!$-q(McPQ*!>o}C!3(*V;>EoPpB+uf6jQiYm(* zcq0f%3kZlq2FU`F1j!;GLWATeG%yG#!4{DwNDECyBuQu_NU|G-*ns3@OU_8nG&wgQ z2#BQF=T`03IA6`TQ?vWcx4Y#ZujNiO|U z224T}YE}0&KXZ;@5M)ym) z3{Frn<4=pL?ZWBOth~xpXF*K6w{7Le#TUFSWa2#V59fCFeApE-AWlZ&{^&%mj z^d2`?vB8?yUhbiAEL9?=oi`S9wQNk9-BHGak)_0KZ+^U5Ltu~>8#f-dIYR8$7anAqQf$bNc1o;7zrX$?M=a4WRXdYUs3v~nYtD^c`aCR|AAi3e=(NrDJ~ z!!>&PaMbO_f*V`wZUJ+haK{M>TM%Qc=W7b5w{(&k5Mr?(E((L1 zhY2N4K4LTebhl0uplS>kXj(*lph-HNV(Q-NWtWJ7^hiucdVWm&;HXSh-W+S%Yjq10xf9eq(#|>s#)CXVsWavdADfxz&tfwl z`VoE@MnR2+v8S3=>x1y390mH?>AeHSq_n_VA)X_<#eC88$Mv>i=o7;W)UzhL$tiLK z6EJ!V_JRgwI=3^d67XK(1A75fONMQjys#V(q4|)=Nu`4o82#F_lFC=#weWgXrv3iZ zsr$f^{!hy1c3O6}-U6wYZH-5FDn7N+FNZ*G{|N3ZZiuo(k&Gh)DHmRL+hwWA^9eEW zXpilJMj%Gm(j&P^@2v4F>MIsXoEZpDwaJv}v}5qalfcEThR)7Y%8rT>M_SA|Og@`o zSmmTlNA`x_v;5EdYa*2i!VP@Gv(B z#Q$}A{qQvi&y|TjWJ*h}LD;h*91y$_WPEj~G>9ci-FI;+C3a>UXE;j5AdDuQOdoFa zzeiyH*I&LrSWx*l zos_117AblvQh1IhKBa5?=@jgv)ebdBdT&E|)@z&@_ge294IT1lhjW2<@}N1|`UfdM zC^(P%_xdXT|Fo{Rnv17F?z<&K7(%4fvZ*F8K7I3)x%Y8VN zwBWhSsILI*)1sb7x2h{5=kQnW+wZUey~b_7&KZ{q;+4AKWtW3=Vr* zWfV%GzVntyU{nEU9+sVZVHHA!^T<2L(C!MOERJH~kHGwhWS2Nq_7w*_6jJ@T+B*UB4A6{%Qm^5~wo%weY)|zpm!jwOL%=J`I5} z#CVn^ysS=lkT7pPn-HYeWdY$JnvB*5p!vUgms;YIn&aI2)%$Xx^-n0m7n%js$>wya zTMB#0WB{c1I~0&ESG{~nzNL6D#QrgP)#}qa7C9673BZ;Al@6Wutqh2xeme#Z*Zr>O zcdU^JS=6pvLf1>x{0_e{u-Y5E=y^;Y73Uf`M`RPzt8d?l!Mn@g7e?b#A zBSNj@axq%#Qu3o1HBOYRHIqNNyg#M$zmPubikHGw57Y|E#~UyR^Cp4q5AGR}E zr_RW5P1E-dDVPIbw$hpi**)+?PmQS{Js03gUr_=Di~@V81UyZ3yJ?)s3O_a4ofX#n z!z)U;X3qBFA_AoG4AnJX!+A#RyW&zLWg`X;wNlvF%9SM~M>TJJCVGUWQLHL&-5RrLKvl@%r(N@g7k0eWj^GAGHaP>_$|6^G0~#AIrzSy!`#mvQOWpDP4A<;+v&{_W99BM! zA-3MS@~mDVv6fj)a+6TCZ86tLU+F>$RTs4R}Qk}PuF)`Tb2q|a$}9W>hh=n`2rw?kNQzxnmvvQ&k+i@Us6Y((S?8eZ z=*;Mv8EPp4vnrBfrR?lBmcR5ME-cSpk2_9z$89=Luad6uachBx>`2JW@Pm@7Ba122 zMk11Pt#42^1SCF-iVN_oCri(@`WWPZ8)LUVQr$|~$5nXObhFzZsU7c5g+E{y77KR? z*Cnx}kDr?`+a?X=nQ9)Y3+yeceI?GwZ5Sik3{EvCVWErlm%(Pe-PNL)zV%|hGj(oh zZp}(|Zf!9Da&3m65Ac-e0tcDK;6E-suKXGFTvzK-6jhFMw$+4|vxohQ_|I3Fq!9_I zinA?SsOQU(PDT0bbe`7FbYQHdm_V;36j{Ac*tP`6W%-8-+WtnOOf^^7i_r zG?$qTg#vh6K40|!_W-A;*@ydUtXv$VDs`r!a)z%1+QBC$F;*vLhUX15IJ}q--k?xv_|T2-#^|PC5tgham1(Ff!gwFSY!}U} zp^m=FiTz@{XgAj0)@E6!utNL>Xbr7Umr~Z%KgCgfb)f-BVVrhoWEXsp=+`Re(zbVT zB8u_AXL!6}PV1(Gm#?eC;;kLV^7+TQx!SX7HQ{^n0np7VjAK$(h4maYA;OezjIG}Z z4AX)=;*|R=xEZM%`4brz03n0ENCTKl!_|es!W{isr4^-4tB+_A#q!bemG#1=@j@PS z{w}fIphr;8##^PS(vKeT1-4z-is`qDkn%N@LyZS6JZ@rhB$u~w5(#&%mENy#y85=k z*seUWugfo#Byijp#xC}d-=4d|b%ykLXN^}0DaRJWIFm1LSPPTUyO6AD zOUElR@AR#m%w=3}$i@YdJ_fD7p0CqV42Odeg5cIcqx3;3uyLeRY&VB(HpRKAdWQ8> z*==Z6NQM)hwd~{2LBH8PLP6XdoF}~;VH;k84l%w@?z2SQJ!vDIC#w^*XAby(sHP?- zZZjz;pB}E#stUrE^MiEZQ~ooyR8<={S>sW8qwDU 支持 DDP / 单卡，AMP，resume，日志，checkpoint） +保存为 train_template_localmodel.py +""" +import torch +import torch.nn as nn +import torch.optim as optim +import torch.backends.cudnn as cudnn +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as tv_models + +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +from torch.sdaa import amp +# from torch.cuda import amp + + +# ---------------------------- +# Helper utilities (self-contained) +# ---------------------------- +class AverageMeter(object): + def __init__(self, name='Meter', fmt=':.4f'): + self.name = name + self.fmt = fmt + self.reset() + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / max(1, self.count) + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} (avg {avg' + self.fmt + '})' + return fmtstr.format(name=self.name, val=self.val, avg=self.avg) + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k + 返回一个 list，每个元素是 tensor（百分比形式） + """ + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + # output: (N, C) -> pred: (maxk, N) + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() # (maxk, N) + correct = pred.eq(target.view(1, -1).expand_as(pred)) # (maxk, N) bool + + res = [] + for k in topk: + # 把前 k 行展平后求和（返回 0-dim tensor），随后换算为百分比 + correct_k = correct[:k].reshape(-1).float().sum() # 注意：不传 keepdim + # 乘以 100.0 / batch_size，保持返回 tensor（和之前代码兼容） + res.append(correct_k.mul_(100.0 / batch_size)) + return res + +def save_checkpoint(state, is_best, save_dir, filename='checkpoint.pth'): + save_path = os.path.join(save_dir, filename) + torch.save(state, save_path) + if is_best: + best_path = os.path.join(save_dir, 'model_best.pth') + torch.save(state, best_path) + +def set_seed(seed, deterministic=False): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + cudnn.deterministic = True + cudnn.benchmark = False + else: + cudnn.deterministic = False + cudnn.benchmark = True + +# ---------------------------- +# Argument parser +# ---------------------------- +def parse_args(): + parser = argparse.ArgumentParser(description='Generic PyTorch training template (DDP/AMP) with LocalModel priority') + parser.add_argument('--name', default='run', type=str, help='experiment name (log/checkpoints dir)') + parser.add_argument('--seed', default=42, type=int, help='random seed') + parser.add_argument('--arch', default='None', type=str, help='model name') + parser.add_argument('--deterministic', action='store_true', help='set cudnn deterministic (may be slower)') + parser.add_argument('--dataset', default='cifar10', choices=['cifar10','cifar100','imagenet','custom'], help='which dataset') + parser.add_argument('--datapath', default='./data', type=str, help='dataset root / imagenet root / custom root') + parser.add_argument('--imagenet_dir', default='./imagenet', type=str, help='if dataset=imagenet, path to imagenet root') + parser.add_argument('--custom_eval_dir', default=None, help='if dataset=custom, provide val dir') + parser.add_argument('--num_workers', default=4, type=int, help='dataloader workers per process') + parser.add_argument('--epochs', default=200, type=int) + parser.add_argument('--steps', default=0, type=int, help='max steps to run (if >0, training will stop when global_step reaches this).') + parser.add_argument('--batch_size', default=128, type=int) + parser.add_argument('--model_name', default='resnet18', help='torchvision model name or python path e.g. mypkg.mymodule.Model (used if no local Model)') + parser.add_argument('--num_classes', default=None, type=int, help='override num classes (auto-detect for common sets)') + parser.add_argument('--pretrained', action='store_true', help='use torchvision pretrained weights when available') + parser.add_argument('--optimizer', default='sgd', choices=['sgd','adam','adamw'], help='optimizer') + parser.add_argument('--lr', '--learning_rate', default=0.1, type=float) + parser.add_argument('--momentum', default=0.9, type=float) + parser.add_argument('--weight_decay', default=5e-4, type=float) + parser.add_argument('--nesterov', action='store_true') + parser.add_argument('--scheduler', default='multistep', choices=['multistep','step','cosine','none'], help='lr scheduler') + parser.add_argument('--milestones', default='100,150', type=str, help='milestones for multistep (comma sep)') + parser.add_argument('--step_size', default=30, type=int, help='step size for StepLR or cosine max epochs') + parser.add_argument('--gamma', default=0.1, type=float) + parser.add_argument('--scheduler_step_per_batch', action='store_true', help='call scheduler.step() per batch (for some schedulers)') + parser.add_argument('--resume', default='', type=str, help='path to checkpoint to resume from') + parser.add_argument('--start_epoch', default=0, type=int) + parser.add_argument('--print_freq', default=100, type=int) + parser.add_argument('--save_freq', default=10, type=int, help='save checkpoint every N epochs (rank0 only)') + parser.add_argument('--amp', action='store_true', default = True,help='use automatic mixed precision (AMP)') + parser.add_argument('--grad_accum_steps', default=1, type=int, help='gradient accumulation steps') + parser.add_argument('--local_rank', default=None, type=int, help='local rank passed by torchrun (if any). Use -1 or None for non-distributed') + parser.add_argument('--cutmix_prob', default=0.0, type=float) + parser.add_argument('--beta', default=1.0, type=float) + parser.add_argument('--seed_sampler', default=False, action='store_true', help='set sampler epoch seeds to make deterministic distributed shuffling') + args = parser.parse_args() + args.milestones = [int(x) for x in args.milestones.split(',')] if args.milestones else [] + return args + +# ---------------------------- +# build model (优先 LocalModel) +# ---------------------------- +def build_model_with_local_priority(args, device=None): + """ + 用参数 args.arch 作为模块名导入 Model() + 如果模块不存在或没有 Model 类，则报错停止。 + """ + try: + # 动态导入模块，比如 args.arch = "rexnet" + mod = importlib.import_module(args.arch) + Model = getattr(mod, "Model") # 从模块中获取 Model 类 + except Exception as e: + raise RuntimeError( + f"无法导入模型模块 '{args.arch}' 或未找到类 Model。" + f"\n错误信息：{e}" + ) + + # 解析数据集类别数 + if args.dataset == 'cifar10': + num_classes = 10 + elif args.dataset == 'cifar100': + num_classes = 100 + else: + print(f"[ERROR] 不支持的数据集类型：{args.dataset}，无法确定类别数。程序终止。") + sys.exit(1) + + + # 实例化 + try: + model = Model(num_classes) + except Exception as e: + raise RuntimeError( + f"Model() 实例化失败，请检查模型构造函数。\n错误信息：{e}" + ) + + return model + +# ---------------------------- +# Data loader factory +# ---------------------------- +def build_dataloaders(args, rank, world_size): + if args.dataset == 'cifar10' or args.dataset == 'cifar100': + mean = (0.4914, 0.4822, 0.4465) + std = (0.2470, 0.2435, 0.2616) if args.dataset == 'cifar10' else (0.2023, 0.1994, 0.2010) + # train_transform = transforms.Compose([ + # transforms.RandomCrop(32, padding=4), + # transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + # test_transform = transforms.Compose([ + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + + train_transform = transforms.Compose([ # 2025/12/3 从visformer模型开始 + transforms.Resize(256), # 先放大到 256 + transforms.RandomCrop(224), # 再随机裁剪为 224（更符合 ImageNet 风格增强） + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + root = args.datapath + if args.dataset == 'cifar10': + train_set = datasets.CIFAR10(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR10(root=root, train=False, download=False, transform=test_transform) + num_classes = 10 + else: + train_set = datasets.CIFAR100(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR100(root=root, train=False, download=False, transform=test_transform) + num_classes = 100 + + elif args.dataset == 'imagenet': + train_dir = os.path.join(args.imagenet_dir, 'train') + val_dir = os.path.join(args.imagenet_dir, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = args.num_classes or 1000 + + elif args.dataset == 'custom': + train_dir = os.path.join(args.datapath, 'train') + val_dir = args.custom_eval_dir or os.path.join(args.datapath, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = len(train_set.classes) + else: + raise ValueError("Unknown dataset") + + if dist.is_initialized() and world_size > 1: + train_sampler = DistributedSampler(train_set, num_replicas=world_size, rank=rank, shuffle=True) + else: + train_sampler = None + + train_loader = DataLoader(train_set, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + num_workers=args.num_workers, + pin_memory=True, + sampler=train_sampler, + drop_last=False) + val_loader = DataLoader(val_set, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + return train_loader, val_loader, num_classes, train_sampler + +# ---------------------------- +# Train & validate +# ---------------------------- +def train_one_epoch(args, epoch, model, criterion, optimizer, train_loader, device, scaler, scheduler=None, train_sampler=None, global_step_start=0, max_global_steps=None): + """ + 现在支持：若 max_global_steps 非 None，则当 global_step 达到该值时提前退出 + 返回: epoch_summary_dict, step_logs_list, global_step_end + step_logs_list: list of dicts with per-step info (for logging to CSV if需要) + """ + batch_time = AverageMeter('Time') + data_time = AverageMeter('Data') + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.train() + end = time.time() + optimizer.zero_grad() + + iters = len(train_loader) + step_logs = [] + global_step = global_step_start + + for i, (images, targets) in enumerate(train_loader): + # check global steps limit + if (max_global_steps is not None) and (global_step >= max_global_steps): + break + + data_time.update(time.time() - end) + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + + if args.amp: + with amp.autocast(): + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + else: + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + + if args.amp: + scaler.scale(loss).backward() + else: + loss.backward() + + if (i + 1) % args.grad_accum_steps == 0: + if args.amp: + scaler.step(optimizer) + scaler.update() + else: + optimizer.step() + optimizer.zero_grad() + if scheduler is not None and args.scheduler_step_per_batch: + scheduler.step() + + with torch.no_grad(): + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + losses.update(loss.item() * args.grad_accum_steps, images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + + batch_time.update(time.time() - end) + end = time.time() + + # increment global step AFTER processing this batch + global_step += 1 + + # per-step print (controlled by print_freq) + # 输出格式调整为：Epoch[23]:step[1/32] step_train_loss 3.0075 acc1 25.95 acc5 54.46 + # 使用 i+1 / iters 更贴近人类可读的“第几步 / 总步数（该 epoch 内）” + if ((global_step % args.print_freq == 0) or (i == iters - 1)) and ((dist.get_rank() if dist.is_initialized() else 0) == 0): + lr = optimizer.param_groups[0]['lr'] + # note: losses.val is 当前 batch 的 loss（经过 grad_accum 处理后还原），losses.avg 是到目前为止的 epoch 平均 + print(f"Epoch[{epoch}]:step[{i+1}/{iters}] step_train_loss {losses.val:.4f} acc1 {top1.val:.2f} acc5 {top5.val:.2f}") + + # collect per-step log + step_logs.append({ + 'epoch': epoch, + 'batch_idx': i, + 'global_step': global_step, + 'lr': optimizer.param_groups[0]['lr'], + 'loss': losses.val, + 'loss_avg': losses.avg, + 'acc1': top1.val, + 'acc1_avg': top1.avg, + 'acc5': top5.val, + 'acc5_avg': top5.avg, + 'time': batch_time.val + }) + + # if reached max_global_steps inside epoch, break (handled at loop start next iter) + if (max_global_steps is not None) and (global_step >= max_global_steps): + # optional message + if (dist.get_rank() if dist.is_initialized() else 0) == 0: + print(f"[Info] 达到 max_global_steps={max_global_steps}，将在 epoch 内提前停止。") + break + + if scheduler is not None and not args.scheduler_step_per_batch: + scheduler.step() + + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]), step_logs, global_step + +def validate(args, model, val_loader, criterion, device): + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.eval() + with torch.no_grad(): + for i, (images, targets) in enumerate(tqdm(val_loader)): + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + outputs = model(images) + loss = criterion(outputs, targets) + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + losses.update(loss.item(), images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]) + +# ---------------------------- +# Main +# ---------------------------- +def main(): + args = parse_args() + + # handle local_rank from env if not provided + local_rank_env = os.environ.get('LOCAL_RANK', None) + if args.local_rank is None and local_rank_env is not None: + args.local_rank = int(local_rank_env) + + distributed = (args.local_rank is not None and args.local_rank != -1) + if distributed: + dist.init_process_group(backend='nccl', init_method='env://') + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + + if distributed: + torch.cuda.set_device(args.local_rank) + device = torch.device('cuda', args.local_rank) + else: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + set_seed(args.seed + (rank if distributed else 0), deterministic=args.deterministic) + + save_dir = os.path.join('models', args.name) + if rank == 0: + os.makedirs(save_dir, exist_ok=True) + with open(os.path.join(save_dir, 'args.json'), 'w') as f: + json.dump(vars(args), f, indent=2) + if distributed: + dist.barrier() + + train_loader, val_loader, auto_num_classes, train_sampler = build_dataloaders(args, rank, world_size) + if args.num_classes is None: + args.num_classes = auto_num_classes + + # 使用本地 Model 优先（LocalModel 已在文件顶部尝试导入） + model = build_model_with_local_priority(args, device) + model.to(device) + + if distributed: + model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) + + criterion = nn.CrossEntropyLoss().to(device) + params = [p for p in model.parameters() if p.requires_grad] + if args.optimizer == 'sgd': + optimizer = optim.SGD(params, lr=args.lr, momentum=args.momentum, + weight_decay=args.weight_decay, nesterov=args.nesterov) + elif args.optimizer == 'adam': + optimizer = optim.Adam(params, lr=args.lr, weight_decay=args.weight_decay) + elif args.optimizer == 'adamw': + optimizer = optim.AdamW(params, lr=args.lr, weight_decay=args.weight_decay) + else: + raise ValueError('Unknown optimizer') + + scheduler = None + if args.scheduler == 'multistep': + scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.milestones, gamma=args.gamma) + elif args.scheduler == 'step': + scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma) + elif args.scheduler == 'cosine': + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) + elif args.scheduler == 'none': + scheduler = None + + scaler = amp.GradScaler() if args.amp else None + + start_epoch = args.start_epoch + best_acc = 0.0 + if args.resume: + if os.path.isfile(args.resume): + ckpt = torch.load(args.resume, map_location='cpu') + model_state = ckpt.get('state_dict', ckpt) + if isinstance(model, DDP): + model.module.load_state_dict(model_state) + else: + model.load_state_dict(model_state) + if 'optimizer' in ckpt: + optimizer.load_state_dict(ckpt['optimizer']) + start_epoch = ckpt.get('epoch', start_epoch) + best_acc = ckpt.get('best_acc', best_acc) + print(f"=> resumed from {args.resume}, start_epoch={start_epoch}") + else: + print(f"=> resume path {args.resume} not found") + + log_columns = ['epoch', 'lr', 'loss', 'acc1', 'acc5', 'val_loss', 'val_acc1', 'val_acc5'] + log_df = pd.DataFrame(columns=log_columns) + # step-level log + step_log_columns = ['epoch', 'batch_idx', 'global_step', 'lr', 'loss', 'loss_avg', 'acc1', 'acc1_avg', 'acc5', 'acc5_avg', 'time'] + step_log_df = pd.DataFrame(columns=step_log_columns) + + total_epochs = args.epochs + # global_step计数器（训练过程中跨epoch持续） + global_step = 0 + + epoch = start_epoch + # loop until either epoch criteria or step criteria met + while True: + if train_sampler is not None: + if args.seed_sampler: + train_sampler.set_epoch(epoch + args.seed) + else: + train_sampler.set_epoch(epoch) + + if rank == 0: + print(f"==== Epoch {epoch}/{total_epochs - 1} ====") + + # 如果传入了 args.steps (>0)，则把剩余允许的 step 数传给 train_one_epoch， + # 否则 max_global_steps=None（按整 epoch 执行完） + if args.steps and args.steps > 0: + max_global_steps = args.steps + else: + max_global_steps = None + + train_log, step_logs, global_step = train_one_epoch( + args, epoch, model, criterion, optimizer, train_loader, device, scaler, + scheduler, train_sampler, global_step_start=global_step, max_global_steps=max_global_steps + ) + + # 如果启用了按 steps 的模式且已经达到上限，直接退出 main（跳过 validate） + if max_global_steps is not None and global_step >= max_global_steps: + if rank == 0: + print(f"[Main] 达到 max_global_steps={max_global_steps}（global_step={global_step}），提前退出训练（跳过验证）。") + # 直接返回 main()，不再执行后续 validate / 保存逻辑 + return + + # 验证并记录 epoch 级别日志（如果在 step 模式下很可能在中间某个 epoch 提前结束，但我们仍做一次 validate） + val_log = validate(args, model, val_loader, criterion, device) + current_lr = optimizer.param_groups[0]['lr'] + + if rank == 0: + # epoch summary print, 格式与示例对齐 + print(f"Epoch[{epoch}]: epoch_train_loss {train_log['loss']:.4f} acc1 {train_log['acc1']:.2f} acc5 {train_log['acc5']:.2f} | " + f"val_loss {val_log['loss']:.4f} acc1 {val_log['acc1']:.2f} acc5 {val_log['acc5']:.2f} lr {current_lr:.6f}") + row = { + 'epoch': epoch, + 'lr': current_lr, + 'loss': train_log['loss'], + 'acc1': train_log['acc1'], + 'acc5': train_log['acc5'], + 'val_loss': val_log['loss'], + 'val_acc1': val_log['acc1'], + 'val_acc5': val_log['acc5'], + } + new_row_df = pd.DataFrame([row]) + log_df = pd.concat([log_df, new_row_df], ignore_index=True) + log_df.to_csv(os.path.join(save_dir, 'log.csv'), index=False) + + is_best = val_log['acc1'] > best_acc + if is_best: + best_acc = val_log['acc1'] + if (epoch % args.save_freq == 0) or is_best or ( (max_global_steps is None) and (epoch == total_epochs - 1) ) : + state = { + 'epoch': epoch, + 'state_dict': model.module.state_dict() if isinstance(model, DDP) else model.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + 'args': vars(args) + } + save_checkpoint(state, is_best, save_dir, filename=f'checkpoint_epoch_{epoch}.pth') + + # increment epoch + epoch += 1 + + # stopping conditions: + # 1) if steps mode enabled and reached steps -> stop + if args.steps and args.steps > 0: + if global_step >= args.steps: + if rank == 0: + print(f"[Main] 已达到指定 steps={args.steps}（global_step={global_step}），训练结束。") + break + + # 2) if steps not used, stop when epoch >= epochs + else: + if epoch >= total_epochs: + if rank == 0: + print(f"[Main] 已达到指定 epochs={total_epochs}（epoch={epoch}），训练结束。") + break + + if dist.is_initialized(): + dist.barrier() + if rank == 0: + print("Training finished. Best val acc1: {:.2f}".format(best_acc)) + +if __name__ == '__main__': + main() \ No newline at end of file From 90fe4d2a29ef138ed38ac86de3a494b8a1a37888 Mon Sep 17 00:00:00 2001 From: wangwl Date: Thu, 11 Dec 2025 07:02:58 +0000 Subject: [PATCH 2/4] add VovNet --- .../VovNet/{Test => }/VovNetV1/coverage.txt | 0 .../VovNet/{Test => }/VovNetV1/vovnetv1.py | 0 .../VovNet/{Test => }/VovNetV1/weloTrainStep.py | 0 .../VovNet/{Test => }/VovNetV2/coverage.txt | 0 .../VovNet/{Test => }/VovNetV2/vovnetv2.py | 0 .../VovNet/{Test => }/VovNetV2/weloTrainStep.py | 0 .../VovNet/{Test => }/vovnetv1_loss.jpg | Bin .../VovNet/{Test => }/vovnetv1_loss.txt | 0 .../VovNet/{Test => }/vovnetv2_loss.jpg | Bin .../VovNet/{Test => }/vovnetv2_loss.txt | 0 .../VovNet/{Test => }/weloTrainStep.py | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename PyTorch/build-in/Classification/VovNet/{Test => }/VovNetV1/coverage.txt (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/VovNetV1/vovnetv1.py (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/VovNetV1/weloTrainStep.py (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/VovNetV2/coverage.txt (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/VovNetV2/vovnetv2.py (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/VovNetV2/weloTrainStep.py (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/vovnetv1_loss.jpg (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/vovnetv1_loss.txt (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/vovnetv2_loss.jpg (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/vovnetv2_loss.txt (100%) rename PyTorch/build-in/Classification/VovNet/{Test => }/weloTrainStep.py (100%) diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/coverage.txt b/PyTorch/build-in/Classification/VovNet/VovNetV1/coverage.txt similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/VovNetV1/coverage.txt rename to PyTorch/build-in/Classification/VovNet/VovNetV1/coverage.txt diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/vovnetv1.py b/PyTorch/build-in/Classification/VovNet/VovNetV1/vovnetv1.py similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/VovNetV1/vovnetv1.py rename to PyTorch/build-in/Classification/VovNet/VovNetV1/vovnetv1.py diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV1/weloTrainStep.py b/PyTorch/build-in/Classification/VovNet/VovNetV1/weloTrainStep.py similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/VovNetV1/weloTrainStep.py rename to PyTorch/build-in/Classification/VovNet/VovNetV1/weloTrainStep.py diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/coverage.txt b/PyTorch/build-in/Classification/VovNet/VovNetV2/coverage.txt similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/VovNetV2/coverage.txt rename to PyTorch/build-in/Classification/VovNet/VovNetV2/coverage.txt diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/vovnetv2.py b/PyTorch/build-in/Classification/VovNet/VovNetV2/vovnetv2.py similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/VovNetV2/vovnetv2.py rename to PyTorch/build-in/Classification/VovNet/VovNetV2/vovnetv2.py diff --git a/PyTorch/build-in/Classification/VovNet/Test/VovNetV2/weloTrainStep.py b/PyTorch/build-in/Classification/VovNet/VovNetV2/weloTrainStep.py similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/VovNetV2/weloTrainStep.py rename to PyTorch/build-in/Classification/VovNet/VovNetV2/weloTrainStep.py diff --git a/PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.jpg b/PyTorch/build-in/Classification/VovNet/vovnetv1_loss.jpg similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.jpg rename to PyTorch/build-in/Classification/VovNet/vovnetv1_loss.jpg diff --git a/PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.txt b/PyTorch/build-in/Classification/VovNet/vovnetv1_loss.txt similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/vovnetv1_loss.txt rename to PyTorch/build-in/Classification/VovNet/vovnetv1_loss.txt diff --git a/PyTorch/build-in/Classification/VovNet/Test/vovnetv2_loss.jpg b/PyTorch/build-in/Classification/VovNet/vovnetv2_loss.jpg similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/vovnetv2_loss.jpg rename to PyTorch/build-in/Classification/VovNet/vovnetv2_loss.jpg diff --git a/PyTorch/build-in/Classification/VovNet/Test/vovnetv2_loss.txt b/PyTorch/build-in/Classification/VovNet/vovnetv2_loss.txt similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/vovnetv2_loss.txt rename to PyTorch/build-in/Classification/VovNet/vovnetv2_loss.txt diff --git a/PyTorch/build-in/Classification/VovNet/Test/weloTrainStep.py b/PyTorch/build-in/Classification/VovNet/weloTrainStep.py similarity index 100% rename from PyTorch/build-in/Classification/VovNet/Test/weloTrainStep.py rename to PyTorch/build-in/Classification/VovNet/weloTrainStep.py From e38705a6adf18476757b277324f190bdaf2f7dc8 Mon Sep 17 00:00:00 2001 From: wangwl Date: Wed, 7 Jan 2026 06:25:45 +0000 Subject: [PATCH 3/4] fix: cleanup code and update --- .../VovNet/VovNetV1/coverage.txt | 3 - .../VovNet/VovNetV2/coverage.txt | 3 - PyTorch/build-in/Classification/VovNet/readme | 65 ++ .../VovNet/requirements_exact.txt | 89 +++ .../Classification/VovNet/vovnetv1_loss.jpg | Bin 34590 -> 0 bytes .../Classification/VovNet/vovnetv1_loss.txt | 29 - .../Classification/VovNet/vovnetv2_loss.jpg | Bin 35744 -> 0 bytes .../Classification/VovNet/vovnetv2_loss.txt | 29 - .../Classification/VovNet/weloTrainStep.py | 647 ------------------ 9 files changed, 154 insertions(+), 711 deletions(-) delete mode 100644 PyTorch/build-in/Classification/VovNet/VovNetV1/coverage.txt delete mode 100644 PyTorch/build-in/Classification/VovNet/VovNetV2/coverage.txt create mode 100644 PyTorch/build-in/Classification/VovNet/readme create mode 100644 PyTorch/build-in/Classification/VovNet/requirements_exact.txt delete mode 100644 PyTorch/build-in/Classification/VovNet/vovnetv1_loss.jpg delete mode 100644 PyTorch/build-in/Classification/VovNet/vovnetv1_loss.txt delete mode 100644 PyTorch/build-in/Classification/VovNet/vovnetv2_loss.jpg delete mode 100644 PyTorch/build-in/Classification/VovNet/vovnetv2_loss.txt delete mode 100644 PyTorch/build-in/Classification/VovNet/weloTrainStep.py diff --git a/PyTorch/build-in/Classification/VovNet/VovNetV1/coverage.txt b/PyTorch/build-in/Classification/VovNet/VovNetV1/coverage.txt deleted file mode 100644 index 18fc47ec9..000000000 --- a/PyTorch/build-in/Classification/VovNet/VovNetV1/coverage.txt +++ /dev/null @@ -1,3 +0,0 @@ -all api: ['_amp_foreach_non_finite_check_and_unscale_', '_amp_update_scale_', '_copy_from', '_has_compatible_shallow_copy_type', '_local_scalar_dense', '_log_softmax', '_log_softmax_backward_data', '_pin_memory', '_reshape_alias', 'add', 'add_', 'addmm', 'as_strided', 'as_strided_', 'cat', 'convolution', 'convolution_backward', 'copy_stride', 'div', 'eq', 'fill_', 'fused_sgd', 'is_pinned', 'linear', 'max_pool2d', 'maxpool2d_backward', 'maxpool2d_forward', 'mean', 'mm', 'mul', 'mul_', 'native_batch_norm', 'native_batch_norm_backward', 'nll_loss_backward', 'nll_loss_forward', 'reciprocal', 'relu_', 'sum', 'threshold_backward', 'topk_out', 'view', 'zero_'], total: 42 -fallback op: [], total: 0 -coverage rate: 100.00% diff --git a/PyTorch/build-in/Classification/VovNet/VovNetV2/coverage.txt b/PyTorch/build-in/Classification/VovNet/VovNetV2/coverage.txt deleted file mode 100644 index 7b95886fe..000000000 --- a/PyTorch/build-in/Classification/VovNet/VovNetV2/coverage.txt +++ /dev/null @@ -1,3 +0,0 @@ -all api: ['_amp_foreach_non_finite_check_and_unscale_', '_amp_update_scale_', '_copy_from', '_has_compatible_shallow_copy_type', '_local_scalar_dense', '_log_softmax', '_log_softmax_backward_data', '_pin_memory', '_reshape_alias', 'add', 'add_', 'addmm', 'as_strided', 'as_strided_', 'bernoulli_', 'cat', 'convolution', 'convolution_backward', 'copy_stride', 'div', 'div_', 'eq', 'fill_', 'fused_sgd', 'hardtanh_', 'hardtanh_backward', 'is_pinned', 'linear', 'lt', 'max_pool2d', 'maxpool2d_backward', 'maxpool2d_forward', 'mean', 'mm', 'mul', 'mul_', 'native_batch_norm', 'native_batch_norm_backward', 'nll_loss_backward', 'nll_loss_forward', 'reciprocal', 'relu_', 'resize_', 'sum', 'threshold_backward', 'topk_out', 'view', 'zero_'], total: 48 -fallback op: ['hardtanh_', 'hardtanh_backward'], total: 2 -coverage rate: 95.83% diff --git a/PyTorch/build-in/Classification/VovNet/readme b/PyTorch/build-in/Classification/VovNet/readme new file mode 100644 index 000000000..81b2537e2 --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/readme @@ -0,0 +1,65 @@ +```markdown +## 1. 模型链接 +- 原始仓库链接： +https://github.com/huggingface/pytorch-image-models?tab=readme-ov-file#models + +## 2. 快速开始 + +使用本模型执行训练的主要流程如下： + +1. **基础环境安装**：介绍训练前需要完成的基础环境检查和安装。 +2. **获取数据集**：介绍如何获取训练所需的数据集。 +3. **构建环境**：介绍如何构建模型运行所需要的环境。 +4. **启动训练**：介绍如何运行训练。 + +### 2.1 基础环境安装 + +请参考主仓库的基础环境安装章节，完成训练前的基础环境检查和安装（如驱动、固件等）。 + +### 2.2 准备数据集 + +#### 2.2.1 获取数据集 + +训练使用 **CIFAR-100** 数据集。该数据集为开源数据集，包含 100 个类别的 60000 张彩色图像。 + +#### 2.2.2 处理数据集 + +请确保数据集已下载并解压。根据训练脚本的默认配置，建议将数据集存放在模型目录的上级 `data` 目录中（即 `../data`），或者根据实际路径修改训练命令中的 `--datapath` 参数。 + +### 2.3 构建环境 + +所使用的环境下需包含 PyTorch 框架虚拟环境。 + +1. 执行以下命令，启动虚拟环境（根据实际环境名称修改）： + + ```bash + conda activate torch_env_py310 + +``` + +2. 安装 Python 依赖。确保已安装项目所需的依赖包： +```bash +pip install -r requirements_exact.txt + +``` + + + +### 2.4 启动训练 + +1. 在构建好的环境中，进入模型训练脚本所在目录。 + +2. 运行训练。该模型支持单机单卡训练。 +执行以下命令启动训练（使用 CIFAR-100 数据集，Batch Size 为 128）： +```bash +python weloTrainStep.py \ + --name train \ + --arch vovnet \ + --print_freq 1 \ + --steps 100 \ + --dataset cifar100 \ + --datapath ../data \ + --batch_size 32 \ + --epochs 100 + +``` diff --git a/PyTorch/build-in/Classification/VovNet/requirements_exact.txt b/PyTorch/build-in/Classification/VovNet/requirements_exact.txt new file mode 100644 index 000000000..7394b3319 --- /dev/null +++ b/PyTorch/build-in/Classification/VovNet/requirements_exact.txt @@ -0,0 +1,89 @@ +addict==2.4.0 +aliyun-python-sdk-core==2.16.0 +aliyun-python-sdk-kms==2.16.5 +anyio==4.11.0 +astunparse==1.6.3 +certifi==2024.12.14 +cffi==2.0.0 +charset-normalizer==3.4.1 +click==8.3.1 +colorama==0.4.6 +contourpy==1.3.2 +crcmod==1.7 +cryptography==46.0.3 +cycler==0.12.1 +einops==0.8.1 +exceptiongroup==1.3.1 +filelock==3.14.0 +fonttools==4.60.1 +fsspec==2024.12.0 +future @ file:///croot/future_1730902796226/work +git-filter-repo==2.47.0 +h11==0.16.0 +hf-xet==1.2.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.1.5 +idna==3.10 +inplace-abn @ git+https://github.com/mapillary/inplace_abn.git@b50bfe9c7cd7116a3ab091a352b48d6ba5ee701c +Jinja2==3.1.5 +jmespath==0.10.0 +joblib==1.5.2 +kiwisolver==1.4.9 +Markdown==3.10 +markdown-it-py==4.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.7 +mdurl==0.1.2 +mmdet==3.3.0 +mmengine==0.10.7 +model-index==0.1.11 +mpmath==1.3.0 +networkx==3.4.2 +numpy==1.23.5 +opencv-python==4.12.0.88 +opendatalab==0.0.10 +openmim==0.3.9 +openxlab==0.1.3 +ordered-set==4.1.0 +oss2==2.17.0 +packaging @ file:///croot/packaging_1734472117206/work +pandas==2.3.3 +pillow==11.1.0 +platformdirs==4.5.1 +pycocotools==2.0.11 +pycparser @ file:///tmp/build/80754af9/pycparser_1636541352034/work +pycryptodome==3.23.0 +Pygments==2.19.2 +pyparsing==3.2.5 +python-dateutil==2.9.0.post0 +pytz==2023.4 +PyYAML @ file:///croot/pyyaml_1728657952215/work +requests==2.28.2 +rich==13.4.2 +safetensors==0.7.0 +scikit-learn==1.7.2 +scipy==1.15.3 +shapely==2.1.2 +shellingham==1.5.4 +six @ file:///tmp/build/80754af9/six_1644875935023/work +sniffio==1.3.1 +sympy==1.13.3 +tabulate==0.9.0 +termcolor==3.2.0 +terminaltables==3.1.10 +threadpoolctl==3.6.0 +timm==1.0.22 +tomli==2.3.0 +torch @ file:///apps/torch-2.4.0a0%2Bgit4451b0e-cp310-cp310-linux_x86_64.whl#sha256=2e472c916044cac5a1a0e0d8b0e12bb943d8522b24ff826c8014dd444dccd378 +torch_sdaa @ file:///apps/torch_sdaa-2.0.0-cp310-cp310-linux_x86_64.whl#sha256=5aa57889b002e1231fbf806642e1353bfa016297bc25178396e89adc2b1f92e7 +torchaudio @ file:///apps/torchaudio-2.0.2%2Bda3eb8d-cp310-cp310-linux_x86_64.whl#sha256=46525c02fb7eaa8dafea860428de3d01e437ba8d6ff2cc228d7c71975ac4054b +torchdata @ file:///apps/torchdata-0.6.1%2Be1feeb2-py3-none-any.whl#sha256=aa2dc1a7732ea68adfad186978049bf68cc1afdbbdd1e17a8024227ab770e433 +torchtext @ file:///apps/torchtext-0.15.2a0%2B4571036-cp310-cp310-linux_x86_64.whl#sha256=7e42c684ba366f97b59ec37488bf95e416cce3892b6589200d2b3ad159ee5788 +torchvision @ file:///apps/torchvision-0.15.1a0%2B42759b1-cp310-cp310-linux_x86_64.whl#sha256=4b904db2d50102415536bc764bbc31c669b90b1b014f90964e9eccaadb2fd9eb +tqdm==4.65.2 +typer-slim==0.20.0 +typing_extensions==4.15.0 +tzdata==2025.2 +urllib3==1.26.20 +yapf==0.43.0 diff --git a/PyTorch/build-in/Classification/VovNet/vovnetv1_loss.jpg b/PyTorch/build-in/Classification/VovNet/vovnetv1_loss.jpg deleted file mode 100644 index a849a202a2e789e3aa3d5f276dc75ceb91e4d703..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 34590 zcmeEu1z22LmUZC{!GcSWU;z@`B?J%d9z0M;a0?WH013ef5FogFkis>%y99T4m!kM9 zZ@PQlOTRbM(>?S0%|BiBov&`mz4x43`>eh8+WVfnsk=qMJvk{^DF7TC9Ka0r1l-L4 zBmk($$SBB2s3<5XXlSVDm;_ju7#Nr&c=$L36r@y?6r|+j)O2hN)HIK1$;lacnI1iU z!o|f!#lSDj$05YV$;I*gAaH1CXqXt7#8_Cw91qAJaQv4acg+A?RCsN~F9>in0C-$D z1YEegHUJd>0EYzY?e`A<=>Z3ifQW>Qf{KQY0s8^?9snK=0RbKn0SO5a5%z0u*!KWL zTqL{)PsEY&Rg6(+90)kxM5dwAJ}qq_R2@E~<1}&fLqoq$L`*_T|B!)^iJ6O=hnJ6E zK;oICl(dYjoSM3Zrk1vjuBn;1h2={tYbR$HS2uSLPye?8fkE%y2S-JJjERl=6rYfu zk(rg9lbe@cR$c+Ftg5c5{o2~r-qG3B-7_*eHa;;q^=*1-d1ZBNePeTL`{?-O^z8iN z^6L8gxZnT?|2C|@jO>ST;ljoRkBEqXi1K}0aPV%hhJcHR^xz3Hp12B%u>(F0#~W0F zr;%x;Eoiiys)vLoj>G8p>A05YkG>D>r;+__1M~a4M)sG1{rkA40hkDIus;t07a$6_ zxMocEMgLd(*ERT;4g8<60lt)XjqKw>w8$}R@mBUu=kI6r>GIpIWXWW19!vLw6**I~ z#E(!q<9x_QPro_y71?Fg(YRUpbFfyb$cJW<$HqPu$u7Jc;fibbK@XF}49|UDm}z#; zHnXdkLGLg-C(Rw-C{Z(tU$VMlv|^+viWP55NA}dTM*I%&N*N_*Cz)-io?l4WI_@je zngmT-q)^Eh7n58xjD7b!wtBU+Z<|kcqOFy&c$6FUx8>jSI-?`Y<8LAZqf@lM9LBXZ zINkx;@NQ#gRLU#wYdO*q&wWPHc&l^@?;r18e zd-xHVLGNX~IiC}w$o{ard3HcKr)zbwY|AYCkO@+Lzaz|)D|p5qu)XzOgYYO zUoXF$C?8+dV(LMlxmAG`JVYBA!Qo@U@cFFpfZTfi(cykf-j1&GHH+1G#8m-Odtnh% zGYL|xn7~GrT-;aCG2i6^^y@v)7MJ=#u+i~zA&EPHp)I5xAWrnbV#prayS!&Ahp%AA z#=7{WG|3>&b#y$%p___vBX5&M>JHG^NIFV^ccSe6;aOD5sg<@=2nR+)CIiZ4sSXX} z>ScG1V5&uUYZM2b$kxe_nGa%kM=3;pS+MemvMS5)(r-g1j6kyV3%FF)tVuHjPeOoa zx0EHI<4Hfj_2N1NG$TT#?Gg~k()lr~wq+e-=z#@5^bjue>LA1|pO*@^o&o5Stb zj<{#fyKhFrpgHB1RIO5tTfIpBO2J{C%)$%jZNPhm-p@{(taR`>qc1){2`Af}){4!> zxcwmB%e-(YUCJnKtouSnRuGCAz(7_v#*RujRynTCBd@~EKUYagBZ6Gwmejs7v zmoU4}()b=6Nb`#Du`B_mQ%?|AJ%igvh?~NM82cPMTk-IeK*{JEtm$Fw-->D8<@@Mx zpd5J@lHWRsQN|Qrwu%q2o0N}%kpkDBogf0qx->XL_bMm~_NNHFWG5bR&k5U~_Qq6$ zEuD1;We^JpZOpJ2PZq!qv^t{@rX$+6a^MJenRTRYLr6dV<9Jn$BGL$KZ63w>92fmd z=_)AUT5%U9+Ic}k>Eo6V?{uX&$~i^^OT|exdm-E6mu1qZm0zmBu@9BqqS@o?90r~o zW4^$?wf1&u7vK%T#K;8= zoFc>c`#sJI1=KEe?*QuK_I9W8EKPW_xm!f?Xvnlx7!Oo4k!)mMHN%sQ1y^C0U~1WZm_Z*CLw#!iUb#^(v0Lg?Ol_KeIj`z61{vs$$92IojZK|eLar{ zFUa`#ToC>JyH4|c7>_6>VqCJf1T-?;JUg=YE4Mgfg1N-wBD=b$;mEON`3NHyLAXZS zwH;|DF4|{v9P`AhA2h;dx^GwRBh!Y*bOFe7i3hn3h|#$0iQPf%N{FLY6QzSzoZ|ogFG(jvp#LVo=Hs2s$;gpa;+T|yGs|41Ane3?f=k-rY!$cnQiabVR z$aXSxVR&D$)kx-UDHfU~kd-x`Wba|S_84dpW`?zHX6!*sSwozP^Stq>DyU*l)0S-= z^5IerrAmg*jjTibYhG6ehcp5$U%$S8o2NpDKTj3K6m;N?xs`>P`4lakhIJ=nYK2^Vf5o>1{86t~!XZER`fCrXL!(}<*)q%+&Ek_49g zBsk!qrsTf)srOl17BLUKS>7H|-V!ZKo~Pxy2cC$R)|dJT>U9;ZMA0ZWZpJ5(8TTS1 zgsh&mGfdU9TQ0G9WglrkWTxxmCTi>S0=~Uc54?Kakch393s_woLP_78-cKr-?smxT zJMJXAvTr1MAM*tpy&CbAwB1=!{&tmR2Ps>CDSO;HT~$>f0siVk0(RQaft{`mbn6sf z@ScQT>~jZJ;}^@}gnaqkW)bZpZ_Pe=GQ zx*!_mcL2Cl)bz=Er4sAudI9*&^YdEsZ{_w??7HX-+cVaOU-#=?7$@@5dYHeA$079^`wVh|FJ#OW0~El7LnB<6wL)($ioEp27%g7 z^0kk(={X)ex0`?CMomKU2tYy!&dGC6ksRTw^On0%4rOI?eMbQo(O+%u%Iv2{n+#WM z**Y-<`s#Lsc_kYcJFFL$v77I36@4|ylsYG!xw(J^zEyHNS13*q!ke{uR5>D$D`n%1 z$in7jIqxehtpvl4d@KCz%7hSBbNgx~mNlWMj7Q{;kz~jh-)K08lU=$QB*-}LN%tr4 z)vu6A(Af;%0X!ra*@P##Hw236=jlgZdlhU}mg{fDg}*RHBcZZACgxg$TYF;^C)!>` z*m%7jzk);Qu4#lLV;85ui*KkHSyqT!A&F=sLu_FpV%WsyrHoMMLoZo;DXL#naZ&`9 zmj`|`z5_gqGkI5qXgiR|`86tYBbU}+7fNz8|Dix4OIsv2YJ2j2SD#R}MwFOuwdVA_ zFZ?}uD=1+s(Ia3zViC&A*4B~POwW3Z{BHV;BkHFk_ojr^15rfI(sCuqDX}jdOKDOS zBor~;gjmqvY$^a;1ggS>_|p~|V$<>-x#8woX_$B*ik)&0s^U+s(-dw)nTDNNfWXkF zx43L!tZgqxDrPNL)e*mR@*|K>ehO3|(mUb~t6ARJL!Gzg)R)Hj_P~8VEdnb`XT_w` zzDG;Qc1!m$1#e639UuTqR+kufc)e+B3iNiu$60 zw|Sla*iT)7%{&w}!;LnwgKz6KcaiimJC@aSJ@AV`a~TK7q4b1((NPqA zRR}ImM?T+uMeRN5eAhOH$kx4Vt*)m=C=q0_8>L0zftn-J5dPs9Z;1RS-A)o&0fQ~iSz zq4%gzKtl!TphZL@hyuWwZ1kr0YPdnPzA4G+4shsuDF_WPz`p|kXzl>ZFYf?vQCb#G znKx4|;fU@4_+k{ur_f`lv%S&*`-c7<075N-eFsR>dUgkBMs2!G5M6>hbh|~bWq<-C zVKj9JMf|5D|8$Z`(K`Stj(^jp#<1uiJUjGZ_@&ruiaWs5;`es|UtG`;7u?WK$Cdhi z3YWnI;!JYwxn8e|L@sNec0JdlDPh0lNl?fvJkrafGz*P&r7M3TlJ#CRROLs!lKv5(0GmI7 zWQHfm10=9<2lypk!F=>D5Xx5oYwLCmvN@kJZ!}PI2lx@NK)*z&RBZpBj{9#o4EiNr zQT>Rm2&q3EHuU=$c2fQrJ@(f32$ZNnY%?>GNpwEu_k znDja{Cb=j_U)wM@YGMh~F*TxiO;}1C1-XzZ^d1d?hKesT0RGDl&hY=dasBU|7XKYR zR{+6ZveEZm8cC>Z+n7+@umYU*e#X5-4+GuFpaBgiFzS{5C+fx7yl~38DSCKXUvNehHI8@nl|-@K!@l*F)s-r zEEw~e_lGermG8`J^k>Yg55~Mm7VWR^pHe~K|HQlo5kNy#sds=yf}dF9%s;*7Z+_}| z0KoaDu8h7xgvS;?(4TR~Xqnfc{W{I?-0#tM25U7Bx3aVl*tAPzdk095^%Y?L2WdP1 zAlV(D3wZ|mwbl(9D`7z4l+1evh`_444RAiVeH2(L0ANfH{@Yr9ZD59dwr>1B8A5vQ zcIOa!EJi>Qfv-kS;LDkVhn{Axx#yAVtBqNX@^J~%c*Bx}W&s2$Vn~0qrWJwUb&r-$ zO0=g?9p?@}ReA@QMeC_~sxx52J zhF8QLi?%`rQoipJTCgd~ye|3$qHzb12!g6*+yO|-bF4(DE1QCW3wYjTb54yorF34G z)UTUYUo1ii1RUg-jl!?6)T2QN5YfK*znna_zC_=fQ|V}S^6i#xKcuR1=}|T&dy!IC zQ;pCX$^tDqzYW;T+LN-NK+g)kYgU@pyLewv1_DOGzG~s84HLZ{PXl|StQkJAy#26x zDbEk{)Fn3-ID7UD8y<;?pd{$PuI*suJg3s+nM_y{PnunVMpz!OSQ6)B>@oL=6dNH% zeWI`SoYK=QnHO-~#DMeu??dA$Nre=&L8-SRF%Iq-binp0%8^O2WJe4KJ=T!I>uB+n zxp6lC`uVKg%NAg1t%#Yfb?@A$&C(~wi8f>fa+xp-PQIHz5UG2B+Qa%fggXG;L95X@M&%t~SYC{R@b5Kn^PS`eBM@H8^9IeO zCI6IoV~@vxZay(Zzy^1oeokAcVyEIIsJLivm1oZ-XI2~#^Ytof9ErDd$u*P~O)@g8*rEjDlI>S#53Y?Dp^NnFn(Qh^!7m+5^`j3t1ei`1Bg-5^JP>cF zcJwry^Sxx|Cmt4wTCx}}NfKxV8cPx=!~Kz(vM2;330(m_Q5*XXfWx5bg+67(Qw`qn zip^CsuNw^)I*y8bRNB(@%!KQaIa-h6v`{mWr^f(dFUt}Yl|AF)f~U8iZ#dcs4Ownv zSE6L>(jsX|TxH?Vg$J!!&oBh@(Dmk8INwB-b`h^k1{2B%0;|3F3${v4f?41X+Lc=i z2D7KF62daLD;L1rnJ-o0TdAK|=zqTVs)mntqR`(bxS5iDbRdYbXm$1q2Xqdo({hVH z52xju3$3f!`RqDK=)a;w$&-`@?3Ar)Iw>S*b0J=o_9Hn&Rf~?25@WDZR-#iz@zUlM zxL?FQR@S~er#jE;*o*0Uo~4X`A^uXf`GRf~+=N#gXy#For06AMdSY2u6-vq%S3~{9 z)R)6GZ@DrflKwqis6VZ5Piq=VS}GhtK65z{H9&iXVIkPX5uu$yas0LF^v4ldOJgTn ztimT!7_m`w4+!95EnHVb8UnOou)CrZJgA&--UIDfZhmH7p3HsnE~c_W3rP5)|JaYr z58FYELYQ!1aJju=bY%TRYMeDYcFi`m!ueek3_hfHfG-AB&|uwfRd;}JE75t-I*_bc z^jHzXeFu0^XMamj0XM87NdWi<8r*>U0#uA=;hb=^>9mY=bLaC>6U5iHX_qAjv}MLU zJoFzLx=uo6!ga>YIKe2)~$#%N1weWg}vm) z5}J5d!FPZ+C%hT%^XaZt>#(GT|1tMcAd9;uqp9|aN`MA2e2OGttt<=e{TVmdJX5;| z8ylR2PQw0-Le%o0RTK@y$B=!MF&_9ewrciy1 zfa5U#!CPct;_3N$#M9Lm6%GM>r^|7J(NOO!$Ry-#R>$oHc~=97OO8_vUNK( z!j22y1omS}w%t7WMsq60)@0Y@YYce<#XgX*fc4xT5G?L(2u(`D4t0-N_^ZR=54$8!C8 zKkfKK1D*hCzviX#2poyv1EDL?jcKEPP)E3k($#s~iP2FiHb#3R;+V^=l4gwY2s>?K zrFK!)7)JTjp4pr>9?z^o#?o}N%*W`HcIMaP0 zEWcfeSqVjCoaykq+>56uZ`*a6>!K=l=v(KDfnz-n#@+sg9(Z{o`=#dQ9Rua~W`J%N zU=z;hDC=ie7_iLx`ayn6#)59IX~t}elLL9=mKKqktgT&ewIH3@BjQT3w*b>{-ObnZ z84E8Hp2bd1V$jwfJaV8bVb)9Qpw$A!`*x~@H2{k2|9m|%zWG8_I#KD5%NLB)y1)oeHXDkB(1ccsz z8X->n(f{Fa{hJoy?U3Vd^VY*#$6rU};Xgzye0@ufc`{J}BsSEbXW6Q{N>m(vYijFu z2cU*cc+FhfvZc=@ZSqC1pB>xj^* z6fmyI+rJ0Tf0auPs~Qr@kB%b zXoiZL+f-^Q&oIorhre~0&&iS_kC26q|~2&OP~TPtoVCli+{p?SikoF*TMv%zqXZ^2b zY7;OD+=Wi&&k8+uI(fwY5J19!iq=fN4rcjy8JRb=wSaRr-cTPnS6p%PjvBAph(iM3 zSW8&;G!@HRzD>X|5@_$>@v5ha4i%3tS(ct}u;(2HB|jgzMvW*5Or>3*)q+Q`OVqi};n70+rG=`BZG z$l9gkfqSak+hZIt4{48LSalejB<+{0M)W!3^-Rw>ihKCBbnZBsY; z+P51C()v_2d!4#_o4?643S;Y%LHP8U9E5)JcqhInef<~GXGO9UCZU;50m|AbV$8A8 zLT(fO%6b}a_WAANU!Vb^wKXoXh#Bmf=msJ+?+U{V*-A#wVDusPAboV=Da5 zTY0J%3#-PiN$3dBcajcPdP2Uqk_^t!nGSuEeB5(HubnX7ZDF@5O;cK!8TC0iEeF;%((%QKLV7>T-tNZiB%w@OLJn!(b++rpi%xO@PWRjHSx(?%Tu*wj8 z>Ml z5@eLTKE8ZavzG}TwcoOVS%uw;f0@Rwn9e_98o}#JthVOW(fd*XgNw|%17j{Im9?ed zK<+5{c-M%z2%Jw7E*5F`7tXvxnOX*9A0$%|p~!>*^C;Fn$A_&XrG<%6`V;1f!R`ua zrU0ZTF*=G%c%f%rt*_0(RN}H6RbR*6*e#oMzF5J)5FZ&R40cQ-)lG4;i?%0gME@fi)1QRjZ!7vL?use z3A(ammt;-(rImZU#d9XM7Sv6~EB{6M&P$0?;YhoA4F;Mo2+v3I5k#cn@- zFCim+YcG2%vV|7;Q`GSn=tG8t4yzuA(29lqsc9@JyqL2YNI#7BR6hAR_xZ)PO=;n< zkRnG|mbRzAVtQanDub;Ysj9e@{cRFqcl@3hPF?;+WI~gUlDUu}ZP+;yGD+H#S*H-( z3#Fzk>j^Hl^yTrQ^wX_NpNHdxtC>j$C4(g^moP4*NBoD(DEvUSejg$P!)f@~yaMgb zN|5y!!@V|rsU{{1S7Aa8@kdIjh@;L$7Oi@f9=i-2kL*k%4S7OW9Q?tNzUF+?i*H&8 zV^(BX^%Tn*tH;wuGQZTzzhf)Le=YxT;yGVePmC*tv`W9(C7eCmSqUDH(~xx!y!+)5 z?~`WZU^Q}gi%?kxOXeE7)x33IZpaT7*ycWf5WM5ss-4*iEdMQz+RvwmcsI#5bhM?^}q zrE68q*=sb{N?HH$gPx@p;;SH(vx4#X+YCB^JHQkJ>X+Vo?!>QTDa0>|J*Ei>r7lJx zO&TY>0UHV}tT16pbf;?+<~viH|K=3hZ_MQ1V}IZspuj$@81xH45IiJ$U3pEq0r_uB zYW!&~kpDHAsrfM#`B%)4P))VzgF=~Cyg!E@^4PX4Uvq&=QmzhLhTF@)CHNVchf$PS zO?9X1WrgdhSG-%d9(GoelKoD5V_+*FQc{_?JyG1fZj9u3{=CifiV$^;*fyTk{jb4{ z*|1>#w}rc0hVrV~D3XWo4VG+=O1E=nj`KD%mop28#y%Sif5@|%q*bnJER9|4vJ~es z(SH!)vT|6I-*~0WoaFPo<@WEbLK2Wf*FhbVS(1V%K||sHrE`f%>LQ?ncgtIfhFAiK2J=Sc6I9Y-FKf? zkX+e{mgUFB?*L_|f{0^m#zq{QoO{}nWgFQmQ12w;6&8#SS!GSZ-W*q9h`jO@PwoKb zwN#_rUai;Uy7!Y-gE(fp=FQvoF0!1)Qnu89q>{x|*}dkjz6Iei%~z3ik1q%doJ)Fb z1yRh;v+)my&NN}=q)8DSCwBnW6(|v9L%9LHIUk=Or?!HE0n96^ zYX6FO1V`8<0c4LSj^QCsBf;vF3wmA`cBA08%-H5o3&|A?A~!uPu#EKAx#rGvC3T#Z zepUW1>5ciW>;Mi!WJ31|a{)MIA*)GEqjxvOtE5g38VK{ey792H{qYj|KkL))=6b&e z=LkRKJ|_8BKv#en(3F>`K z(_?N!5vByd(RWH1E_OKc#t|bTiX)8@oEJ?Xcsq{a_h)syrurpV1{tvV0!DnxGZ1Z(_ zjN=2v4L9dug*(Y$@p|!PEPB}qXM8SZj!6Nh1LCtO52{cXXU)>QoJTRkAM=3yL>3yO zXJiVRJ%YVVvD*5tTB}Ma!oj-5O3FheT)>TS2u}u^lS@wY+s~dQ7kZEzGFM8j5xeXg z?%wVkgA2biq4Jcxm9y+E11AZM%QUH?6jlYQ#aryaU@-}RI3nd+)rp5KKT>@iQX#w< zqwTJv+KvlK>nFrm)lT8MlZ6%apWPGW89=j%#>{LeD6Tl%0d^XEvKNq1-mJbAzT$;w zsjUAEir)YH3@VmaJ90iWT(GLttUSogg52lH1k;m z#f%4J6~__1CtWAg!-!{<7>Mn^7bP1`T^2si*n&WfFySk(fW2J?)TQGs|N3RHQQQ2P zZ4V_sDbMsPj+&>^l`NWHUXwW5)w9~{Vbwp6Qo#3)=aE{YcV;UXK(L*=1Hcz;0|v&y z{eluqj^6#dMH~ZRTmiI#Z^WAz;t*edb`&oV#r{+o;Mo##lqBUEcA2N< ziH0&jebi0o5OC!Vc%h8+BOij5Sd1Ltk$Fu_x}a1EG$#9e?M8{>bi3E_%Q8{e6sD+^%Cwhq$gPC9Xhfs!yEmyBLirmoV&ej)DjMrO)WB` zboKs1T9y&&>~;wK(Zd4n1OA&wC_TPD2(i~kdR}Zsp))7YhUeZ@yoWIV#@+;Ze;{O) zBh3#Ujk47|fqLi^FNzcm;SA&&=w3Ey2k|1$cYK>-stEAsR~xIl@x%s?f1A>^8bW=v zm+tof@+22V6iRf4`dK%E8PI4y0Qvj~Q372zQuNU`+QxFaccU&VPEJmT7H||ZQYxY^ zU5<}WJU?W)8_5jM^i@`M`_Es|LscfqF(74$e z_f6Ju7nXdcErij1gw&tYbrz9zc|PtBoR6N{vPs*4M=J-YI{0|oV{ZAYVR#>n-vk6Q& z#DPRNVS%RDX6^uztj;*lqQ*d2nK|GN&@Fg4dF!{=bgDXpBKad?{w?qI@8{(!xGXoQ z4y?47Ub+e*h3_7{^%dIgBJR;ucC34^&nB|e?*?w4caQ2+X0(5eU#$OWKy6@T`23b* zRvEd;A53y)g#OJ?-@`(BK3yhbDk{H@xvOAGz!z4kXI%wMI(%wS60icC@%rGASyva= zQ%xMHvQKA*=z1kLDfyFO{p}&kJyhxt|7loA0ecW5fMJGtZ%&^+D{~UhyDOp;!c4*D zVOKH5W*JHwtB&2`>E-h&7pzn|`|h+Z=apSKSS@eHe-iwXIY2od0C)j6U8>MiP(IAw z2_;?dD3{}kO=#+cQo?S&uTYV+G5Ix#l(6q|dj8oc_4r;KuYms7C5Jfyf!inD*IEkC zW9gEK0yVJ6gEc;0p!hLw^~TCQQx6i-mzvR+b^~JdvDPvv3s5ObNy_V2`K@`$*gInc zKD6a9d8efLzD5Fa2awx-nMbL^MX)hQ0E<&{AN|?ZnUnL(xUo2v#}3VuK3?;xTrU@^ zsPW~scj$4XF`zNA4ogc{Cv3=8MJ@BnR_?u8L5%DGQGz|o8fSzk>2An(u^QS0Qx`Eg zZkG;rbiqkz43v616jZ&C3$7wE`GjdrDi%8Wu`y9!pC zt^qG6bcO33&3an5ud8Zx*}rQlrSE=C1k08M%gDYL2;M7)1z}qmw zF*@N1dbHAYu?3ZiAtVI1xw#I{KAp*klwisOnvH5nDRAw=jQu~B31Ec(JL&&tG`Nyq zL)uBPUxj6Wv-&@nbo;9k@gId|=f6}9Xd0ZIb{A@E>k;+^WG2$`i9;WPSp=102bHT6 zl>_hNF#$o`P4C6`2IV@yb7@2t3yW1eS>_&K=eXr_u9xY-Q*D37mtT8jLC`wPDo%4c zzmYXk8tY_nv2PJQzz$*Y3CP%6B-0? zVaBr7bS=-}NqgBS|876cfsTV}$@rFeRtvXl4=0gKftb=H?0*OZc9Kh5v&IL4WSg?Nx%G{aD*})m_oNZQHHvqQ)K=o@*O4|9&_?rsy2-xHI=FnuEYkpy~>&CuO~8~ zZz|D{0@@Z*m-e7QlY}eRZ2|v=v&GX`wdxWGqOB23i&gsTBIm!2fitF}vUS~X<|Kcq z_e~dcHQ~D+^E40Mns&Y)3|s94ty66*WgySmb<}W&)LBH&JL*1X*X;+KS{9QpK@FO! zKwJEqM4ps71xevE$OsJOs(w3_@@KDC0LszXjtvER+eLq!(+E;}~ zsH@~Y8H`(;fpZJD)k0;CyJ}tN74?Tn!-{J(H7U^4A`@IpH^0c6d*x1Z*|E0G+pvd! z3?!kZvYtv0nO$-%gx$j8@*<6e!37G-1pWaFs$ow1kC^(){HgvaFDsLpBLo=Z(np9wSGe$d)8{B~{uM{H`8J#h2+UiIGoSN^y6jaBO{z@OuTv zON(D4p}+Q-*^VLQmq{9@P-Ni&uJir1VyfMf-M^ zbqP{|HoA+;@~p278q2c=jh$=4G&-a;gx_5tg9DVxg|*f;O$9tpB&4sd<6aiqf1S>1 zeSfN;(6nr6d|Xa-WlMF~CG6e-x;`6GSH#Bdt5#6t6_+GXkNw^;bL*<;sRgSMl=i}? zPucp|g(7MxMb_h9Q)#iokWIvGfA??-nN?M*i_&eLdvk4%%h()}gQ^Qn_s65`y;UJp zo3`wp30Aflpx|41uHT~Jzjg(F&(Hoh6|DT6%Kg?hzY+0t_shz8+H$q?2L3k@?2roI zlz63$6gyycRdw+8G|M|}F4a_L<5}ED`A4!s?3nqaysCvcX()aypAMZ%OeUce)h6bn zS-0JGB^Zu5kn7-?m7Mh^_DfJ-=VIO(4A9UaVfn#~g$jP(J#AUQXHptN!x?GL4Bz#6 z5i#MO@RJen%iJVxR?ngolOMT)E|FwNfk>jACXbq&hpgq(|MPj-p--lgct zu2rv`{%rBY^m;piiUBg*&j|k?Vf&AWV}CF0F4k&`(6_3cNLgkPw8}YtQ&k^~G&_j9 zO77OOaa^1qT&)x{`@S_vdt(;@Of?jZJ3sTYS?qMoroqV(@&OOQqau>~&*vLi#R`0s z5}c8-RDE*Oh8UiSV3WeTeCj4F#-hx6pzj;k_z*0P3M5oh14o@GCZJ}$ z%rd{C&?Baeyws>=HYa4PZp@NW8t!8`tmK#r;>{|=S3J^?~037l8JE0{T(A`HPv%0cQbF-F0T?a0wybhhoO|uKEg_` zepOfaJ;Cee=lb1%471--ziK#YR*08~mzT}73S22RBG-)f3Jtc#EnO7gtU$nfHEseZ zRH?k zp^$6Fy|HDp%ADDY#07HTTXIHEvJXDg}m^M+#om0_%tl4REAC#|`u|3c; z7^9rszV^7mpre%OpOqlNJyjX>GOhI}?g03g^^an$1*6+t&k4LRv$VU$-i-w7=81-I}&W?h~40JnKdigbF%t#O^ zgNCkgxAfF6qVg6%Mwbt{8YZ^RXGWoL)_+D-gd%f&BYES(>6xzFN?x?J4h@G&@u^!c z%_0dWAkwTc!+SH07mK{PTRIvcnGKy5O$DT?A5Cu0TyCuuBvnSa>BrI{>&ppBWlWmG`<$nn|FVZVYv%_Nbxad7F2tMllGx8+#raGo^{`N* ze!Vk+e3me;B;Z$7685Y(&Ec826#Ijs+phhljh%6w`5Ei`c`jBQ3J4(uXT%xb?daFtqXsJToV zh_WydwV@*LjAzF=jtEdpR~;tgEA5E#e8j~Zc}~)N0T$L2V5zFfs=0YL^e^S^-D>4uu{!0?edH3pN3yUx5lyC{D%pyaHZz1UxuY~QS}Qw$qo?W}AkQ;i z)NsnpSU4xDTh$m@G^abiCvsL_&%3nEB3GOwGZxN8G0d66Y#%Gv2h&M@wGQ*ovHJh? z>+h!p{G>4cP}2BcyWQ}=NUZ>3w1{S;Yy0&ywswXWtyQ7f`lYkOI0uF!LS2O?1DhIecD1>AjEWs*AH0^F$-( zC~WP$`x)l>xjUFc@3e2xk5Y<6a7gte1ZIG)S_04ZnmLP2Om|oO&$i^Jx*-DBpxC9o z^hSDnqm!wv&Fs!cPEuvhwvWE_lzsLS4izZlp#{1d@`k2`WPC1*AvQhOQTi!wkxEdZ zS|(9vhWAZRNl2|1ovV~QBeG}0rE8LvjV|HaQhk+)%$tr*l}#K_MyC`Q%?W7n>M)LM*Jd{hz26>(7*l^@@O@n@@=@${ z!q%`P)~>B{&PIaFA+ghx^WK%}M&A9=&`Eh02pEFWjYcTF_saR#Zo(uFmRa7k$Mp zB+thAr5Y5NWF=^v7Y$I2jRQ44j zq%fzg8RMio@x}?ZsNVU!-wt+W2435&v~y$V!NDD1r)VrjfuRX>^7xhWncryD6)19K zSo@mjoh`U}j)%M9R7hV}zO@6Cdr=hpU4gPFjf$#;QGm?UKSc}hG3(LBJ~PEdrjezg zZN{Z%Nk1kj+~v7-wSMEnGV3a%n1t^2}706b~NEj%(&dX$?EidC!nkvgbcL zewbOghsU;-Oe5tTJXWa-4ogj)(a+$$Kx@TmKGCxNJgxEeg2{=#(8BD+SFy*A0+tjY z(qy>^S@AiK1K9mx%`C~EXPh*Bj9~H^VvVG^q1E+=IpOx?{Gq_keZ$9Fqg7cV29gz1 zg_FvyT>Yi%A;yGtmZPF8(v@rWNnCLTthb&YrcUU)_7@cFtYBG~n%~O8{6&oajV#T7 z`TO7P_C5~qtt&guHwg#&to~6Y2Z{>i8N~U44RjG#mC3R3B#vv~jT+ar?R@f5@D;BV z$Ew5N{#>9a`O!%3Wg1rvSA~M9T#9N1C!2BLaItFy8m5Xk9i}IxOyMAct^)A3*rdFs zba<^6D@S)WPb-v%(ed7Ku>KQthcI`NR18DjUoL~QEM%EAPsejnRJ@I2=i)7N8xVy< zXR-ev?;iGdEA81o%1#{Tx;1=oJGv?dI@gU_=(u+O zq__QLc1w%ll5ORI(r~Tm>~gL3Pt)}|?_+5$>3o^AHxMkXijYRe+i6fUQTb;d5^q={ zq6Ke$fBJ}bMqg3@N{Pl+*U1nVRPeP z7mZ+-O(~>w<8XPgKv+dvPjGDQgIRqZ*j%JE1XIs!2)bb>d6Jg$WLP3T>1XYeC&)ovYt)2zs?J%R zyA_7I*pg?7Y`F|D^;=q>@7PToaj`Uqq%w=5w@Ve;#T1Q#+P?LCESx)-v~P}LGL7eL zr;*9IjCn(Fon@P;qjR-Bfr}RJTJzNV+S4UJ@+C4_e_TdE>OSVbR+NPT$%)v8Qi&nO8y zoaXE;C#sIjo!^|}7(a{-tJ-Utt4Wj81dlyBa5@Km%9`eQ`k2mJQV}L;6%u=qPegt+ zD(PJjA8l#}}Zm`DC+G9HrNoItN?P)`Fma7ma>%;EI3@TCyiQk}h<;Io7>O-~wI^a3bqAUs9GbP*2Y&8t(+zZPy5d`yFZHhFj8FSu=)0ifeA;A`1e3&!U!|S}BbS+r$dbj8NC?-aR+b)0L4#1L8 zonHOvweWIoA!U!?9vd-5)x8f7_*ZK9DUf7smj(&+$aCJANX%T=Ec#$NNNC{8&I}qf zVTsEnDimBdcqQN+_Eo+q=bqb`JKk35noC#ON(a7 z<$@g*^m;B%!9A}$87mGpfS&W6rjoDDr;!_k!EII39VM84rMi$6B zklW^7MzQw$9dN`LIce>^wDPdO{RWCn;Sf&pw#8~{pgIw}X7Q^lJK{-n*KAcED-Oo> z=R!Yfq7Yawt_tC%5a3qO6Hn$>k~Qgmf9r*pGX-^aYN<>;ussE>mGrEvCCK*c-dHyc zDkX(|4PJe48I%=Evrn=&Nlp=RB`U_0vCI?wnaeS}9n_IFCdaHGD9QkpWHeM{Dk$NG zTAE5askl#kg)KyqK5aD6SO7tjVPiZWqqG};o^ z=*8*n96$c}(Y|lb7CSk310`IDOG+vobVz9geHhD5;qGYq?hJ2!8STl{UiK1Q@eosc z-9u+}O_s|l@lXt&ENq7)fBL~;hv+L!v7@$&Nkgg7bOl#wBM~#-s++VLuBaxXhM5x- z7ErSSRN7S1A#~-uGA6xoXg@ttk;F8Hjv8uG)VWe5+{~f9tqjkBIfR7hMSgOr-rd1e zMQ`3%pwyO|C!3kRe?Gi(Ng+wtTNQRXoD}OBQoU{J?eQ8`uol z&86gu`B>%-)T4IeSp@0&ba?yENHndxsB19_8GCNvUYe>i%ra()Fpt0gdZ~FKx+j+& zZpzNyfKnNhG7HO4Z|4?kLGct`hm^d{Y1yhhLOIQXNrb^v`|i>wW2Scibm5Tt#xtn# zUdFfEK}cA3@EWM{weFk{Cp-+ze1fjj&*Pv>vBw0p!$l9O*bu|m{5eV-Q9OmC_Zv^q z%e<*(2f|GJ zme_m+HuX~u>A;FtGmT&4cPMk%nhHphOJ6PMm%?F3@UCI zBDyAG*QQoWL-3R)33+4RXgp;jc!BAhqoN?iV#dHEJk>(*;xTF}FXoepR8n)_NVEHz zOJjkWxNPEQ1F))SKFR$zit-jJwZj!V-$JT*-qQv7b@FG~(zNX2wHJ-eJeVzAT_lg* zNnS4=R@t>OXw)daw$rN2Gpk8Xv{o(~uDBI3<<2T=e_)X1kKeiMosWDt5VEUN_%X^+ z@f^0)(?oi7s|qltMQ@m(lrVZ-&F##bY>u*3#o4RQftGqYvy_jXbmS5=@fdr!=FlAG3KRv)fM zyS+LYd`YRrZ!zD>aLAn}sWhH@e09k^V~J|iepvT^{^ z1b6&6b*=M?sP)pX`nLd@l5q_mde`LZ!j)WIO>eGhvXdbCS|^X$ zf9pIC8#Tve+~~&+r9A91^@(8k2OY<;ZeWd`(r^h_qz_b{WnV-ZYUiYcSxInnPILeH zf{9-o8b6mD5f)x7j5NPJcbL%9^YwMFP|>87Lp>K7^u14M#=%eANOAAI>Du!!KMn+H zA&CT*hvjANxh^Q}c-L%!VfC!2ovT1f159(1K%Q%siF^PBO&hLWT)zLF^$(^?-*t6= z*Xgw(uK_eEtHcc=Z|~-UXWnbM-}#3oWwd=Ee{^;K#{-$@k8TK*SPx$7yD*8J+e>p( zlP^7}CuOE{$cndKFvnZuH0a1d`Yg)^ETM$Ts#G2g7b3?9j*fHsa`9YQms>MmUz@qN z3Z`xWMol>1Ku&N+NzE1@RI_7K1qs@}{Av~8cj!X;#1n^Vg4r0pJhYc)b3;w zg$+f^g~dz+qVpjDO}zY-?c2wqE7771*RKau(N9r=8l|V?=mFNx?^x|! z159X-sC7M@pd&a=8xW^40Fb!3FGyiC$h%`*=ymWy!Q>>zHT)@QJ4Xo)`+1mO#P#oF zGd8@i8G)9-@2p6LYaVo5wrbpQC&{J(3FQ)c8D`A!aDB2nEYrTdlXmJ(GOf8wfzjc~ z0q$&7Xod|V;+}lgcH6DXpQe95YrB7{>;FRC`7(MR{&u(+zi(GpTCn4Paszu8a zko4-y;6M#YJ0EdYTh*p`931!fZLi8s*6zKnqFMWN%lAIw;a7E$8MZ?;>VU;FcWUkP z^F|W~tzIKewIz$ORZqV$d?*h)$Y;SuRdcarYtLs9r$lR|QbFMn(m)Kfa5tWlvn*No9mT9dnyu~y`hQ$-2b=^=( z9~E43!vllu8lOh)nk{~3Q>T|*m2w<)7JkmbWJe0lu+85Vl$24?aeLi{=rmI>G-Nex zmbj5=jX3eJF4{&|>E^8OG5KRuoPz7blYZoQK03%_&l38Y%c1W>z>ZOnc^*$+q4ecE za`3JJC$TAi_FDKx13V~g@V>qIqAWzXumXlG#CtY5j)$mC{tko~g_Z->JxbmnXJn5e z3rw1_B=(C{_Vjb#R%aZs;am7>Q%)%$o>yd=)i$^)Z9t{gB$Uh%%PASIO61isBd$gM zz&6!ciT-cC1b8%Sb@K2OxUnt=+UCjK4|b4?45Ww;r;{9?yC#ZxN$S&7<7zRjgWRs-inH7krlq~*`OziDie z4-P{DWgD@PS*E&Q+zR0aDE>*F!%@0X+-&eavqK104WH-`1gSt=sGicvdWkjs5V zA>0`1mgT3a8HZR3lNVjF zt@vs?Q_AYfkG@4(o{WGG^B}${i+5L-Sr*aX+*_>n0sKzev2LlJzA{(kM`TKKTFCuC zD2*8%YgEQ5V*$}m${1G8)-p5*rMkB2L7He4e*6psi`Ny;LbqNG3-xq|sr2+&M`c+} zCLsLK36Z9yrMba&rv>0DK>7M5iq8{ALi=TzY|TS2k^=mxG`F!KrwbDP;V5 zj0h7%3s(+T4=$EUF0}iIguQX&KN`g(Ruas5Try$ij!b?XO@+dIOg40nOyh<`AYOrGSB30Pk z6e^*v)jg{P%B?6#75ScRX_scFyZiFH!80T~K(lWl5ATh{cs~FulV??KWqR&(nnG(t zOzoonw(>XXX}^>A(O^PUGT8{geKPlR+uLa;KWSVrw!|lCRqiMhok)RQ!Jx z1M#oU_xs(?8(qIV+Ptpd_qDLoB7%#oA5Tk$%%;Gq_+oLSi2=|vACX6^-A^qmOAB6i z*t(Q~o%HtGlT3%?8ATKg4T;_K+&X~QB@6{tRbE+>W>sc|=;iUm!S|rF@_sm@%5dcq(PJEEd;CCkcg~=nh`nUe_f8ACkx z0o;Gl29;l2+yJ{d7DQ_i>KT6PwmCJEaA~R@m_|39f7Spr=}TUT4N`n!=yMo<_^$Y& zR^ey3H|_@C_+<*T#)&&7CpHPRH_WGg0a~=YQ2OIMUxX(Go{kd_Ylrg2pzIf7*%4BG zO#4cU^KF&gCpF@gW#S2N0phq~SR(Frjr0MG?=xT*2HU53@$?c?Ayy-+m%EPa@!t6k z=4oh0wf1Xn1DFioG>WOBsM&OvO*BXzZ#Ay0h-93FT-BjT__~>0Z2)mvlft=Zn5XBf zI+q7#+Rq+VYq_B4?4ab(WE>yz0sum%`{w5^;BL9u+hEqPWZ>@Z4{pIB!JPz$AOV60cTaHW5L^O*4iF%CfB*^Z?$$UVcyM=jY20bL z|2}8t%$+%R?*EP4nLFp+X`Xua)7^XTuBu(N*89F|RRN!dF9YxuWff!rNJvNk3&bA) zJ_nEnV4$I)qoHD;qoZSDV%*0j#=*wI!X_sq!Xu`pprxUvprU%nz{UKKj)R_xibar> zTIMBS#FpX3qYY_aBgwky9`-F|)9;@$m}? z3JHryKbMh}lUGpG)Y8_`)zddHx3ILbwz0KyadmU|@O@OioSD%&x4it#52@ZSU+JpPZhZ zUtC^Y-~7}rBmna7n)Qd4{iBljOL@U1W&|J1afTlSYV%>Qp{*&iDAAMKh2U?U?T9v(6Q zKmu@i!;<;_{=dt=`{3VW;Q!1R5K8;l!aX5Mj~2)E)z-n~B4pl(p|Imxfl~hVvD`q7 z3U6ADar0F`MMa#s*~FMwp;ZsMBF*_i=K=t`?t#mHzb?LJkYsbO^OuaR#NBWs7d|rG=Ww*$2bP!jtXW)9nZhP_Rx(Kzi zxP1n7HP5i;F==PEda~g>Sf8?)(XT4> z=~&jUdh?#9TSc|U@DPgcoDN&~qC|C7K$TXX)*jPVgsPy_`&*ouk=!$-bRP?S z^?A^beTu2>T%OUy6<@VW4s)ATjn|+CZ9YFm0aEs8^F~0bsf!M#NxT&%IrtaE9M1aU z>T0ZA^-1JWib!5t;4Yso)i}}Xk3m_F={u@`qx_Y2(FV=o14NI%YVed$e8SZeP+81# zGrE!k!$>wtd$2Jtikd4Pw}ty;swU7ZupnEjOmR7g+Lzi?%3;)&gKOfM)I4IjzcxAz zJ~_dDiF;?~tcoXmZV?n#RTuu@QYd{^6M+a(x$jggEOkl=oP-quXG7xsvpO zySQ$mQ>#sa(2a$!>(sG$nDsG|sJe?+u=F@zw7qo!DidRja9V$qBQ2{ajThu19ruUo zgKdHseqg{vS$61@x} zFx_qF$}n@&J73^gBwG*Dj-2bgTYG>;9~IXFpvos3;yWb6A%&5ji3`Ig2R{Z|yf81Fa%0a5gPoy(OcmCrihB(Q*t9u7>E? zb8SMyt`yP1@(do7U6S7mdb)Vzkm-d6i~>5ml)C~1z|_;QK|f1dD+|kMdIoW=h;ROR z;qIj#?a^{@0G4Xk*lQZ=gqIl=Jv5qBpftq|w;s#jP7fH;b5P|~m|C!!20GRo=P7Bi z%HcCOAh#N6+bb%(nwkkq*nvCLr}7COK{26d=@)d5_(DBgIkhLMNu9GO6xbAVyN)R$ zEB?|v_h9B!?+eo(aJDUDANjC zDYyTmoATPBg)}6t9QS@5ilf|{^OV9}uyq#&SD-m}!X^V4Tue-~&P2>jA2GPsvvuDt z?R^bI+OT#!aG~GhViJ6hU0H?Q<^DqMH-%YHLr19`I+Z4u*6h2N<#h=k53BXQaVQFRFi6%-E5P{z3q#O2FsZ$2XF|Df`v9}B>wxb)0Z?Z{y?kSP7zQr zHmhxM^%!{`y?C?e=kbu7oC82kQ<^GWx>C-@+r8}FyA$=-R zsdfA0FzCC-G4{1WLi~tfWY&J6({=3i6l?l|T=w=7PR*UF$AxlfnkeD?YmVAck$looYT^wPL+K_G$xCK>M^Q3)v}+qYZT>`hjF63D@1M} zQK)H+QkvoQ2psTMnuSYjihoO_q-l|H?45Vfc5Ri>PD0enSD56q_9tX~8%P^|rU??A zV3L-b&97^CG@d%9c=B%&lm&@QRH7@3394mKUdxkNnTeaUa(SyE7yB~GlwL^~)mNXE z)F>$dXI{Yp&lAi(f>G=TlX$$=BmAYbUxdw39GP|G3)rD)F}S6m{t(F_<2UY3>6LTm2(AM zy0WwimS4CP9p1Jwz)b`kDJqw$g7UkI)4~YiR)Fxf=E{t`=l^qC- zkh&w_isbCD8LghTUe`h??-oX;n)(u?Olo+{A6dV;2f6Rc;*H*(PAQ@wZAIkWu3mkCt{^}_#(3l7g0MNk!t2S`J2lTe3 zGq&xtD?g8<=4L z83arnMwk5g&OiUilmr}rffvxatvw=fbdMXx6m|9N9W@*vRT=^Zye9x1^C1oYd|%n0 ze!^`C4)~6|e0W3RstN^!!2uyz|E~JKd;Q-7=70M%#hc=TxZkXcN3ZIg^}H~o5v)*v zy-rI}copD}E=+~XwiJBkt%_?Skf@wpd~8@iV-XSW&QSG4JSRjVLjBkFO77PN3b6gN zk<9V}y#`32-(1BkUz{6y!+>3v5P5K(EZ-LoeQMOJ|(h5?83d zgI)rE1igl!k5bVP$jbpCDt=@68F^tJ3Ep9O;lYr8hrE>k2zl`$?3%S->$c%A=;;2> z$V*xj2Z6j60}#kd{U_u#_D9I8AA!8cmmO{%oY6w>{T_J@A%lj&>2Sa@@z1Dn?ti@I zem~bd5a9ZARmS{K?%_%v8qIlPwJqw?f1hQ3;m`3Cg0)>lT3cBPYTYNbhXXR>-;1#Q zt+1V~HjC=iHo?j8iD7WyBoj_`5SydRT;ggI9XOGu%sL17X{j{F&EkazA&}Rk{iz(d z|Jp(*;u_sAoDeA;_a=0$g-(*s>;A6$P2gSeRZT@V}xEVmK zE`uoQgC`7xqUzU`Y%v9ml#xQ@kY|MIdN=4FA5L-3zqNPpMw)Q~GA~B7=3kbC7RYqY z#FDeJr)P$;DgGvdUu6%`E*{D{hJ)ht@@Zib=5RpiPm-^4jSu>U?8F&uoD1!=FB@Wh zw8H_c2*VrktGB@YHR#?k96(QN%e@W?zD^PU=^{$qVPEZj(5tH=STG(mPY%X!tup#f z(jG)S)xvOV*|7DF)=d8L=-IVjPg@(5#dS5Iqm}lAqJRm8eg*p4{D&Xsd}~s7W0kg6 zUxqnalXUH1|6TtDnW;fqG|2hw54;vxV&@sVp7Ap=U7gxd)abfM*<0gz_vmXoTrP()GrIqbQQ~TOW zE$|glYfKr1(z0@{12@Bs*ht?B0xB9(0~J4gLymcj1W7`)Ov~O>@>(|pDmg(%y7;g$ z0&$I@O0@W(2__BfWnnmDL;xWAV!JO;crWx~7BDz^4z$-l9K~4IG0IbzOchPYZ~(($yXgg1EgUeSWQgdyV+ix@5Ol!Y6CI}T-(eT%>VqKy zRg#;%q+h~QEtG>_gj=ebJ=MXc!4KiGXl$4`H=G-@XRH?F{ox0RJFe>@Adz+yH^W@k zm6HLuG+cg|G94|c-l-|kS;UY+KcV3qi(~sSXd=*yD!E3j;$o;HFseTzW46y2;>d|pjt0wmwg#za4t#rb_AS%PIF#m1Z!Qn z=C+E`8r^s>+gW!v-`dUsTe}G`lvPxU0WsgXKKX1!D=H4W*}G0bB?()B1MuaFyBUa{ z+t;wW)NW4PWSr9!`T*^!J{uW-&9=xc^$%q!U}4D&1^ku@8^RYyZMV)lY_fSF+|Akz z_a)_ZC?GX7_og<5Dm)5gSDGE96DNt{E;=4-YG9b{z(4+^IDp0E88Y7M^IbNvV*Uvm z)2cKpz09Ickgce*0?s%OgE?34!$)%U5Sf+?Yqv*A-gQilJUR$H%H{c%Z|QM@KTIgIEbciy}mUYlr2 zN_%t&`$lSaa4J5sZcSMM_7HQ4oLMb@XS>!lR?X?TBX9|G^Zyc=MTK!BEsk{G*;&2^ z1;U289tt=FqHRe{j& zed&IjB}aJ4@BrHpO=Ute2a<>Iae05Dxf+1HKD% zJ-p)xY7qI8sqpO>Jvb32f6ghO4lJ6BT6iHM^wWIP@C0?R7Zt}(J)tz#nA@nUd4qsp zZ+*1jfHa?mHpDfN*&|ESN*Bf^b6>d7k3r>&g`lAvdvaDT1!$VX6&g=i*jDg>2f}1J zsO$OC9%D?rn|s>R(pNa=96UAMnT?$<=zf61Tid zJB4s>wyw#gF+*>auFI~*y;*>n8EzGc^5clhCV^S{h@gIPYRhcMM>89$670 z-pVRksalP3Yn~Pu6)}z4V1EJ?+(nlGSCju#hlx*)Ri4rvQX{S(nZ_h4S0{Bf6pYtT zzlx7-Ra?3*<14gVYynuEJJ6FpEZgFps>BG zzOS{OGSl-g^{UfVV3FlZ69*JHu>ka`!c47PM)^~U!diihSLIBGZrw9V;=?#o47?G|gDWFNUeG}7>Z zf>n02sd@HUs@`wyagyD!4;jHRk^OV?=?oR_Fe+@^nhDHg0BZ4j-1j7iDx3b&!|Awm zQ{O3yDa6i;YemHRNd*adY2uvR_~IwGivlWTrP{(?)jdNKIxh|WZXk9MlC5Cx<5>B0 z@T8ki@g%2#%8cLR`U|$3(BAO<$v^55(Vw2AD&k4Lr*wb=XhuOza6td0;Cs+|$`ugS z=Du{p3Vg$01NkC^6uJ43<}1S_^B;d_gPfTe$^!U1%3to&fAQ_u-1wf}x0-#;RSEkI zHDG~Db~qp%T$pw`Pt*>31RJyEaLBynDzWdl*lN91+RcMrFuRvSC08~k;@OMzeftUx z-%Vb!kJBFfd<}ILBV(^bECj(pyVRWC9bK0qke%`mc4Wl8?4=a=AhO(*yBAPpr0uoa zon0Jd*c+6dcgG{Tctz)t?~51_T(;S&~M?JpQL2`A?4? z+Z5l^hyUYRDyDL2j>D;;F|)RK@PulPbh@YCVNUFe@znCiwglMUjCdpb<~OKegd=LH|Qt+3)W8o2kg#m1JjQ1ps}quplPl-I6#Ke6%SU@ z5`@ru0dPR?)1#?7e@N?@#xS~!46z#0-&AzQ4OU$HYO9B4pL3agLphvvYP;-og5gZ? zbAu(#SFs?rZVJWFrOMq07rjK~bDMn1_fM;nUnL2>UTX$;@aeJ^8QaY!Aheir*9GP3 zp5=1Vm?0`w`o^b~bSrI0vR%yKslp@~E7T@yMn}ylce9xb0#~^wUBtbV)vurJV`_jX z0A;_T2c`5~UU$i#=5~O%>0D@Kc@dDj!AGkH^swsOA*gq13K0AU0`ZHn`Q?c118P6o zbEM}8TD(DSncWdTv%iz8v6#IY6cfROh|0EFj+Z>^(eA0EMwZ)4Q-!sN?3RfI<_vTQ z63xjl3{!O+y#FlTT?3>Dov+fB)yzE{K(}(5QeK7VEYyTKyIz$XbJnJXH!ZuJif0L# z!U4u^UgAX#CAMm&bF?wq8>(7wc7584Cc2g20OH^fC>+q3WmjpYdeA0OR_<^*TDn`f zp*UaN3QDzo8(J{7~X79%&Pd<)!CvhAdQkTq*ZjJ`LR>YB#8!{WN~g=`MU& z0?G03O|K-7LzYLgU^5N_|7apaHxVvHFuct}s9=5{2v}%)GitCKVtes4B)gh0dnaXP zrry2WzQe_@&L79l#z2$;Ed-b|*Vs$lVBY@;I@l4@@6*cwDSS>0BCC%rbD4$xRy+|}`TnmN`v zk&E`i;r5&N@zPN#2*|>;+qIYoOMv3seYP0QW53zCnI6k5Y|EGA%+|$KZusF|ICLWL zc8#Wqq6Wcv6?eWWa&h$|xND}=?>Y6o!d(%c+M>JxOAneWOXFfOvF^G(o&4M+m2G9F z;z7f9ufV}HFBrvy#e&P~Y>t0?igzuHp6mThIJqH2H0&dzNs+&x zFWC^yM60!PLao51`dx39KIyhB&>%iW)7ayjC(jeCd^R@*LlPra*Gu{>``CYB-+KSj zU1#d(T30Q*Ao+cJ#>H9Tq?g(lp_4UVHV^E8HM#Q9RvuV!B(usjePhVp`U>G~txdKKd~UB}zCC!; z^4c0`Jw;!(rhhW=!Dnh7ao z@m!3Yj6g%DAh$Gp)^;Y4Bn4EeXIojUOPKcfM4Pxmeb;GNdpa-FCM!k|)zY|k&T(%v zlm8W~P;P0cYaUqve-@?Nn*?_kLxoGGh%AQ;GX8*N!{CJP<&xYtdve~k=ovc8zIvrG z^`>|T(0SF^A!5725Y0_b)7U)-64iadpT~H5C!v|>r4@E=TDI+`r7mOf`gd~mPm=X- zuON);1bBE#Vwv`u*%tvF%8kLWP=jsSJ3}8F7Z?&85aY2@c8fX(+RjPG4gJMZ{EbIk zmo7Nq1PLKrmY1f$0Xd(hF`X37UrYkxk{4wB4M ze@1?B!x)%zISa`+v1mh8|EPK8T3E^aQD?cEs@5J${CSSNh0$3zNO3l^+|$!wfL+uJ1Z=ddl^* zz}C5wBEYd#lsN-ic==*TvFkWfrG>I)ToIJLbBVt|V)7OpTRa&Sr>D!PJ0n(-bv|aw z^F)y=p~i3#;swl$i!2RY$65LaRlz8i9dX^0f64eYEZGp?rVOwMb=u4yR5%`?H`H z;EI@kf7i78E8Oybg*$Wv0{Aix_pPJXV`i6ElGi`K(Y=`*ur@NKG`1QY@9=+`zSMMB zFQ@vWyvXJU@0&^Y+cW6CgW^eedfZ{zLfM9kY`p|g8&`C`;Z^1U78!b+4`Lej$t&!g zo1Ts~{IIBC7;e&Fu~|^8O#MKU*-$VPuPbaPohm?xxmpj7^fVIHU~zpl9MHqh=)8D8 z8e)G~&HCl5YhM7xRzXOMg~ZTVX>MrrdaR_%yJHQ6l$T!q8yp$`tWlvR4~TI@_gX1| zDiqp@3y#b#I??#OOJGMaF^F}fS9=HPuwZ50K+Y%Ma{PwwBI9hrGx>(EZ-H)lwu_q~2)gN}95tRfo zPJGaqGY%k%sGGf$ws7V}SkR#T!P;2Lm${A~y2vbuBH2X>>K3^MfO!em`?ktOvx+-A zWohTmxv?3xiBzgIDR$+-!sapkOB!#BfrnS|Q0cMK2ATXhv8o#f(=W7Tl71vP zir-R|;bGOV(a;-IALmI^^U~boRr`&Zoae;R1UgKiYF_1{r4ke)#rZC>J4O0C zJHwM;j^eTA$(Cjn?>15gw-u%52_Yp__7BBEsgSPC8YtE|*B!7uYrSpj#@s?)l=sdG z4=rk*L$181OpUb@n)_?Ll0Ta^{zWe3FRoZJ>BO_lEQGF6@&|G~`rYjEbn`e7qepYHKGY*A-Fa9vhZ2b|cZhknQJJVOdC|pa^+oxQ zo}NL!(DF77ImP(Ov#^~~rskSPddPz@f^S`FC%N@Y9PG@{A4~47n{Vp#WXd+C4#%J& z{LOSYE{cg=Ywy=s)(VBT+jsO^NGEgZ6DdqnJYzw~I)PkJc%A zB0blE#1hztW?P7nyvm)$hR z(RQ@JvfzMm5s{F#wwZ+Ano|CcSHsb0!_J?k(0-o+PX<2Fnr^Z9Xl$nArt_)HF;s~F zHsI1=lEYaR>KaA~BI#TuD|=unz)OW%5~o%k_kvBJ;AKHT@&Vb7u}Vz|$O1#|LMO+U zX(6pba&x~+{F0u0h*sJtr)lTyY6OyNE0&#s))V7C{`pG1VuH%MR7=XxRFzF5tM_8} zeox5%9)kT}gcI%$j5F7aZYo`Z`}+G(Vxm?|I&O91bX%$-VO>Cd-~F)g8=CGT{uG=hxa#yB-$3^6m% z6xoTUIfdY(x_zGriT1?RJuj!XM7_;^O-1>5dh666WJK_y#E_)-D(J_W+_+pwbzNU& z>D;Xw91uT(w@M4aJ3SmhA8by#21O6>_j22`fVNIVey}x1Kw7daaW;zmqF9p7&gy$_$6uRe3ia5} z&?mFw3wJw7_?-ir&86a357v!7G{&=7+jKZ9P$VPM;nm$a*P-Vj3#Fs~DbU*sInMi7 zt?&{&YCg$9t3lC{nmh6a{|83n$LK2I<$GSb_J4I0mR!MH4E!`TIkl!St`8?tl{?vht|zA*KC1bMp=Ioafr2k93DF-9_-v1eLaW&E zG{p_~9qR0o5L%TvHcC6!?4ZA5{v~t(1 zM|OE^G*&SmMSIB6eSWoz(N^m4ZA{cl^hRso>seoLt?;g{u;y9Y9eZA>I%rk zy=eA;=A`@QG*^+`yhB;_>${~RT!Ym8HSUBZH~?jLJ2SLn`u~M!{uRFe=TPHcU^>-* zmaFT3o#6gA36h%Q{Z9JLV{@GChRAHJyBG&zY(T>gjn$#667_6ov>yxfrCD_V{*NUO z^>1F_>ph!&tt;;G%~|XGS;r@K(gBL1D$^*a;QH%&W99{4(<3ph@T=kCTU-Xl8bCKP}E)rJVw2WV%0&K|iD}>~=A-1(A-II?=3` zI25!zQJUTyx#`V5Eva1x7Bb7bQq3=9{+^f^-o4uV;_ zkaR(2-Cp0&15f8srN+F9iPtNX*)MZh-EWrP)l9aDdI%h=61|Dk!VMlLh*lSQ!*UV>7)sx8JLORB9s5)ZTzJ) zc{JL-`T-Szdy?k?#H#i);xV(VjS~bIUHTm`Q1CTa zwp*>eNP(EnBcG(@3PIU(OVtposk2reO1LlSfxE#ev~Ym=P2y}?Gr8Q>CFt9ef`cEt z*FK13EtS%Lb7*QPU58HkKupz1=Oax0OUVPW9>1$>ZLKdww%;v0jN=q7$`$i`AL?4D+H^Kdm5eO(VF?n^eVP`7nu@wVgjI+S#>#u!5Z) zRu@9an+jmgaUcIfR?Hu%cz@Any=Dq4p4nuUi00S(e28GFH?8%=$?|W6R9R7s`L%;U&kMvbN`6D)Tz;s$K0}XB+>KT)1cPvW$`4!3WNF=vxG~f zmBve}eI%h`MJAX?-)9!%DZfyy7)o?J^_t9y|H4O_xO2Er^1?fhFnj|JXoj2>7iu}B zWEFcgLhju5ZB?j_nyM3I5W)V^lz|4;ik_IRO4+vPAZ@?pP&d1j{tJW$`Sa2L|Dn(S zIrH$I2*<~sU2?sb0mGr|s_hOqovYOB+Lfc#d#dHU^`Y!CH0H^#Ux&XeAg7{fE-K4> z|Cpci-mT72cmv~Rsfl;s$pP2o{G%A28Mphmq^t@vuAISZoy5OuYjGW$sh`LMnbx*eI z!;4PcqX9^N5Mo*Igt2vF`6KLbiK*x98|J%CQ%9xo^%D392$Iw>tJBmZ{pglt1`e>S z*O8FEV^_ov4Xy|s%PWHtv7IJJ)H;~7q#fZHK7Ey*(;E*#OUAH?JN?ot8h&F{6Boc@ z`Ou#S<4`$~Wh-zs4!LU$JKIsla^@Q!nU!JK#4U#7LqfL~7tEvegcqDL+%6|@fZsKB z#I42pQtLGw&;}jwVnL^;3*8p}{x3xS|Cd0TKMRHQZ#KUq4u*K-wMh7o-&eKT%{zLi z6G%K}yKrAYLH~;{;rNo)(HWk&%s4r?4n03wNcd8ewyWlVbF~5N$^7iNuATKzPcRj0 z>`j?6Evey;t2ed+ys3xjMCo$(bba-Z-i_ln*nXU0N7G`&sc^v7(Rguntje)?C=Dyh*g- zgIxiSjEI_k4G6h?$=IHz9G^ybHyr|FS^AY^3H6$AZEAeT+@A;@ks)~=_;b8iOWEn3 z!bbH-5gg!Jj((CoYZ$?F!I=48k{q2b)a-xCGXHZd@Grvz{nralOaF7m+Fz|i{v~{t zDm5$1#KvP3C{7dQ=+bbgO2yIM$eaaAGq1_9tEG7~Hn}-n9Fi>hM%-qup?n{xSl_JK z&U4p%ZhwDoUnCZ9k(E3YWKGio2f!NU68T9vE)QW=qLsPV8DNJK!K|v@W%re-1yRWaqn7Bhhe*tyJcgvru|cL z4sH$}WHGazx#5W9^fzR_X7#%(=6RwIl5DRjM>`dkVpNAr`v+o;48eA}v@s>~_71utW3mZ6-yMvKF4}55LkkDn-vjtE{ofv&xyh z@fCqq3pBIxyRI$H!YJOIb1Cqt%Z@70(Qz+Yy#(AOX4#kOEiQ%`>Qh*?F-GH#obxl* z#+n1IE}{*~X66qp%=C!5%Z9b{S0jlr3E3WQ7d}RnSQ4mJDl=_#=rL_jI9O6i+q(XW ztk~4jbb9_YZ9(=n6Z}5T1P?Vz!){|SBF|LOvc6h@fgUwd@{TRFaaD*9#I-0id#IlF zk(2JW-$yn5vKS7i@*G0Qy~aMTa%4e482y!a1#cwSM#)&5Z!fA2OZvAW()csHOK0_r zJv4*cFXCt4eBS5&dWw^+u-O=pyv6J{q`C9L^ufYT%fB*x`M?6=%5OB>0KqK)jk$|x~M(pU|AOk zmOg8n_l5(KZ84bYb252SWYajbXQoGt@{a;1Q(ZEEQ}K!-(gMuk-hc%O1JkP#h)`Fw zZ)z)_!_`IJ{wj=|yYJ?<6JpWA(mHf=8<`ur(KJNW_q{FS8=i+PS=5hM7=f{p`j+7_ z&ioIrHXD>DceuiIQC10^jMu?64l7~M2p_o)5!tx3+TF*`+Y8_H!2wJ3X`=;mS99$i zLNi6r5Q}J1zeXW2#l2$N#RnsN#gxhds7`PICzXT4K&2g|>+36zwCd?O<sZ<+|CsYbcAVt@UroPvOS;6^933K?7U+VP8U*|`!%6f4I^Azqg<*CbIDyx zhmV${-msAzKPX6-e{{^54|H~Hh%1bxh2nT2lB|UC_CL3vgSavvMxc)+TZkn3$q{u! zW68H~V&`7OO0BVN&!8U{0I4Hl>;78W^p~7J_2-t20n`8760^T!bCQ2d5dN=`tb;a3 zXHU4fb{ARKVEg%GzOwbr(rp4tNdwp*t6lY!5d}*84{(56LN4Ce>5j2BiDLoF>z))L z)m2%)pBjTU!$2KZI$}%Z5qE7ruTNXg`J<`r;g8e2czPS2LUr!feBX8WOO``#J<6X^DvH7&O z&B5sxB@Haa8_dk^wq*H5&k=nX6G~B2GtrqYpjoX;@HqOhuP*(S;q>UV2`3~*Rg~uW zhke&ih)CK)>k^Z~%UXfih}LK~{20+Ka(eWs&@Q!@*kqd8xjZPc%^(*Wk%6gi>(Pq= zV7owKOYcbHD2(P+V?eO<;jUPb0JTGhmtB&voMhsS-XQ8k5>6$akvo4G>X+e+`Y%Qc zs&)l&el5;*d!eniZ&A-H=MCig4HrA4dgpL|mC&SVhAlwm$n68M}<$PhI5y zGo1H3v^Pt6=>0KXS4fLPkm#)7f!@Lbp^UY3keajZ-i}7jgZ-~6JO)#z2mf+(3=!2Rf;RGQ@;WktTR@pFCWE&-BFokfm)?aq1wx5 zZ-)ApGBf=*u|6+3@Hv-VW;wQ>6#(fGIc&0)G93FP&w3e_k)C16xCpk0j@UCj={Pk( zLAXlBV((E#*s0(PA6`V@zn?UIWZqD*vi;$FWbnh#dd<(;>mMWJ6=u2CtJP&BU;S09 z5!J`}Qv?;1FE7j6x}LdTM-9741in;S)91>v8gUH^%kPFZ8xf!&FS)qgIO!Dh7R{;L zVQ*-#o-COx_^y={QbdAtZCM*&H)KLRIu2j%h@FA4o}$X7)BL>?V}0CJ`wi9yu=?ow zwUeSn+9TtZ6yEv8c6+|kX7YF6^QK%}(sJagv$JE~=t97YXSb!MjJgl@v)?a5)+g@{`=J~hP}guo}qkDH?cpqD2!jX9T0^=bMW`9tbJl0L*9`k!auu)OMX0RcI;h(Z(`7Qp;v*Z`5d|ymjLEM;K)lNQ}u| z7GVeI-I7ajY9=g|ZI!J8Az4f*@^`bort1Vfv9gu2yW8!LKCe@l&r4G|H~z>lqs-oV zD$6^dSwE@)lrQ1jiWyzN)_f}4!Oyb)-os-`ILcA=Xp(ovNwNY{@qTEedC~* z534=j7s$g@?&^gXOvc4l2~Jc5-FXJXnPt<`Nhu`?7h@Y6s?y>^(qpgnL?47~;(CDl zXU==`%Umge1q0*Uj3rCZix10b`G*G88lGM%qREBrdZC3z(7{c62xKwwli8qQkN>K^ zdDoGT`{kj|Yi`InyMy>l7?XRlKZbZv@r2T?FjvV7_e*Xd6J%As8{{;vwx)s8`?Pdv z9kF_rxS*G%#(D`lb^Tzwcn_0bMnP9Ch|t_LUf%HT)s1_83IDC_flTr7fVK0I`_sU= z#%=FWy^pp7Ik(%mAI>)>7i#zma~2cw5V7*^jfk4=rV4i2aqR~741pS19Of)xr9<>? ztG#?rpnXsl=_5&Tt@hh+F^QqN`mC$MQJCl{UpRZRY%v>Qq+hXhvDf z-NJ9tS~WQ6iW8Rl4#f{sWhkW^PNQB-h88APpP%kp>khN+|uUrNE>hQ`@ zNGqhhG4&h3Ymz}kAY-*&X6qyJobBDqF@m0+{ZWEJpVl~IW5MsU;stLiW(?)L^f10> zSZR2o&+QoqHpH!yGEv&I0h0k9hZuR=%GMm(6bdI|z85Wun-$+D3#Y0Le0re^rG~ZT z)s;23{593;lHqS{-Xot=?|n&-AAk46Ys#1Q!-DOpfc2P?`779&#n}xgx|tIK^)fI9 zSra)mUdFFJlQ&BJ;3m{oP<^L3r5P|#`)(~ry8Y1^U2ZCy=sbAAWbg(KF!f@~C8;}g zAKje=nYug%t?h+3fKctiUODd1 z;viDXGW)wFtX!bjpaySRSo3Sd=H6p`At-qCMkZiP;G4#E^D}A*3TkF=Ijswf$%ryB ziuu{hdi6rjmLsb<$?7W2kuok1-&b#mLAIWa%A1$+)#GLd7ld(^h#eKc%tu+%EqjNO zkOXT7SvM_R4pHx%t|uPA{MpC1ef&k(zWOD~%pG1Tbw;+A1?+i?QuoU*+W>2ep9@sj zAS4vy>-(v>CB#}1uWN4=YHy#5dy+qKpsYCC)1Mw)AJ=$VY-2plmfvgU2q?7z(QG@; zR|6SNrGg2Kr7=Ct#RG%S&`KYFhLJckYdpS6iA)VG02WM}8YCqd_AtK3PS8c+o*FwY z_AaB^)*1Vvv2l>>b6DABabHY<%2$icCALzM?Nex@wVaGgx#wZK2RgE*{9Iie`IGC= zhIh&t*LaU7_P-J+3CxTq`ydSnDgr$#*s2#tP2c8lj2L!|4$#R_gjY2J0GCV^Z$zzE z`i!mSD?&TxIF3RZP7jxE4#5oA`V9HzhBp|%t zlv~l^e4BS^2)!kZ!uO2aHXHh0nJ&@o=2)=?L!Jy)ylLCnvfwwERR*pni4^AK+uiua z(cJF!*mtW@e`DSjN&8-|61SS7n|AJ=k=?6(81$|G3InTTs?+XuM@p{OW4oE+?DhrG25L91w!go`xJz-mW%h`hXKMp!B?l8*G*^}pE zu39`rugcDuPO)eYStR;(m0hxCP5ZFH(br{!%Utp=ZdR)iSYCPXpvbw^r)5okTx-98 zkd&z(Z=58UK)*mgfj#Ily>;7^!#HfClV4`=A_D1p>R)GnzlxdT_!K zs{UC3i?wSknMRVVtppEy`_e#*;=|O5)Kp^N+Tx3H>%Q|H|a}k)b)K#;=#Ht zms6(#H=W8H5c`r0S!$-q(McPQ*!>o}C!3(*V;>EoPpB+uf6jQiYm(* zcq0f%3kZlq2FU`F1j!;GLWATeG%yG#!4{DwNDECyBuQu_NU|G-*ns3@OU_8nG&wgQ z2#BQF=T`03IA6`TQ?vWcx4Y#ZujNiO|U z224T}YE}0&KXZ;@5M)ym) z3{Frn<4=pL?ZWBOth~xpXF*K6w{7Le#TUFSWa2#V59fCFeApE-AWlZ&{^&%mj z^d2`?vB8?yUhbiAEL9?=oi`S9wQNk9-BHGak)_0KZ+^U5Ltu~>8#f-dIYR8$7anAqQf$bNc1o;7zrX$?M=a4WRXdYUs3v~nYtD^c`aCR|AAi3e=(NrDJ~ z!!>&PaMbO_f*V`wZUJ+haK{M>TM%Qc=W7b5w{(&k5Mr?(E((L1 zhY2N4K4LTebhl0uplS>kXj(*lph-HNV(Q-NWtWJ7^hiucdVWm&;HXSh-W+S%Yjq10xf9eq(#|>s#)CXVsWavdADfxz&tfwl z`VoE@MnR2+v8S3=>x1y390mH?>AeHSq_n_VA)X_<#eC88$Mv>i=o7;W)UzhL$tiLK z6EJ!V_JRgwI=3^d67XK(1A75fONMQjys#V(q4|)=Nu`4o82#F_lFC=#weWgXrv3iZ zsr$f^{!hy1c3O6}-U6wYZH-5FDn7N+FNZ*G{|N3ZZiuo(k&Gh)DHmRL+hwWA^9eEW zXpilJMj%Gm(j&P^@2v4F>MIsXoEZpDwaJv}v}5qalfcEThR)7Y%8rT>M_SA|Og@`o zSmmTlNA`x_v;5EdYa*2i!VP@Gv(B z#Q$}A{qQvi&y|TjWJ*h}LD;h*91y$_WPEj~G>9ci-FI;+C3a>UXE;j5AdDuQOdoFa zzeiyH*I&LrSWx*l zos_117AblvQh1IhKBa5?=@jgv)ebdBdT&E|)@z&@_ge294IT1lhjW2<@}N1|`UfdM zC^(P%_xdXT|Fo{Rnv17F?z<&K7(%4fvZ*F8K7I3)x%Y8VN zwBWhSsILI*)1sb7x2h{5=kQnW+wZUey~b_7&KZ{q;+4AKWtW3=Vr* zWfV%GzVntyU{nEU9+sVZVHHA!^T<2L(C!MOERJH~kHGwhWS2Nq_7w*_6jJ@T+B*UB4A6{%Qm^5~wo%weY)|zpm!jwOL%=J`I5} z#CVn^ysS=lkT7pPn-HYeWdY$JnvB*5p!vUgms;YIn&aI2)%$Xx^-n0m7n%js$>wya zTMB#0WB{c1I~0&ESG{~nzNL6D#QrgP)#}qa7C9673BZ;Al@6Wutqh2xeme#Z*Zr>O zcdU^JS=6pvLf1>x{0_e{u-Y5E=y^;Y73Uf`M`RPzt8d?l!Mn@g7e?b#A zBSNj@axq%#Qu3o1HBOYRHIqNNyg#M$zmPubikHGw57Y|E#~UyR^Cp4q5AGR}E zr_RW5P1E-dDVPIbw$hpi**)+?PmQS{Js03gUr_=Di~@V81UyZ3yJ?)s3O_a4ofX#n z!z)U;X3qBFA_AoG4AnJX!+A#RyW&zLWg`X;wNlvF%9SM~M>TJJCVGUWQLHL&-5RrLKvl@%r(N@g7k0eWj^GAGHaP>_$|6^G0~#AIrzSy!`#mvQOWpDP4A<;+v&{_W99BM! zA-3MS@~mDVv6fj)a+6TCZ86tLU+F>$RTs4R}Qk}PuF)`Tb2q|a$}9W>hh=n`2rw?kNQzxnmvvQ&k+i@Us6Y((S?8eZ z=*;Mv8EPp4vnrBfrR?lBmcR5ME-cSpk2_9z$89=Luad6uachBx>`2JW@Pm@7Ba122 zMk11Pt#42^1SCF-iVN_oCri(@`WWPZ8)LUVQr$|~$5nXObhFzZsU7c5g+E{y77KR? z*Cnx}kDr?`+a?X=nQ9)Y3+yeceI?GwZ5Sik3{EvCVWErlm%(Pe-PNL)zV%|hGj(oh zZp}(|Zf!9Da&3m65Ac-e0tcDK;6E-suKXGFTvzK-6jhFMw$+4|vxohQ_|I3Fq!9_I zinA?SsOQU(PDT0bbe`7FbYQHdm_V;36j{Ac*tP`6W%-8-+WtnOOf^^7i_r zG?$qTg#vh6K40|!_W-A;*@ydUtXv$VDs`r!a)z%1+QBC$F;*vLhUX15IJ}q--k?xv_|T2-#^|PC5tgham1(Ff!gwFSY!}U} zp^m=FiTz@{XgAj0)@E6!utNL>Xbr7Umr~Z%KgCgfb)f-BVVrhoWEXsp=+`Re(zbVT zB8u_AXL!6}PV1(Gm#?eC;;kLV^7+TQx!SX7HQ{^n0np7VjAK$(h4maYA;OezjIG}Z z4AX)=;*|R=xEZM%`4brz03n0ENCTKl!_|es!W{isr4^-4tB+_A#q!bemG#1=@j@PS z{w}fIphr;8##^PS(vKeT1-4z-is`qDkn%N@LyZS6JZ@rhB$u~w5(#&%mENy#y85=k z*seUWugfo#Byijp#xC}d-=4d|b%ykLXN^}0DaRJWIFm1LSPPTUyO6AD zOUElR@AR#m%w=3}$i@YdJ_fD7p0CqV42Odeg5cIcqx3;3uyLeRY&VB(HpRKAdWQ8> z*==Z6NQM)hwd~{2LBH8PLP6XdoF}~;VH;k84l%w@?z2SQJ!vDIC#w^*XAby(sHP?- zZZjz;pB}E#stUrE^MiEZQ~ooyR8<={S>sW8qwDU 支持 DDP / 单卡，AMP，resume，日志，checkpoint） -保存为 train_template_localmodel.py -""" -import torch -import torch.nn as nn -import torch.optim as optim -import torch.backends.cudnn as cudnn -import torchvision.transforms as transforms -import torchvision.datasets as datasets -import torchvision.models as tv_models - -import torch.distributed as dist -from torch.nn.parallel import DistributedDataParallel as DDP -from torch.utils.data import DataLoader -from torch.utils.data.distributed import DistributedSampler - -from torch.sdaa import amp -# from torch.cuda import amp - - -# ---------------------------- -# Helper utilities (self-contained) -# ---------------------------- -class AverageMeter(object): - def __init__(self, name='Meter', fmt=':.4f'): - self.name = name - self.fmt = fmt - self.reset() - def reset(self): - self.val = 0 - self.avg = 0 - self.sum = 0 - self.count = 0 - def update(self, val, n=1): - self.val = val - self.sum += val * n - self.count += n - self.avg = self.sum / max(1, self.count) - def __str__(self): - fmtstr = '{name} {val' + self.fmt + '} (avg {avg' + self.fmt + '})' - return fmtstr.format(name=self.name, val=self.val, avg=self.avg) - -def accuracy(output, target, topk=(1,)): - """Computes the precision@k for the specified values of k - 返回一个 list，每个元素是 tensor（百分比形式） - """ - with torch.no_grad(): - maxk = max(topk) - batch_size = target.size(0) - - # output: (N, C) -> pred: (maxk, N) - _, pred = output.topk(maxk, 1, True, True) - pred = pred.t() # (maxk, N) - correct = pred.eq(target.view(1, -1).expand_as(pred)) # (maxk, N) bool - - res = [] - for k in topk: - # 把前 k 行展平后求和（返回 0-dim tensor），随后换算为百分比 - correct_k = correct[:k].reshape(-1).float().sum() # 注意：不传 keepdim - # 乘以 100.0 / batch_size，保持返回 tensor（和之前代码兼容） - res.append(correct_k.mul_(100.0 / batch_size)) - return res - -def save_checkpoint(state, is_best, save_dir, filename='checkpoint.pth'): - save_path = os.path.join(save_dir, filename) - torch.save(state, save_path) - if is_best: - best_path = os.path.join(save_dir, 'model_best.pth') - torch.save(state, best_path) - -def set_seed(seed, deterministic=False): - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - torch.cuda.manual_seed_all(seed) - if deterministic: - cudnn.deterministic = True - cudnn.benchmark = False - else: - cudnn.deterministic = False - cudnn.benchmark = True - -# ---------------------------- -# Argument parser -# ---------------------------- -def parse_args(): - parser = argparse.ArgumentParser(description='Generic PyTorch training template (DDP/AMP) with LocalModel priority') - parser.add_argument('--name', default='run', type=str, help='experiment name (log/checkpoints dir)') - parser.add_argument('--seed', default=42, type=int, help='random seed') - parser.add_argument('--arch', default='None', type=str, help='model name') - parser.add_argument('--deterministic', action='store_true', help='set cudnn deterministic (may be slower)') - parser.add_argument('--dataset', default='cifar10', choices=['cifar10','cifar100','imagenet','custom'], help='which dataset') - parser.add_argument('--datapath', default='./data', type=str, help='dataset root / imagenet root / custom root') - parser.add_argument('--imagenet_dir', default='./imagenet', type=str, help='if dataset=imagenet, path to imagenet root') - parser.add_argument('--custom_eval_dir', default=None, help='if dataset=custom, provide val dir') - parser.add_argument('--num_workers', default=4, type=int, help='dataloader workers per process') - parser.add_argument('--epochs', default=200, type=int) - parser.add_argument('--steps', default=0, type=int, help='max steps to run (if >0, training will stop when global_step reaches this).') - parser.add_argument('--batch_size', default=128, type=int) - parser.add_argument('--model_name', default='resnet18', help='torchvision model name or python path e.g. mypkg.mymodule.Model (used if no local Model)') - parser.add_argument('--num_classes', default=None, type=int, help='override num classes (auto-detect for common sets)') - parser.add_argument('--pretrained', action='store_true', help='use torchvision pretrained weights when available') - parser.add_argument('--optimizer', default='sgd', choices=['sgd','adam','adamw'], help='optimizer') - parser.add_argument('--lr', '--learning_rate', default=0.1, type=float) - parser.add_argument('--momentum', default=0.9, type=float) - parser.add_argument('--weight_decay', default=5e-4, type=float) - parser.add_argument('--nesterov', action='store_true') - parser.add_argument('--scheduler', default='multistep', choices=['multistep','step','cosine','none'], help='lr scheduler') - parser.add_argument('--milestones', default='100,150', type=str, help='milestones for multistep (comma sep)') - parser.add_argument('--step_size', default=30, type=int, help='step size for StepLR or cosine max epochs') - parser.add_argument('--gamma', default=0.1, type=float) - parser.add_argument('--scheduler_step_per_batch', action='store_true', help='call scheduler.step() per batch (for some schedulers)') - parser.add_argument('--resume', default='', type=str, help='path to checkpoint to resume from') - parser.add_argument('--start_epoch', default=0, type=int) - parser.add_argument('--print_freq', default=100, type=int) - parser.add_argument('--save_freq', default=10, type=int, help='save checkpoint every N epochs (rank0 only)') - parser.add_argument('--amp', action='store_true', default = True,help='use automatic mixed precision (AMP)') - parser.add_argument('--grad_accum_steps', default=1, type=int, help='gradient accumulation steps') - parser.add_argument('--local_rank', default=None, type=int, help='local rank passed by torchrun (if any). Use -1 or None for non-distributed') - parser.add_argument('--cutmix_prob', default=0.0, type=float) - parser.add_argument('--beta', default=1.0, type=float) - parser.add_argument('--seed_sampler', default=False, action='store_true', help='set sampler epoch seeds to make deterministic distributed shuffling') - args = parser.parse_args() - args.milestones = [int(x) for x in args.milestones.split(',')] if args.milestones else [] - return args - -# ---------------------------- -# build model (优先 LocalModel) -# ---------------------------- -def build_model_with_local_priority(args, device=None): - """ - 用参数 args.arch 作为模块名导入 Model() - 如果模块不存在或没有 Model 类，则报错停止。 - """ - try: - # 动态导入模块，比如 args.arch = "rexnet" - mod = importlib.import_module(args.arch) - Model = getattr(mod, "Model") # 从模块中获取 Model 类 - except Exception as e: - raise RuntimeError( - f"无法导入模型模块 '{args.arch}' 或未找到类 Model。" - f"\n错误信息：{e}" - ) - - # 解析数据集类别数 - if args.dataset == 'cifar10': - num_classes = 10 - elif args.dataset == 'cifar100': - num_classes = 100 - else: - print(f"[ERROR] 不支持的数据集类型：{args.dataset}，无法确定类别数。程序终止。") - sys.exit(1) - - - # 实例化 - try: - model = Model(num_classes) - except Exception as e: - raise RuntimeError( - f"Model() 实例化失败，请检查模型构造函数。\n错误信息：{e}" - ) - - return model - -# ---------------------------- -# Data loader factory -# ---------------------------- -def build_dataloaders(args, rank, world_size): - if args.dataset == 'cifar10' or args.dataset == 'cifar100': - mean = (0.4914, 0.4822, 0.4465) - std = (0.2470, 0.2435, 0.2616) if args.dataset == 'cifar10' else (0.2023, 0.1994, 0.2010) - # train_transform = transforms.Compose([ - # transforms.RandomCrop(32, padding=4), - # transforms.RandomHorizontalFlip(), - # transforms.ToTensor(), - # transforms.Normalize(mean, std), - # ]) - # test_transform = transforms.Compose([ - # transforms.ToTensor(), - # transforms.Normalize(mean, std), - # ]) - - train_transform = transforms.Compose([ # 2025/12/3 从visformer模型开始 - transforms.Resize(256), # 先放大到 256 - transforms.RandomCrop(224), # 再随机裁剪为 224（更符合 ImageNet 风格增强） - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize(mean, std), - ]) - test_transform = transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize(mean, std), - ]) - root = args.datapath - if args.dataset == 'cifar10': - train_set = datasets.CIFAR10(root=root, train=True, download=False, transform=train_transform) - val_set = datasets.CIFAR10(root=root, train=False, download=False, transform=test_transform) - num_classes = 10 - else: - train_set = datasets.CIFAR100(root=root, train=True, download=False, transform=train_transform) - val_set = datasets.CIFAR100(root=root, train=False, download=False, transform=test_transform) - num_classes = 100 - - elif args.dataset == 'imagenet': - train_dir = os.path.join(args.imagenet_dir, 'train') - val_dir = os.path.join(args.imagenet_dir, 'val') - train_transform = transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), - ]) - test_transform = transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), - ]) - train_set = datasets.ImageFolder(train_dir, train_transform) - val_set = datasets.ImageFolder(val_dir, test_transform) - num_classes = args.num_classes or 1000 - - elif args.dataset == 'custom': - train_dir = os.path.join(args.datapath, 'train') - val_dir = args.custom_eval_dir or os.path.join(args.datapath, 'val') - train_transform = transforms.Compose([ - transforms.RandomResizedCrop(224), - transforms.RandomHorizontalFlip(), - transforms.ToTensor(), - ]) - test_transform = transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor(), - ]) - train_set = datasets.ImageFolder(train_dir, train_transform) - val_set = datasets.ImageFolder(val_dir, test_transform) - num_classes = len(train_set.classes) - else: - raise ValueError("Unknown dataset") - - if dist.is_initialized() and world_size > 1: - train_sampler = DistributedSampler(train_set, num_replicas=world_size, rank=rank, shuffle=True) - else: - train_sampler = None - - train_loader = DataLoader(train_set, - batch_size=args.batch_size, - shuffle=(train_sampler is None), - num_workers=args.num_workers, - pin_memory=True, - sampler=train_sampler, - drop_last=False) - val_loader = DataLoader(val_set, - batch_size=args.batch_size, - shuffle=False, - num_workers=args.num_workers, - pin_memory=True) - - return train_loader, val_loader, num_classes, train_sampler - -# ---------------------------- -# Train & validate -# ---------------------------- -def train_one_epoch(args, epoch, model, criterion, optimizer, train_loader, device, scaler, scheduler=None, train_sampler=None, global_step_start=0, max_global_steps=None): - """ - 现在支持：若 max_global_steps 非 None，则当 global_step 达到该值时提前退出 - 返回: epoch_summary_dict, step_logs_list, global_step_end - step_logs_list: list of dicts with per-step info (for logging to CSV if需要) - """ - batch_time = AverageMeter('Time') - data_time = AverageMeter('Data') - losses = AverageMeter('Loss') - top1 = AverageMeter('Acc@1') - top5 = AverageMeter('Acc@5') - - model.train() - end = time.time() - optimizer.zero_grad() - - iters = len(train_loader) - step_logs = [] - global_step = global_step_start - - for i, (images, targets) in enumerate(train_loader): - # check global steps limit - if (max_global_steps is not None) and (global_step >= max_global_steps): - break - - data_time.update(time.time() - end) - images = images.to(device, non_blocking=True) - targets = targets.to(device, non_blocking=True) - - if args.amp: - with amp.autocast(): - outputs = model(images) - loss = criterion(outputs, targets) / args.grad_accum_steps - else: - outputs = model(images) - loss = criterion(outputs, targets) / args.grad_accum_steps - - if args.amp: - scaler.scale(loss).backward() - else: - loss.backward() - - if (i + 1) % args.grad_accum_steps == 0: - if args.amp: - scaler.step(optimizer) - scaler.update() - else: - optimizer.step() - optimizer.zero_grad() - if scheduler is not None and args.scheduler_step_per_batch: - scheduler.step() - - with torch.no_grad(): - acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) - losses.update(loss.item() * args.grad_accum_steps, images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - - batch_time.update(time.time() - end) - end = time.time() - - # increment global step AFTER processing this batch - global_step += 1 - - # per-step print (controlled by print_freq) - # 输出格式调整为：Epoch[23]:step[1/32] step_train_loss 3.0075 acc1 25.95 acc5 54.46 - # 使用 i+1 / iters 更贴近人类可读的“第几步 / 总步数（该 epoch 内）” - if ((global_step % args.print_freq == 0) or (i == iters - 1)) and ((dist.get_rank() if dist.is_initialized() else 0) == 0): - lr = optimizer.param_groups[0]['lr'] - # note: losses.val is 当前 batch 的 loss（经过 grad_accum 处理后还原），losses.avg 是到目前为止的 epoch 平均 - print(f"Epoch[{epoch}]:step[{i+1}/{iters}] step_train_loss {losses.val:.4f} acc1 {top1.val:.2f} acc5 {top5.val:.2f}") - - # collect per-step log - step_logs.append({ - 'epoch': epoch, - 'batch_idx': i, - 'global_step': global_step, - 'lr': optimizer.param_groups[0]['lr'], - 'loss': losses.val, - 'loss_avg': losses.avg, - 'acc1': top1.val, - 'acc1_avg': top1.avg, - 'acc5': top5.val, - 'acc5_avg': top5.avg, - 'time': batch_time.val - }) - - # if reached max_global_steps inside epoch, break (handled at loop start next iter) - if (max_global_steps is not None) and (global_step >= max_global_steps): - # optional message - if (dist.get_rank() if dist.is_initialized() else 0) == 0: - print(f"[Info] 达到 max_global_steps={max_global_steps}，将在 epoch 内提前停止。") - break - - if scheduler is not None and not args.scheduler_step_per_batch: - scheduler.step() - - return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]), step_logs, global_step - -def validate(args, model, val_loader, criterion, device): - losses = AverageMeter('Loss') - top1 = AverageMeter('Acc@1') - top5 = AverageMeter('Acc@5') - - model.eval() - with torch.no_grad(): - for i, (images, targets) in enumerate(tqdm(val_loader)): - images = images.to(device, non_blocking=True) - targets = targets.to(device, non_blocking=True) - outputs = model(images) - loss = criterion(outputs, targets) - acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) - losses.update(loss.item(), images.size(0)) - top1.update(acc1.item(), images.size(0)) - top5.update(acc5.item(), images.size(0)) - return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]) - -# ---------------------------- -# Main -# ---------------------------- -def main(): - args = parse_args() - - # handle local_rank from env if not provided - local_rank_env = os.environ.get('LOCAL_RANK', None) - if args.local_rank is None and local_rank_env is not None: - args.local_rank = int(local_rank_env) - - distributed = (args.local_rank is not None and args.local_rank != -1) - if distributed: - dist.init_process_group(backend='nccl', init_method='env://') - rank = dist.get_rank() - world_size = dist.get_world_size() - else: - rank = 0 - world_size = 1 - - if distributed: - torch.cuda.set_device(args.local_rank) - device = torch.device('cuda', args.local_rank) - else: - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') - - set_seed(args.seed + (rank if distributed else 0), deterministic=args.deterministic) - - save_dir = os.path.join('models', args.name) - if rank == 0: - os.makedirs(save_dir, exist_ok=True) - with open(os.path.join(save_dir, 'args.json'), 'w') as f: - json.dump(vars(args), f, indent=2) - if distributed: - dist.barrier() - - train_loader, val_loader, auto_num_classes, train_sampler = build_dataloaders(args, rank, world_size) - if args.num_classes is None: - args.num_classes = auto_num_classes - - # 使用本地 Model 优先（LocalModel 已在文件顶部尝试导入） - model = build_model_with_local_priority(args, device) - model.to(device) - - if distributed: - model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) - - criterion = nn.CrossEntropyLoss().to(device) - params = [p for p in model.parameters() if p.requires_grad] - if args.optimizer == 'sgd': - optimizer = optim.SGD(params, lr=args.lr, momentum=args.momentum, - weight_decay=args.weight_decay, nesterov=args.nesterov) - elif args.optimizer == 'adam': - optimizer = optim.Adam(params, lr=args.lr, weight_decay=args.weight_decay) - elif args.optimizer == 'adamw': - optimizer = optim.AdamW(params, lr=args.lr, weight_decay=args.weight_decay) - else: - raise ValueError('Unknown optimizer') - - scheduler = None - if args.scheduler == 'multistep': - scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.milestones, gamma=args.gamma) - elif args.scheduler == 'step': - scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma) - elif args.scheduler == 'cosine': - scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) - elif args.scheduler == 'none': - scheduler = None - - scaler = amp.GradScaler() if args.amp else None - - start_epoch = args.start_epoch - best_acc = 0.0 - if args.resume: - if os.path.isfile(args.resume): - ckpt = torch.load(args.resume, map_location='cpu') - model_state = ckpt.get('state_dict', ckpt) - if isinstance(model, DDP): - model.module.load_state_dict(model_state) - else: - model.load_state_dict(model_state) - if 'optimizer' in ckpt: - optimizer.load_state_dict(ckpt['optimizer']) - start_epoch = ckpt.get('epoch', start_epoch) - best_acc = ckpt.get('best_acc', best_acc) - print(f"=> resumed from {args.resume}, start_epoch={start_epoch}") - else: - print(f"=> resume path {args.resume} not found") - - log_columns = ['epoch', 'lr', 'loss', 'acc1', 'acc5', 'val_loss', 'val_acc1', 'val_acc5'] - log_df = pd.DataFrame(columns=log_columns) - # step-level log - step_log_columns = ['epoch', 'batch_idx', 'global_step', 'lr', 'loss', 'loss_avg', 'acc1', 'acc1_avg', 'acc5', 'acc5_avg', 'time'] - step_log_df = pd.DataFrame(columns=step_log_columns) - - total_epochs = args.epochs - # global_step计数器（训练过程中跨epoch持续） - global_step = 0 - - epoch = start_epoch - # loop until either epoch criteria or step criteria met - while True: - if train_sampler is not None: - if args.seed_sampler: - train_sampler.set_epoch(epoch + args.seed) - else: - train_sampler.set_epoch(epoch) - - if rank == 0: - print(f"==== Epoch {epoch}/{total_epochs - 1} ====") - - # 如果传入了 args.steps (>0)，则把剩余允许的 step 数传给 train_one_epoch， - # 否则 max_global_steps=None（按整 epoch 执行完） - if args.steps and args.steps > 0: - max_global_steps = args.steps - else: - max_global_steps = None - - train_log, step_logs, global_step = train_one_epoch( - args, epoch, model, criterion, optimizer, train_loader, device, scaler, - scheduler, train_sampler, global_step_start=global_step, max_global_steps=max_global_steps - ) - - # 如果启用了按 steps 的模式且已经达到上限，直接退出 main（跳过 validate） - if max_global_steps is not None and global_step >= max_global_steps: - if rank == 0: - print(f"[Main] 达到 max_global_steps={max_global_steps}（global_step={global_step}），提前退出训练（跳过验证）。") - # 直接返回 main()，不再执行后续 validate / 保存逻辑 - return - - # 验证并记录 epoch 级别日志（如果在 step 模式下很可能在中间某个 epoch 提前结束，但我们仍做一次 validate） - val_log = validate(args, model, val_loader, criterion, device) - current_lr = optimizer.param_groups[0]['lr'] - - if rank == 0: - # epoch summary print, 格式与示例对齐 - print(f"Epoch[{epoch}]: epoch_train_loss {train_log['loss']:.4f} acc1 {train_log['acc1']:.2f} acc5 {train_log['acc5']:.2f} | " - f"val_loss {val_log['loss']:.4f} acc1 {val_log['acc1']:.2f} acc5 {val_log['acc5']:.2f} lr {current_lr:.6f}") - row = { - 'epoch': epoch, - 'lr': current_lr, - 'loss': train_log['loss'], - 'acc1': train_log['acc1'], - 'acc5': train_log['acc5'], - 'val_loss': val_log['loss'], - 'val_acc1': val_log['acc1'], - 'val_acc5': val_log['acc5'], - } - new_row_df = pd.DataFrame([row]) - log_df = pd.concat([log_df, new_row_df], ignore_index=True) - log_df.to_csv(os.path.join(save_dir, 'log.csv'), index=False) - - is_best = val_log['acc1'] > best_acc - if is_best: - best_acc = val_log['acc1'] - if (epoch % args.save_freq == 0) or is_best or ( (max_global_steps is None) and (epoch == total_epochs - 1) ) : - state = { - 'epoch': epoch, - 'state_dict': model.module.state_dict() if isinstance(model, DDP) else model.state_dict(), - 'best_acc': best_acc, - 'optimizer': optimizer.state_dict(), - 'args': vars(args) - } - save_checkpoint(state, is_best, save_dir, filename=f'checkpoint_epoch_{epoch}.pth') - - # increment epoch - epoch += 1 - - # stopping conditions: - # 1) if steps mode enabled and reached steps -> stop - if args.steps and args.steps > 0: - if global_step >= args.steps: - if rank == 0: - print(f"[Main] 已达到指定 steps={args.steps}（global_step={global_step}），训练结束。") - break - - # 2) if steps not used, stop when epoch >= epochs - else: - if epoch >= total_epochs: - if rank == 0: - print(f"[Main] 已达到指定 epochs={total_epochs}（epoch={epoch}），训练结束。") - break - - if dist.is_initialized(): - dist.barrier() - if rank == 0: - print("Training finished. Best val acc1: {:.2f}".format(best_acc)) - -if __name__ == '__main__': - main() \ No newline at end of file From 8a5eda9646d95ec66184852b936137fab0726be1 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 8 Jan 2026 10:31:16 +0000 Subject: [PATCH 4/4] fix: rename files and update code --- PyTorch/build-in/Classification/VovNet/{readme => readme.md} | 0 .../VovNet/{requirements_exact.txt => requirements.txt} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename PyTorch/build-in/Classification/VovNet/{readme => readme.md} (100%) rename PyTorch/build-in/Classification/VovNet/{requirements_exact.txt => requirements.txt} (100%) diff --git a/PyTorch/build-in/Classification/VovNet/readme b/PyTorch/build-in/Classification/VovNet/readme.md similarity index 100% rename from PyTorch/build-in/Classification/VovNet/readme rename to PyTorch/build-in/Classification/VovNet/readme.md diff --git a/PyTorch/build-in/Classification/VovNet/requirements_exact.txt b/PyTorch/build-in/Classification/VovNet/requirements.txt similarity index 100% rename from PyTorch/build-in/Classification/VovNet/requirements_exact.txt rename to PyTorch/build-in/Classification/VovNet/requirements.txt