From 2a8612ef87bd89ad38d4479752766138e7fd3857 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 9 Jan 2026 08:59:47 +0000 Subject: [PATCH] add TResNet --- .../Classification/TResNet/models/__init__.py | 0 .../TResNet/models/tresnet/__init__.py | 1 + .../models/tresnet/layers/anti_aliasing.py | 62 ++ .../TResNet/models/tresnet/layers/avg_pool.py | 19 + .../models/tresnet/layers/space_to_depth.py | 53 ++ .../tresnet/layers/squeeze_and_excite.py | 39 + .../TResNet/models/tresnet/tresnet.py | 226 ++++++ .../build-in/Classification/TResNet/readme.md | 65 ++ .../Classification/TResNet/requirements.txt | 72 ++ .../Classification/TResNet/tresnet.py | 23 + .../Classification/TResNet/weloTrainStep.py | 748 ++++++++++++++++++ 11 files changed, 1308 insertions(+) create mode 100644 PyTorch/build-in/Classification/TResNet/models/__init__.py create mode 100644 PyTorch/build-in/Classification/TResNet/models/tresnet/__init__.py create mode 100644 PyTorch/build-in/Classification/TResNet/models/tresnet/layers/anti_aliasing.py create mode 100644 PyTorch/build-in/Classification/TResNet/models/tresnet/layers/avg_pool.py create mode 100644 PyTorch/build-in/Classification/TResNet/models/tresnet/layers/space_to_depth.py create mode 100644 PyTorch/build-in/Classification/TResNet/models/tresnet/layers/squeeze_and_excite.py create mode 100644 PyTorch/build-in/Classification/TResNet/models/tresnet/tresnet.py create mode 100644 PyTorch/build-in/Classification/TResNet/readme.md create mode 100644 PyTorch/build-in/Classification/TResNet/requirements.txt create mode 100644 PyTorch/build-in/Classification/TResNet/tresnet.py create mode 100644 PyTorch/build-in/Classification/TResNet/weloTrainStep.py diff --git a/PyTorch/build-in/Classification/TResNet/models/__init__.py b/PyTorch/build-in/Classification/TResNet/models/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/PyTorch/build-in/Classification/TResNet/models/tresnet/__init__.py b/PyTorch/build-in/Classification/TResNet/models/tresnet/__init__.py new file mode 100644 index 000000000..f637efe25 --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/models/tresnet/__init__.py @@ -0,0 +1 @@ +from .tresnet import TResnetM, TResnetL, TResnetXL diff --git a/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/anti_aliasing.py b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/anti_aliasing.py new file mode 100644 index 000000000..d83aae7d2 --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/anti_aliasing.py @@ -0,0 +1,62 @@ +import torch +import torch.nn.parallel +import numpy as np +import torch.nn as nn +import torch.nn.functional as F + + +class AntiAliasDownsampleLayer(nn.Module): + def __init__(self, remove_aa_jit: bool = False, filt_size: int = 3, stride: int = 2, + channels: int = 0): + super(AntiAliasDownsampleLayer, self).__init__() + if not remove_aa_jit: + self.op = DownsampleJIT(filt_size, stride, channels) + else: + self.op = Downsample(filt_size, stride, channels) + + def forward(self, x): + return self.op(x) + + +@torch.jit.script +class DownsampleJIT(object): + def __init__(self, filt_size: int = 3, stride: int = 2, channels: int = 0): + self.stride = stride + self.filt_size = filt_size + self.channels = channels + + assert self.filt_size == 3 + assert stride == 2 + a = torch.tensor([1., 2., 1.]) + + filt = (a[:, None] * a[None, :]).clone().detach() + filt = filt / torch.sum(filt) + self.filt = filt[None, None, :, :].repeat((self.channels, 1, 1, 1)).cuda().half() + + def __call__(self, input: torch.Tensor): + if input.dtype != self.filt.dtype: + self.filt = self.filt.float() + input_pad = F.pad(input, (1, 1, 1, 1), 'reflect') + return F.conv2d(input_pad, self.filt, stride=2, padding=0, groups=input.shape[1]) + + +class Downsample(nn.Module): + def __init__(self, filt_size=3, stride=2, channels=None): + super(Downsample, self).__init__() + self.filt_size = filt_size + self.stride = stride + self.channels = channels + + + assert self.filt_size == 3 + a = torch.tensor([1., 2., 1.]) + + filt = (a[:, None] * a[None, :]) + filt = filt / torch.sum(filt) + + # self.filt = filt[None, None, :, :].repeat((self.channels, 1, 1, 1)) + self.register_buffer('filt', filt[None, None, :, :].repeat((self.channels, 1, 1, 1))) + + def forward(self, input): + input_pad = F.pad(input, (1, 1, 1, 1), 'reflect') + return F.conv2d(input_pad, self.filt, stride=self.stride, padding=0, groups=input.shape[1]) diff --git a/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/avg_pool.py b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/avg_pool.py new file mode 100644 index 000000000..bd8c88bef --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/avg_pool.py @@ -0,0 +1,19 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + + +class FastGlobalAvgPool2d(nn.Module): + def __init__(self, flatten=False): + super(FastGlobalAvgPool2d, self).__init__() + self.flatten = flatten + + def forward(self, x): + if self.flatten: + in_size = x.size() + return x.view((in_size[0], in_size[1], -1)).mean(dim=2) + else: + return x.view(x.size(0), x.size(1), -1).mean(-1).view(x.size(0), x.size(1), 1, 1) + + diff --git a/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/space_to_depth.py b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/space_to_depth.py new file mode 100644 index 000000000..70bf7db94 --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/space_to_depth.py @@ -0,0 +1,53 @@ +import torch +import torch.nn as nn + + +class SpaceToDepth(nn.Module): + def __init__(self, block_size=4): + super().__init__() + assert block_size == 4 + self.bs = block_size + + def forward(self, x): + N, C, H, W = x.size() + x = x.view(N, C, H // self.bs, self.bs, W // self.bs, self.bs) # (N, C, H//bs, bs, W//bs, bs) + x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # (N, bs, bs, C, H//bs, W//bs) + x = x.view(N, C * (self.bs ** 2), H // self.bs, W // self.bs) # (N, C*bs^2, H//bs, W//bs) + return x + + +@torch.jit.script +class SpaceToDepthJit(object): + def __call__(self, x: torch.Tensor): + # assuming hard-coded that block_size==4 for acceleration + N, C, H, W = x.size() + x = x.view(N, C, H // 4, 4, W // 4, 4) # (N, C, H//bs, bs, W//bs, bs) + x = x.permute(0, 3, 5, 1, 2, 4).contiguous() # (N, bs, bs, C, H//bs, W//bs) + x = x.view(N, C * 16, H // 4, W // 4) # (N, C*bs^2, H//bs, W//bs) + return x + + +class SpaceToDepthModule(nn.Module): + def __init__(self, remove_model_jit=False): + super().__init__() + if not remove_model_jit: + self.op = SpaceToDepthJit() + else: + self.op = SpaceToDepth() + + def forward(self, x): + return self.op(x) + + +class DepthToSpace(nn.Module): + + def __init__(self, block_size): + super().__init__() + self.bs = block_size + + def forward(self, x): + N, C, H, W = x.size() + x = x.view(N, self.bs, self.bs, C // (self.bs ** 2), H, W) # (N, bs, bs, C//bs^2, H, W) + x = x.permute(0, 3, 4, 1, 5, 2).contiguous() # (N, C//bs^2, H, bs, W, bs) + x = x.view(N, C // (self.bs ** 2), H * self.bs, W * self.bs) # (N, C//bs^2, H * bs, W * bs) + return x \ No newline at end of file diff --git a/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/squeeze_and_excite.py b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/squeeze_and_excite.py new file mode 100644 index 000000000..e67dc63a4 --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/models/tresnet/layers/squeeze_and_excite.py @@ -0,0 +1,39 @@ +import torch.nn as nn +import torch.nn.functional as F +from models.tresnet.layers.avg_pool import FastGlobalAvgPool2d + + +class Flatten(nn.Module): + def forward(self, x): + return x.view(x.size(0), -1) + + +class SEModule(nn.Module): + + def __init__(self, channels, reduction_channels, inplace=True): + super(SEModule, self).__init__() + self.avg_pool = FastGlobalAvgPool2d() + self.fc1 = nn.Conv2d(channels, reduction_channels, kernel_size=1, padding=0, bias=True) + self.relu = nn.ReLU(inplace=inplace) + self.fc2 = nn.Conv2d(reduction_channels, channels, kernel_size=1, padding=0, bias=True) + # self.activation = hard_sigmoid(inplace=inplace) + self.activation = nn.Sigmoid() + + def forward(self, x): + x_se = self.avg_pool(x) + x_se2 = self.fc1(x_se) + x_se2 = self.relu(x_se2) + x_se = self.fc2(x_se2) + x_se = self.activation(x_se) + return x * x_se + +class hard_sigmoid(nn.Module): + def __init__(self, inplace=True): + super(hard_sigmoid, self).__init__() + self.inplace = inplace + + def forward(self, x): + if self.inplace: + return x.add_(3.).clamp_(0., 6.).div_(6.) + else: + return F.relu6(x + 3.) / 6. \ No newline at end of file diff --git a/PyTorch/build-in/Classification/TResNet/models/tresnet/tresnet.py b/PyTorch/build-in/Classification/TResNet/models/tresnet/tresnet.py new file mode 100644 index 000000000..085d08df7 --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/models/tresnet/tresnet.py @@ -0,0 +1,226 @@ +from functools import partial + +import torch +import torch.nn as nn +from collections import OrderedDict +from models.tresnet.layers.anti_aliasing import AntiAliasDownsampleLayer +from .layers.avg_pool import FastGlobalAvgPool2d +from .layers.squeeze_and_excite import SEModule +from models.tresnet.layers.space_to_depth import SpaceToDepthModule +from inplace_abn import InPlaceABN + + +def IABN2Float(module: nn.Module) -> nn.Module: + "If `module` is IABN don't use half precision." + if isinstance(module, InPlaceABN): + module.float() + for child in module.children(): IABN2Float(child) + return module + + +def conv2d_ABN(ni, nf, stride, activation="leaky_relu", kernel_size=3, activation_param=1e-2, groups=1): + return nn.Sequential( + nn.Conv2d(ni, nf, kernel_size=kernel_size, stride=stride, padding=kernel_size // 2, groups=groups, + bias=False), + InPlaceABN(num_features=nf, activation=activation, activation_param=activation_param) + ) + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, inplanes, planes, stride=1, downsample=None, use_se=True, anti_alias_layer=None): + super(BasicBlock, self).__init__() + if stride == 1: + self.conv1 = conv2d_ABN(inplanes, planes, stride=1, activation_param=1e-3) + else: + if anti_alias_layer is None: + self.conv1 = conv2d_ABN(inplanes, planes, stride=2, activation_param=1e-3) + else: + self.conv1 = nn.Sequential(conv2d_ABN(inplanes, planes, stride=1, activation_param=1e-3), + anti_alias_layer(channels=planes, filt_size=3, stride=2)) + + self.conv2 = conv2d_ABN(planes, planes, stride=1, activation="identity") + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + reduce_layer_planes = max(planes * self.expansion // 4, 64) + self.se = SEModule(planes * self.expansion, reduce_layer_planes) if use_se else None + + def forward(self, x): + if self.downsample is not None: + residual = self.downsample(x) + else: + residual = x + + out = self.conv1(x) + out = self.conv2(out) + + if self.se is not None: out = self.se(out) + + out += residual + + out = self.relu(out) + + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1, downsample=None, use_se=True, anti_alias_layer=None): + super(Bottleneck, self).__init__() + self.conv1 = conv2d_ABN(inplanes, planes, kernel_size=1, stride=1, activation="leaky_relu", + activation_param=1e-3) + if stride == 1: + self.conv2 = conv2d_ABN(planes, planes, kernel_size=3, stride=1, activation="leaky_relu", + activation_param=1e-3) + else: + if anti_alias_layer is None: + self.conv2 = conv2d_ABN(planes, planes, kernel_size=3, stride=2, activation="leaky_relu", + activation_param=1e-3) + else: + self.conv2 = nn.Sequential(conv2d_ABN(planes, planes, kernel_size=3, stride=1, + activation="leaky_relu", activation_param=1e-3), + anti_alias_layer(channels=planes, filt_size=3, stride=2)) + + self.conv3 = conv2d_ABN(planes, planes * self.expansion, kernel_size=1, stride=1, + activation="identity") + + self.relu = nn.ReLU(inplace=True) + self.downsample = downsample + self.stride = stride + + reduce_layer_planes = max(planes * self.expansion // 8, 64) + self.se = SEModule(planes, reduce_layer_planes) if use_se else None + + def forward(self, x): + if self.downsample is not None: + residual = self.downsample(x) + else: + residual = x + + out = self.conv1(x) + out = self.conv2(out) + if self.se is not None: out = self.se(out) + + out = self.conv3(out) + out = out + residual # no inplace + out = self.relu(out) + + return out + + +class TResNet(nn.Module): + + def __init__(self, layers, in_chans=3, num_classes=1000, width_factor=1.0, remove_aa_jit=False): + super(TResNet, self).__init__() + + # JIT layers + space_to_depth = SpaceToDepthModule() + anti_alias_layer = partial(AntiAliasDownsampleLayer, remove_aa_jit=remove_aa_jit) + global_pool_layer = FastGlobalAvgPool2d(flatten=True) + + # TResnet stages + self.inplanes = int(64 * width_factor) + self.planes = int(64 * width_factor) + conv1 = conv2d_ABN(in_chans * 16, self.planes, stride=1, kernel_size=3) + layer1 = self._make_layer(BasicBlock, self.planes, layers[0], stride=1, use_se=True, + anti_alias_layer=anti_alias_layer) # 56x56 + layer2 = self._make_layer(BasicBlock, self.planes * 2, layers[1], stride=2, use_se=True, + anti_alias_layer=anti_alias_layer) # 28x28 + layer3 = self._make_layer(Bottleneck, self.planes * 4, layers[2], stride=2, use_se=True, + anti_alias_layer=anti_alias_layer) # 14x14 + layer4 = self._make_layer(Bottleneck, self.planes * 8, layers[3], stride=2, use_se=False, + anti_alias_layer=anti_alias_layer) # 7x7 + + # body + self.body = nn.Sequential(OrderedDict([ + ('SpaceToDepth', space_to_depth), + ('conv1', conv1), + ('layer1', layer1), + ('layer2', layer2), + ('layer3', layer3), + ('layer4', layer4)])) + + # head + self.embeddings = [] + self.global_pool = nn.Sequential(OrderedDict([('global_pool_layer', global_pool_layer)])) + self.num_features = (self.planes * 8) * Bottleneck.expansion + fc = nn.Linear(self.num_features, num_classes) + self.head = nn.Sequential(OrderedDict([('fc', fc)])) + + # model initilization + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='leaky_relu') + elif isinstance(m, nn.BatchNorm2d) or isinstance(m, InPlaceABN): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # residual connections special initialization + for m in self.modules(): + if isinstance(m, BasicBlock): + m.conv2[1].weight = nn.Parameter(torch.zeros_like(m.conv2[1].weight)) # BN to zero + if isinstance(m, Bottleneck): + m.conv3[1].weight = nn.Parameter(torch.zeros_like(m.conv3[1].weight)) # BN to zero + if isinstance(m, nn.Linear): m.weight.data.normal_(0, 0.01) + + def _make_layer(self, block, planes, blocks, stride=1, use_se=True, anti_alias_layer=None): + downsample = None + if stride != 1 or self.inplanes != planes * block.expansion: + layers = [] + if stride == 2: + # avg pooling before 1x1 conv + layers.append(nn.AvgPool2d(kernel_size=2, stride=2, ceil_mode=True, count_include_pad=False)) + layers += [conv2d_ABN(self.inplanes, planes * block.expansion, kernel_size=1, stride=1, + activation="identity")] + downsample = nn.Sequential(*layers) + + layers = [] + layers.append(block(self.inplanes, planes, stride, downsample, use_se=use_se, + anti_alias_layer=anti_alias_layer)) + self.inplanes = planes * block.expansion + for i in range(1, blocks): layers.append( + block(self.inplanes, planes, use_se=use_se, anti_alias_layer=anti_alias_layer)) + return nn.Sequential(*layers) + + def forward(self, x): + x = self.body(x) + self.embeddings = self.global_pool(x) + logits = self.head(self.embeddings) + return logits + + +def TResnetM(model_params): + """ Constructs a medium TResnet model. + """ + in_chans = 3 + num_classes = model_params['num_classes'] + remove_aa_jit = model_params['remove_aa_jit'] + model = TResNet(layers=[3, 4, 11, 3], num_classes=num_classes, in_chans=in_chans, + remove_aa_jit=remove_aa_jit) + return model + + +def TResnetL(model_params): + """ Constructs a large TResnet model. + """ + in_chans = 3 + num_classes = model_params['num_classes'] + remove_aa_jit = model_params['remove_aa_jit'] + model = TResNet(layers=[4, 5, 18, 3], num_classes=num_classes, in_chans=in_chans, width_factor=1.2, + remove_aa_jit=remove_aa_jit) + return model + + +def TResnetXL(model_params): + """ Constructs an extra-large TResnet model. + """ + in_chans = 3 + num_classes = model_params['num_classes'] + remove_aa_jit = model_params['remove_aa_jit'] + model = TResNet(layers=[4, 5, 24, 3], num_classes=num_classes, in_chans=in_chans, width_factor=1.3, + remove_aa_jit=remove_aa_jit) + + return model diff --git a/PyTorch/build-in/Classification/TResNet/readme.md b/PyTorch/build-in/Classification/TResNet/readme.md new file mode 100644 index 000000000..d65d1d4ad --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/readme.md @@ -0,0 +1,65 @@ +```markdown +## 1. 模型链接 +- 原始仓库链接： +https://github.com/huggingface/pytorch-image-models?tab=readme-ov-file#models + +## 2. 快速开始 + +使用本模型执行训练的主要流程如下： + +1. **基础环境安装**：介绍训练前需要完成的基础环境检查和安装。 +2. **获取数据集**：介绍如何获取训练所需的数据集。 +3. **构建环境**：介绍如何构建模型运行所需要的环境。 +4. **启动训练**：介绍如何运行训练。 + +### 2.1 基础环境安装 + +请参考主仓库的基础环境安装章节，完成训练前的基础环境检查和安装（如驱动、固件等）。 + +### 2.2 准备数据集 + +#### 2.2.1 获取数据集 + +训练使用 **CIFAR-100** 数据集。该数据集为开源数据集，包含 100 个类别的 60000 张彩色图像。 + +#### 2.2.2 处理数据集 + +请确保数据集已下载并解压。根据训练脚本的默认配置，建议将数据集存放在模型目录的上级 `data` 目录中（即 `../data`），或者根据实际路径修改训练命令中的 `--datapath` 参数。 + +### 2.3 构建环境 + +所使用的环境下需包含 PyTorch 框架虚拟环境。 + +1. 执行以下命令，启动虚拟环境（根据实际环境名称修改）： + + ```bash + conda activate torch_env_py310 + +``` + +2. 安装 Python 依赖。确保已安装项目所需的依赖包： +```bash +pip install -r requirements.txt + +``` + + + +### 2.4 启动训练 + +1. 在构建好的环境中，进入模型训练脚本所在目录。 + +2. 运行训练。该模型支持单机单卡训练。 +执行以下命令启动训练（使用 CIFAR-100 数据集，Batch Size 为 128）： +```bash +python weloTrainStep.py \ + --name train \ + --arch tresnet \ + --print_freq 1 \ + --steps 100 \ + --dataset cifar100 \ + --datapath ../data \ + --batch_size 32 \ + --epochs 100 + +``` diff --git a/PyTorch/build-in/Classification/TResNet/requirements.txt b/PyTorch/build-in/Classification/TResNet/requirements.txt new file mode 100644 index 000000000..23ac8e3c4 --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/requirements.txt @@ -0,0 +1,72 @@ +addict==2.4.0 +aliyun-python-sdk-core==2.16.0 +aliyun-python-sdk-kms==2.16.5 +anyio==4.11.0 +astunparse==1.6.3 +certifi==2024.12.14 +cffi==2.0.0 +charset-normalizer==3.4.1 +click==8.3.1 +colorama==0.4.6 +contourpy==1.3.2 +crcmod==1.7 +cryptography==46.0.3 +cycler==0.12.1 +einops==0.8.1 +exceptiongroup==1.3.1 +filelock==3.14.0 +fonttools==4.60.1 +fsspec==2024.12.0 +git-filter-repo==2.47.0 +h11==0.16.0 +hf-xet==1.2.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.1.5 +idna==3.10 +Jinja2==3.1.5 +jmespath==0.10.0 +joblib==1.5.2 +kiwisolver==1.4.9 +Markdown==3.10 +markdown-it-py==4.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.7 +mdurl==0.1.2 +model-index==0.1.11 +mpmath==1.3.0 +networkx==3.4.2 +numpy +opendatalab==0.0.10 +openxlab==0.1.3 +ordered-set==4.1.0 +oss2==2.17.0 +pandas==2.3.3 +pillow==11.1.0 +platformdirs==4.5.1 +pycocotools==2.0.11 +pycryptodome==3.23.0 +Pygments==2.19.2 +pyparsing==3.2.5 +python-dateutil==2.9.0.post0 +pytz==2023.4 +requests==2.28.2 +rich==13.4.2 +safetensors==0.7.0 +scipy==1.15.3 +shapely==2.1.2 +shellingham==1.5.4 +sniffio==1.3.1 +sympy==1.13.3 +tabulate==0.9.0 +termcolor==3.2.0 +terminaltables==3.1.10 +threadpoolctl==3.6.0 +timm==1.0.22 +tomli==2.3.0 +tqdm==4.65.2 +typer-slim==0.20.0 +typing_extensions==4.15.0 +tzdata==2025.2 +urllib3==1.26.20 +yapf==0.43.0 diff --git a/PyTorch/build-in/Classification/TResNet/tresnet.py b/PyTorch/build-in/Classification/TResNet/tresnet.py new file mode 100644 index 000000000..77cec464c --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/tresnet.py @@ -0,0 +1,23 @@ +import models.tresnet as tresnet + +# --- replace the bottom part with this: --- + +def Model(num_classes=1000, variant='M', remove_aa_jit=False): + """ + Simple factory returning a TResNet variant defined in this file. + variant: 'M' (medium), 'L' (large), 'XL' (extra-large) + """ + model_params = {'num_classes': num_classes, 'remove_aa_jit': remove_aa_jit} + if variant == 'M': + return tresnet.TResnetM(model_params) + elif variant == 'L': + return tresnet.TResnetL(model_params) + elif variant == 'XL': + return tresnet.TResnetXL(model_params) + else: + raise ValueError(f"Unknown variant: {variant}") + +# 测试 +if __name__ == "__main__": + m = Model(num_classes=100, variant='M') + print(m) \ No newline at end of file diff --git a/PyTorch/build-in/Classification/TResNet/weloTrainStep.py b/PyTorch/build-in/Classification/TResNet/weloTrainStep.py new file mode 100644 index 000000000..dda3ae604 --- /dev/null +++ b/PyTorch/build-in/Classification/TResNet/weloTrainStep.py @@ -0,0 +1,748 @@ +#!/usr/bin/env python3 +# coding: utf-8 + +import os +import random +import sys +import time +import json +import argparse +from collections import OrderedDict +from pathlib import Path +import numpy as np +import pandas as pd +from tqdm import tqdm +import importlib +from inplace_abn import InPlaceABN + +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" # 强烈推荐在 shell/最顶端设置 +os.environ["PYTHONHASHSEED"] = "12345" +os.environ["OMP_NUM_THREADS"] = "1" +os.environ["MKL_NUM_THREADS"] = "1" + +def ensure_cublas_workspace(config=":4096:8"): + """ + 尝试为 cuBLAS 设置可复现 workspace。强烈建议在主脚本入口处（import torch 之前） + 通过 export 设置该 env。此函数会在运行时设置，但如果 torch 已经被 import， + 则可能为时已晚——函数会打印提醒。 + """ + already = os.environ.get("CUBLAS_WORKSPACE_CONFIG") + if already: + print(f"[seed_utils] CUBLAS_WORKSPACE_CONFIG 已存在：{already}") + else: + os.environ["CUBLAS_WORKSPACE_CONFIG"] = config + print(f"[seed_utils] 已设置 CUBLAS_WORKSPACE_CONFIG={config} （注意：请在 import torch 前设置以保证生效）") + +def set_global_seed(seed: int = 42, set_threads: bool = True): + """ + 统一随机性设置。注意：若希望完全发挥效果，请在主脚本入口（import torch 之前） + 先调用 ensure_cublas_workspace(...) 或在 shell 中 export CUBLAS_WORKSPACE_CONFIG。 + """ + ensure_cublas_workspace() # 会设置 env 并提醒 + os.environ["PYTHONHASHSEED"] = str(seed) + + if set_threads: + os.environ["OMP_NUM_THREADS"] = "1" + os.environ["MKL_NUM_THREADS"] = "1" + + random.seed(seed) + np.random.seed(seed) + + # 现在导入 torch（晚导入以便前面 env 生效） + import torch + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + # 强制确定性（如果存在不确定性算子，PyTorch 会报错并提示） + try: + torch.use_deterministic_algorithms(True) + except Exception as e: + print("[seed_utils] 设置 deterministic 模式时出错：", e) + print("[seed_utils] 请确认 CUBLAS_WORKSPACE_CONFIG 已在 import torch 之前设置。") + + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + + if set_threads: + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + + print(f"[seed_utils] 全局 seed 已设置为 {seed}") + +set_global_seed(2025) + +""" +通用训练模版（优先从本地导入 Model -> 支持 DDP / 单卡，AMP，resume，日志，checkpoint） +保存为 train_template_localmodel.py +""" +import torch +import torch.nn as nn +import torch.optim as optim +import torch.backends.cudnn as cudnn +import torchvision.transforms as transforms +import torchvision.datasets as datasets +import torchvision.models as tv_models + +import torch.distributed as dist +from torch.nn.parallel import DistributedDataParallel as DDP +from torch.utils.data import DataLoader +from torch.utils.data.distributed import DistributedSampler + +from torch.sdaa import amp +# from torch.cuda import amp + + +# ---------------------------- +# Helper utilities (self-contained) +# ---------------------------- +class AverageMeter(object): + def __init__(self, name='Meter', fmt=':.4f'): + self.name = name + self.fmt = fmt + self.reset() + def reset(self): + self.val = 0 + self.avg = 0 + self.sum = 0 + self.count = 0 + def update(self, val, n=1): + self.val = val + self.sum += val * n + self.count += n + self.avg = self.sum / max(1, self.count) + def __str__(self): + fmtstr = '{name} {val' + self.fmt + '} (avg {avg' + self.fmt + '})' + return fmtstr.format(name=self.name, val=self.val, avg=self.avg) + +def accuracy(output, target, topk=(1,)): + """Computes the precision@k for the specified values of k + 返回一个 list，每个元素是 tensor（百分比形式） + """ + with torch.no_grad(): + maxk = max(topk) + batch_size = target.size(0) + + # output: (N, C) -> pred: (maxk, N) + _, pred = output.topk(maxk, 1, True, True) + pred = pred.t() # (maxk, N) + correct = pred.eq(target.view(1, -1).expand_as(pred)) # (maxk, N) bool + + res = [] + for k in topk: + # 把前 k 行展平后求和（返回 0-dim tensor），随后换算为百分比 + correct_k = correct[:k].reshape(-1).float().sum() # 注意：不传 keepdim + # 乘以 100.0 / batch_size，保持返回 tensor（和之前代码兼容） + res.append(correct_k.mul_(100.0 / batch_size)) + return res + +def save_checkpoint(state, is_best, save_dir, filename='checkpoint.pth'): + save_path = os.path.join(save_dir, filename) + torch.save(state, save_path) + if is_best: + best_path = os.path.join(save_dir, 'model_best.pth') + torch.save(state, best_path) + +def set_seed(seed, deterministic=False): + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + if deterministic: + cudnn.deterministic = True + cudnn.benchmark = False + else: + cudnn.deterministic = False + cudnn.benchmark = True + +# ---------------------------- +# Argument parser +# ---------------------------- +def parse_args(): + parser = argparse.ArgumentParser(description='Generic PyTorch training template (DDP/AMP) with LocalModel priority') + parser.add_argument('--name', default='run', type=str, help='experiment name (log/checkpoints dir)') + parser.add_argument('--seed', default=42, type=int, help='random seed') + parser.add_argument('--arch', default='None', type=str, help='model name') + parser.add_argument('--deterministic', action='store_true', help='set cudnn deterministic (may be slower)') + parser.add_argument('--dataset', default='cifar10', choices=['cifar10','cifar100','imagenet','custom'], help='which dataset') + parser.add_argument('--datapath', default='./data', type=str, help='dataset root / imagenet root / custom root') + parser.add_argument('--imagenet_dir', default='./imagenet', type=str, help='if dataset=imagenet, path to imagenet root') + parser.add_argument('--custom_eval_dir', default=None, help='if dataset=custom, provide val dir') + parser.add_argument('--num_workers', default=4, type=int, help='dataloader workers per process') + parser.add_argument('--epochs', default=200, type=int) + parser.add_argument('--steps', default=0, type=int, help='max steps to run (if >0, training will stop when global_step reaches this).') + parser.add_argument('--batch_size', default=128, type=int) + parser.add_argument('--model_name', default='resnet18', help='torchvision model name or python path e.g. mypkg.mymodule.Model (used if no local Model)') + parser.add_argument('--num_classes', default=None, type=int, help='override num classes (auto-detect for common sets)') + parser.add_argument('--pretrained', action='store_true', help='use torchvision pretrained weights when available') + parser.add_argument('--optimizer', default='sgd', choices=['sgd','adam','adamw'], help='optimizer') + parser.add_argument('--lr', '--learning_rate', default=0.1, type=float) + parser.add_argument('--momentum', default=0.9, type=float) + parser.add_argument('--weight_decay', default=5e-4, type=float) + parser.add_argument('--nesterov', action='store_true') + parser.add_argument('--scheduler', default='multistep', choices=['multistep','step','cosine','none'], help='lr scheduler') + parser.add_argument('--milestones', default='100,150', type=str, help='milestones for multistep (comma sep)') + parser.add_argument('--step_size', default=30, type=int, help='step size for StepLR or cosine max epochs') + parser.add_argument('--gamma', default=0.1, type=float) + parser.add_argument('--scheduler_step_per_batch', action='store_true', help='call scheduler.step() per batch (for some schedulers)') + parser.add_argument('--resume', default='', type=str, help='path to checkpoint to resume from') + parser.add_argument('--start_epoch', default=0, type=int) + parser.add_argument('--print_freq', default=100, type=int) + parser.add_argument('--save_freq', default=10, type=int, help='save checkpoint every N epochs (rank0 only)') + parser.add_argument('--amp', action='store_true', default = True,help='use automatic mixed precision (AMP)') + parser.add_argument('--grad_accum_steps', default=1, type=int, help='gradient accumulation steps') + parser.add_argument('--local_rank', default=None, type=int, help='local rank passed by torchrun (if any). Use -1 or None for non-distributed') + parser.add_argument('--cutmix_prob', default=0.0, type=float) + parser.add_argument('--beta', default=1.0, type=float) + parser.add_argument('--seed_sampler', default=False, action='store_true', help='set sampler epoch seeds to make deterministic distributed shuffling') + args = parser.parse_args() + args.milestones = [int(x) for x in args.milestones.split(',')] if args.milestones else [] + return args + +# ---------------------------- +# build model (优先 LocalModel) +# ---------------------------- +def build_model_with_local_priority(args, device=None): + """ + 用参数 args.arch 作为模块名导入 Model() + 如果模块不存在或没有 Model 类，则报错停止。 + """ + try: + # 动态导入模块，比如 args.arch = "rexnet" + mod = importlib.import_module(args.arch) + Model = getattr(mod, "Model") # 从模块中获取 Model 类 + except Exception as e: + raise RuntimeError( + f"无法导入模型模块 '{args.arch}' 或未找到类 Model。" + f"\n错误信息：{e}" + ) + + # 解析数据集类别数 + if args.dataset == 'cifar10': + num_classes = 10 + elif args.dataset == 'cifar100': + num_classes = 100 + else: + print(f"[ERROR] 不支持的数据集类型：{args.dataset}，无法确定类别数。程序终止。") + sys.exit(1) + + + # 实例化 + try: + model = Model(num_classes) + except Exception as e: + raise RuntimeError( + f"Model() 实例化失败，请检查模型构造函数。\n错误信息：{e}" + ) + + return model + +# ---------------------------- +# Data loader factory +# ---------------------------- +def build_dataloaders(args, rank, world_size): + if args.dataset == 'cifar10' or args.dataset == 'cifar100': + mean = (0.4914, 0.4822, 0.4465) + std = (0.2470, 0.2435, 0.2616) if args.dataset == 'cifar10' else (0.2023, 0.1994, 0.2010) + # train_transform = transforms.Compose([ + # transforms.RandomCrop(32, padding=4), + # transforms.RandomHorizontalFlip(), + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + # test_transform = transforms.Compose([ + # transforms.ToTensor(), + # transforms.Normalize(mean, std), + # ]) + + train_transform = transforms.Compose([ # 2025/12/3 从visformer模型开始 + transforms.Resize(256), # 先放大到 256 + transforms.RandomCrop(224), # 再随机裁剪为 224（更符合 ImageNet 风格增强） + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(mean, std), + ]) + root = args.datapath + if args.dataset == 'cifar10': + train_set = datasets.CIFAR10(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR10(root=root, train=False, download=False, transform=test_transform) + num_classes = 10 + else: + train_set = datasets.CIFAR100(root=root, train=True, download=False, transform=train_transform) + val_set = datasets.CIFAR100(root=root, train=False, download=False, transform=test_transform) + num_classes = 100 + + elif args.dataset == 'imagenet': + train_dir = os.path.join(args.imagenet_dir, 'train') + val_dir = os.path.join(args.imagenet_dir, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize((0.485,0.456,0.406), (0.229,0.224,0.225)), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = args.num_classes or 1000 + + elif args.dataset == 'custom': + train_dir = os.path.join(args.datapath, 'train') + val_dir = args.custom_eval_dir or os.path.join(args.datapath, 'val') + train_transform = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + ]) + test_transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + ]) + train_set = datasets.ImageFolder(train_dir, train_transform) + val_set = datasets.ImageFolder(val_dir, test_transform) + num_classes = len(train_set.classes) + else: + raise ValueError("Unknown dataset") + + if dist.is_initialized() and world_size > 1: + train_sampler = DistributedSampler(train_set, num_replicas=world_size, rank=rank, shuffle=True) + else: + train_sampler = None + + train_loader = DataLoader(train_set, + batch_size=args.batch_size, + shuffle=(train_sampler is None), + num_workers=args.num_workers, + pin_memory=True, + sampler=train_sampler, + drop_last=False) + val_loader = DataLoader(val_set, + batch_size=args.batch_size, + shuffle=False, + num_workers=args.num_workers, + pin_memory=True) + + return train_loader, val_loader, num_classes, train_sampler + +# ---------------------------- +# Train & validate +# ---------------------------- +def train_one_epoch(args, epoch, model, criterion, optimizer, train_loader, device, scaler, scheduler=None, train_sampler=None, global_step_start=0, max_global_steps=None): + """ + 现在支持：若 max_global_steps 非 None，则当 global_step 达到该值时提前退出 + 返回: epoch_summary_dict, step_logs_list, global_step_end + step_logs_list: list of dicts with per-step info (for logging to CSV if需要) + """ + batch_time = AverageMeter('Time') + data_time = AverageMeter('Data') + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.train() + end = time.time() + optimizer.zero_grad() + + iters = len(train_loader) + step_logs = [] + global_step = global_step_start + + for i, (images, targets) in enumerate(train_loader): + # check global steps limit + if (max_global_steps is not None) and (global_step >= max_global_steps): + break + + data_time.update(time.time() - end) + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + + if args.amp: + with amp.autocast(): + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + else: + outputs = model(images) + loss = criterion(outputs, targets) / args.grad_accum_steps + + if args.amp: + scaler.scale(loss).backward() + else: + loss.backward() + + # 每当累积步满足 grad_accum_steps 就 step + if (i + 1) % args.grad_accum_steps == 0: + if args.amp: + scaler.step(optimizer) + scaler.update() + else: + optimizer.step() + optimizer.zero_grad() + if scheduler is not None and args.scheduler_step_per_batch: + scheduler.step() + + with torch.no_grad(): + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + losses.update(loss.item() * args.grad_accum_steps, images.size(0)) + top1.update(acc1.item(), images.size(0)) + top5.update(acc5.item(), images.size(0)) + + batch_time.update(time.time() - end) + end = time.time() + + # increment global step AFTER processing this batch + global_step += 1 + + # per-step print (controlled by print_freq) + if ((global_step % args.print_freq == 0) or (i == iters - 1)) and ((dist.get_rank() if dist.is_initialized() else 0) == 0): + lr = optimizer.param_groups[0]['lr'] + print(f"Epoch[{epoch}]:step[{i+1}/{iters}] step_train_loss {losses.val:.4f} acc1 {top1.val:.2f} acc5 {top5.val:.2f}") + + # collect per-step log + step_logs.append({ + 'epoch': epoch, + 'batch_idx': i, + 'global_step': global_step, + 'lr': optimizer.param_groups[0]['lr'], + 'loss': losses.val, + 'loss_avg': losses.avg, + 'acc1': top1.val, + 'acc1_avg': top1.avg, + 'acc5': top5.val, + 'acc5_avg': top5.avg, + 'time': batch_time.val + }) + + # if reached max_global_steps inside epoch, break (handled at loop start next iter) + if (max_global_steps is not None) and (global_step >= max_global_steps): + if (dist.get_rank() if dist.is_initialized() else 0) == 0: + print(f"[Info] 达到 max_global_steps={max_global_steps}，将在 epoch 内提前停止。") + break + + # --- flush remaining grads if needed (handle gradient accumulation leftovers) --- + processed_batches = global_step - global_step_start # 实际处理的 batch 数 + if args.grad_accum_steps > 1 and (processed_batches % args.grad_accum_steps) != 0: + # only step if there are gradients + grads_present = any((p.grad is not None and p.requires_grad) for p in model.parameters()) + if grads_present: + if args.amp: + try: + scaler.step(optimizer) + scaler.update() + except Exception as e: + # 防御性：若 scaler.step 因某些原因失败，尝试普通 step（只在极端情况下） + print("[Warning] scaler.step 失败，尝试普通 optimizer.step():", e) + optimizer.step() + else: + optimizer.step() + optimizer.zero_grad() + if scheduler is not None and args.scheduler_step_per_batch: + scheduler.step() + if (dist.get_rank() if dist.is_initialized() else 0) == 0: + print(f"[Info] flushed remaining gradients after early stop (processed_batches={processed_batches}, grad_accum={args.grad_accum_steps}).") + + if scheduler is not None and not args.scheduler_step_per_batch: + scheduler.step() + + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]), step_logs, global_step + +def validate(args, model, val_loader, criterion, device, max_batches=None): + """ + Validate on the val_loader. + If max_batches is not None, only process up to that many batches (useful for quick checks). + Returns an OrderedDict with loss/acc1/acc5 (averaged over processed samples). + """ + losses = AverageMeter('Loss') + top1 = AverageMeter('Acc@1') + top5 = AverageMeter('Acc@5') + + model.eval() + processed_batches = 0 + processed_samples = 0 + with torch.no_grad(): + for i, (images, targets) in enumerate(tqdm(val_loader)): + images = images.to(device, non_blocking=True) + targets = targets.to(device, non_blocking=True) + outputs = model(images) + loss = criterion(outputs, targets) + acc1, acc5 = accuracy(outputs, targets, topk=(1,5)) + batch_n = images.size(0) + losses.update(loss.item(), batch_n) + top1.update(acc1.item(), batch_n) + top5.update(acc5.item(), batch_n) + + processed_batches += 1 + processed_samples += batch_n + + if (max_batches is not None) and (processed_batches >= max_batches): + break + + # 如果没处理任何样本，避免除0（不太可能，但防御性） + if processed_samples == 0: + return OrderedDict([('loss', 0.0), ('acc1', 0.0), ('acc5', 0.0)]) + return OrderedDict([('loss', losses.avg), ('acc1', top1.avg), ('acc5', top5.avg)]) + +# ---------------------------- +# Main +# ---------------------------- +def main(): + args = parse_args() + + # handle local_rank from env if not provided + local_rank_env = os.environ.get('LOCAL_RANK', None) + if args.local_rank is None and local_rank_env is not None: + args.local_rank = int(local_rank_env) + + distributed = (args.local_rank is not None and args.local_rank != -1) + if distributed: + dist.init_process_group(backend='nccl', init_method='env://') + rank = dist.get_rank() + world_size = dist.get_world_size() + else: + rank = 0 + world_size = 1 + + if distributed: + torch.cuda.set_device(args.local_rank) + device = torch.device('cuda', args.local_rank) + else: + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + + set_seed(args.seed + (rank if distributed else 0), deterministic=args.deterministic) + + save_dir = os.path.join('models', args.name) + if rank == 0: + os.makedirs(save_dir, exist_ok=True) + with open(os.path.join(save_dir, 'args.json'), 'w') as f: + json.dump(vars(args), f, indent=2) + if distributed: + dist.barrier() + + train_loader, val_loader, auto_num_classes, train_sampler = build_dataloaders(args, rank, world_size) + if args.num_classes is None: + args.num_classes = auto_num_classes + + # 使用本地 Model 优先（LocalModel 已在文件顶部尝试导入） + model = build_model_with_local_priority(args, device) + model.to(device) + + # ========================================================================== + # [终极方案] 移除 InplaceABN，替换为原生 PyTorch 层 + # 解决所有 Segfault 和驱动不兼容问题 + # ========================================================================== + if InPlaceABN is not None: + def replace_iabn_with_standard_layers(module): + for name, child in module.named_children(): + if isinstance(child, InPlaceABN): + # 1. 获取原层参数 + num_features = child.num_features + eps = child.eps + momentum = child.momentum + affine = child.affine + track_running_stats = child.track_running_stats + + # 2. 获取激活函数配置 (TResNet 通常是用 leaky_relu) + act_name = getattr(child, 'activation', 'leaky_relu') + act_param = getattr(child, 'activation_param', 0.01) + + # 3. 创建原生 BatchNorm2d + bn = nn.BatchNorm2d(num_features, eps=eps, momentum=momentum, + affine=affine, track_running_stats=track_running_stats) + + # 迁移权重 (state_dict key 通常兼容) + bn.load_state_dict(child.state_dict(), strict=False) + + # 4. 创建激活函数 (InplaceABN 是融合层，所以拆分为 BN + Act) + layers = [bn] + if act_name == 'leaky_relu': + layers.append(nn.LeakyReLU(negative_slope=act_param, inplace=True)) + elif act_name == 'relu': + layers.append(nn.ReLU(inplace=True)) + elif act_name == 'identity' or act_name == 'none': + pass # 无激活 + else: + # 默认兜底 + layers.append(nn.LeakyReLU(negative_slope=0.01, inplace=True)) + + # 5. 替换为 Sequential (BN + Act) + # 注意：将层转回当前设备 + replacement = nn.Sequential(*layers).to(device) + setattr(module, name, replacement) + + if rank == 0: + print(f" -> Replaced {name} with BatchNorm2d + {act_name}") + else: + # 递归处理 + replace_iabn_with_standard_layers(child) + + print("[Setup] 正在移除 InplaceABN 并替换为标准 PyTorch 层...") + replace_iabn_with_standard_layers(model) + if rank == 0: + print("[Setup] 替换完成。模型现在使用原生算子，硬件兼容性 100%。") + # ========================================================================== + + if distributed: + model = DDP(model, device_ids=[args.local_rank], output_device=args.local_rank, find_unused_parameters=True) + + criterion = nn.CrossEntropyLoss().to(device) + params = [p for p in model.parameters() if p.requires_grad] + if args.optimizer == 'sgd': + optimizer = optim.SGD(params, lr=args.lr, momentum=args.momentum, + weight_decay=args.weight_decay, nesterov=args.nesterov) + elif args.optimizer == 'adam': + optimizer = optim.Adam(params, lr=args.lr, weight_decay=args.weight_decay) + elif args.optimizer == 'adamw': + optimizer = optim.AdamW(params, lr=args.lr, weight_decay=args.weight_decay) + else: + raise ValueError('Unknown optimizer') + + scheduler = None + if args.scheduler == 'multistep': + scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=args.milestones, gamma=args.gamma) + elif args.scheduler == 'step': + scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=args.step_size, gamma=args.gamma) + elif args.scheduler == 'cosine': + scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=args.epochs) + elif args.scheduler == 'none': + scheduler = None + + scaler = amp.GradScaler() if args.amp else None + + start_epoch = args.start_epoch + best_acc = 0.0 + if args.resume: + if os.path.isfile(args.resume): + ckpt = torch.load(args.resume, map_location='cpu') + model_state = ckpt.get('state_dict', ckpt) + if isinstance(model, DDP): + model.module.load_state_dict(model_state) + else: + model.load_state_dict(model_state) + if 'optimizer' in ckpt: + optimizer.load_state_dict(ckpt['optimizer']) + start_epoch = ckpt.get('epoch', start_epoch) + best_acc = ckpt.get('best_acc', best_acc) + print(f"=> resumed from {args.resume}, start_epoch={start_epoch}") + else: + print(f"=> resume path {args.resume} not found") + + log_columns = ['epoch', 'lr', 'loss', 'acc1', 'acc5', 'val_loss', 'val_acc1', 'val_acc5'] + log_df = pd.DataFrame(columns=log_columns) + # step-level log + step_log_columns = ['epoch', 'batch_idx', 'global_step', 'lr', 'loss', 'loss_avg', 'acc1', 'acc1_avg', 'acc5', 'acc5_avg', 'time'] + step_log_df = pd.DataFrame(columns=step_log_columns) + + total_epochs = args.epochs + # global_step计数器（训练过程中跨epoch持续） + global_step = 0 + + epoch = start_epoch + # loop until either epoch criteria or step criteria met + while True: + if train_sampler is not None: + if args.seed_sampler: + train_sampler.set_epoch(epoch + args.seed) + else: + train_sampler.set_epoch(epoch) + + if rank == 0: + print(f"==== Epoch {epoch}/{total_epochs - 1} ====") + + # 如果传入了 args.steps (>0)，则把剩余允许的 step 数传给 train_one_epoch， + # 否则 max_global_steps=None（按整 epoch 执行完） + if args.steps and args.steps > 0: + max_global_steps = args.steps + else: + max_global_steps = None + + train_log, step_logs, global_step = train_one_epoch( + args, epoch, model, criterion, optimizer, train_loader, device, scaler, + scheduler, train_sampler, global_step_start=global_step, max_global_steps=max_global_steps + ) + + # 如果启用了按 steps 的模式且已经达到上限，标记需要在做一次验证后退出 + if max_global_steps is not None and global_step >= max_global_steps: + if rank == 0: + print(f"[Main] 达到 max_global_steps={max_global_steps}（global_step={global_step}），将在完成验证后退出训练。") + # 我们不 return 立刻退出；后面的 validate / 保存逻辑会执行一次，然后 main 返回/结束 + end_due_to_steps = True + else: + end_due_to_steps = False + + # 验证并记录 epoch 级别日志（如果在 step 模式下很可能在中间某个 epoch 提前结束，但我们仍做一次 validate） + val_log = validate(args, model, val_loader, criterion, device, args.batch_size) + current_lr = optimizer.param_groups[0]['lr'] + + if rank == 0: + # epoch summary print, 格式与示例对齐 + print(f"Epoch[{epoch}]: epoch_train_loss {train_log['loss']:.4f} acc1 {train_log['acc1']:.2f} acc5 {train_log['acc5']:.2f} | " + f"val_loss {val_log['loss']:.4f} acc1 {val_log['acc1']:.2f} acc5 {val_log['acc5']:.2f} lr {current_lr:.6f}") + row = { + 'epoch': epoch, + 'lr': current_lr, + 'loss': train_log['loss'], + 'acc1': train_log['acc1'], + 'acc5': train_log['acc5'], + 'val_loss': val_log['loss'], + 'val_acc1': val_log['acc1'], + 'val_acc5': val_log['acc5'], + } + new_row_df = pd.DataFrame([row]) + log_df = pd.concat([log_df, new_row_df], ignore_index=True) + log_df.to_csv(os.path.join(save_dir, 'log.csv'), index=False) + + is_best = val_log['acc1'] > best_acc + if is_best: + best_acc = val_log['acc1'] + if (epoch % args.save_freq == 0) or is_best or ( (max_global_steps is None) and (epoch == total_epochs - 1) ) : + state = { + 'epoch': epoch, + 'state_dict': model.module.state_dict() if isinstance(model, DDP) else model.state_dict(), + 'best_acc': best_acc, + 'optimizer': optimizer.state_dict(), + 'args': vars(args) + } + save_checkpoint(state, is_best, save_dir, filename=f'checkpoint_epoch_{epoch}.pth') + + # 如果是因为 steps 模式达到上限，则在完成 validation / 保存后退出训练 + if end_due_to_steps: + if rank == 0: + print(f"[Main] 已在 steps 模式下完成最后一次验证并保存，训练结束（global_step={global_step}）。") + break + + # increment epoch + epoch += 1 + + # stopping conditions: + # 1) if steps mode enabled and reached steps -> stop + if args.steps and args.steps > 0: + if global_step >= args.steps: + if rank == 0: + print(f"[Main] 已达到指定 steps={args.steps}（global_step={global_step}），训练结束。") + break + + # 2) if steps not used, stop when epoch >= epochs + else: + if epoch >= total_epochs: + if rank == 0: + print(f"[Main] 已达到指定 epochs={total_epochs}（epoch={epoch}），训练结束。") + break + + if dist.is_initialized(): + dist.barrier() + if rank == 0: + print("Training finished. Best val acc1: {:.2f}".format(best_acc)) + +if __name__ == '__main__': + main() \ No newline at end of file