BUG: The size of tensor mismatch

[Jingwei-Bao]

rendered_video_size: 512896
ref_image_size:7681344

Sampling with RFSampler for 51 steps: 0%| | 0/51 [00:00<?, ?it/s]
[rank0]: Traceback (most recent call last):
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sample_video.py", line 529, in
[rank0]: sampling_main(args, model_cls=Engine)
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sample_video.py", line 476, in sampling_main
[rank0]: samples_z = sample_func(
[rank0]: ^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/utils/_contextlib.py", line 120, in decorate_context
[rank0]: return func(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/diffusion_video.py", line 569, in sample
[rank0]: samples = self.sampler(denoiser, randn, cond, uc=uc)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sgm/modules/diffusionmodules/sampling.py", line 971, in call
[rank0]: x = self.sampler_step(
[rank0]: ^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sgm/modules/diffusionmodules/sampling.py", line 961, in sampler_step
[rank0]: output = self.denoise(x, denoiser, sigma, cond, uc, scale=scale, fps=fps).to(torch.float32)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sgm/modules/diffusionmodules/sampling.py", line 956, in denoise
[rank0]: denoised = denoiser(*self.guider.prepare_inputs(x, sigma, cond, uc), **additional_model_inputs).to(torch.float32)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/diffusion_video.py", line 555, in
[rank0]: denoiser = lambda input, sigma, c, **additional_model_inputs: self.denoiser(
[rank0]: ^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sgm/modules/diffusionmodules/denoiser.py", line 42, in forward
[rank0]: model_output = network(input * c_in, c_noise, cond, **additional_model_inputs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sgm/modules/diffusionmodules/wrappers.py", line 39, in forward
[rank0]: return self.diffusion_model(
[rank0]: ^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/dit_video_crossattn_sc_xc.py", line 1592, in forward
[rank0]: output = super().forward(**kwargs)[0]
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sat/model/base_model.py", line 138, in forward
[rank0]: return self.transformer(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sat/model/transformer.py", line 713, in forward
[rank0]: layer_ret = self.hooks['layer_forward'](*args,
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/dit_video_crossattn_sc_xc.py", line 1033, in layer_forward
[rank0]: attention_output = layer.attention(attention_input, mask, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1775, in _wrapped_call_impl
[rank0]: return self._call_impl(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/miniconda3/envs/scail/lib/python3.11/site-packages/torch/nn/modules/module.py", line 1786, in _call_impl
[rank0]: return forward_call(*args, **kwargs)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/sat/model/transformer.py", line 111, in forward
[rank0]: return self.hooks['attention_forward'](hidden_states, mask, **kw_args)
[rank0]: ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/dit_video_crossattn_sc_xc.py", line 1092, in attention_forward
[rank0]: context_layer = attention_fn(
[rank0]: ^^^^^^^^^^^^^
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/dit_video_crossattn_sc_xc.py", line 681, in attention_fn
[rank0]: self.rotary(
[rank0]: File "/mnt/data_nas_own/aigc/SCAIL/dit_video_crossattn_sc_xc.py", line 557, in rotary
[rank0]: return t * freqs_cos + rotate_half(t) * freqs_sin
[rank0]: ~~^~~~~~~~~~~
[rank0]: RuntimeError: The size of tensor a (120064) must match the size of tensor b (37632) at non-singleton dimension 2

[Jingwei-Bao]

rendered_video_size: 512✖️896
ref_image_size:768✖️1344

[gukush]

I've run into same error. It was due to too long duration of the rendered.mp4 video. I decided to split the video into separate parts as well as downsample the frames in each part.