Question about video model finetuning

[h6kplus]

Nice work! I really appreciate your effort on creating a unified framework.

I'm currently tring to apply these algorithms with my own reward on Wan2.1-1.3B but I'm finding difficulty even for training with rewards like pickscore. Maybe can you share some successful configs for training? Thank you so much!

And also, is there any plan for implementing some video based scores like video align? I can also try implement it and send pull requests

Again thank you for sharing your work!

[Jayce-Ping]

Hi, this is my config for Wan2.1-1.3B-Diffusers (some system info is deleted):

{
  "log": {
    "value": {
      "verbose": true,
      "save_dir": "saves",
      "save_freq": 100,
      "save_model_only": true
    }
  },
  "data": {
    "value": {
      "cache_dir": "~/.cache/flow_factory/datasets",
      "image_dir": null,
      "video_dir": null,
      "dataset_dir": "dataset/pickscore",
      "force_reprocess": false,
      "max_dataset_size": 1024,
      "enable_preprocess": true,
      "dataloader_num_workers": 16,
      "preprocess_parallelism": "local",
      "preprocessing_batch_size": 8
    }
  },
  "eval": {
    "value": {
      "seed": 42,
      "width": 1280,
      "height": 720,
      "eval_freq": 20,
      "num_frames": 41,
      "resolution": [
        720,
        1280
      ],
      "guidance_scale": 5,
      "num_inference_steps": 50,
      "per_device_batch_size": 1
    }
  },
  "model": {
    "value": {
      "lora_rank": 64,
      "lora_alpha": 128,
      "model_type": "wan2_t2v",
      "resume_path": null,
      "resume_type": null,
      "attn_backend": "_flash_3_hub",
      "finetune_type": "lora",
      "target_modules": "default",
      "target_components": [
        "transformer"
      ],
      "model_name_or_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
      "master_weight_dtype": "bfloat16"
    }
  },
      "cli_version": "0.25.0",
      "python_version": "3.12.12"
    }
  },
  "reward": {
    "value": {
      "reward_0": {
        "name": "pick_score",
        "dtype": "bfloat16",
        "device": "cuda",
        "weight": 1,
        "batch_size": 16,
        "reward_model": "PickScore"
      }
    }
  },
  "project": {
    "value": "Flow-Factory"
  },
  "launcher": {
    "value": "accelerate"
  },
  "run_name": {
    "value": "wan2_t2v_lora_grpo_20260310_141220"
  },
  "training": {
    "value": {
      "seed": 42,
      "width": 720,
      "height": 480,
      "kl_beta": 0,
      "kl_type": "v-based",
      "nft_beta": 1,
      "ema_decay": 0.99,
      "adam_betas": [
        0.9,
        0.999
      ],
      "clip_range": [
        -0.0001,
        0.0001
      ],
      "ema_device": "cuda",
      "global_std": true,
      "group_size": 32,
      "num_frames": 13,
      "resolution": [
        480,
        720
      ],
      "time_shift": 3,
      "ema_kl_beta": 0,
      "adam_epsilon": 1e-8,
      "trainer_type": "grpo",
      "learning_rate": 0.0003,
      "max_grad_norm": 1,
      "adv_clip_range": [
        -5,
        5
      ],
      "guidance_scale": 1,
      "timestep_range": [
        0,
        0.9
      ],
      "num_inner_epochs": 1,
      "ref_param_device": "cuda",
      "adam_weight_decay": 0.0001,
      "ema_decay_schedule": "power",
      "ema_update_interval": 4,
      "num_inference_steps": 16,
      "num_train_timesteps": 16,
      "advantage_aggregation": "sum",
      "num_batches_per_epoch": 12,
      "per_device_batch_size": 4,
      "time_sampling_strategy": "discrete",
      "gradient_step_per_epoch": 2,
      "gradient_accumulation_steps": 18,
      "unique_sample_num_per_epoch": 48,
      "enable_gradient_checkpointing": false
    }
  },
  "scheduler": {
    "value": {
      "seed": 42,
      "sde_steps": [
        0,
        1,
        2,
        3,
        4
      ],
      "noise_level": 0.8,
      "dynamics_type": "CPS",
      "num_sde_steps": 3
    }
  },
  "config_file": {
    "value": "multinode_examples/fsdp2_wan.yaml"
  },
  "eval_reward": {
    "value": {
      "reward_0": {
        "name": "pick_score",
        "dtype": "bfloat16",
        "device": "cuda",
        "weight": 1,
        "batch_size": 32,
        "reward_model": "PickScore"
      }
    }
  },
  "num_processes": {
    "value": 1
  },
  "logging_backend": {
    "value": "wandb"
  },
  "mixed_precision": {
    "value": "bf16"
  },
  "main_process_port": {
    "value": 29500
  }
}

And also, is there any plan for implementing some video based scores like video align? I can also try implement it and send pull requests

No plan for specific reward yet. Since adding a reward model is quite labor-intensive, especially to make the enviroment compatible. One solution is to wrap the reward model as a server (check here and using the remote_reward_model. But if you can add new reward models without breaking the enviroment, I would really like to merge it!

[Henglin-Liu]

您好，这是我的 Wan2.1-1.3B-Diffusers 配置（已删除部分系统信息）：

{
"log": {
"value": {
"verbose": true,
"save_dir": "saves",
"save_freq": 100,
"save_model_only": true
}
},
"data": {
"value": {
"cache_dir": "~/.cache/flow_factory/datasets",
"image_dir": null,
"video_dir": null,
"dataset_dir": "dataset/pickscore",
"force_reprocess": false,
"max_dataset_size": 1024,
"enable_preprocess": true,
"dataloader_num_workers": 16,
"preprocess_parallelism": "local",
"preprocessing_batch_size": 8
}
},
"eval": {
"value": {
"seed": 42,
"width": 1280,
"height": 720,
"eval_freq": 20,
"num_frames": 41,
"resolution": [
720,
1280
],
"guidance_scale": 5,
"num_inference_steps": 50,
"per_device_batch_size": 1
}
},
"model": {
"value": {
"lora_rank": 64,
"lora_alpha": 128,
"model_type": "wan2_t2v",
"resume_path": null,
"resume_type": null,
"attn_backend": "_flash_3_hub",
"finetune_type": "lora",
"target_modules": "default",
"target_components": [
"transformer"
],
"model_name_or_path": "Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
"master_weight_dtype": "bfloat16"
}
},
"cli_version": "0.25.0",
"python_version": "3.12.12"
}
},
"reward": {
"value": {
"reward_0": {
"name": "pick_score",
"dtype": "bfloat16",
"device": "cuda",
"weight": 1,
"batch_size": 16,
"reward_model": "PickScore"
}
}
},
"project": {
"value": "Flow-Factory"
},
"launcher": {
"value": "accelerate"
},
"run_name": {
"value": "wan2_t2v_lora_grpo_20260310_141220"
},
"training": {
"value": {
"seed": 42,
"width": 720,
"height": 480,
"kl_beta": 0,
"kl_type": "v-based",
"nft_beta": 1,
"ema_decay": 0.99,
"adam_betas": [
0.9,
0.999
],
"clip_range": [
-0.0001,
0.0001
],
"ema_device": "cuda",
"global_std": true,
"group_size": 32,
"num_frames": 13,
"resolution": [
480,
720
],
"time_shift": 3,
"ema_kl_beta": 0,
"adam_epsilon": 1e-8,
"trainer_type": "grpo",
"learning_rate": 0.0003,
"max_grad_norm": 1,
"adv_clip_range": [
-5,
5
],
"guidance_scale": 1,
"timestep_range": [
0,
0.9
],
"num_inner_epochs": 1,
"ref_param_device": "cuda",
"adam_weight_decay": 0.0001,
"ema_decay_schedule": "power",
"ema_update_interval": 4,
"num_inference_steps": 16,
"num_train_timesteps": 16,
"advantage_aggregation": "sum",
"num_batches_per_epoch": 12,
"per_device_batch_size": 4,
"time_sampling_strategy": "discrete",
"gradient_step_per_epoch": 2,
"gradient_accumulation_steps": 18,
"unique_sample_num_per_epoch": 48,
"enable_gradient_checkpointing": false
}
},
"scheduler": {
"value": {
"seed": 42,
"sde_steps": [
0,
1,
2,
3,
4
],
"noise_level": 0.8,
"dynamics_type": "CPS",
"num_sde_steps": 3
}
},
"config_file": {
"value": "multinode_examples/fsdp2_wan.yaml"
},
"eval_reward": {
"value": {
"reward_0": {
"name": "pick_score",
"dtype": "bfloat16",
"device": "cuda",
"weight": 1,
"batch_size": 32,
"reward_model": "PickScore"
}
}
},
"num_processes": {
"value": 1
},
"logging_backend": {
"value": "wandb"
},
"mixed_precision": {
"value": "bf16"
},
"main_process_port": {
"value": 29500
}
}

另外，是否有计划实现类似视频对齐之类的基于视频的评分功能？我可以尝试实现并提交 pull request。

目前还没有具体的奖励方案。因为添加奖励模型非常耗费精力，尤其是在确保环境兼容性方面。一种解决方案是将奖励模型封装成一个服务器（参见此处并使用remote_reward_model）。但是，如果您能在不破坏环境的前提下添加新的奖励模型，我非常乐意将其合并！

想问问这个配置训练的reward曲线大概是什么样的呢

[Jayce-Ping]

@Henglin-Liu