From 9fae1e44162ea809a535a808209c97f45edc3fdd Mon Sep 17 00:00:00 2001 From: Jinliang Li Date: Wed, 4 Mar 2026 00:46:33 -0800 Subject: [PATCH] change default data-parallel-sharding-strategy from no_shard to optim_grads_params --- megatron/training/arguments.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/megatron/training/arguments.py b/megatron/training/arguments.py index 6c1cf20c5ac..16d3d0f64ba 100644 --- a/megatron/training/arguments.py +++ b/megatron/training/arguments.py @@ -2593,7 +2593,7 @@ def _add_distributed_args(parser): group.add_argument('--create-all-gather-group', action='store_true', help='Create a separate process group for all-gather operations ' 'to overlap reduce-scatter and all-gather operations.') - group.add_argument('--data-parallel-sharding-strategy', type=str, default='no_shard', + group.add_argument('--data-parallel-sharding-strategy', type=str, default='optim_grads_params', choices=['no_shard', 'optim', 'optim_grads', 'optim_grads_params'], help='Sharding strategy of data parallelism.') group.add_argument('--outer-dp-sharding-strategy', type=str, default='no_shard',