En

Author: baijin.xh

tests/DeviceMesh/test_device_mesh.py

@@ -24,40 +24,40 @@ def test_dp_rank_only(self):
 
     def test_tp_rank_only(self):
         mesh = DeviceMesh.from_sizes(tp_size=4)
-        # from_sizes 默认 dp_size=1，维度顺序是 (dp, tp)
+        # from_sizes default dp_size=1, dimension order (dp, tp)
         mesh_array = mesh.mesh.reshape(1, 4)
 
         for tp_idx in range(4):
             global_rank = int(mesh_array[0, tp_idx])
             with patch.object(Platform, 'get_rank', return_value=global_rank):
                 assert mesh.tp_rank == tp_idx
-                assert mesh.dp_rank == 0  # dp 默认是 1，所以 dp_rank 总是 0
+                assert mesh.dp_rank == 0  # dp default is 1, so dp_rank is always 0
                 assert mesh.pp_rank is None
                 assert mesh.fsdp_rank is None
 
     def test_pp_rank_only(self):
         mesh = DeviceMesh.from_sizes(pp_size=4)
-        # from_sizes 维度顺序是 (pp, dp)，默认 dp_size=1
+        # from_sizes dimension order (pp, dp), default dp_size=1
         mesh_array = mesh.mesh.reshape(4, 1)
 
         for pp_idx in range(4):
             global_rank = int(mesh_array[pp_idx, 0])
             with patch.object(Platform, 'get_rank', return_value=global_rank):
                 assert mesh.pp_rank == pp_idx
-                assert mesh.dp_rank == 0  # dp 默认是 1，所以 dp_rank 总是 0
+                assert mesh.dp_rank == 0  # dp default is 1, so dp_rank is always 0
                 assert mesh.tp_rank is None
                 assert mesh.fsdp_rank is None
 
     def test_fsdp_rank_only(self):
         mesh = DeviceMesh.from_sizes(fsdp_size=4)
-        # from_sizes 维度顺序是 (fsdp, dp)，默认 dp_size=1
+        # from_sizes dimension order (fsdp, dp), default dp_size=1
         mesh_array = mesh.mesh.reshape(4, 1)
 
         for fsdp_idx in range(4):
             global_rank = int(mesh_array[fsdp_idx, 0])
             with patch.object(Platform, 'get_rank', return_value=global_rank):
                 assert mesh.fsdp_rank == fsdp_idx
-                assert mesh.dp_rank == 0  # dp 默认是 1，所以 dp_rank 总是 0
+                assert mesh.dp_rank == 0  # dp default is 1, so dp_rank is always 0
                 assert mesh.tp_rank is None
                 assert mesh.pp_rank is None
 
@@ -77,7 +77,7 @@ def test_dp_tp_combination(self):
 
     def test_dp_fsdp_combination(self):
         mesh = DeviceMesh.from_sizes(dp_size=2, fsdp_size=4)
-        # from_sizes 维度顺序是 (fsdp, dp)
+        # from_sizes dimension order (fsdp, dp)
         mesh_array = mesh.mesh.reshape(4, 2)
 
         for fsdp_idx in range(4):
@@ -91,7 +91,7 @@ def test_dp_fsdp_combination(self):
 
     def test_tp_pp_combination(self):
         mesh = DeviceMesh.from_sizes(tp_size=2, pp_size=4)
-        # from_sizes 维度顺序是 (pp, dp, tp)，默认 dp_size=1
+        # from_sizes dimension order (pp, dp, tp), default dp_size=1
         mesh_array = mesh.mesh.reshape(4, 1, 2)
 
         for pp_idx in range(4):
@@ -100,12 +100,12 @@ def test_tp_pp_combination(self):
                 with patch.object(Platform, 'get_rank', return_value=global_rank):
                     assert mesh.pp_rank == pp_idx
                     assert mesh.tp_rank == tp_idx
-                    assert mesh.dp_rank == 0  # dp 默认是 1，所以 dp_rank 总是 0
+                    assert mesh.dp_rank == 0  # dp default is 1, so dp_rank is always 0
                     assert mesh.fsdp_rank is None
 
     def test_dp_tp_pp_combination(self):
         mesh = DeviceMesh.from_sizes(dp_size=2, tp_size=2, pp_size=2)
-        # from_sizes 维度顺序是 (pp, dp, tp)
+        # from_sizes dimension order (pp, dp, tp)
         mesh_array = mesh.mesh.reshape(2, 2, 2)
 
         for pp_idx in range(2):
@@ -120,7 +120,7 @@ def test_dp_tp_pp_combination(self):
 
     def test_dp_fsdp_tp_combination(self):
         mesh = DeviceMesh.from_sizes(dp_size=2, fsdp_size=2, tp_size=2)
-        # from_sizes 维度顺序是 (fsdp, dp, tp)
+        # from_sizes dimension order (fsdp, dp, tp)
         mesh_array = mesh.mesh.reshape(2, 2, 2)
 
         for fsdp_idx in range(2):
@@ -135,7 +135,7 @@ def test_dp_fsdp_tp_combination(self):
 
     def test_all_dimensions_combination(self):
         mesh = DeviceMesh.from_sizes(dp_size=2, fsdp_size=2, tp_size=2, pp_size=2)
-        # from_sizes 维度顺序是 (fsdp, pp, dp, tp)
+        # from_sizes dimension order (fsdp, pp, dp, tp)
         mesh_array = mesh.mesh.reshape(2, 2, 2, 2)
 
         for fsdp_idx in range(2):
@@ -197,13 +197,13 @@ def test_data_rank_with_fsdp_only(self):
 
     def test_data_rank_with_dp_fsdp(self):
         mesh = DeviceMesh.from_sizes(dp_size=2, fsdp_size=3)
-        # from_sizes 维度顺序是 (fsdp, dp)
+        # from_sizes dimension order (fsdp, dp)
         mesh_array = mesh.mesh.reshape(3, 2)
 
         for fsdp_idx in range(3):
             for dp_idx in range(2):
                 global_rank = int(mesh_array[fsdp_idx, dp_idx])
                 with patch.object(Platform, 'get_rank', return_value=global_rank):
-                    # data_rank 的计算公式: dp_rank * fsdp_world_size + fsdp_rank
+                    # data_rank formula: dp_rank * fsdp_world_size + fsdp_rank
                     expected_data_rank = dp_idx * 3 + fsdp_idx
                     assert mesh.data_rank == expected_data_rank

tests/dataloader/test_dataloader.py

@@ -50,7 +50,7 @@ def create_dataset():
 
 
 class TestDataCollator:
-    """测试data_collator（InputProcessor）功能"""
+    """Test data_collator (InputProcessor) functionality"""
 
     @pytest.mark.skipif(SKIP_MODEL_DOWNLOAD, reason='Skipping tests that require model download')
     def test_dataloader_with_collator(self):

tests/dataset/test_lazy.py

@@ -21,25 +21,25 @@ def convert_to_messages(example):
 class TestLazyDataset:
 
     def test_lazy_dataset_basic(self):
-        # 基本功能测试
+        # Basic functionality test
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
 
         assert len(dataset) == 4
-        assert dataset.do_encode
-        assert dataset.do_check
+        assert not dataset.do_encode
+        assert not dataset.do_check
 
         item = dataset[0]
         assert 'text' in item
         assert item['text'] == 'Hello world'
 
     def test_lazy_dataset_encode_flag(self):
-        # 懒加载编码标志测试
+        # Lazy encode flag test
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
         dataset.map(convert_to_messages)
 
-        assert dataset.do_encode
+        assert not dataset.do_encode
 
         try:
             dataset.set_template('Template', model_id='ms://Qwen/Qwen2.5-0.5B-Instruct', max_length=128)
@@ -48,12 +48,14 @@ def test_lazy_dataset_encode_flag(self):
 
         dataset.encode()
 
-        assert dataset.do_encode
+        # Lazy load: encode() only sets flag, actual encoding on access; raw dataset has no input_ids
         assert 'messages' in dataset.dataset[0]
         assert 'input_ids' not in dataset.dataset[0]
+        item = dataset[0]
+        assert 'input_ids' in item
 
     def test_lazy_dataset_encode_on_access(self):
-        # 懒加载编码执行测试
+        # Lazy encode execution test
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
         dataset.map(convert_to_messages)
@@ -71,12 +73,12 @@ def test_lazy_dataset_encode_on_access(self):
         assert len(item['input_ids']) > 0
 
     def test_lazy_dataset_check_flag(self):
-        # 懒加载检查标志测试,验证check()只设置标志，不实际执行检查
+        # Lazy check flag test: check() only sets flag, does not execute check
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
         dataset.map(convert_to_messages)
 
-        assert dataset.do_check
+        assert not dataset.do_check
 
         try:
             dataset.set_template('Template', model_id='ms://Qwen/Qwen2.5-0.5B-Instruct', max_length=128)
@@ -85,10 +87,12 @@ def test_lazy_dataset_check_flag(self):
 
         dataset.check()
 
-        assert dataset.do_check
+        # Lazy load: check() only sets flag, actual check on access
+        item = dataset[0]
+        assert item is not None
 
     def test_lazy_dataset_check_on_access(self):
-        # 懒加载检查执行测试,验证在访问数据时才执行检查
+        # Lazy check execution test: check runs on data access
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
         dataset.map(convert_to_messages)
@@ -105,7 +109,7 @@ def test_lazy_dataset_check_on_access(self):
         assert 'messages' in item or item is None
 
     def test_lazy_dataset_encode_requires_template(self):
-        # 编码要求模板测试,验证未设置模板时抛出异常
+        # Encode requires template: raises when template not set
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
 
@@ -121,7 +125,7 @@ def test_lazy_dataset_check_requires_template(self):
 
     @pytest.mark.skipif(SKIP_MODEL_DOWNLOAD, reason='Skipping tests that require model download')
     def test_lazy_dataset_no_split_strategy(self):
-        # 编码不支持split策略测试,验证未设置模板时抛出异常
+        # Encode does not support split strategy: raises when template not set
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
         dataset.map(convert_to_messages)
@@ -136,7 +140,7 @@ def test_lazy_dataset_no_split_strategy(self):
             dataset.encode()
 
     def test_lazy_dataset_multiple_items(self):
-        # 验证多个数据项的懒加载编码
+        # Lazy encode for multiple items
         csv_path = str(TEST_DATA_DIR / 'test.csv')
         dataset = LazyDataset(dataset_meta=DatasetMeta(dataset_id=csv_path))
         dataset.map(convert_to_messages)