Merge remote-tracking branch 'origin/dev' into kernels_unittest_fix_ljl

Author: meichangsu1

cookbook/client/tinker/megatron/server_config.yaml

@@ -48,6 +48,7 @@ applications:
         max_model_len: 16000           # Maximum sequence length the engine supports
         gpu_memory_utilization: 0.85   # Fraction of GPU memory to use (0.0-1.0)
         enable_lora: true             # Allow loading LoRA adapters during inference
+        max_loras: 5                  # Max allowed loras working on vLLM at the same time
       device_group:                   # Logical device group for the sampler
         name: sampler
         gpus_per_worker: 1

tests/sampler/test_30b_weight_sync.py

@@ -17,11 +17,10 @@
 import argparse
 import datetime
 import os
+import pytest
 import sys
 import time
 
-import pytest
-
 os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
 os.environ['VLLM_LOGGING_LEVEL'] = 'WARNING'
 os.environ['NCCL_CUMEM_ENABLE'] = '0'

tests/sampler/test_megatron_weight_sync.py

@@ -30,11 +30,10 @@
 import argparse
 import logging
 import os
+import pytest
 import sys
 import time
 
-import pytest
-
 # Must set before importing anything
 os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
 os.environ['VLLM_LOGGING_LEVEL'] = 'WARNING'

tests/sampler/test_sampler_e2e.py

@@ -20,11 +20,10 @@
 
 import argparse
 import os
+import pytest
 import sys
 import traceback
 
-import pytest
-
 # Set environment variables before imports
 os.environ.setdefault('TRUST_REMOTE_CODE', '1')
 

tests/sampler/test_weight_sync.py

@@ -26,11 +26,10 @@
 import argparse
 import logging
 import os
+import pytest
 import sys
 import time
 
-import pytest
-
 # Must set before importing anything
 os.environ['VLLM_WORKER_MULTIPROC_METHOD'] = 'spawn'
 os.environ['VLLM_LOGGING_LEVEL'] = 'WARNING'