change server_config

Author: tastelikefeet

cookbook/client/tinker/megatron/server_config.yaml

@@ -1,4 +1,4 @@
-# Twinkle Server Configuration - Tinker-Compatible Megatron Backend
+# Twinkle Server Configuration - Tinker-Compatible Transformers Backend
 
 # Server protocol type: "tinker" enables the Tinker-compatible API
 server_type: tinker
@@ -31,23 +31,30 @@ applications:
         ray_actor_options:
           num_cpus: 0.1                  # CPU resources allocated to this actor
 
-  # 2. Model Service - Hosts the base model for training (Megatron backend)
-  #    This is the actual model worker that performs forward/backward passes.
-  - name: models-Qwen2.5-0.5B-Instruct
-    route_prefix: /api/v1/model/Qwen/Qwen2.5-0.5B-Instruct   # REST path for this model
+  # 2. Model Service (commented out) - Would host the base model for training.
+  #    Uncomment and configure if you need a training model worker.
+  - name: models-Qwen3-30B-A3B-Instruct-2507
+    route_prefix: /api/v1/model/Qwen/Qwen3-30B-A3B-Instruct-2507
     import_path: model
     args:
-      use_megatron: true                               # Use Megatron-LM backend (not HuggingFace)
-      model_id: "ms://Qwen/Qwen2.5-0.5B-Instruct"     # ModelScope model identifier to load
-      nproc_per_node: 2               # Number of GPU processes per node
-      device_group:                   # Logical device group for this model
+      use_megatron: true                          # Use HuggingFace Transformers backend
+      model_id: "ms://Qwen/Qwen3-30B-A3B-Instruct-2507" # ModelScope model identifier
+      nproc_per_node: 4                            # Number of GPU processes per node
+      device_group:
         name: model
-        ranks: [0, 1]                 # GPU rank indices to use
+        ranks: [0, 1, 2, 3]                              # GPU rank indices
         device_type: cuda
-      device_mesh:                    # Distributed training mesh configuration
+      device_mesh:
         device_type: cuda
-        mesh: [0, 1]                  # Device indices in the mesh
-        mesh_dim_names: ['dp']        # Mesh dimension names: 'dp' = data parallel
+        dp_size: 2
+        tp_size: 2
+
+      queue_config:
+        rps_limit: 100                             # Max requests per second
+        tps_limit: 10000                           # Max tokens per second
+      adapter_config:
+        per_token_adapter_limit: 30                # Max concurrent LoRA adapters
+        adapter_timeout: 1800                      # Seconds before idle adapter unload
     deployments:
       - name: ModelManagement
         autoscaling_config:
@@ -56,3 +63,35 @@ applications:
           target_ongoing_requests: 16
         ray_actor_options:
           num_cpus: 0.1
+
+  # 3. Sampler Service - Runs inference / sampling using vLLM engine
+  #    Used for generating text from the model (e.g., evaluating LoRA results).
+  - name: sampler-Qwen3-30B-A3B-Instruct-2507
+    route_prefix: /api/v1/sampler/Qwen/Qwen3-30B-A3B-Instruct-2507
+    import_path: sampler
+    args:
+      model_id: "ms://Qwen/Qwen3-30B-A3B-Instruct-2507"   # ModelScope model identifier
+      nproc_per_node: 4               # Number of GPU processes per node
+      sampler_type: vllm              # Inference engine: 'vllm' (fast) or 'torch' (TorchSampler)
+      engine_args:                    # vLLM engine-specific settings
+        max_model_len: 4096           # Maximum sequence length the engine supports
+        gpu_memory_utilization: 0.7   # Fraction of GPU memory to use (0.0-1.0)
+        enable_lora: true             # Allow loading LoRA adapters during inference
+      device_group:                   # Logical device group for the sampler
+        name: sampler
+        gpus_per_worker: 2
+        ranks: [4,5,6,7]                    # GPU rank indices to use
+        device_type: cuda
+      device_mesh:
+        device_type: cuda
+        dp_size: 4
+        tp_size: 1
+    deployments:
+      - name: SamplerManagement
+        autoscaling_config:
+          min_replicas: 1
+          max_replicas: 1
+          target_ongoing_requests: 16
+        ray_actor_options:
+          num_cpus: 0.1
+          num_gpus: 1                 # Sampler needs a full GPU for inference

src/twinkle/server/twinkle/model.py

@@ -154,7 +154,10 @@ class ModelManagement(AdapterManagerMixin):
         def __init__(self, nproc_per_node: int, device_group: Dict[str, Any], device_mesh: Dict[str, Any]):
             self.device_group = DeviceGroup(**device_group)
             twinkle.initialize(mode='ray', nproc_per_node=nproc_per_node, groups=[self.device_group], lazy_collect=False)
-            self.device_mesh = DeviceMesh(**device_mesh)
+            if 'mesh_dim_names' in device_mesh:
+                self.device_mesh = DeviceMesh(**device_mesh)
+            else:
+                self.device_mesh = DeviceMesh.from_sizes(**device_mesh)
             if use_megatron:
                 from twinkle.model import MultiLoraMegatronModel
                 self.model = MultiLoraMegatronModel(

src/twinkle/server/twinkle/processor.py

@@ -61,7 +61,10 @@ def __init__(self, nproc_per_node: int, ncpu_proc_per_node:int, device_group: Di
             self.device_group = DeviceGroup(**device_group)
             twinkle.initialize(mode='ray', nproc_per_node=nproc_per_node, groups=[self.device_group],
                                lazy_collect=False, ncpu_proc_per_node=ncpu_proc_per_node)
-            self.device_mesh = DeviceMesh(**device_mesh)
+            if 'mesh_dim_names' in device_mesh:
+                self.device_mesh = DeviceMesh(**device_mesh)
+            else:
+                self.device_mesh = DeviceMesh.from_sizes(**device_mesh)
             self.resource_dict = {}
             self.resource_records: Dict[str, int] = {}
             self.hb_thread = threading.Thread(target=self.countdown, daemon=True)

src/twinkle/server/twinkle/sampler.py

@@ -274,7 +274,10 @@ def __init__(self, nproc_per_node: int, device_group: Dict[str, Any],
                                nproc_per_node=nproc_per_node,
                                groups=[self.device_group],
                                lazy_collect=False)
-            self.device_mesh = DeviceMesh(**device_mesh)
+            if 'mesh_dim_names' in device_mesh:
+                self.device_mesh = DeviceMesh(**device_mesh)
+            else:
+                self.device_mesh = DeviceMesh.from_sizes(**device_mesh)
             self.sampler_type = sampler_type
 
             # Initialize sampler based on type