update io

Author: Yunnglin

client_tools/client_generator.py

@@ -347,7 +347,7 @@ class {class_name}({inheritance}):
     def __init__({init_params}):
         from twinkle_client.http import get_base_url
 
-        self.server_url = f'{{get_base_url()}}/processors/twinkle'
+        self.server_url = f'{{get_base_url()}}/processor/twinkle'
         response = http_post(
             url=f'{{self.server_url}}/create',
             json_data={{
@@ -466,7 +466,7 @@ def __init__(self, model_id: str, **kwargs):
         self.model_id = model_id
         if '://' in model_id:
             model_id = model_id.split('://')[1]
-        self.server_url = f'{self.server_url}/models/{model_id}/twinkle'
+        self.server_url = f'{self.server_url}/model/{model_id}/twinkle'
         self.adapter_name = None
         response = http_post(
             url=f'{self.server_url}/create',
@@ -743,7 +743,7 @@ def __init__(self, model_id: str, **kwargs):
         self.adapter_name = None
         if '://' in model_id:
             model_id = model_id.split('://')[1]
-        self.server_url = f'{self.server_url}/samplers/{model_id}/twinkle'
+        self.server_url = f'{self.server_url}/sampler/{model_id}/twinkle'
         response = http_post(
             url=f'{self.server_url}/create',
             json_data=kwargs

cookbook/client/server/transformer/server_config.yaml

@@ -100,3 +100,26 @@ applications:
   #         runtime_env:
   #           env_vars:
   #             TWINKLE_TRUST_REMOTE_CODE: "0"
+
+  # 4. Processor Service - Runs inference / sampling using vLLM engine
+  - name: processor
+    route_prefix: /api/v1/processor
+    import_path: processor
+    args:
+      nproc_per_node: 2               # 每节点处理器 worker 数
+      ncpu_proc_per_node: 2           # 每节点 CPU 进程数
+      device_group:
+        name: model
+        ranks: 2
+        device_type: CPU
+      device_mesh:
+        device_type: CPU
+        dp_size: 2                    # 数据并行大小
+    deployments:
+      - name: ProcessorManagement
+        autoscaling_config:
+          min_replicas: 1
+          max_replicas: 1
+          target_ongoing_requests: 128
+        ray_actor_options:
+          num_cpus: 0.1

src/twinkle/server/common/__init__.py

@@ -1,6 +1,6 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
+from .checkpoint_factory import create_checkpoint_manager, create_training_run_manager
 from .datum import datum_to_input_feature, extract_rl_feature, input_feature_to_datum
-from .io_utils import create_checkpoint_manager, create_training_run_manager, validate_ownership, validate_user_path
 from .router import StickyLoraRequestRouter
 from .serialize import deserialize_object, serialize_object
 
@@ -10,8 +10,6 @@
     'input_feature_to_datum',
     'create_checkpoint_manager',
     'create_training_run_manager',
-    'validate_user_path',
-    'validate_ownership',
     'StickyLoraRequestRouter',
     'deserialize_object',
     'serialize_object',

src/twinkle/server/common/checkpoint_factory.py

@@ -0,0 +1,39 @@
+# Copyright (c) ModelScope Contributors. All rights reserved.
+"""
+Factory functions for creating checkpoint and training-run manager instances.
+
+Use these functions as the entry point rather than instantiating managers directly:
+
+    from twinkle.server.common.checkpoint_factory import (
+        create_checkpoint_manager,
+        create_training_run_manager,
+    )
+"""
+from twinkle.server.common.tinker_checkpoint import TinkerCheckpointManager, TinkerTrainingRunManager
+from twinkle.server.common.twinkle_checkpoint import TwinkleCheckpointManager, TwinkleTrainingRunManager
+
+
+def create_training_run_manager(token: str, client_type: str = 'twinkle'):
+    """Create a TrainingRunManager for the given token.
+
+    Args:
+        token: User authentication token.
+        client_type: 'tinker' or 'twinkle' (default 'twinkle').
+    """
+    if client_type == 'tinker':
+        return TinkerTrainingRunManager(token)
+    return TwinkleTrainingRunManager(token)
+
+
+def create_checkpoint_manager(token: str, client_type: str = 'twinkle'):
+    """Create a CheckpointManager for the given token.
+
+    Args:
+        token: User authentication token.
+        client_type: 'tinker' or 'twinkle' (default 'twinkle').
+    """
+    if client_type == 'tinker':
+        run_mgr = TinkerTrainingRunManager(token)
+        return TinkerCheckpointManager(token, run_mgr)
+    run_mgr = TwinkleTrainingRunManager(token)
+    return TwinkleCheckpointManager(token, run_mgr)

src/twinkle/server/common/io_utils.py

@@ -1,14 +1,14 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
 """
-Tinker-specific IO managers for training runs and checkpoints.
+Tinker-specific checkpoint and training-run managers.
 
 Uses ``tinker.types`` models for all serialization and response construction.
 """
 from datetime import datetime
 from tinker import types as tinker_types
 from typing import Any, Dict, List, Optional
 
-from twinkle.server.utils.io_utils import TRAIN_RUN_INFO_FILENAME, BaseCheckpointManager, BaseTrainingRunManager
+from twinkle.server.utils.checkpoint_base import TRAIN_RUN_INFO_FILENAME, BaseCheckpointManager, BaseTrainingRunManager
 
 
 class TinkerTrainingRunManager(BaseTrainingRunManager):

…twinkle/server/common/tinker_io_utils.py …inkle/server/common/tinker_checkpoint.pysrc/twinkle/server/common/tinker_io_utils.py renamed to src/twinkle/server/common/tinker_checkpoint.py src/twinkle/server/common/tinker_io_utils.py renamed to src/twinkle/server/common/tinker_checkpoint.py

@@ -1,19 +1,17 @@
 # Copyright (c) ModelScope Contributors. All rights reserved.
 """
-Twinkle-specific IO managers for training runs and checkpoints.
+Twinkle-specific checkpoint and training-run managers.
 
 Uses ``twinkle_client.types.training`` models for all serialization and response construction.
 """
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
-from twinkle.server.utils.io_utils import (TRAIN_RUN_INFO_FILENAME, BaseCheckpointManager, BaseTrainingRunManager,
-                                           validate_ownership)
-from twinkle_client.types.training import Checkpoint as TwinkleCheckpoint
-from twinkle_client.types.training import (CheckpointsListResponse, CreateModelRequest, Cursor,
-                                           ParsedCheckpointTwinklePath)
-from twinkle_client.types.training import TrainingRun as TwinkleTrainingRun
-from twinkle_client.types.training import TrainingRunsResponse, WeightsInfoResponse
+from twinkle.server.utils.checkpoint_base import (TRAIN_RUN_INFO_FILENAME, BaseCheckpointManager,
+                                                  BaseTrainingRunManager, validate_ownership)
+from twinkle_client.types.training import (Checkpoint, CheckpointsListResponse, CreateModelRequest, Cursor,
+                                           ParsedCheckpointTwinklePath, TrainingRun, TrainingRunsResponse,
+                                           WeightsInfoResponse)
 
 
 class TwinkleTrainingRunManager(BaseTrainingRunManager):
@@ -25,7 +23,7 @@ def train_run_info_filename(self) -> str:
 
     def _create_training_run(self, model_id: str, run_config: CreateModelRequest) -> Dict[str, Any]:
         lora_config = run_config.lora_config
-        train_run_data = TwinkleTrainingRun(
+        train_run_data = TrainingRun(
             training_run_id=model_id,
             base_model=run_config.base_model,
             model_owner=self.token,
@@ -44,14 +42,14 @@ def _create_training_run(self, model_id: str, run_config: CreateModelRequest) ->
             new_data['train_attn'] = lora_config.train_attn
         return new_data
 
-    def _parse_training_run(self, data: Dict[str, Any]) -> TwinkleTrainingRun:
-        return TwinkleTrainingRun(**data)
+    def _parse_training_run(self, data: Dict[str, Any]) -> TrainingRun:
+        return TrainingRun(**data)
 
-    def _create_training_runs_response(self, runs: List[TwinkleTrainingRun], limit: int, offset: int,
+    def _create_training_runs_response(self, runs: List[TrainingRun], limit: int, offset: int,
                                        total: int) -> TrainingRunsResponse:
         return TrainingRunsResponse(training_runs=runs, cursor=Cursor(limit=limit, offset=offset, total_count=total))
 
-    def get_with_permission(self, model_id: str) -> Optional[TwinkleTrainingRun]:
+    def get_with_permission(self, model_id: str) -> Optional[TrainingRun]:
         run = self.get(model_id)
         if run and validate_ownership(self.token, run.model_owner):
             return run
@@ -82,7 +80,7 @@ def _create_checkpoint(self,
                            train_mlp=None,
                            train_attn=None,
                            user_metadata=None) -> Dict[str, Any]:
-        checkpoint = TwinkleCheckpoint(
+        checkpoint = Checkpoint(
             checkpoint_id=checkpoint_id,
             checkpoint_type=checkpoint_type,
             time=datetime.now(),
@@ -98,15 +96,15 @@ def _create_checkpoint(self,
             user_metadata=user_metadata)
         return checkpoint.model_dump(mode='json')
 
-    def _parse_checkpoint(self, data: Dict[str, Any]) -> TwinkleCheckpoint:
+    def _parse_checkpoint(self, data: Dict[str, Any]) -> Checkpoint:
         data = data.copy()
         if 'tinker_path' in data and 'twinkle_path' not in data:
             data['twinkle_path'] = data.pop('tinker_path')
         elif 'twinkle_path' not in data and 'path' in data:
             data['twinkle_path'] = data.pop('path')
-        return TwinkleCheckpoint(**data)
+        return Checkpoint(**data)
 
-    def get(self, model_id: str, checkpoint_id: str) -> Optional[TwinkleCheckpoint]:
+    def get(self, model_id: str, checkpoint_id: str) -> Optional[Checkpoint]:
         data = self._read_ckpt_info(model_id, checkpoint_id)
         if not data:
             return None
@@ -116,7 +114,7 @@ def get(self, model_id: str, checkpoint_id: str) -> Optional[TwinkleCheckpoint]:
                 data['twinkle_path'] = f"{self.path_prefix}{model_id}/{data['checkpoint_id']}"
         return self._parse_checkpoint(data)
 
-    def _create_checkpoints_response(self, checkpoints: List[TwinkleCheckpoint]) -> CheckpointsListResponse:
+    def _create_checkpoints_response(self, checkpoints: List[Checkpoint]) -> CheckpointsListResponse:
         return CheckpointsListResponse(checkpoints=checkpoints, cursor=None)
 
     def _create_parsed_path(self, path, training_run_id, checkpoint_type, checkpoint_id) -> ParsedCheckpointTwinklePath:

…winkle/server/common/twinkle_io_utils.py …nkle/server/common/twinkle_checkpoint.pysrc/twinkle/server/common/twinkle_io_utils.py renamed to src/twinkle/server/common/twinkle_checkpoint.py src/twinkle/server/common/twinkle_io_utils.py renamed to src/twinkle/server/common/twinkle_checkpoint.py

@@ -17,7 +17,7 @@
     from .server import GatewayServer
 
 from twinkle.hub import HubOperation
-from twinkle.server.common.io_utils import create_checkpoint_manager, create_training_run_manager
+from twinkle.server.common.checkpoint_factory import create_checkpoint_manager, create_training_run_manager
 from twinkle.server.utils.task_queue import QueueState
 from twinkle.server.utils.validation import get_token_from_request
 from twinkle.utils.logger import get_logger

src/twinkle/server/gateway/tinker_gateway_handlers.py

@@ -12,7 +12,8 @@
 if TYPE_CHECKING:
     from .server import GatewayServer
 
-from twinkle.server.common.io_utils import create_checkpoint_manager, create_training_run_manager, validate_user_path
+from twinkle.server.common.checkpoint_factory import create_checkpoint_manager, create_training_run_manager
+from twinkle.server.utils.checkpoint_base import validate_user_path
 from twinkle.server.utils.validation import get_token_from_request
 from twinkle.utils.logger import get_logger
 from twinkle_client.types.server import DeleteCheckpointResponse, HealthResponse, WeightsInfoRequest

src/twinkle/server/gateway/twinkle_gateway_handlers.py

@@ -118,7 +118,7 @@ def load(self, checkpoint_dir: str, **kwargs):
         token = kwargs.pop('token', None)
         if not token:
             raise ValueError('Token is required for loading checkpoints')
-        from twinkle.server.common.io_utils import create_checkpoint_manager
+        from twinkle.server.common.checkpoint_factory import create_checkpoint_manager
         checkpoint_manager = create_checkpoint_manager(token, client_type='tinker')
         resolved = checkpoint_manager.resolve_load_path(checkpoint_dir)
         if resolved.is_twinkle_path: