Python 3.12

pyproject.toml

@@ -16,6 +16,8 @@ classifiers = [
   "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
   "Operating System :: POSIX :: Linux",
   "Operating System :: Microsoft :: Windows",
   "Operating System :: MacOS :: MacOS X",
@@ -49,10 +51,10 @@ exclude = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.9"
+python = "^3.12"
 fastapi = "0.104.*"
 uvicorn = "0.24.*"
-numpy = "1.24.3"
+numpy = "1.26.*"
 pandas = "1.5.3"
 aiofiles = "*"
 tqdm = "*"
@@ -70,10 +72,11 @@ httpx = "0.21.0"
 starlette = "0.27.0"
 firebase_admin = "*"
 pyrebase4 = "4.7.1"
-pydantic = { version = "1.10.13", extras = ["email", "dotenv"] }
+pydantic = { version = "^2.0", extras = ["email", "dotenv"] }
+pydantic-settings = "^2.0.0"
 python-jose = { version = "*", extras = ["cryptography"] }
 alembic = "1.9.2"
-sqlmodel = "0.0.12"
+sqlmodel = "^0.0.14"
 pymysql = "1.1.0"
 fastapi_pagination = "0.12.13"
 plotly = "5.18.0"

studio/app/Snakefile

@@ -10,7 +10,7 @@ if config.get('type') == "EDIT_ROI":
         conda:
             EditRoiUtils.conda(config)
         script:
-            f"{DIRPATH.APP_DIR}/optinist/core/rules/edit_ROI.py"
+            DIRPATH.APP_DIR + "/optinist/core/rules/edit_ROI.py"
 else:
     rule all:
         input: [join_filepath([DIRPATH.OUTPUT_DIR, x]) for x in config["last_output"]]
@@ -28,7 +28,7 @@ else:
                 params:
                     name = details
                 script:
-                    f"{DIRPATH.APP_DIR}/common/core/rules/data.py"
+                    DIRPATH.APP_DIR + "/common/core/rules/data.py"
         else:
             rule:
                 name:
@@ -42,4 +42,4 @@ else:
                 conda:
                     SmkUtils.conda(details)
                 script:
-                    f"{DIRPATH.APP_DIR}/common/core/rules/func.py"
+                    DIRPATH.APP_DIR + "/common/core/rules/func.py"

studio/app/common/core/auth/auth_config.py

@@ -1,4 +1,5 @@
-from pydantic import BaseSettings, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 from studio.app.dir_path import DIRPATH
 
@@ -9,11 +10,12 @@ class AuthConfig(BaseSettings):
     )
     SECRET_KEY: str = Field(default="123456", env="SECRET_KEY")
     USE_FIREBASE_TOKEN: bool = Field(default=True, env="USE_FIREBASE_TOKEN")
-    ALGORITHM = "HS256"
+    ALGORITHM: str = "HS256"
 
     class Config:
         env_file = f"{DIRPATH.CONFIG_DIR}/.env"
         env_file_encoding = "utf-8"
+        extra = "allow"
 
 
 AUTH_CONFIG = AuthConfig()

studio/app/common/core/experiment/experiment_builder.py

@@ -1,4 +1,6 @@
 from studio.app.common.core.experiment.experiment import ExptConfig
+from studio.app.optinist.schemas.nwb import NWBParams
+from studio.app.common.core.snakemake.smk import SmkParam
 
 
 class ExptConfigBuilder:
@@ -58,11 +60,27 @@
         return self
 
     def set_nwbfile(self, nwbfile) -> "ExptConfigBuilder":
-        self._nwbfile = nwbfile
+        # Convert dict to NWBParams if needed
+        if isinstance(nwbfile, dict):
+            try:
+                self._nwbfile = NWBParams(**nwbfile)
+            except Exception:
+                # If conversion fails, store as dict
+                self._nwbfile = nwbfile
+        else:
+            self._nwbfile = nwbfile
         return self
 
     def set_snakemake(self, snakemake) -> "ExptConfigBuilder":
-        self._snakemake = snakemake
+        # Convert dict to SmkParam if needed
+        if isinstance(snakemake, dict):
+            try:
+                self._snakemake = SmkParam(**snakemake)
+            except Exception:
+                # If conversion fails, store as dict
+                self._snakemake = snakemake
+        else:
+            self._snakemake = snakemake
         return self
 
     def set_data_usage(self, data_usage) -> "ExptConfigBuilder":
@@ -82,4 +100,4 @@
             nwb=self._nwbfile,
             snakemake=self._snakemake,
             data_usage=self._data_usage,
-        )
+        )

studio/app/common/core/experiment/experiment_writer.py

@@ -57,9 +57,12 @@
 
         self.build_function_from_nodeDict()
 
+        config = self.builder.build()
         # Write EXPERIMENT_YML
         self._write_raw(
-            self.workspace_id, self.unique_id, config=asdict(self.builder.build())
+            self.workspace_id, self.unique_id, config=self._convert_config_to_dict(
+                config
+                )
         )
 
     @classmethod
@@ -73,6 +76,19 @@
             auto_file_lock=auto_file_lock,
         )
 
+    @classmethod
+    def _convert_config_to_dict(self, config):
+        """Convert ExptConfig to dictionary, handling Pydantic v2 models"""
+        config_dict = asdict(config)
+
+        # Convert Pydantic v2 models to dictionaries
+        if hasattr(config.nwb, 'model_dump'):
+            config_dict['nwb'] = config.nwb.model_dump()
+        if hasattr(config.snakemake, 'model_dump'):
+            config_dict['snakemake'] = config.snakemake.model_dump()
+
+        return config_dict
+
     def overwrite(self, update_params: dict) -> None:
         expt_filepath = ExptConfigReader.get_config_yaml_path(
             self.workspace_id, self.unique_id
@@ -85,7 +101,10 @@
             config = ExptConfigReader.read(self.workspace_id, self.unique_id)
 
             # Merge overwrite params
-            config_merged = differential_deep_merge(asdict(config), update_params)
+            config_merged = differential_deep_merge(
+                self._convert_config_to_dict(config),
+                update_params
+                )
 
             # Overwrite experiment config
             __class__._write_raw(
@@ -382,3 +401,4 @@
         except Exception as e:
             logger.error(f"Failed to update experiment.yml: {e}")
             return False
+

studio/app/common/core/mode.py

@@ -1,4 +1,5 @@
-from pydantic import BaseSettings, Field
+from pydantic import Field
+from pydantic_settings import BaseSettings
 
 from studio.app.dir_path import DIRPATH
 
@@ -20,6 +21,7 @@ def reset_mode(self, is_standalone: bool):
     class Config:
         env_file = f"{DIRPATH.CONFIG_DIR}/.env"
         env_file_encoding = "utf-8"
+        extra = "allow"
 
 
 MODE = Mode()

studio/app/common/core/snakemake/smk.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass, field
+from dataclasses import dataclass
 from typing import Dict, List, Union
 
 from pydantic import BaseModel
@@ -29,11 +29,10 @@ class ForceRun(BaseModel):
     name: str
 
 
-@dataclass
-class SmkParam:
+class SmkParam(BaseModel):
     use_conda: bool
     cores: int
     forceall: bool
     forcetargets: bool
     lock: bool
-    forcerun: List[ForceRun] = field(default_factory=list)
+    forcerun: List[ForceRun] = []

studio/app/common/core/snakemake/smk_utils.py

@@ -157,7 +157,18 @@
         config_copy = copy.deepcopy(config)
         nwb_template = config_copy.get("nwb_template", {})
 
-        template_str = json.dumps(nwb_template, sort_keys=True) if nwb_template else ""
+        # Handle Pydantic models by converting to dict
+        if hasattr(nwb_template, 'model_dump'):
+            # Pydantic v2
+            nwb_template_dict = nwb_template.model_dump()
+        elif hasattr(nwb_template, 'dict'):
+            # Pydantic v1 fallback
+            nwb_template_dict = nwb_template.dict()
+        else:
+            # Regular dict or other type
+            nwb_template_dict = nwb_template
+
+        template_str = json.dumps(nwb_template_dict, sort_keys=True) if nwb_template_dict else ""
 
         # Check each rule and convert matching nwbfiles to references
         for rule_name, rule in config_copy.get("rules", {}).items():

studio/app/common/core/utils/config_handler.py

@@ -1,5 +1,6 @@
 import os
 
+from pydantic import BaseModel
 import yaml
 from filelock import FileLock
 
@@ -49,6 +50,17 @@
 
         config_path = join_filepath([dirname, filename])
 
+        def convert_to_serializable(obj):
+            if isinstance(obj, BaseModel):
+                return obj.model_dump()
+            elif isinstance(obj, dict):
+                return {k: convert_to_serializable(v) for k, v in obj.items()}
+            elif isinstance(obj, list):
+                return [convert_to_serializable(item) for item in obj]
+            return obj
+
+        config = convert_to_serializable(config)
+
         if auto_file_lock:
             # Exclusive control for parallel updates from multiple processes.
             lock_path = FileLockUtils.get_lockfile_path(config_path)

studio/app/common/core/workflow/workflow.py

@@ -128,7 +128,7 @@ class NodeItem(BaseModel):
 class OutputPath:
     path: str
     type: str
-    max_index: int = None
+    max_index: Optional[int] = None
     data_shape: Optional[list] = field(default_factory=list)
 
 
@@ -186,9 +186,9 @@ class NodeData:
     param: dict
     path: Union[str, List]
     type: str
-    fileType: str = None
-    hdf5Path: str = None
-    matPath: str = None
+    fileType: Optional[str] = None
+    hdf5Path: Optional[str] = None
+    matPath: Optional[str] = None
     dataFilterParam: Union[DataFilterParam, dict, None] = field(
         default_factory=lambda: DataFilterParam(dim1=[], roi=[])
     )
@@ -199,17 +199,17 @@ class NodeData:
 
 @dataclass
 class NodePosition:
-    x: int
-    y: int
+    x: float
+    y: float
 
 
 @dataclass
 class Style:
-    border: str = None
-    height: int = None
-    padding: int = None
-    width: int = None
-    borderRadius: int = None
+    border: Optional[str] = None
+    height: Optional[int] = None
+    padding: Optional[int] = None
+    width: Optional[int] = None
+    borderRadius: Optional[int] = None
 
 
 @dataclass
@@ -234,7 +234,7 @@ class Edge:
 
 
 class RunItem(BaseModel):
-    name: str = None
+    name: Optional[str] = None
     nodeDict: Dict[str, Node] = {}
     edgeDict: Dict[str, Edge] = {}
     snakemakeParam: dict = {}

studio/app/common/core/workflow/workflow_params.py

@@ -1,15 +1,35 @@
 from studio.app.common.core.logger import AppLogger
 from studio.app.common.core.utils.config_handler import ConfigReader
 from studio.app.common.core.utils.filepath_finder import find_param_filepath
+from studio.app.optinist.schemas.nwb import NWBParams
+from studio.app.common.core.snakemake.smk import SmkParam
 
 logger = AppLogger.get_logger()
 
 
 def get_typecheck_params(message_params, name):
     default_params = ConfigReader.read(find_param_filepath(name))
     if message_params != {} and message_params is not None:
-        return check_types(nest2dict(message_params), default_params)
-    return default_params
+        params = check_types(nest2dict(message_params), default_params)
+    else:
+        params = default_params
+
+    if name == "nwb":
+        try:
+            validated_params = NWBParams(**params)
+            return validated_params
+        except Exception as e:
+            logger.error(f"Error converting params to NWBParams: {e}")
+            return params
+    elif name == "snakemake":
+        try:
+            validated_params = SmkParam(**params)
+            return validated_params
+        except Exception as e:
+            logger.error(f"Error converting params to SmkParam: {e}")
+            return params
+
+    return params
 
 
 def check_types(params, default_params):