From 59447be051f41381d97d3a57f8ca571862ab5830 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Thu, 2 Nov 2023 19:55:12 +0900 Subject: [PATCH 01/10] refactor to use docval --- frontend/src/api/nwb/NWB.ts | 4 +- frontend/src/api/params/Params.ts | 4 +- frontend/src/api/snakemake/Snakemake.ts | 4 +- .../AlgorithmNode/AlgorithmNodeActions.ts | 4 +- .../slice/AlgorithmNode/AlgorithmNodeSlice.ts | 6 +- .../slice/FlowElement/FlowElementActions.ts | 4 +- frontend/src/store/slice/NWB/NWBAction.ts | 4 +- frontend/src/store/slice/NWB/NWBSlice.ts | 4 +- .../store/slice/Snakemake/SnakemakeAction.ts | 4 +- .../store/slice/Snakemake/SnakemakeSlice.ts | 4 +- frontend/src/utils/param/ParamType.ts | 3 + frontend/src/utils/param/ParamUtils.ts | 70 --- .../app/common/core/experiment/experiment.py | 12 +- .../core/experiment/experiment_reader.py | 63 ++- studio/app/common/core/param/param.py | 66 +++ studio/app/common/core/param/param_utils.py | 104 ++++ studio/app/common/core/rules/runner.py | 12 +- studio/app/common/core/snakemake/smk.py | 18 +- .../app/common/core/snakemake/snakemake.yaml | 5 - .../common/core/snakemake/snakemake_rule.py | 9 +- .../app/common/core/utils/filepath_finder.py | 9 +- studio/app/common/core/workflow/workflow.py | 5 +- .../common/core/workflow/workflow_params.py | 38 -- .../common/core/workflow/workflow_reader.py | 37 +- .../common/core/workflow/workflow_runner.py | 16 +- studio/app/common/core/wrapper/wrapper.py | 25 + .../app/common/core/wrapper/wrapper_utils.py | 20 + studio/app/common/routers/algolist.py | 32 +- studio/app/common/routers/params.py | 18 +- studio/app/common/routers/workflow.py | 4 +- studio/app/common/schemas/algolist.py | 4 +- studio/app/common/schemas/files.py | 2 +- studio/app/common/schemas/workflow.py | 3 +- studio/app/const.py | 4 +- studio/app/dir_path.py | 6 - studio/app/optinist/core/nwb/nwb.py | 65 ++- studio/app/optinist/core/nwb/nwb.yaml | 25 - studio/app/optinist/routers/nwb.py | 13 +- studio/app/optinist/wrappers/__init__.py | 13 +- .../app/optinist/wrappers/caiman/__init__.py | 19 +- studio/app/optinist/wrappers/caiman/cnmf.py | 468 +++++++++++------- .../wrappers/caiman/motion_correction.py | 190 ++++--- .../wrappers/caiman/params/caiman_cnmf.yaml | 21 - .../wrappers/caiman/params/caiman_mc.yaml | 19 - 44 files changed, 890 insertions(+), 570 deletions(-) create mode 100644 studio/app/common/core/param/param.py create mode 100644 studio/app/common/core/param/param_utils.py delete mode 100644 studio/app/common/core/snakemake/snakemake.yaml delete mode 100644 studio/app/common/core/workflow/workflow_params.py create mode 100644 studio/app/common/core/wrapper/wrapper.py create mode 100644 studio/app/common/core/wrapper/wrapper_utils.py delete mode 100644 studio/app/optinist/core/nwb/nwb.yaml delete mode 100644 studio/app/optinist/wrappers/caiman/params/caiman_cnmf.yaml delete mode 100644 studio/app/optinist/wrappers/caiman/params/caiman_mc.yaml diff --git a/frontend/src/api/nwb/NWB.ts b/frontend/src/api/nwb/NWB.ts index 0a5c32685..de57f2f98 100644 --- a/frontend/src/api/nwb/NWB.ts +++ b/frontend/src/api/nwb/NWB.ts @@ -1,8 +1,8 @@ import { BASE_URL } from "const/API" import axios from "utils/axios" -import { ParamDTO } from "utils/param/ParamType" +import { ParamMap } from "utils/param/ParamType" -export async function getNWBParamsApi(): Promise { +export async function getNWBParamsApi(): Promise { const response = await axios.get(`${BASE_URL}/nwb`) return response.data } diff --git a/frontend/src/api/params/Params.ts b/frontend/src/api/params/Params.ts index 6e2b015fc..7a739ffd0 100644 --- a/frontend/src/api/params/Params.ts +++ b/frontend/src/api/params/Params.ts @@ -1,8 +1,8 @@ import { BASE_URL } from "const/API" import axios from "utils/axios" -import { ParamDTO } from "utils/param/ParamType" +import { ParamMap } from "utils/param/ParamType" -export async function getAlgoParamsApi(algoName: string): Promise { +export async function getAlgoParamsApi(algoName: string): Promise { const response = await axios.get(`${BASE_URL}/params/${algoName}`) return response.data } diff --git a/frontend/src/api/snakemake/Snakemake.ts b/frontend/src/api/snakemake/Snakemake.ts index 01dc28bc4..0f734d550 100644 --- a/frontend/src/api/snakemake/Snakemake.ts +++ b/frontend/src/api/snakemake/Snakemake.ts @@ -1,8 +1,8 @@ import { BASE_URL } from "const/API" import axios from "utils/axios" -import { ParamDTO } from "utils/param/ParamType" +import { ParamMap } from "utils/param/ParamType" -export async function getSnakemakeParamsApi(): Promise { +export async function getSnakemakeParamsApi(): Promise { const response = await axios.get(`${BASE_URL}/snakemake`) return response.data } diff --git a/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeActions.ts b/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeActions.ts index 95c36db66..bb52d1b77 100644 --- a/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeActions.ts +++ b/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeActions.ts @@ -2,10 +2,10 @@ import { createAsyncThunk } from "@reduxjs/toolkit" import { getAlgoParamsApi } from "api/params/Params" import { ALGORITHM_NODE_SLICE_NAME } from "store/slice/AlgorithmNode/AlgorithmNodeType" -import { ParamDTO } from "utils/param/ParamType" +import { ParamMap } from "utils/param/ParamType" export const getAlgoParams = createAsyncThunk< - ParamDTO, + ParamMap, { nodeId: string; algoName: string } >( `${ALGORITHM_NODE_SLICE_NAME}/getAlgoParams`, diff --git a/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeSlice.ts b/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeSlice.ts index 05153a9d9..d858ae985 100644 --- a/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeSlice.ts +++ b/frontend/src/store/slice/AlgorithmNode/AlgorithmNodeSlice.ts @@ -19,7 +19,7 @@ import { importWorkflowConfig, fetchWorkflow, } from "store/slice/Workflow/WorkflowActions" -import { convertToParamMap, getChildParam } from "utils/param/ParamUtils" +import { getChildParam } from "utils/param/ParamUtils" const initialState: AlgorithmNode = {} @@ -50,7 +50,7 @@ export const algorithmNodeSlice = createSlice({ builder .addCase(getAlgoParams.fulfilled, (state, action) => { const { nodeId } = action.meta.arg - state[nodeId].params = convertToParamMap(action.payload) + state[nodeId].params = action.payload }) .addCase(addAlgorithmNode.fulfilled, (state, action) => { const { node, functionPath, name, runAlready } = action.meta.arg @@ -59,7 +59,7 @@ export const algorithmNodeSlice = createSlice({ state[node.id] = { functionPath, name, - params: convertToParamMap(params), + params: params, isUpdated: runAlready ?? false, } } diff --git a/frontend/src/store/slice/FlowElement/FlowElementActions.ts b/frontend/src/store/slice/FlowElement/FlowElementActions.ts index 82834c41c..563b8f153 100644 --- a/frontend/src/store/slice/FlowElement/FlowElementActions.ts +++ b/frontend/src/store/slice/FlowElement/FlowElementActions.ts @@ -8,13 +8,13 @@ import { NodeData, } from "store/slice/FlowElement/FlowElementType" import { FILE_TYPE } from "store/slice/InputNode/InputNodeType" -import { ParamDTO } from "utils/param/ParamType" +import { ParamMap } from "utils/param/ParamType" export type AddingNodeData = Omit, "position"> & Partial, "position">> export const addAlgorithmNode = createAsyncThunk< - ParamDTO, + ParamMap, { node: AddingNodeData functionPath: string diff --git a/frontend/src/store/slice/NWB/NWBAction.ts b/frontend/src/store/slice/NWB/NWBAction.ts index c5041962d..4da8e9d15 100644 --- a/frontend/src/store/slice/NWB/NWBAction.ts +++ b/frontend/src/store/slice/NWB/NWBAction.ts @@ -2,9 +2,9 @@ import { createAsyncThunk } from "@reduxjs/toolkit" import { getNWBParamsApi } from "api/nwb/NWB" import { NWB_SLICE_NAME } from "store/slice/NWB/NWBType" -import { ParamDTO } from "utils/param/ParamType" +import { ParamMap } from "utils/param/ParamType" -export const getNWBParams = createAsyncThunk( +export const getNWBParams = createAsyncThunk( `${NWB_SLICE_NAME}/getNWBParams`, async (_, thunkAPI) => { const { rejectWithValue } = thunkAPI diff --git a/frontend/src/store/slice/NWB/NWBSlice.ts b/frontend/src/store/slice/NWB/NWBSlice.ts index f9d6f414c..125681ae1 100644 --- a/frontend/src/store/slice/NWB/NWBSlice.ts +++ b/frontend/src/store/slice/NWB/NWBSlice.ts @@ -3,7 +3,7 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit" import { clearFlowElements } from "store/slice/FlowElement/FlowElementSlice" import { getNWBParams } from "store/slice/NWB/NWBAction" import { NWBType, NWB_SLICE_NAME } from "store/slice/NWB/NWBType" -import { convertToParamMap, getChildParam } from "utils/param/ParamUtils" +import { getChildParam } from "utils/param/ParamUtils" const initialState: NWBType = { params: {}, } @@ -29,7 +29,7 @@ export const nwbSlice = createSlice({ extraReducers: (builder) => { builder .addCase(getNWBParams.fulfilled, (state, action) => { - state.params = convertToParamMap(action.payload) + state.params = action.payload }) .addCase(clearFlowElements, () => initialState) }, diff --git a/frontend/src/store/slice/Snakemake/SnakemakeAction.ts b/frontend/src/store/slice/Snakemake/SnakemakeAction.ts index 698bd18e8..c3a0fe241 100644 --- a/frontend/src/store/slice/Snakemake/SnakemakeAction.ts +++ b/frontend/src/store/slice/Snakemake/SnakemakeAction.ts @@ -2,9 +2,9 @@ import { createAsyncThunk } from "@reduxjs/toolkit" import { getSnakemakeParamsApi } from "api/snakemake/Snakemake" import { SNAKEMAKE_SLICE_NAME } from "store/slice/Snakemake/SnakemakeType" -import { ParamDTO } from "utils/param/ParamType" +import { ParamMap } from "utils/param/ParamType" -export const getSnakemakeParams = createAsyncThunk( +export const getSnakemakeParams = createAsyncThunk( `${SNAKEMAKE_SLICE_NAME}/getSnakemakeParams`, async (_, thunkAPI) => { const { rejectWithValue } = thunkAPI diff --git a/frontend/src/store/slice/Snakemake/SnakemakeSlice.ts b/frontend/src/store/slice/Snakemake/SnakemakeSlice.ts index d45603fac..9d9766f0b 100644 --- a/frontend/src/store/slice/Snakemake/SnakemakeSlice.ts +++ b/frontend/src/store/slice/Snakemake/SnakemakeSlice.ts @@ -6,7 +6,7 @@ import { SnakemakeType, SNAKEMAKE_SLICE_NAME, } from "store/slice/Snakemake/SnakemakeType" -import { convertToParamMap, getChildParam } from "utils/param/ParamUtils" +import { getChildParam } from "utils/param/ParamUtils" const initialState: SnakemakeType = { params: {}, @@ -33,7 +33,7 @@ export const SnakemakeSlice = createSlice({ extraReducers: (builder) => { builder .addCase(getSnakemakeParams.fulfilled, (state, action) => { - state.params = convertToParamMap(action.payload) + state.params = action.payload }) .addCase(clearFlowElements, () => initialState) }, diff --git a/frontend/src/utils/param/ParamType.ts b/frontend/src/utils/param/ParamType.ts index 0e47bdc19..7fa82c068 100644 --- a/frontend/src/utils/param/ParamType.ts +++ b/frontend/src/utils/param/ParamType.ts @@ -13,6 +13,9 @@ export type ParamParent = { export type ParamChild = { type: "child" + // FIXME: remove ? after update test code + dataType?: string | string[] + doc?: string value: unknown path: string } diff --git a/frontend/src/utils/param/ParamUtils.ts b/frontend/src/utils/param/ParamUtils.ts index 43dc58857..6e88c608b 100644 --- a/frontend/src/utils/param/ParamUtils.ts +++ b/frontend/src/utils/param/ParamUtils.ts @@ -1,5 +1,4 @@ import { - ParamDTO, ParamChild, ParamParent, ParamType, @@ -33,72 +32,3 @@ export function isParamChild(param: ParamType): param is ParamChild { export function isParamParent(param: ParamType): param is ParamParent { return param.type === "parent" } - -function isDictObject(value: unknown): value is { [key: string]: unknown } { - return value !== null && typeof value === "object" && !Array.isArray(value) -} - -const PATH_SEPARATOR = "/" - -export function convertToParamMap(dto: ParamDTO, keyList?: string[]): ParamMap { - const ParamMap: ParamMap = {} - Object.entries(dto).forEach(([name, value]) => { - const kList = keyList ?? [] - if (isDictObject(value)) { - kList.push(name) - ParamMap[name] = { - type: "parent", - children: convertToParamMap(value, kList), - } - } else { - ParamMap[name] = { - type: "child", - value, - path: kList.concat([name]).join(PATH_SEPARATOR), - } - } - }) - return ParamMap -} - -export function equalsParamMap(a: ParamMap, b: ParamMap) { - if (a === b) { - return true - } - const aArray = Object.keys(a) - const bArray = Object.keys(b) - return ( - aArray.length === bArray.length && - aArray.every((aKey) => { - const aValue = a[aKey] - const bValue = b[aKey] - return equalsParam(aValue, bValue) - }) - ) -} - -function equalsParam(a: ParamType, b: ParamType): boolean { - if (a === b) { - return true - } - if (isParamChild(a) && isParamChild(b)) { - return equalsParamChild(a, b) - } else if (isParamParent(a) && isParamParent(b)) { - const aArray = Object.keys(a) - const bArray = Object.keys(b) - return ( - aArray.length === bArray.length && - aArray.every((aKey) => { - const aValue = a.children[aKey] - const bValue = b.children[aKey] - return equalsParam(aValue, bValue) - }) - ) - } else { - return false - } -} - -function equalsParamChild(a: ParamChild, b: ParamChild) { - return a.path === b.path && a.value === b.value -} diff --git a/studio/app/common/core/experiment/experiment.py b/studio/app/common/core/experiment/experiment.py index 6d5c34dd7..a44f70c96 100644 --- a/studio/app/common/core/experiment/experiment.py +++ b/studio/app/common/core/experiment/experiment.py @@ -1,9 +1,9 @@ -from dataclasses import dataclass -from typing import Dict, Optional +from typing import Dict, Optional, Union -from studio.app.common.core.snakemake.smk import SmkParam +from pydantic.dataclasses import dataclass + +from studio.app.common.core.param.param import ParamChild, ParamParent from studio.app.common.core.workflow.workflow import OutputPath -from studio.app.optinist.schemas.nwb import NWBParams @dataclass @@ -28,5 +28,5 @@ class ExptConfig: success: Optional[str] hasNWB: bool function: Dict[str, ExptFunction] - nwb: NWBParams - snakemake: SmkParam + nwb: Dict[str, Union[ParamChild, ParamParent]] + snakemake: Dict[str, Union[ParamChild, ParamParent]] diff --git a/studio/app/common/core/experiment/experiment_reader.py b/studio/app/common/core/experiment/experiment_reader.py index d706beed3..ef5749f41 100644 --- a/studio/app/common/core/experiment/experiment_reader.py +++ b/studio/app/common/core/experiment/experiment_reader.py @@ -1,8 +1,10 @@ -from typing import Dict +from typing import Dict, Union import yaml from studio.app.common.core.experiment.experiment import ExptConfig, ExptFunction +from studio.app.common.core.param.param import ParamChild, ParamParent +from studio.app.common.core.param.param_utils import ParamUtils from studio.app.common.core.workflow.workflow import OutputPath @@ -21,10 +23,65 @@ def read(cls, filepath) -> ExptConfig: success=config.get("success", "running"), hasNWB=config["hasNWB"], function=cls.read_function(config["function"]), - nwb=config.get("nwb"), - snakemake=config.get("snakemake"), + nwb=cls.read_params(config.get("nwb"), "nwb"), + snakemake=cls.read_params(config.get("snakemake"), "snakemake"), ) + @classmethod + def read_params(cls, config, kind) -> Dict[str, Union[ParamChild, ParamParent]]: + default_params = ParamUtils.get_default_params(kind) + try: + # NOTE: older version params has differernt format + assert ( + isinstance(list(config.items())[0][1], dict) + and "type" in list(config.items())[0][1] + ) + return ParamUtils.merge_params(config, default_params) + except AssertionError: + param_tree = cls.convert_to_param_tree(config) + return ParamUtils.merge_params(param_tree, default_params) + + @classmethod + def convert_to_param_tree(cls, config): + """ + This function is for converting older version params to new version params. + Strictly speaking, new version has other attributes like doc, dataType. + But, these values are not updated by user and filled by merge_params function. + So we don't need to add them here. + - older version format + { + "param1": "value1", + "section1": { + "param2": "value2", + } + } + - new version format + { + "param1": { + "type": "child", + "value": "value1" + }, + "section1": { + "type": "parent", + "children": { + "param2": { + "type": "child", + "value": "value2" + } + } + } + } + """ + for k, v in config.items(): + if isinstance(v, dict): + config[k] = { + "children": cls.convert_to_param_tree(v), + "type": "parent", + } + else: + config[k] = {"type": "child", "value": v} + return config + @classmethod def read_function(cls, config) -> Dict[str, ExptFunction]: return { diff --git a/studio/app/common/core/param/param.py b/studio/app/common/core/param/param.py new file mode 100644 index 000000000..78c888644 --- /dev/null +++ b/studio/app/common/core/param/param.py @@ -0,0 +1,66 @@ +from typing import Any, Dict, List, Optional, Union + +from pydantic.dataclasses import dataclass + + +@dataclass +class ParamChild: + path: str + value: Any + dataType: Optional[Union[str, List]] = None + doc: Optional[str] = None + type: str = "child" + + +@dataclass +class ParamParent: + children: Dict[str, Union["ParamParent", ParamChild]] + type: str = "parent" + + +class Param: + def __init__( + self, + name: str, + type: Union[type, List[type]], + doc: Optional[str] = None, + section: Optional[str] = None, + **kwargs, + ): + self.name = name + self.type = type + self.doc = doc + self.section = section + if "default" in kwargs: + self.default = kwargs["default"] + + @property + def is_none(self): + if "default" in self.__dict__: + return self.default is None + else: + return False + + def docval_dict(self): + docval_dict = { + "name": self.name, + "type": tuple(self.type) if isinstance(self.type, List) else self.type, + "doc": self.doc, + } + if "default" in self.__dict__: + docval_dict["default"] = self.default + return docval_dict + + def json_dict(self): + json_dict = { + "dataType": [t.__name__ for t in self.type] + if isinstance(self.type, List) + else self.type.__name__, + "doc": self.doc, + "path": self.name + if self.section is None + else f"{self.section}/{self.name}", + "type": "child", + "value": self.default if "default" in self.__dict__ else None, + } + return json_dict diff --git a/studio/app/common/core/param/param_utils.py b/studio/app/common/core/param/param_utils.py new file mode 100644 index 000000000..b04435fce --- /dev/null +++ b/studio/app/common/core/param/param_utils.py @@ -0,0 +1,104 @@ +from dataclasses import asdict + +from studio.app.common.core.param.param import ParamChild, ParamParent +from studio.app.common.core.snakemake.smk import SnakemakeParams +from studio.app.common.core.wrapper.wrapper_utils import WrapperUtils +from studio.app.optinist.core.nwb.nwb import NWBParams + + +class ParamUtils: + @classmethod + def get_default_params(cls, name): + if name == "snakemake": + params = SnakemakeParams.PARAMS + elif name == "nwb": + params = NWBParams.PARAMS + else: + wrapper = WrapperUtils.find_wrapper_by_name(name=name) + if wrapper is None: + return None + params = wrapper._DEFAULT_PARAMS + + result = {} + for param in params: + if param.section is None: + result[param.name] = param.json_dict() + else: + sections = param.section.split("/") + current_section = result + for section in sections: + if section not in current_section: + current_section[section] = {"type": "parent", "children": {}} + current_section = current_section[section]["children"] + else: + current_section[param.name] = param.json_dict() + return result + + @classmethod + def merge_params(cls, source, destination): + for key, value in source.items(): + if key in destination: + if isinstance(value, dict): + cls.merge_params(value, destination[key]) + else: + destination[key] = value + else: + destination[key] = value + return destination + + @classmethod + def convert_to_plane_dict(cls, params): + for k, v in params.items(): + if isinstance(v, ParamParent): + params[k] = { + "type": "parent", + "children": cls.convert_to_plane_dict(v.children), + } + elif isinstance(v, ParamChild): + params[k] = asdict(v) + return params + + @classmethod + def get_type_fixed_params(cls, params, name): + default_params = cls.get_default_params(name) + if params != {} and params is not None: + params = cls.merge_params(params, default_params) + return cls.fix_param_value_type(params) + else: + return default_params + + @classmethod + def fix_param_value_type(cls, params): + for k, v in params.items(): + if v["type"] == "parent": + params[k]["children"] = cls.fix_param_value_type(v["children"]) + if v["type"] == "child" and v["value"] is not None: + if v["dataType"] == "str": + v["value"] = str(v["value"]) + elif v["dataType"] == "int": + v["value"] = int(v["value"]) + elif v["dataType"] == "float": + v["value"] = float(v["value"]) + elif v["dataType"] == "bool": + v["value"] = bool(v["value"]) + return params + + @classmethod + def get_key_value_params(cls, params): + key_value_params = {} + for k, v in params.items(): + if v["type"] == "parent": + key_value_params[k] = cls.get_key_value_params(v["children"]) + elif v["type"] == "child": + key_value_params[k] = v["value"] + return key_value_params + + @classmethod + def get_flatten_params(cls, params): + flatten_params = {} + for k, v in params.items(): + if isinstance(v, dict): + flatten_params.update(cls.get_flatten_params(v)) + else: + flatten_params[k] = v + return flatten_params diff --git a/studio/app/common/core/rules/runner.py b/studio/app/common/core/rules/runner.py index 04394541e..54c60d657 100644 --- a/studio/app/common/core/rules/runner.py +++ b/studio/app/common/core/rules/runner.py @@ -8,6 +8,7 @@ from filelock import FileLock from studio.app.common.core.experiment.experiment_reader import ExptConfigReader +from studio.app.common.core.param.param_utils import ParamUtils from studio.app.common.core.snakemake.smk import Rule from studio.app.common.core.utils.config_handler import ConfigWriter from studio.app.common.core.utils.filepath_creater import join_filepath @@ -122,11 +123,14 @@ def save_all_nwb(cls, save_path, all_nwbfile): @classmethod def execute_function(cls, path, params, nwb_params, output_dir, input_info): wrapper = cls.dict2leaf(wrapper_dict, path.split("/")) - func = copy.deepcopy(wrapper["function"]) - output_info = func( - params=params, nwbfile=nwb_params, output_dir=output_dir, **input_info + wrapper = copy.deepcopy(wrapper["function"]) + flatten_params = ParamUtils.get_flatten_params(params) + output_info = ( + wrapper.set_output_dir(output_dir) + .set_nwb_params(nwb_params) + .func(**flatten_params, **input_info) ) - del func + del wrapper gc.collect() return output_info diff --git a/studio/app/common/core/snakemake/smk.py b/studio/app/common/core/snakemake/smk.py index f9aac4063..d7af5a785 100644 --- a/studio/app/common/core/snakemake/smk.py +++ b/studio/app/common/core/snakemake/smk.py @@ -1,7 +1,9 @@ -from dataclasses import dataclass, field from typing import Dict, List, Union -from pydantic import BaseModel +from pydantic import BaseModel, Field +from pydantic.dataclasses import dataclass + +from studio.app.common.core.param.param import Param @dataclass @@ -34,4 +36,14 @@ class SmkParam: forceall: bool forcetargets: bool lock: bool - forcerun: List[ForceRun] = field(default_factory=list) + forcerun: List[ForceRun] = Field(default_factory=list) + + +class SnakemakeParams: + PARAMS = [ + Param(name="use_conda", type=bool, default=True), + Param(name="cores", type=int, default=2), + Param(name="forceall", type=bool, default=False), + Param(name="forcetargets", type=bool, default=True), + Param(name="lock", type=bool, default=False), + ] diff --git a/studio/app/common/core/snakemake/snakemake.yaml b/studio/app/common/core/snakemake/snakemake.yaml deleted file mode 100644 index 3de99490b..000000000 --- a/studio/app/common/core/snakemake/snakemake.yaml +++ /dev/null @@ -1,5 +0,0 @@ -use_conda: True -cores: 2 -forceall: False -forcetargets: True -lock: False diff --git a/studio/app/common/core/snakemake/snakemake_rule.py b/studio/app/common/core/snakemake/snakemake_rule.py index 79b38c711..122317215 100644 --- a/studio/app/common/core/snakemake/snakemake_rule.py +++ b/studio/app/common/core/snakemake/snakemake_rule.py @@ -1,10 +1,10 @@ from typing import Dict +from studio.app.common.core.param.param_utils import ParamUtils from studio.app.common.core.snakemake.smk import Rule from studio.app.common.core.snakemake.smk_builder import RuleBuilder from studio.app.common.core.utils.filepath_creater import get_pickle_file from studio.app.common.core.workflow.workflow import Edge, Node, NodeType -from studio.app.common.core.workflow.workflow_params import get_typecheck_params class SmkRule: @@ -76,7 +76,12 @@ def algo(self, nodeDict: Dict[str, Node]) -> Rule: return_arg_names[return_name] = arg_name - params = get_typecheck_params(self._node.data.param, self._node.data.label) + params = ParamUtils.get_key_value_params( + ParamUtils.get_type_fixed_params( + ParamUtils.convert_to_plane_dict(self._node.data.param), + self._node.data.label, + ) + ) algo_output = get_pickle_file( self._workspace_id, self._unique_id, self._node.id, self._node.data.label ) diff --git a/studio/app/common/core/utils/filepath_finder.py b/studio/app/common/core/utils/filepath_finder.py index 31c19ab4d..f17bb37d8 100644 --- a/studio/app/common/core/utils/filepath_finder.py +++ b/studio/app/common/core/utils/filepath_finder.py @@ -3,7 +3,7 @@ from typing import Optional from studio.app.common.core.utils.filepath_creater import join_filepath -from studio.app.dir_path import CORE_PARAM_PATH, DIRPATH +from studio.app.dir_path import DIRPATH def find_filepath(name, category) -> Optional[str]: @@ -18,12 +18,5 @@ def find_filepath(name, category) -> Optional[str]: return filepaths[0] if len(filepaths) > 0 else None -def find_param_filepath(name: str): - if name in CORE_PARAM_PATH.__members__: - return CORE_PARAM_PATH[name].value - else: - return find_filepath(name, "params") - - def find_condaenv_filepath(name: str): return find_filepath(name, "conda") diff --git a/studio/app/common/core/workflow/workflow.py b/studio/app/common/core/workflow/workflow.py index 645f06f04..fb13ac893 100644 --- a/studio/app/common/core/workflow/workflow.py +++ b/studio/app/common/core/workflow/workflow.py @@ -1,8 +1,9 @@ -from dataclasses import dataclass from typing import Dict, List, Union from pydantic import BaseModel +from pydantic.dataclasses import dataclass +from studio.app.common.core.param.param import ParamChild, ParamParent from studio.app.common.core.snakemake.smk import ForceRun @@ -52,7 +53,7 @@ class Message: @dataclass class NodeData: label: str - param: dict + param: Dict[str, Union[ParamParent, ParamChild]] path: Union[str, List] type: str fileType: str = None diff --git a/studio/app/common/core/workflow/workflow_params.py b/studio/app/common/core/workflow/workflow_params.py deleted file mode 100644 index 88e67627d..000000000 --- a/studio/app/common/core/workflow/workflow_params.py +++ /dev/null @@ -1,38 +0,0 @@ -from studio.app.common.core.utils.config_handler import ConfigReader -from studio.app.common.core.utils.filepath_finder import find_param_filepath - - -def get_typecheck_params(message_params, name): - default_params = ConfigReader.read(find_param_filepath(name)) - if message_params != {} and message_params is not None: - return check_types(nest2dict(message_params), default_params) - return default_params - - -def check_types(params, default_params): - for key in params.keys(): - if isinstance(params[key], dict): - params[key] = check_types(params[key], default_params[key]) - else: - if not isinstance(type(params[key]), type(default_params[key])): - data_type = type(default_params[key]) - p = params[key] - if isinstance(data_type, str): - params[key] = str(p) - elif isinstance(data_type, float): - params[key] = float(p) - elif isinstance(data_type, int): - params[key] = int(p) - - return params - - -def nest2dict(value): - nwb_dict = {} - for _k, _v in value.items(): - if _v["type"] == "child": - nwb_dict[_k] = _v["value"] - elif _v["type"] == "parent": - nwb_dict[_k] = nest2dict(_v["children"]) - - return nwb_dict diff --git a/studio/app/common/core/workflow/workflow_reader.py b/studio/app/common/core/workflow/workflow_reader.py index d46209993..5f0705aba 100644 --- a/studio/app/common/core/workflow/workflow_reader.py +++ b/studio/app/common/core/workflow/workflow_reader.py @@ -2,6 +2,7 @@ import yaml +from studio.app.common.core.param.param_utils import ParamUtils from studio.app.common.core.workflow.workflow import ( Edge, Node, @@ -23,19 +24,53 @@ def read(cls, filepath) -> WorkflowConfig: edgeDict=cls.read_edgeDict(config["edgeDict"]), ) + @classmethod + def read_binary(cls, file) -> WorkflowConfig: + return WorkflowConfig( + nodeDict=cls.read_nodeDict(file["nodeDict"]), + edgeDict=cls.read_edgeDict(file["edgeDict"]), + ) + @classmethod def read_nodeDict(cls, config) -> Dict[str, Node]: return { key: Node( id=key, type=value["type"], - data=NodeData(**value["data"]), + data=cls.read_nodeData(value["data"]), position=NodePosition(**value["position"]), style=Style(**value["style"]), ) for key, value in config.items() } + @classmethod + def read_nodeData(cls, config) -> Dict[str, NodeData]: + default_params = ( + ParamUtils.get_default_params(config["label"]) + if config["type"] == "algorithm" + else None + ) + + if default_params is not None: + return NodeData( + label=config["label"], + path=config["path"], + type=config["type"], + param=ParamUtils.merge_params(config["param"], default_params), + fileType=config.get("fileType"), + hdf5Path=config.get("hdf5Path"), + ) + else: + return NodeData( + label=config["label"], + path=config["path"], + type=config["type"], + param=config["param"], + fileType=config.get("fileType"), + hdf5Path=config.get("hdf5Path"), + ) + @classmethod def read_edgeDict(cls, config) -> Dict[str, Edge]: return { diff --git a/studio/app/common/core/workflow/workflow_runner.py b/studio/app/common/core/workflow/workflow_runner.py index f72a2d877..f4fc7fc3d 100644 --- a/studio/app/common/core/workflow/workflow_runner.py +++ b/studio/app/common/core/workflow/workflow_runner.py @@ -2,6 +2,7 @@ from typing import Dict, List from studio.app.common.core.experiment.experiment_writer import ExptConfigWriter +from studio.app.common.core.param.param_utils import ParamUtils from studio.app.common.core.snakemake.smk import FlowConfig, Rule, SmkParam from studio.app.common.core.snakemake.snakemake_executor import ( delete_dependencies, @@ -11,7 +12,6 @@ from studio.app.common.core.snakemake.snakemake_rule import SmkRule from studio.app.common.core.snakemake.snakemake_writer import SmkConfigWriter from studio.app.common.core.workflow.workflow import NodeType, RunItem -from studio.app.common.core.workflow.workflow_params import get_typecheck_params from studio.app.common.core.workflow.workflow_reader import WorkflowConfigReader from studio.app.common.core.workflow.workflow_writer import WorkflowConfigWriter @@ -35,15 +35,17 @@ def __init__(self, workspace_id: str, unique_id: str, runItem: RunItem) -> None: self.workspace_id, self.unique_id, self.runItem.name, - nwbfile=get_typecheck_params(self.runItem.nwbParam, "nwb"), - snakemake=get_typecheck_params(self.runItem.snakemakeParam, "snakemake"), + nwbfile=ParamUtils.get_type_fixed_params(self.runItem.nwbParam, "nwb"), + snakemake=ParamUtils.get_type_fixed_params( + self.runItem.snakemakeParam, "snakemake" + ), ).write() def run_workflow(self, background_tasks): self.set_smk_config() - snakemake_params: SmkParam = get_typecheck_params( - self.runItem.snakemakeParam, "snakemake" + snakemake_params: SmkParam = ParamUtils.get_key_value_params( + ParamUtils.get_type_fixed_params(self.runItem.snakemakeParam, "snakemake") ) snakemake_params = SmkParamReader.read(snakemake_params) snakemake_params.forcerun = self.runItem.forceRunList @@ -72,7 +74,9 @@ def set_smk_config(self): def rulefile(self): endNodeList = self.get_endNodeList() - nwbfile = get_typecheck_params(self.runItem.nwbParam, "nwb") + nwbfile = ParamUtils.get_key_value_params( + ParamUtils.get_type_fixed_params(self.runItem.nwbParam, "nwb") + ) rule_dict: Dict[str, Rule] = {} last_outputs = [] diff --git a/studio/app/common/core/wrapper/wrapper.py b/studio/app/common/core/wrapper/wrapper.py new file mode 100644 index 000000000..0e57fc52f --- /dev/null +++ b/studio/app/common/core/wrapper/wrapper.py @@ -0,0 +1,25 @@ +from typing import List + +from studio.app.common.core.param.param import Param + + +class Wrapper: + _INPUT_NODES: List[Param] = [] + _OUTPUT_NODES: List[Param] = [] + _DEFAULT_PARAMS: List[Param] = [] + + def set_output_dir(self, output_dir: str): + self.output_dir = output_dir + self.function_id = output_dir.split("/")[-1] + return self + + def set_nwb_params(self, nwb_params): + self.nwb_params = nwb_params + return self + + @staticmethod + def docval_params(params: List[Param]): + return [param.docval_dict() for param in params] + + def func(self, **kwargs): + pass diff --git a/studio/app/common/core/wrapper/wrapper_utils.py b/studio/app/common/core/wrapper/wrapper_utils.py new file mode 100644 index 000000000..5f79d5d01 --- /dev/null +++ b/studio/app/common/core/wrapper/wrapper_utils.py @@ -0,0 +1,20 @@ +from typing import Any, Dict + +from studio.app.common.core.wrapper.wrapper import Wrapper +from studio.app.wrappers import wrapper_dict + + +class WrapperUtils: + @classmethod + def find_wrapper_by_name( + cls, + name: str, + wrapper_dict: Dict[str, Any] = wrapper_dict, + ) -> Wrapper: + if name in wrapper_dict: + return wrapper_dict[name]["function"] + else: + for v in wrapper_dict.values(): + if isinstance(v, dict) and "function" not in v: + return cls.find_wrapper_by_name(name, v) + return None diff --git a/studio/app/common/routers/algolist.py b/studio/app/common/routers/algolist.py index b5526c173..e57ae6350 100644 --- a/studio/app/common/routers/algolist.py +++ b/studio/app/common/routers/algolist.py @@ -1,10 +1,10 @@ -import inspect -from typing import Dict, List, ValuesView +from typing import Dict, List from fastapi import APIRouter +from studio.app.common.core.param.param import Param +from studio.app.common.core.wrapper.wrapper import Wrapper from studio.app.common.schemas.algolist import Algo, AlgoList, Arg, Return -from studio.app.const import NOT_DISPLAY_ARGS_LIST from studio.app.wrappers import wrapper_dict router = APIRouter() @@ -12,7 +12,7 @@ class NestDictGetter: @classmethod - def get_nest_dict(cls, parent_value, parent_key: str) -> Dict[str, Algo]: + def get_nest_dict(cls, parent_value, parent_key: str) -> AlgoList: algo_dict = {} for key, value in parent_value.items(): algo_dict[key] = {} @@ -21,35 +21,25 @@ def get_nest_dict(cls, parent_value, parent_key: str) -> Dict[str, Algo]: value, cls._parent_key(parent_key, key) ) else: - sig = inspect.signature(value["function"]) - returns_list = None - if sig.return_annotation is not inspect._empty: - returns_list = cls._return_list(sig.return_annotation.items()) + wrapper: Wrapper = value["function"] algo_dict[key] = Algo( - args=cls._args_list(sig.parameters.values()), - returns=returns_list, - parameter=value["parameter"] if "parameter" in value else None, + args=cls._args_list(wrapper._INPUT_NODES), + returns=cls._return_list(wrapper._OUTPUT_NODES), path=cls._parent_key(parent_key, key), ) return algo_dict @classmethod - def _args_list(cls, arg_params: ValuesView[inspect.Parameter]) -> List[Arg]: + def _args_list(cls, arg_params: List[Param]) -> List[Arg]: return [ - Arg( - name=x.name, - type=x.annotation.__name__, - isNone=x.default is None, - ) - for x in arg_params - if x.name not in NOT_DISPLAY_ARGS_LIST + Arg(name=x.name, type=x.type.__name__, isNone=x.is_none) for x in arg_params ] @classmethod - def _return_list(cls, return_params: ValuesView[inspect.Parameter]) -> List[Return]: - return [Return(name=k, type=v.__name__) for k, v in return_params] + def _return_list(cls, return_params: List[Param]) -> List[Return]: + return [Return(name=x.name, type=x.type.__name__) for x in return_params] @classmethod def _parent_key(cls, parent_key: str, key: str) -> str: diff --git a/studio/app/common/routers/params.py b/studio/app/common/routers/params.py index ec1eeda78..b0cb81edc 100644 --- a/studio/app/common/routers/params.py +++ b/studio/app/common/routers/params.py @@ -1,22 +1,18 @@ -from typing import Any, Dict +from typing import Dict, Union from fastapi import APIRouter -from studio.app.common.core.utils.config_handler import ConfigReader -from studio.app.common.core.utils.filepath_finder import find_param_filepath -from studio.app.common.schemas.params import SnakemakeParams +from studio.app.common.core.param.param import ParamChild, ParamParent +from studio.app.common.core.param.param_utils import ParamUtils router = APIRouter(tags=["params"]) -@router.get("/params/{name}", response_model=Dict[str, Any]) +@router.get("/params/{name}", response_model=Dict[str, Union[ParamChild, ParamParent]]) async def get_params(name: str): - filepath = find_param_filepath(name) - config = ConfigReader.read(filepath) - return config + return ParamUtils.get_default_params(name) -@router.get("/snakemake", response_model=SnakemakeParams) +@router.get("/snakemake", response_model=Dict[str, Union[ParamChild, ParamParent]]) async def get_snakemake_params(): - filepath = find_param_filepath("snakemake") - return ConfigReader.read(filepath) + return ParamUtils.get_default_params("snakemake") diff --git a/studio/app/common/routers/workflow.py b/studio/app/common/routers/workflow.py index 5727646a5..459f9d30a 100644 --- a/studio/app/common/routers/workflow.py +++ b/studio/app/common/routers/workflow.py @@ -12,7 +12,7 @@ from studio.app.common.core.workspace.workspace_dependencies import ( is_workspace_available, ) -from studio.app.common.schemas.workflow import WorkflowConfig, WorkflowWithResults +from studio.app.common.schemas.workflow import WorkflowWithResults from studio.app.dir_path import DIRPATH router = APIRouter(prefix="/workflow", tags=["workflow"]) @@ -84,6 +84,6 @@ async def import_workflow_config(file: UploadFile = File(...)): contents = yaml.safe_load(await file.read()) if contents is None: raise HTTPException(status_code=400, detail="Invalid yaml file") - return WorkflowConfig(**contents) + return WorkflowConfigReader.read_binary(contents) except Exception as e: raise HTTPException(status_code=400, detail=f"Parsing yaml failed: {str(e)}") diff --git a/studio/app/common/schemas/algolist.py b/studio/app/common/schemas/algolist.py index e97a0f1aa..b8eeb8978 100644 --- a/studio/app/common/schemas/algolist.py +++ b/studio/app/common/schemas/algolist.py @@ -1,7 +1,7 @@ -from dataclasses import dataclass from typing import Dict, List, Union from pydantic import BaseModel +from pydantic.dataclasses import dataclass @dataclass @@ -21,7 +21,6 @@ class Return: class Algo: args: List[Arg] returns: List[Return] - parameter: str = None path: str = None @@ -36,7 +35,6 @@ class AlgoList(BaseModel): "caiman_mc": { "args": [{"name": "image", "type": "ImageData", "isNone": False}], "returns": [{"name": "mc_images", "type": "ImageData"}], - "parameter": None, "path": "caiman/caiman_mc", } } diff --git a/studio/app/common/schemas/files.py b/studio/app/common/schemas/files.py index 556f4f678..c8ad964b9 100644 --- a/studio/app/common/schemas/files.py +++ b/studio/app/common/schemas/files.py @@ -1,6 +1,6 @@ -from dataclasses import dataclass from typing import List +from pydantic.dataclasses import dataclass from pydantic.dataclasses import dataclass as pydantic_dataclass diff --git a/studio/app/common/schemas/workflow.py b/studio/app/common/schemas/workflow.py index 686001789..a681dc2da 100644 --- a/studio/app/common/schemas/workflow.py +++ b/studio/app/common/schemas/workflow.py @@ -1,6 +1,7 @@ -from dataclasses import dataclass from typing import Dict, Optional +from pydantic.dataclasses import dataclass + from studio.app.common.core.experiment.experiment import ExptFunction from studio.app.common.core.workflow.workflow import Edge, Node diff --git a/studio/app/const.py b/studio/app/const.py index 0192415f1..579a778c2 100644 --- a/studio/app/const.py +++ b/studio/app/const.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from pydantic.dataclasses import dataclass @dataclass @@ -13,6 +13,4 @@ class FILETYPE: ACCEPT_CSV_EXT = [".csv"] ACCEPT_HDF5_EXT = [".hdf5", ".nwb", ".HDF5", ".NWB"] -NOT_DISPLAY_ARGS_LIST = ["params", "output_dir", "nwbfile", "kwargs"] - DATE_FORMAT = "%Y-%m-%d %H:%M:%S" diff --git a/studio/app/dir_path.py b/studio/app/dir_path.py index edaac3c2b..2a858d75b 100644 --- a/studio/app/dir_path.py +++ b/studio/app/dir_path.py @@ -1,5 +1,4 @@ import os -from enum import Enum _DEFAULT_DIR = "/tmp/studio" _ENV_DIR = os.environ.get("OPTINIST_DIR") @@ -35,8 +34,3 @@ class DIRPATH: FIREBASE_PRIVATE_PATH = f"{CONFIG_DIR}/auth/firebase_private.json" FIREBASE_CONFIG_PATH = f"{CONFIG_DIR}/auth/firebase_config.json" - - -class CORE_PARAM_PATH(Enum): - nwb = f"{DIRPATH.APP_DIR}/optinist/core/nwb/nwb.yaml" - snakemake = f"{DIRPATH.APP_DIR}/common/core/snakemake/snakemake.yaml" diff --git a/studio/app/optinist/core/nwb/nwb.py b/studio/app/optinist/core/nwb/nwb.py index f522bf6c1..3b7dabd33 100644 --- a/studio/app/optinist/core/nwb/nwb.py +++ b/studio/app/optinist/core/nwb/nwb.py @@ -1,4 +1,6 @@ -from dataclasses import dataclass +from pydantic.dataclasses import dataclass + +from studio.app.common.core.param.param import Param @dataclass @@ -11,3 +13,64 @@ class NWBDATASET: FLUORESCENCE: str = "FLUORESCENCE" BEHAVIOR: str = "BEHAVIOR" IMAGE_SERIES: str = "image_series" + + +class NWBParams: + PARAMS = [ + Param(name="session_description", type=str, default="optinist"), + Param(name="identifier", type=str, default="optinist"), + Param(name="experiment_description", type=str, default=None), + Param(name="name", type=str, default="Microscope device", section="device"), + Param( + name="description", + type=str, + default="Microscope Information", + section="device", + ), + Param( + name="manufacturer", + type=str, + default="Microscope Manufacture", + section="device", + ), + Param( + name="name", type=str, default="OpticalChannel", section="optical_channel" + ), + Param( + name="description", + type=str, + default="optical channel", + section="optical_channel", + ), + Param( + name="emission_lambda", type=float, default=500.0, section="optical_channel" + ), + Param(name="name", type=str, default="ImagingPlane", section="imaging_plane"), + Param( + name="description", type=str, default="standard", section="imaging_plane" + ), + Param(name="imaging_rate", type=float, default=30.0, section="imaging_plane"), + Param( + name="excitation_lambda", type=float, default=600.0, section="imaging_plane" + ), + Param(name="indicator", type=str, default="GCaMap", section="imaging_plane"), + Param(name="location", type=str, default="V1", section="imaging_plane"), + Param(name="starting_time", type=int, default=0, section="image_series"), + Param( + name="starting_frame", + type=list, + default=[ + 0, + ], + section="image_series", + ), + Param( + name="name", + type=str, + default="PlaneSegmentation", + section="ophys/plane_segmentation", + ), + Param( + name="description", type=str, default="", section="ophys/plane_segmentation" + ), + ] diff --git a/studio/app/optinist/core/nwb/nwb.yaml b/studio/app/optinist/core/nwb/nwb.yaml deleted file mode 100644 index c8f868b54..000000000 --- a/studio/app/optinist/core/nwb/nwb.yaml +++ /dev/null @@ -1,25 +0,0 @@ -session_description: 'optinist' -identifier: 'optinist' -experiment_description: 'None' -device: - name: 'Microscope device' - description: 'Microscope Information' - manufacturer: 'Microscope Manufacture' -optical_channel: - name: 'OpticalChannel' - description: 'optical channel' - emission_lambda: 500.0 -imaging_plane: - name: 'ImagingPlane' - description: 'standard' - imaging_rate: 30.0 - excitation_lambda: 600.0 - indicator: 'GCaMap' - location: 'V1' -image_series: - starting_time: 0 - starting_frame: [0,] -ophys: - plane_segmentation: - name: 'PlaneSegmentation' - description: '' diff --git a/studio/app/optinist/routers/nwb.py b/studio/app/optinist/routers/nwb.py index ff172be5b..e16565852 100644 --- a/studio/app/optinist/routers/nwb.py +++ b/studio/app/optinist/routers/nwb.py @@ -1,24 +1,25 @@ from glob import glob +from typing import Dict, Union from fastapi import APIRouter, Depends from fastapi.responses import FileResponse -from studio.app.common.core.utils.config_handler import ConfigReader +from studio.app.common.core.param.param import ParamChild, ParamParent +from studio.app.common.core.param.param_utils import ParamUtils from studio.app.common.core.utils.filepath_creater import join_filepath -from studio.app.common.core.utils.filepath_finder import find_param_filepath from studio.app.common.core.workspace.workspace_dependencies import ( is_workspace_available, ) from studio.app.dir_path import DIRPATH -from studio.app.optinist.schemas.nwb import NWBParams router = APIRouter() -@router.get("/nwb", response_model=NWBParams, tags=["params"]) +@router.get( + "/nwb", tags=["params"], response_model=Dict[str, Union[ParamChild, ParamParent]] +) async def get_nwb_params(): - filepath = find_param_filepath("nwb") - return ConfigReader.read(filepath) + return ParamUtils.get_default_params("nwb") @router.get( diff --git a/studio/app/optinist/wrappers/__init__.py b/studio/app/optinist/wrappers/__init__.py index c829fdf5d..5d62855a8 100644 --- a/studio/app/optinist/wrappers/__init__.py +++ b/studio/app/optinist/wrappers/__init__.py @@ -1,10 +1,11 @@ from studio.app.optinist.wrappers.caiman import caiman_wrapper_dict -from studio.app.optinist.wrappers.lccd import lccd_wrapper_dict -from studio.app.optinist.wrappers.optinist import optinist_wrapper_dict -from studio.app.optinist.wrappers.suite2p import suite2p_wrapper_dict + +# from studio.app.optinist.wrappers.lccd import lccd_wrapper_dict +# from studio.app.optinist.wrappers.optinist import optinist_wrapper_dict +# from studio.app.optinist.wrappers.suite2p import suite2p_wrapper_dict wrapper_dict = {} wrapper_dict.update(**caiman_wrapper_dict) -wrapper_dict.update(**suite2p_wrapper_dict) -wrapper_dict.update(**lccd_wrapper_dict) -wrapper_dict.update(**optinist_wrapper_dict) +# wrapper_dict.update(**suite2p_wrapper_dict) +# wrapper_dict.update(**lccd_wrapper_dict) +# wrapper_dict.update(**optinist_wrapper_dict) diff --git a/studio/app/optinist/wrappers/caiman/__init__.py b/studio/app/optinist/wrappers/caiman/__init__.py index e0a3d5cc1..c43d929c4 100644 --- a/studio/app/optinist/wrappers/caiman/__init__.py +++ b/studio/app/optinist/wrappers/caiman/__init__.py @@ -1,20 +1,21 @@ -from studio.app.optinist.wrappers.caiman.cnmf import caiman_cnmf -from studio.app.optinist.wrappers.caiman.cnmfe import caiman_cnmfe -from studio.app.optinist.wrappers.caiman.motion_correction import caiman_mc +from studio.app.optinist.wrappers.caiman.cnmf import CaimanCnmf +from studio.app.optinist.wrappers.caiman.motion_correction import CaimanMc + +# from studio.app.optinist.wrappers.caiman.cnmfe import caiman_cnmfe caiman_wrapper_dict = { "caiman": { "caiman_mc": { - "function": caiman_mc, + "function": CaimanMc(), "conda_name": "caiman", }, "caiman_cnmf": { - "function": caiman_cnmf, - "conda_name": "caiman", - }, - "caiman_cnmfe": { - "function": caiman_cnmfe, + "function": CaimanCnmf(), "conda_name": "caiman", }, + # "caiman_cnmfe": { + # "function": caiman_cnmfe, + # "conda_name": "caiman", + # }, } } diff --git a/studio/app/optinist/wrappers/caiman/cnmf.py b/studio/app/optinist/wrappers/caiman/cnmf.py index 59f190d32..a810856df 100644 --- a/studio/app/optinist/wrappers/caiman/cnmf.py +++ b/studio/app/optinist/wrappers/caiman/cnmf.py @@ -1,8 +1,11 @@ import gc import numpy as np +from hdmf.utils import docval, popargs +from studio.app.common.core.param.param import Param from studio.app.common.core.utils.filepath_creater import join_filepath +from studio.app.common.core.wrapper.wrapper import Wrapper from studio.app.common.dataclass import ImageData from studio.app.optinist.core.nwb.nwb import NWBDATASET from studio.app.optinist.dataclass import CaimanCnmfData, FluoData, IscellData, RoiData @@ -63,204 +66,293 @@ def get_roi(A, thr, thr_method, swap_dim, dims): return ims -def caiman_cnmf( - images: ImageData, output_dir: str, params: dict = None, **kwargs -) -> dict(fluorescence=FluoData, iscell=IscellData): - import scipy - from caiman import local_correlations, stop_server - from caiman.cluster import setup_cluster - from caiman.mmapping import prepare_shape - from caiman.paths import memmap_frames_filename - from caiman.source_extraction.cnmf import cnmf - from caiman.source_extraction.cnmf.params import CNMFParams - - function_id = output_dir.split("/")[-1] - print("start caiman_cnmf:", function_id) - - # flatten params segments. - params_flatten = {} - for params_segment in params.values(): - params_flatten.update(params_segment) - params = params_flatten - - Ain = params.pop("Ain", None) - do_refit = params.pop("do_refit", None) - thr = params.pop("thr", None) - - file_path = images.path - if isinstance(file_path, list): - file_path = file_path[0] - - images = images.data - - # np.arrayをmmapへ変換 - order = "C" - dims = images.shape[1:] - T = images.shape[0] - shape_mov = (np.prod(dims), T) - - dir_path = join_filepath(file_path.split("/")[:-1]) - basename = file_path.split("/")[-1] - fname_tot = memmap_frames_filename(basename, dims, T, order) - - mmap_images = np.memmap( - join_filepath([dir_path, fname_tot]), - mode="w+", - dtype=np.float32, - shape=prepare_shape(shape_mov), - order=order, - ) - - mmap_images = np.reshape(mmap_images.T, [T] + list(dims), order="F") - mmap_images[:] = images[:] - - del images - gc.collect() - - nwbfile = kwargs.get("nwbfile", {}) - fr = nwbfile.get("imaging_plane", {}).get("imaging_rate", 30) - - if params is None: - ops = CNMFParams() - else: - ops = CNMFParams(params_dict={**params, "fr": fr}) - - if "dview" in locals(): - stop_server(dview=dview) # noqa: F821 - - c, dview, n_processes = setup_cluster( - backend="local", n_processes=None, single_thread=True - ) - - cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=ops) - cnm = cnm.fit(mmap_images) - - if do_refit: - cnm = cnm.refit(mmap_images, dview=dview) - - stop_server(dview=dview) - - # contours plot - Cn = local_correlations(mmap_images.transpose(1, 2, 0)) - Cn[np.isnan(Cn)] = 0 - - thr_method = "nrg" - swap_dim = False - - iscell = np.concatenate( - [ - np.ones(cnm.estimates.A.shape[-1]), - np.zeros(cnm.estimates.b.shape[-1] if cnm.estimates.b is not None else 0), - ] - ).astype(bool) - - ims = get_roi(cnm.estimates.A, thr, thr_method, swap_dim, dims) - ims = np.stack(ims) - cell_roi = np.nanmax(ims, axis=0).astype(float) - cell_roi[cell_roi == 0] = np.nan - cell_roi -= 1 - - if cnm.estimates.b is not None and cnm.estimates.b.size != 0: - non_cell_roi_ims = get_roi( - scipy.sparse.csc_matrix(cnm.estimates.b), thr, thr_method, swap_dim, dims +class CaimanCnmf(Wrapper): + _INPUT_NODES = [Param(name="images", type=ImageData)] + _OUTPUT_NODES = [ + Param(name="fluorescence", type=FluoData), + Param(name="iscell", type=IscellData), + ] + _DEFAULT_PARAMS = [ + # TODO: need to support 2D array type. + Param( + name="Ain", + type=list, + default=None, + section="init_params", + doc="possibility to seed with predetermined binary masks", + ), + Param(name="do_refit", type=bool, default=False, section="init_params"), + Param( + name="K", + type=int, + default=4, + section="init_params", + doc="upper bound on number of components per patch, in general None", + ), + Param( + name="gSig", + type=list, + default=[4, 4], + section="init_params", + doc="gaussian width of a 2D gaussian kernel, which approximates a neuron", + ), + Param( + name="ssub", + type=int, + default=1, + section="init_params", + doc=( + "downsampling factor in space for initialization. " + "Increase if you have memory problems. " + "You can pass them here as boolean vectors." + ), + ), + Param( + name="tsub", + type=int, + default=1, + section="init_params", + doc=( + "downsampling factor in time for initialization. " + "Increase if you have memory problems." + ), + ), + Param( + name="nb", + type=int, + default=2, + section="init_params", + doc=( + "number of background components (rank) if positive, " + "else exact ring model with following settings. " + "gnb= 0: Return background as b and W " + "gnb=-1: Return full rank background B " + "gnb<-1: Don't return background" + ), + ), + Param( + name="method_init", type=str, default="greedy_roi", section="init_params" + ), + Param( + name="p", + type=int, + default=1, + section="preprocess_params", + doc="order of the autoregressive system", + ), + Param(name="rf", type=[int, list], default=None, section="patch_params"), + Param(name="stride", type=int, default=6, section="patch_params"), + Param(name="thr", type=float, default=0.9, section="merge_params"), + Param( + name="merge_thr", + type=float, + default=0.85, + section="merge_params", + doc="merging threshold, max correlation allowed", + ), + ] + + @docval(*Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS])) + def func(self, **kwargs): + """caiman_cnmf + + TODO: Add documentation for this function + """ + import scipy + from caiman import local_correlations, stop_server + from caiman.cluster import setup_cluster + from caiman.mmapping import prepare_shape + from caiman.paths import memmap_frames_filename + from caiman.source_extraction.cnmf import cnmf + from caiman.source_extraction.cnmf.params import CNMFParams + + print("start caiman_cnmf:", self.function_id) + + Ain, do_refit, thr, images = popargs("Ain", "do_refit", "thr", "images", kwargs) + + file_path = images.path + if isinstance(file_path, list): + file_path = file_path[0] + + images = images.data + + # np.arrayをmmapへ変換 + order = "C" + dims = images.shape[1:] + T = images.shape[0] + shape_mov = (np.prod(dims), T) + + dir_path = join_filepath(file_path.split("/")[:-1]) + basename = file_path.split("/")[-1] + fname_tot = memmap_frames_filename(basename, dims, T, order) + + mmap_images = np.memmap( + join_filepath([dir_path, fname_tot]), + mode="w+", + dtype=np.float32, + shape=prepare_shape(shape_mov), + order=order, ) - non_cell_roi_ims = np.stack(non_cell_roi_ims) - non_cell_roi = np.nanmax(non_cell_roi_ims, axis=0).astype(float) - else: - non_cell_roi_ims = None - non_cell_roi = np.zeros(dims) - non_cell_roi[non_cell_roi == 0] = np.nan - - all_roi = np.nanmax(np.stack([cell_roi, non_cell_roi]), axis=0) - - # NWBの追加 - nwbfile = {} - # NWBにROIを追加 - roi_list = [] - n_cells = cnm.estimates.A.shape[-1] - for i in range(n_cells): - kargs = {} - kargs["image_mask"] = cnm.estimates.A.T[i].T.toarray().reshape(dims) - if hasattr(cnm.estimates, "accepted_list"): - kargs["accepted"] = i in cnm.estimates.accepted_list - if hasattr(cnm.estimates, "rejected_list"): - kargs["rejected"] = i in cnm.estimates.rejected_list - roi_list.append(kargs) - - # backgroundsを追加 - if cnm.estimates.b is not None: - for bg in cnm.estimates.b.T: + + mmap_images = np.reshape(mmap_images.T, [T] + list(dims), order="F") + mmap_images[:] = images[:] + + del images + gc.collect() + + fr = self.nwb_params.get("imaging_plane", {}).get("imaging_rate", 30) + + if kwargs is None: + ops = CNMFParams() + else: + ops = CNMFParams(params_dict={**kwargs, "fr": fr}) + + if "dview" in locals(): + stop_server(dview=dview) # noqa: F821 + + c, dview, n_processes = setup_cluster( + backend="local", n_processes=None, single_thread=True + ) + + cnm = cnmf.CNMF(n_processes=n_processes, dview=dview, Ain=Ain, params=ops) + cnm = cnm.fit(mmap_images) + + if do_refit: + cnm = cnm.refit(mmap_images, dview=dview) + + stop_server(dview=dview) + + # contours plot + Cn = local_correlations(mmap_images.transpose(1, 2, 0)) + Cn[np.isnan(Cn)] = 0 + + thr_method = "nrg" + swap_dim = False + + iscell = np.concatenate( + [ + np.ones(cnm.estimates.A.shape[-1]), + np.zeros( + cnm.estimates.b.shape[-1] if cnm.estimates.b is not None else 0 + ), + ] + ).astype(bool) + + ims = get_roi(cnm.estimates.A, thr, thr_method, swap_dim, dims) + ims = np.stack(ims) + cell_roi = np.nanmax(ims, axis=0).astype(float) + cell_roi[cell_roi == 0] = np.nan + cell_roi -= 1 + + if cnm.estimates.b is not None and cnm.estimates.b.size != 0: + non_cell_roi_ims = get_roi( + scipy.sparse.csc_matrix(cnm.estimates.b), + thr, + thr_method, + swap_dim, + dims, + ) + non_cell_roi_ims = np.stack(non_cell_roi_ims) + non_cell_roi = np.nanmax(non_cell_roi_ims, axis=0).astype(float) + else: + non_cell_roi_ims = None + non_cell_roi = np.zeros(dims) + non_cell_roi[non_cell_roi == 0] = np.nan + + all_roi = np.nanmax(np.stack([cell_roi, non_cell_roi]), axis=0) + + # NWBの追加 + nwbfile = {} + # NWBにROIを追加 + roi_list = [] + n_cells = cnm.estimates.A.shape[-1] + for i in range(n_cells): kargs = {} - kargs["image_mask"] = bg.reshape(dims) + kargs["image_mask"] = cnm.estimates.A.T[i].T.toarray().reshape(dims) if hasattr(cnm.estimates, "accepted_list"): - kargs["accepted"] = False + kargs["accepted"] = i in cnm.estimates.accepted_list if hasattr(cnm.estimates, "rejected_list"): - kargs["rejected"] = False + kargs["rejected"] = i in cnm.estimates.rejected_list roi_list.append(kargs) - nwbfile[NWBDATASET.ROI] = {function_id: roi_list} - - # iscellを追加 - nwbfile[NWBDATASET.COLUMN] = { - function_id: { - "name": "iscell", - "discription": "two columns - iscell & probcell", - "data": iscell, + # backgroundsを追加 + if cnm.estimates.b is not None: + for bg in cnm.estimates.b.T: + kargs = {} + kargs["image_mask"] = bg.reshape(dims) + if hasattr(cnm.estimates, "accepted_list"): + kargs["accepted"] = False + if hasattr(cnm.estimates, "rejected_list"): + kargs["rejected"] = False + roi_list.append(kargs) + + nwbfile[NWBDATASET.ROI] = {self.function_id: roi_list} + + # iscellを追加 + nwbfile[NWBDATASET.COLUMN] = { + self.function_id: { + "name": "iscell", + "discription": "two columns - iscell & probcell", + "data": iscell, + } } - } - # Fluorescence - n_rois = len(cnm.estimates.C) - n_bg = len(cnm.estimates.f) if cnm.estimates.f is not None else 0 + # Fluorescence + n_rois = len(cnm.estimates.C) + n_bg = len(cnm.estimates.f) if cnm.estimates.f is not None else 0 - fluorescence = ( - np.concatenate( - [ - cnm.estimates.C, - cnm.estimates.f, - ] + fluorescence = ( + np.concatenate( + [ + cnm.estimates.C, + cnm.estimates.f, + ] + ) + if cnm.estimates.f is not None + else cnm.estimates.C ) - if cnm.estimates.f is not None - else cnm.estimates.C - ) - - nwbfile[NWBDATASET.FLUORESCENCE] = { - function_id: { - "Fluorescence": { - "table_name": "ROIs", - "region": list(range(n_rois + n_bg)), - "name": "Fluorescence", - "data": fluorescence, - "unit": "lumens", + + nwbfile[NWBDATASET.FLUORESCENCE] = { + self.function_id: { + "Fluorescence": { + "table_name": "ROIs", + "region": list(range(n_rois + n_bg)), + "name": "Fluorescence", + "data": fluorescence, + "unit": "lumens", + } } } - } - - cnmf_data = {} - cnmf_data["fluorescence"] = fluorescence - cnmf_data["im"] = ( - np.concatenate([ims, non_cell_roi_ims], axis=0) - if non_cell_roi_ims is not None - else ims - ) - cnmf_data["is_cell"] = iscell.astype(bool) - cnmf_data["images"] = mmap_images - - info = { - "images": ImageData( - np.array(Cn * 255, dtype=np.uint8), - output_dir=output_dir, - file_name="images", - ), - "fluorescence": FluoData(fluorescence, file_name="fluorescence"), - "iscell": IscellData(iscell, file_name="iscell"), - "all_roi": RoiData(all_roi, output_dir=output_dir, file_name="all_roi"), - "cell_roi": RoiData(cell_roi, output_dir=output_dir, file_name="cell_roi"), - "non_cell_roi": RoiData( - non_cell_roi, output_dir=output_dir, file_name="non_cell_roi" - ), - "nwbfile": nwbfile, - "cnmf_data": CaimanCnmfData(cnmf_data), - } - return info + cnmf_data = {} + cnmf_data["fluorescence"] = fluorescence + cnmf_data["im"] = ( + np.concatenate([ims, non_cell_roi_ims], axis=0) + if non_cell_roi_ims is not None + else ims + ) + cnmf_data["is_cell"] = iscell.astype(bool) + cnmf_data["images"] = mmap_images + + info = { + "images": ImageData( + np.array(Cn * 255, dtype=np.uint8), + output_dir=self.output_dir, + file_name="images", + ), + "fluorescence": FluoData(fluorescence, file_name="fluorescence"), + "iscell": IscellData(iscell, file_name="iscell"), + "all_roi": RoiData( + all_roi, output_dir=self.output_dir, file_name="all_roi" + ), + "cell_roi": RoiData( + cell_roi, output_dir=self.output_dir, file_name="cell_roi" + ), + "non_cell_roi": RoiData( + non_cell_roi, output_dir=self.output_dir, file_name="non_cell_roi" + ), + "nwbfile": nwbfile, + "cnmf_data": CaimanCnmfData(cnmf_data), + } + + return info diff --git a/studio/app/optinist/wrappers/caiman/motion_correction.py b/studio/app/optinist/wrappers/caiman/motion_correction.py index 3da5c4ddc..1c2999da8 100644 --- a/studio/app/optinist/wrappers/caiman/motion_correction.py +++ b/studio/app/optinist/wrappers/caiman/motion_correction.py @@ -1,86 +1,120 @@ +from hdmf.utils import docval, popargs + +from studio.app.common.core.param.param import Param +from studio.app.common.core.wrapper.wrapper import Wrapper from studio.app.common.dataclass import ImageData from studio.app.optinist.core.nwb.nwb import NWBDATASET from studio.app.optinist.dataclass import RoiData -def caiman_mc( - image: ImageData, output_dir: str, params: dict = None, **kwargs -) -> dict(mc_images=ImageData): - import numpy as np - from caiman import load_memmap, save_memmap, stop_server - from caiman.base.rois import extract_binary_masks_from_structural_channel - from caiman.cluster import setup_cluster - from caiman.motion_correction import MotionCorrect - from caiman.source_extraction.cnmf.params import CNMFParams - - function_id = output_dir.split("/")[-1] - print("start caiman motion_correction:", function_id) - - opts = CNMFParams() - - if params is not None: - opts.change_params(params_dict=params) - - c, dview, n_processes = setup_cluster( - backend="local", n_processes=None, single_thread=True - ) - - mc = MotionCorrect(image.path, dview=dview, **opts.get_group("motion")) - - mc.motion_correct(save_movie=True) - border_to_0 = 0 if mc.border_nan == "copy" else mc.border_to_0 - - # memory mapping - fname_new = save_memmap( - mc.mmap_file, base_name="memmap_", order="C", border_to_0=border_to_0 - ) - - stop_server(dview=dview) - - # now load the file - Yr, dims, T = load_memmap(fname_new) - - images = np.array(Yr.T.reshape((T,) + dims, order="F")) - - meanImg = images.mean(axis=0) - rois = ( - extract_binary_masks_from_structural_channel( - meanImg, gSig=7, expand_method="dilation" - )[0] - .reshape(meanImg.shape[0], meanImg.shape[1], -1) - .transpose(2, 0, 1) - ) - - rois = rois.astype(np.float) - - for i, _ in enumerate(rois): - rois[i] *= i + 1 - - rois = np.nanmax(rois, axis=0) - rois[rois == 0] = np.nan - rois -= 1 - - xy_trans_data = ( - (np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) - if params["pw_rigid"] - else np.array(mc.shifts_rig) - ) - - mc_images = ImageData(images, output_dir=output_dir, file_name="mc_images") - - nwbfile = {} - nwbfile[NWBDATASET.MOTION_CORRECTION] = { - function_id: { - "mc_data": mc_images, - "xy_trans_data": xy_trans_data, +class CaimanMc(Wrapper): + _INPUT_NODES = [Param(name="image", type=ImageData)] + _OUTPUT_NODES = [Param(name="mc_images", type=ImageData)] + _DEFAULT_PARAMS = [ + Param(name="border_nan", type=str, default="copy"), + Param(name="gSig_filt", type=list, default=None), + Param(name="is3D", type=bool, default=False), + Param(name="max_deviation_rigid", type=int, default=3), + Param(name="max_shifts", type=list, default=[6, 6]), + Param(name="min_mov", type=float, default=None), + Param(name="niter_rig", type=int, default=1), + Param(name="nonneg_movie", type=bool, default=True), + Param(name="num_frames_split", type=int, default=80), + Param(name="num_splits_to_process_els", type=int, default=None), + Param(name="num_splits_to_process_rig", type=int, default=None), + Param(name="overlaps", type=list, default=[32, 32]), + Param(name="pw_rigid", type=bool, default=False), + Param(name="shifts_opencv", type=bool, default=True), + Param(name="splits_els", type=int, default=14), + Param(name="splits_rig", type=int, default=14), + Param(name="strides", type=list, default=[96, 96]), + Param(name="upsample_factor_grid", type=int, default=4), + Param(name="use_cuda", type=bool, default=False), + ] + + @docval(*Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS])) + def func(self, **kwargs): + """caiman_mc + + TODO: Add documentation for this function + """ + import numpy as np + from caiman import load_memmap, save_memmap, stop_server + from caiman.base.rois import extract_binary_masks_from_structural_channel + from caiman.cluster import setup_cluster + from caiman.motion_correction import MotionCorrect + from caiman.source_extraction.cnmf.params import CNMFParams + + print("start caiman motion_correction:", self.function_id) + + image = popargs("image", kwargs) + opts = CNMFParams() + + if kwargs is not None: + opts.change_params(params_dict=kwargs) + + c, dview, n_processes = setup_cluster( + backend="local", n_processes=None, single_thread=True + ) + + mc = MotionCorrect(image.path, dview=dview, **opts.get_group("motion")) + + mc.motion_correct(save_movie=True) + border_to_0 = 0 if mc.border_nan == "copy" else mc.border_to_0 + + # memory mapping + fname_new = save_memmap( + mc.mmap_file, base_name="memmap_", order="C", border_to_0=border_to_0 + ) + + stop_server(dview=dview) + + # now load the file + Yr, dims, T = load_memmap(fname_new) + + images = np.array(Yr.T.reshape((T,) + dims, order="F")) + + meanImg = images.mean(axis=0) + rois = ( + extract_binary_masks_from_structural_channel( + meanImg, gSig=7, expand_method="dilation" + )[0] + .reshape(meanImg.shape[0], meanImg.shape[1], -1) + .transpose(2, 0, 1) + ) + + rois = rois.astype(np.float) + + for i, _ in enumerate(rois): + rois[i] *= i + 1 + + rois = np.nanmax(rois, axis=0) + rois[rois == 0] = np.nan + rois -= 1 + + xy_trans_data = ( + (np.array(mc.x_shifts_els), np.array(mc.y_shifts_els)) + if kwargs.get("pw_rigid", False) + else np.array(mc.shifts_rig) + ) + + mc_images = ImageData(images, output_dir=self.output_dir, file_name="mc_images") + + nwbfile = {} + nwbfile[NWBDATASET.MOTION_CORRECTION] = { + self.function_id: { + "mc_data": mc_images, + "xy_trans_data": xy_trans_data, + } } - } - info = { - "mc_images": mc_images, - "meanImg": ImageData(meanImg, output_dir=output_dir, file_name="meanImg"), - "rois": RoiData(rois, output_dir=output_dir, file_name="rois"), - "nwbfile": nwbfile, - } + info = { + "mc_images": mc_images, + "meanImg": ImageData( + meanImg, output_dir=self.output_dir, file_name="meanImg" + ), + "rois": RoiData(rois, output_dir=self.output_dir, file_name="rois"), + "nwbfile": nwbfile, + } - return info + return info diff --git a/studio/app/optinist/wrappers/caiman/params/caiman_cnmf.yaml b/studio/app/optinist/wrappers/caiman/params/caiman_cnmf.yaml deleted file mode 100644 index 4803d65e7..000000000 --- a/studio/app/optinist/wrappers/caiman/params/caiman_cnmf.yaml +++ /dev/null @@ -1,21 +0,0 @@ -init_params: - # Ain: null # TBD: need to support 2D array type. - do_refit: False - - K: 4 - gSig: [4, 4] - ssub: 1 - tsub: 1 - nb: 2 - method_init: "greedy_roi" - -preprocess_params: - p: 1 - -patch_params: - rf: - stride: 6 - -merge_params: - thr: 0.9 - merge_thr: 0.85 diff --git a/studio/app/optinist/wrappers/caiman/params/caiman_mc.yaml b/studio/app/optinist/wrappers/caiman/params/caiman_mc.yaml deleted file mode 100644 index 6f6eb462f..000000000 --- a/studio/app/optinist/wrappers/caiman/params/caiman_mc.yaml +++ /dev/null @@ -1,19 +0,0 @@ -border_nan: 'copy' -gSig_filt: null -is3D: False -max_deviation_rigid: 3 -max_shifts: [6, 6] -min_mov: null -niter_rig: 1 -nonneg_movie: True -num_frames_split: 80 -num_splits_to_process_els: null -num_splits_to_process_rig: null -overlaps: [32, 32] -pw_rigid: False -shifts_opencv: True -splits_els: 14 -splits_rig: 14 -strides: [96, 96] -upsample_factor_grid: 4 -use_cuda: False From 334742f865ee6faea8c42a6880e5d9ae002017e5 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Thu, 2 Nov 2023 19:55:58 +0900 Subject: [PATCH 02/10] add settings to generate docs --- Makefile | 2 +- docs/conf.py | 45 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b9af2f8d3..d7f9a815e 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ build_frontend: docs: rm -rf docs/_build/ # pip install -e '.[doc]' - # sphinx-apidoc -f -o ./docs/_build/modules ./studio + sphinx-apidoc -f --no-toc -o ./docs/modules ./studio sphinx-autobuild -b html docs docs/_build --port 8001 .PHONY: dockerhub diff --git a/docs/conf.py b/docs/conf.py index 32269f2ea..65721b726 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -5,6 +5,7 @@ # https://www.sphinx-doc.org/en/master/usage/configuration.html import os +import sys from datetime import datetime from sphinx_pyproject import SphinxConfig @@ -16,6 +17,8 @@ # documentation root, use os.path.abspath to make it absolute, like shown here. # +sys.path.insert(0, os.path.abspath("../studio")) + # -- Project information ----------------------------------------------------- config = SphinxConfig("../pyproject.toml", globalns=globals()) @@ -96,3 +99,45 @@ "includehidden": True, "titles_only": False, } + + +# include_modules = [ +# # CaImAn +# "caiman_cnmf", +# "caiman_cnmfe", +# "caiman_mc", +# "foobar", +# # Suite2p +# "suite2p_file_convert", +# "suite2p_registration", +# "suite2p_roi", +# "suite2p_spike_deconv", +# # LCCD +# "lccd_detect", +# # OptiNiSt: Basic neural analysis +# "ETA", +# # OptiNiSt: Dimension reduction +# "CCA", +# "dpca_fit", +# "PCA", +# "TSNE", +# # OptiNiSt: Neural decoding +# "GLM", +# "LDA", +# "SVM", +# # OptiNiSt: Neural population analysis +# "correlation", +# "cross_correlation", +# "Granger", +# ] + + +# def skip(app, what, name, obj, would_skip, options): +# if name in include_modules: +# return False +# else: +# return True + + +# def setup(app): +# app.connect("autodoc-skip-member", skip) From 7bc95194bc23cfe28ea09273c7eb6f3f1bf9450e Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Thu, 2 Nov 2023 19:56:20 +0900 Subject: [PATCH 03/10] generate module docs --- docs/modules/studio.app.common.core.auth.rst | 45 +++++++ .../studio.app.common.core.experiment.rst | 53 ++++++++ docs/modules/studio.app.common.core.rst | 43 +++++++ docs/modules/studio.app.common.core.rules.rst | 45 +++++++ .../studio.app.common.core.snakemake.rst | 69 +++++++++++ docs/modules/studio.app.common.core.users.rst | 21 ++++ docs/modules/studio.app.common.core.utils.rst | 61 +++++++++ .../studio.app.common.core.workflow.rst | 61 +++++++++ docs/modules/studio.app.common.dataclass.rst | 117 ++++++++++++++++++ docs/modules/studio.app.common.models.rst | 37 ++++++ docs/modules/studio.app.common.routers.rst | 109 ++++++++++++++++ docs/modules/studio.app.common.rst | 23 ++++ docs/modules/studio.app.common.schemas.rst | 93 ++++++++++++++ .../studio.app.common.wrappers.dummy.rst | 21 ++++ docs/modules/studio.app.common.wrappers.rst | 18 +++ .../studio.app.optinist.core.edit_ROI.rst | 37 ++++++ ...core.edit_ROI.wrappers.caiman_edit_roi.rst | 45 +++++++ ...t.core.edit_ROI.wrappers.lccd_edit_roi.rst | 45 +++++++ ...io.app.optinist.core.edit_ROI.wrappers.rst | 20 +++ ...ore.edit_ROI.wrappers.suite2p_edit_roi.rst | 45 +++++++ docs/modules/studio.app.optinist.core.nwb.rst | 37 ++++++ docs/modules/studio.app.optinist.core.rst | 20 +++ .../studio.app.optinist.core.rules.rst | 21 ++++ .../modules/studio.app.optinist.dataclass.rst | 77 ++++++++++++ docs/modules/studio.app.optinist.routers.rst | 37 ++++++ docs/modules/studio.app.optinist.rst | 22 ++++ docs/modules/studio.app.optinist.schemas.rst | 37 ++++++ .../studio.app.optinist.wrappers.caiman.rst | 37 ++++++ ...app.optinist.wrappers.lccd.lccd_python.rst | 53 ++++++++ .../studio.app.optinist.wrappers.lccd.rst | 29 +++++ ...rappers.optinist.basic_neural_analysis.rst | 29 +++++ ....wrappers.optinist.dimension_reduction.rst | 45 +++++++ ...nist.wrappers.optinist.neural_decoding.rst | 37 ++++++ ...rs.optinist.neural_population_analysis.rst | 37 ++++++ .../studio.app.optinist.wrappers.optinist.rst | 32 +++++ docs/modules/studio.app.optinist.wrappers.rst | 21 ++++ .../studio.app.optinist.wrappers.suite2p.rst | 45 +++++++ docs/modules/studio.app.rst | 46 +++++++ docs/modules/studio.rst | 18 +++ 39 files changed, 1688 insertions(+) create mode 100644 docs/modules/studio.app.common.core.auth.rst create mode 100644 docs/modules/studio.app.common.core.experiment.rst create mode 100644 docs/modules/studio.app.common.core.rst create mode 100644 docs/modules/studio.app.common.core.rules.rst create mode 100644 docs/modules/studio.app.common.core.snakemake.rst create mode 100644 docs/modules/studio.app.common.core.users.rst create mode 100644 docs/modules/studio.app.common.core.utils.rst create mode 100644 docs/modules/studio.app.common.core.workflow.rst create mode 100644 docs/modules/studio.app.common.dataclass.rst create mode 100644 docs/modules/studio.app.common.models.rst create mode 100644 docs/modules/studio.app.common.routers.rst create mode 100644 docs/modules/studio.app.common.rst create mode 100644 docs/modules/studio.app.common.schemas.rst create mode 100644 docs/modules/studio.app.common.wrappers.dummy.rst create mode 100644 docs/modules/studio.app.common.wrappers.rst create mode 100644 docs/modules/studio.app.optinist.core.edit_ROI.rst create mode 100644 docs/modules/studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.rst create mode 100644 docs/modules/studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.rst create mode 100644 docs/modules/studio.app.optinist.core.edit_ROI.wrappers.rst create mode 100644 docs/modules/studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.rst create mode 100644 docs/modules/studio.app.optinist.core.nwb.rst create mode 100644 docs/modules/studio.app.optinist.core.rst create mode 100644 docs/modules/studio.app.optinist.core.rules.rst create mode 100644 docs/modules/studio.app.optinist.dataclass.rst create mode 100644 docs/modules/studio.app.optinist.routers.rst create mode 100644 docs/modules/studio.app.optinist.rst create mode 100644 docs/modules/studio.app.optinist.schemas.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.caiman.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.lccd.lccd_python.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.lccd.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.optinist.basic_neural_analysis.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.optinist.dimension_reduction.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.optinist.neural_decoding.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.optinist.neural_population_analysis.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.optinist.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.rst create mode 100644 docs/modules/studio.app.optinist.wrappers.suite2p.rst create mode 100644 docs/modules/studio.app.rst create mode 100644 docs/modules/studio.rst diff --git a/docs/modules/studio.app.common.core.auth.rst b/docs/modules/studio.app.common.core.auth.rst new file mode 100644 index 000000000..ce84638fe --- /dev/null +++ b/docs/modules/studio.app.common.core.auth.rst @@ -0,0 +1,45 @@ +studio.app.common.core.auth package +=================================== + +Submodules +---------- + +studio.app.common.core.auth.auth module +--------------------------------------- + +.. automodule:: studio.app.common.core.auth.auth + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.auth.auth\_config module +----------------------------------------------- + +.. automodule:: studio.app.common.core.auth.auth_config + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.auth.auth\_dependencies module +----------------------------------------------------- + +.. automodule:: studio.app.common.core.auth.auth_dependencies + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.auth.security module +------------------------------------------- + +.. automodule:: studio.app.common.core.auth.security + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core.auth + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.core.experiment.rst b/docs/modules/studio.app.common.core.experiment.rst new file mode 100644 index 000000000..9873208ff --- /dev/null +++ b/docs/modules/studio.app.common.core.experiment.rst @@ -0,0 +1,53 @@ +studio.app.common.core.experiment package +========================================= + +Submodules +---------- + +studio.app.common.core.experiment.experiment module +--------------------------------------------------- + +.. automodule:: studio.app.common.core.experiment.experiment + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.experiment.experiment\_builder module +------------------------------------------------------------ + +.. automodule:: studio.app.common.core.experiment.experiment_builder + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.experiment.experiment\_reader module +----------------------------------------------------------- + +.. automodule:: studio.app.common.core.experiment.experiment_reader + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.experiment.experiment\_utils module +---------------------------------------------------------- + +.. automodule:: studio.app.common.core.experiment.experiment_utils + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.experiment.experiment\_writer module +----------------------------------------------------------- + +.. automodule:: studio.app.common.core.experiment.experiment_writer + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core.experiment + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.core.rst b/docs/modules/studio.app.common.core.rst new file mode 100644 index 000000000..efb5fe6a0 --- /dev/null +++ b/docs/modules/studio.app.common.core.rst @@ -0,0 +1,43 @@ +studio.app.common.core package +============================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.common.core.auth + studio.app.common.core.experiment + studio.app.common.core.rules + studio.app.common.core.snakemake + studio.app.common.core.users + studio.app.common.core.utils + studio.app.common.core.workflow + +Submodules +---------- + +studio.app.common.core.logger module +------------------------------------ + +.. automodule:: studio.app.common.core.logger + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.mode module +---------------------------------- + +.. automodule:: studio.app.common.core.mode + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.core.rules.rst b/docs/modules/studio.app.common.core.rules.rst new file mode 100644 index 000000000..ce37dac66 --- /dev/null +++ b/docs/modules/studio.app.common.core.rules.rst @@ -0,0 +1,45 @@ +studio.app.common.core.rules package +==================================== + +Submodules +---------- + +studio.app.common.core.rules.data module +---------------------------------------- + +.. automodule:: studio.app.common.core.rules.data + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.rules.file\_writer module +------------------------------------------------ + +.. automodule:: studio.app.common.core.rules.file_writer + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.rules.func module +---------------------------------------- + +.. automodule:: studio.app.common.core.rules.func + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.rules.runner module +------------------------------------------ + +.. automodule:: studio.app.common.core.rules.runner + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core.rules + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.core.snakemake.rst b/docs/modules/studio.app.common.core.snakemake.rst new file mode 100644 index 000000000..52e6264df --- /dev/null +++ b/docs/modules/studio.app.common.core.snakemake.rst @@ -0,0 +1,69 @@ +studio.app.common.core.snakemake package +======================================== + +Submodules +---------- + +studio.app.common.core.snakemake.smk module +------------------------------------------- + +.. automodule:: studio.app.common.core.snakemake.smk + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.snakemake.smk\_builder module +---------------------------------------------------- + +.. automodule:: studio.app.common.core.snakemake.smk_builder + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.snakemake.smk\_utils module +-------------------------------------------------- + +.. automodule:: studio.app.common.core.snakemake.smk_utils + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.snakemake.snakemake\_executor module +----------------------------------------------------------- + +.. automodule:: studio.app.common.core.snakemake.snakemake_executor + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.snakemake.snakemake\_reader module +--------------------------------------------------------- + +.. automodule:: studio.app.common.core.snakemake.snakemake_reader + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.snakemake.snakemake\_rule module +------------------------------------------------------- + +.. automodule:: studio.app.common.core.snakemake.snakemake_rule + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.snakemake.snakemake\_writer module +--------------------------------------------------------- + +.. automodule:: studio.app.common.core.snakemake.snakemake_writer + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core.snakemake + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.core.users.rst b/docs/modules/studio.app.common.core.users.rst new file mode 100644 index 000000000..2da33a154 --- /dev/null +++ b/docs/modules/studio.app.common.core.users.rst @@ -0,0 +1,21 @@ +studio.app.common.core.users package +==================================== + +Submodules +---------- + +studio.app.common.core.users.crud\_users module +----------------------------------------------- + +.. automodule:: studio.app.common.core.users.crud_users + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core.users + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.core.utils.rst b/docs/modules/studio.app.common.core.utils.rst new file mode 100644 index 000000000..7926aa9bf --- /dev/null +++ b/docs/modules/studio.app.common.core.utils.rst @@ -0,0 +1,61 @@ +studio.app.common.core.utils package +==================================== + +Submodules +---------- + +studio.app.common.core.utils.config\_handler module +--------------------------------------------------- + +.. automodule:: studio.app.common.core.utils.config_handler + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.utils.file\_reader module +------------------------------------------------ + +.. automodule:: studio.app.common.core.utils.file_reader + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.utils.filepath\_creater module +----------------------------------------------------- + +.. automodule:: studio.app.common.core.utils.filepath_creater + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.utils.filepath\_finder module +---------------------------------------------------- + +.. automodule:: studio.app.common.core.utils.filepath_finder + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.utils.json\_writer module +------------------------------------------------ + +.. automodule:: studio.app.common.core.utils.json_writer + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.utils.pickle\_handler module +--------------------------------------------------- + +.. automodule:: studio.app.common.core.utils.pickle_handler + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core.utils + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.core.workflow.rst b/docs/modules/studio.app.common.core.workflow.rst new file mode 100644 index 000000000..e363bc26e --- /dev/null +++ b/docs/modules/studio.app.common.core.workflow.rst @@ -0,0 +1,61 @@ +studio.app.common.core.workflow package +======================================= + +Submodules +---------- + +studio.app.common.core.workflow.workflow module +----------------------------------------------- + +.. automodule:: studio.app.common.core.workflow.workflow + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.workflow.workflow\_builder module +-------------------------------------------------------- + +.. automodule:: studio.app.common.core.workflow.workflow_builder + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.workflow.workflow\_reader module +------------------------------------------------------- + +.. automodule:: studio.app.common.core.workflow.workflow_reader + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.workflow.workflow\_result module +------------------------------------------------------- + +.. automodule:: studio.app.common.core.workflow.workflow_result + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.workflow.workflow\_runner module +------------------------------------------------------- + +.. automodule:: studio.app.common.core.workflow.workflow_runner + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.core.workflow.workflow\_writer module +------------------------------------------------------- + +.. automodule:: studio.app.common.core.workflow.workflow_writer + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.core.workflow + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.dataclass.rst b/docs/modules/studio.app.common.dataclass.rst new file mode 100644 index 000000000..e07f0ddd2 --- /dev/null +++ b/docs/modules/studio.app.common.dataclass.rst @@ -0,0 +1,117 @@ +studio.app.common.dataclass package +=================================== + +Submodules +---------- + +studio.app.common.dataclass.bar module +-------------------------------------- + +.. automodule:: studio.app.common.dataclass.bar + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.base module +--------------------------------------- + +.. automodule:: studio.app.common.dataclass.base + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.csv module +-------------------------------------- + +.. automodule:: studio.app.common.dataclass.csv + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.heatmap module +------------------------------------------ + +.. automodule:: studio.app.common.dataclass.heatmap + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.histogram module +-------------------------------------------- + +.. automodule:: studio.app.common.dataclass.histogram + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.html module +--------------------------------------- + +.. automodule:: studio.app.common.dataclass.html + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.image module +---------------------------------------- + +.. automodule:: studio.app.common.dataclass.image + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.line module +--------------------------------------- + +.. automodule:: studio.app.common.dataclass.line + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.pie module +-------------------------------------- + +.. automodule:: studio.app.common.dataclass.pie + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.polar module +---------------------------------------- + +.. automodule:: studio.app.common.dataclass.polar + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.scatter module +------------------------------------------ + +.. automodule:: studio.app.common.dataclass.scatter + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.timeseries module +--------------------------------------------- + +.. automodule:: studio.app.common.dataclass.timeseries + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.dataclass.utils module +---------------------------------------- + +.. automodule:: studio.app.common.dataclass.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.dataclass + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.models.rst b/docs/modules/studio.app.common.models.rst new file mode 100644 index 000000000..798057f5d --- /dev/null +++ b/docs/modules/studio.app.common.models.rst @@ -0,0 +1,37 @@ +studio.app.common.models package +================================ + +Submodules +---------- + +studio.app.common.models.base module +------------------------------------ + +.. automodule:: studio.app.common.models.base + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.models.user module +------------------------------------ + +.. automodule:: studio.app.common.models.user + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.models.workspace module +----------------------------------------- + +.. automodule:: studio.app.common.models.workspace + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.models + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.routers.rst b/docs/modules/studio.app.common.routers.rst new file mode 100644 index 000000000..97589baf5 --- /dev/null +++ b/docs/modules/studio.app.common.routers.rst @@ -0,0 +1,109 @@ +studio.app.common.routers package +================================= + +Submodules +---------- + +studio.app.common.routers.algolist module +----------------------------------------- + +.. automodule:: studio.app.common.routers.algolist + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.auth module +------------------------------------- + +.. automodule:: studio.app.common.routers.auth + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.experiment module +------------------------------------------- + +.. automodule:: studio.app.common.routers.experiment + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.files module +-------------------------------------- + +.. automodule:: studio.app.common.routers.files + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.outputs module +---------------------------------------- + +.. automodule:: studio.app.common.routers.outputs + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.params module +--------------------------------------- + +.. automodule:: studio.app.common.routers.params + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.run module +------------------------------------ + +.. automodule:: studio.app.common.routers.run + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.users\_admin module +--------------------------------------------- + +.. automodule:: studio.app.common.routers.users_admin + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.users\_me module +------------------------------------------ + +.. automodule:: studio.app.common.routers.users_me + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.users\_search module +---------------------------------------------- + +.. automodule:: studio.app.common.routers.users_search + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.workflow module +----------------------------------------- + +.. automodule:: studio.app.common.routers.workflow + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.routers.workspace module +------------------------------------------ + +.. automodule:: studio.app.common.routers.workspace + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.routers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.rst b/docs/modules/studio.app.common.rst new file mode 100644 index 000000000..343bf6adf --- /dev/null +++ b/docs/modules/studio.app.common.rst @@ -0,0 +1,23 @@ +studio.app.common package +========================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.common.core + studio.app.common.dataclass + studio.app.common.models + studio.app.common.routers + studio.app.common.schemas + studio.app.common.wrappers + +Module contents +--------------- + +.. automodule:: studio.app.common + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.schemas.rst b/docs/modules/studio.app.common.schemas.rst new file mode 100644 index 000000000..c31b122cb --- /dev/null +++ b/docs/modules/studio.app.common.schemas.rst @@ -0,0 +1,93 @@ +studio.app.common.schemas package +================================= + +Submodules +---------- + +studio.app.common.schemas.algolist module +----------------------------------------- + +.. automodule:: studio.app.common.schemas.algolist + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.auth module +------------------------------------- + +.. automodule:: studio.app.common.schemas.auth + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.base module +------------------------------------- + +.. automodule:: studio.app.common.schemas.base + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.experiment module +------------------------------------------- + +.. automodule:: studio.app.common.schemas.experiment + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.files module +-------------------------------------- + +.. automodule:: studio.app.common.schemas.files + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.outputs module +---------------------------------------- + +.. automodule:: studio.app.common.schemas.outputs + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.params module +--------------------------------------- + +.. automodule:: studio.app.common.schemas.params + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.users module +-------------------------------------- + +.. automodule:: studio.app.common.schemas.users + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.workflow module +----------------------------------------- + +.. automodule:: studio.app.common.schemas.workflow + :members: + :undoc-members: + :show-inheritance: + +studio.app.common.schemas.workspace module +------------------------------------------ + +.. automodule:: studio.app.common.schemas.workspace + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.schemas + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.wrappers.dummy.rst b/docs/modules/studio.app.common.wrappers.dummy.rst new file mode 100644 index 000000000..54d542719 --- /dev/null +++ b/docs/modules/studio.app.common.wrappers.dummy.rst @@ -0,0 +1,21 @@ +studio.app.common.wrappers.dummy package +======================================== + +Submodules +---------- + +studio.app.common.wrappers.dummy.dummy module +--------------------------------------------- + +.. automodule:: studio.app.common.wrappers.dummy.dummy + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.common.wrappers.dummy + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.common.wrappers.rst b/docs/modules/studio.app.common.wrappers.rst new file mode 100644 index 000000000..b629e0ab5 --- /dev/null +++ b/docs/modules/studio.app.common.wrappers.rst @@ -0,0 +1,18 @@ +studio.app.common.wrappers package +================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.common.wrappers.dummy + +Module contents +--------------- + +.. automodule:: studio.app.common.wrappers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.edit_ROI.rst b/docs/modules/studio.app.optinist.core.edit_ROI.rst new file mode 100644 index 000000000..fe274bb38 --- /dev/null +++ b/docs/modules/studio.app.optinist.core.edit_ROI.rst @@ -0,0 +1,37 @@ +studio.app.optinist.core.edit\_ROI package +========================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.optinist.core.edit_ROI.wrappers + +Submodules +---------- + +studio.app.optinist.core.edit\_ROI.edit\_ROI module +--------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.edit_ROI + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.utils module +----------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core.edit_ROI + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.rst b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.rst new file mode 100644 index 000000000..42a7ae722 --- /dev/null +++ b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.rst @@ -0,0 +1,45 @@ +studio.app.optinist.core.edit\_ROI.wrappers.caiman\_edit\_roi package +===================================================================== + +Submodules +---------- + +studio.app.optinist.core.edit\_ROI.wrappers.caiman\_edit\_roi.add\_roi module +----------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.add_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.caiman\_edit\_roi.delete\_roi module +-------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.delete_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.caiman\_edit\_roi.merge\_roi module +------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.merge_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.caiman\_edit\_roi.utils module +-------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.rst b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.rst new file mode 100644 index 000000000..a0e3d59df --- /dev/null +++ b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.rst @@ -0,0 +1,45 @@ +studio.app.optinist.core.edit\_ROI.wrappers.lccd\_edit\_roi package +=================================================================== + +Submodules +---------- + +studio.app.optinist.core.edit\_ROI.wrappers.lccd\_edit\_roi.add\_roi module +--------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.add_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.lccd\_edit\_roi.delete\_roi module +------------------------------------------------------------------------------ + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.delete_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.lccd\_edit\_roi.merge\_roi module +----------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.merge_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.lccd\_edit\_roi.utils module +------------------------------------------------------------------------ + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.rst b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.rst new file mode 100644 index 000000000..58f70c9a4 --- /dev/null +++ b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.rst @@ -0,0 +1,20 @@ +studio.app.optinist.core.edit\_ROI.wrappers package +=================================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.optinist.core.edit_ROI.wrappers.caiman_edit_roi + studio.app.optinist.core.edit_ROI.wrappers.lccd_edit_roi + studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.rst b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.rst new file mode 100644 index 000000000..6673887c1 --- /dev/null +++ b/docs/modules/studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.rst @@ -0,0 +1,45 @@ +studio.app.optinist.core.edit\_ROI.wrappers.suite2p\_edit\_roi package +====================================================================== + +Submodules +---------- + +studio.app.optinist.core.edit\_ROI.wrappers.suite2p\_edit\_roi.add\_roi module +------------------------------------------------------------------------------ + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.add_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.suite2p\_edit\_roi.delete\_roi module +--------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.delete_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.suite2p\_edit\_roi.merge\_roi module +-------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.merge_roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.edit\_ROI.wrappers.suite2p\_edit\_roi.utils module +--------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core.edit_ROI.wrappers.suite2p_edit_roi + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.nwb.rst b/docs/modules/studio.app.optinist.core.nwb.rst new file mode 100644 index 000000000..591660cd2 --- /dev/null +++ b/docs/modules/studio.app.optinist.core.nwb.rst @@ -0,0 +1,37 @@ +studio.app.optinist.core.nwb package +==================================== + +Submodules +---------- + +studio.app.optinist.core.nwb.nwb module +--------------------------------------- + +.. automodule:: studio.app.optinist.core.nwb.nwb + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.nwb.nwb\_creater module +------------------------------------------------ + +.. automodule:: studio.app.optinist.core.nwb.nwb_creater + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.core.nwb.optinist\_data module +-------------------------------------------------- + +.. automodule:: studio.app.optinist.core.nwb.optinist_data + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core.nwb + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.rst b/docs/modules/studio.app.optinist.core.rst new file mode 100644 index 000000000..6cc1caee1 --- /dev/null +++ b/docs/modules/studio.app.optinist.core.rst @@ -0,0 +1,20 @@ +studio.app.optinist.core package +================================ + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.optinist.core.edit_ROI + studio.app.optinist.core.nwb + studio.app.optinist.core.rules + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.core.rules.rst b/docs/modules/studio.app.optinist.core.rules.rst new file mode 100644 index 000000000..5b6fa7902 --- /dev/null +++ b/docs/modules/studio.app.optinist.core.rules.rst @@ -0,0 +1,21 @@ +studio.app.optinist.core.rules package +====================================== + +Submodules +---------- + +studio.app.optinist.core.rules.edit\_ROI module +----------------------------------------------- + +.. automodule:: studio.app.optinist.core.rules.edit_ROI + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.core.rules + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.dataclass.rst b/docs/modules/studio.app.optinist.dataclass.rst new file mode 100644 index 000000000..addae58b2 --- /dev/null +++ b/docs/modules/studio.app.optinist.dataclass.rst @@ -0,0 +1,77 @@ +studio.app.optinist.dataclass package +===================================== + +Submodules +---------- + +studio.app.optinist.dataclass.behavior module +--------------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.behavior + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.dataclass.caiman module +------------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.caiman + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.dataclass.fluo module +----------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.fluo + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.dataclass.iscell module +------------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.iscell + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.dataclass.lccd module +----------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.lccd + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.dataclass.nwb module +---------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.nwb + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.dataclass.roi module +---------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.dataclass.suite2p module +-------------------------------------------- + +.. automodule:: studio.app.optinist.dataclass.suite2p + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.dataclass + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.routers.rst b/docs/modules/studio.app.optinist.routers.rst new file mode 100644 index 000000000..f6414205a --- /dev/null +++ b/docs/modules/studio.app.optinist.routers.rst @@ -0,0 +1,37 @@ +studio.app.optinist.routers package +=================================== + +Submodules +---------- + +studio.app.optinist.routers.hdf5 module +--------------------------------------- + +.. automodule:: studio.app.optinist.routers.hdf5 + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.routers.nwb module +-------------------------------------- + +.. automodule:: studio.app.optinist.routers.nwb + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.routers.roi module +-------------------------------------- + +.. automodule:: studio.app.optinist.routers.roi + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.routers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.rst b/docs/modules/studio.app.optinist.rst new file mode 100644 index 000000000..af104fc34 --- /dev/null +++ b/docs/modules/studio.app.optinist.rst @@ -0,0 +1,22 @@ +studio.app.optinist package +=========================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.optinist.core + studio.app.optinist.dataclass + studio.app.optinist.routers + studio.app.optinist.schemas + studio.app.optinist.wrappers + +Module contents +--------------- + +.. automodule:: studio.app.optinist + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.schemas.rst b/docs/modules/studio.app.optinist.schemas.rst new file mode 100644 index 000000000..f48e6fb4e --- /dev/null +++ b/docs/modules/studio.app.optinist.schemas.rst @@ -0,0 +1,37 @@ +studio.app.optinist.schemas package +=================================== + +Submodules +---------- + +studio.app.optinist.schemas.hdf5 module +--------------------------------------- + +.. automodule:: studio.app.optinist.schemas.hdf5 + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.schemas.nwb module +-------------------------------------- + +.. automodule:: studio.app.optinist.schemas.nwb + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.schemas.roi module +-------------------------------------- + +.. automodule:: studio.app.optinist.schemas.roi + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.schemas + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.caiman.rst b/docs/modules/studio.app.optinist.wrappers.caiman.rst new file mode 100644 index 000000000..84c9cd4f8 --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.caiman.rst @@ -0,0 +1,37 @@ +studio.app.optinist.wrappers.caiman package +=========================================== + +Submodules +---------- + +studio.app.optinist.wrappers.caiman.cnmf module +----------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.caiman.cnmf + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.caiman.cnmfe module +------------------------------------------------ + +.. automodule:: studio.app.optinist.wrappers.caiman.cnmfe + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.caiman.motion\_correction module +------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.caiman.motion_correction + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.caiman + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.lccd.lccd_python.rst b/docs/modules/studio.app.optinist.wrappers.lccd.lccd_python.rst new file mode 100644 index 000000000..8ac15a25c --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.lccd.lccd_python.rst @@ -0,0 +1,53 @@ +studio.app.optinist.wrappers.lccd.lccd\_python package +====================================================== + +Submodules +---------- + +studio.app.optinist.wrappers.lccd.lccd\_python.blob\_detector module +-------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.lccd.lccd_python.blob_detector + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.lccd.lccd\_python.lccd module +---------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.lccd.lccd_python.lccd + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.lccd.lccd\_python.oval\_filter module +------------------------------------------------------------------ + +.. automodule:: studio.app.optinist.wrappers.lccd.lccd_python.oval_filter + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.lccd.lccd\_python.roi\_integration module +---------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.lccd.lccd_python.roi_integration + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.lccd.lccd\_python.utils module +----------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.lccd.lccd_python.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.lccd.lccd_python + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.lccd.rst b/docs/modules/studio.app.optinist.wrappers.lccd.rst new file mode 100644 index 000000000..d73946228 --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.lccd.rst @@ -0,0 +1,29 @@ +studio.app.optinist.wrappers.lccd package +========================================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.optinist.wrappers.lccd.lccd_python + +Submodules +---------- + +studio.app.optinist.wrappers.lccd.lccd\_detection module +-------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.lccd.lccd_detection + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.lccd + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.optinist.basic_neural_analysis.rst b/docs/modules/studio.app.optinist.wrappers.optinist.basic_neural_analysis.rst new file mode 100644 index 000000000..b9222ad5a --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.optinist.basic_neural_analysis.rst @@ -0,0 +1,29 @@ +studio.app.optinist.wrappers.optinist.basic\_neural\_analysis package +===================================================================== + +Submodules +---------- + +studio.app.optinist.wrappers.optinist.basic\_neural\_analysis.cell\_grouping module +----------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.basic_neural_analysis.cell_grouping + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.basic\_neural\_analysis.eta module +------------------------------------------------------------------------ + +.. automodule:: studio.app.optinist.wrappers.optinist.basic_neural_analysis.eta + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.basic_neural_analysis + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.optinist.dimension_reduction.rst b/docs/modules/studio.app.optinist.wrappers.optinist.dimension_reduction.rst new file mode 100644 index 000000000..a658d053b --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.optinist.dimension_reduction.rst @@ -0,0 +1,45 @@ +studio.app.optinist.wrappers.optinist.dimension\_reduction package +================================================================== + +Submodules +---------- + +studio.app.optinist.wrappers.optinist.dimension\_reduction.cca module +--------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.dimension_reduction.cca + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.dimension\_reduction.dpca\_fit module +--------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.dimension_reduction.dpca_fit + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.dimension\_reduction.pca module +--------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.dimension_reduction.pca + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.dimension\_reduction.tsne module +---------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.dimension_reduction.tsne + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.dimension_reduction + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.optinist.neural_decoding.rst b/docs/modules/studio.app.optinist.wrappers.optinist.neural_decoding.rst new file mode 100644 index 000000000..b055b8fd3 --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.optinist.neural_decoding.rst @@ -0,0 +1,37 @@ +studio.app.optinist.wrappers.optinist.neural\_decoding package +============================================================== + +Submodules +---------- + +studio.app.optinist.wrappers.optinist.neural\_decoding.glm module +----------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_decoding.glm + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.neural\_decoding.lda module +----------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_decoding.lda + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.neural\_decoding.svm module +----------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_decoding.svm + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_decoding + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.optinist.neural_population_analysis.rst b/docs/modules/studio.app.optinist.wrappers.optinist.neural_population_analysis.rst new file mode 100644 index 000000000..9521b9932 --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.optinist.neural_population_analysis.rst @@ -0,0 +1,37 @@ +studio.app.optinist.wrappers.optinist.neural\_population\_analysis package +========================================================================== + +Submodules +---------- + +studio.app.optinist.wrappers.optinist.neural\_population\_analysis.correlation module +------------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_population_analysis.correlation + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.neural\_population\_analysis.cross\_correlation module +-------------------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_population_analysis.cross_correlation + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.optinist.neural\_population\_analysis.granger module +--------------------------------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_population_analysis.granger + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.neural_population_analysis + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.optinist.rst b/docs/modules/studio.app.optinist.wrappers.optinist.rst new file mode 100644 index 000000000..3e406fc43 --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.optinist.rst @@ -0,0 +1,32 @@ +studio.app.optinist.wrappers.optinist package +============================================= + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.optinist.wrappers.optinist.basic_neural_analysis + studio.app.optinist.wrappers.optinist.dimension_reduction + studio.app.optinist.wrappers.optinist.neural_decoding + studio.app.optinist.wrappers.optinist.neural_population_analysis + +Submodules +---------- + +studio.app.optinist.wrappers.optinist.utils module +-------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.optinist.utils + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.optinist + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.rst b/docs/modules/studio.app.optinist.wrappers.rst new file mode 100644 index 000000000..36c03b240 --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.rst @@ -0,0 +1,21 @@ +studio.app.optinist.wrappers package +==================================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.optinist.wrappers.caiman + studio.app.optinist.wrappers.lccd + studio.app.optinist.wrappers.optinist + studio.app.optinist.wrappers.suite2p + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.optinist.wrappers.suite2p.rst b/docs/modules/studio.app.optinist.wrappers.suite2p.rst new file mode 100644 index 000000000..5736871e6 --- /dev/null +++ b/docs/modules/studio.app.optinist.wrappers.suite2p.rst @@ -0,0 +1,45 @@ +studio.app.optinist.wrappers.suite2p package +============================================ + +Submodules +---------- + +studio.app.optinist.wrappers.suite2p.file\_convert module +--------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.suite2p.file_convert + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.suite2p.registration module +-------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.suite2p.registration + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.suite2p.roi module +----------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.suite2p.roi + :members: + :undoc-members: + :show-inheritance: + +studio.app.optinist.wrappers.suite2p.spike\_deconv module +--------------------------------------------------------- + +.. automodule:: studio.app.optinist.wrappers.suite2p.spike_deconv + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app.optinist.wrappers.suite2p + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.app.rst b/docs/modules/studio.app.rst new file mode 100644 index 000000000..f272e95a4 --- /dev/null +++ b/docs/modules/studio.app.rst @@ -0,0 +1,46 @@ +studio.app package +================== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app.common + studio.app.optinist + +Submodules +---------- + +studio.app.const module +----------------------- + +.. automodule:: studio.app.const + :members: + :undoc-members: + :show-inheritance: + +studio.app.dir\_path module +--------------------------- + +.. automodule:: studio.app.dir_path + :members: + :undoc-members: + :show-inheritance: + +studio.app.wrappers module +-------------------------- + +.. automodule:: studio.app.wrappers + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: studio.app + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/modules/studio.rst b/docs/modules/studio.rst new file mode 100644 index 000000000..e00ca86dd --- /dev/null +++ b/docs/modules/studio.rst @@ -0,0 +1,18 @@ +studio package +============== + +Subpackages +----------- + +.. toctree:: + :maxdepth: 4 + + studio.app + +Module contents +--------------- + +.. automodule:: studio + :members: + :undoc-members: + :show-inheritance: From b03859002199a9b44f10bc8e3e7b5bacc1a4a4af Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:25:12 +0900 Subject: [PATCH 04/10] downgrade sphinx --- docs/requirements.txt | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index ed1f751a9..ed34d0f0f 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,4 +1,4 @@ -sphinx==7.1.2 +sphinx<7.0.0 sphinxcontrib-apidoc sphinx_rtd_theme sphinx-prompt diff --git a/pyproject.toml b/pyproject.toml index 8cb305a6d..b81a248c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,7 +65,7 @@ dev = [ "pre-commit", ] doc = [ - "sphinx==7.1.2", + "sphinx<7.0.0", "sphinxcontrib-apidoc", "sphinx_rtd_theme", "sphinx-prompt", From d4a71ccea8d127728bcdfad3fe980169c12bbb13 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:31:45 +0900 Subject: [PATCH 05/10] set 'function' as const --- studio/app/common/core/experiment/experiment_reader.py | 5 +++-- studio/app/common/core/rules/runner.py | 4 ++-- studio/app/common/core/wrapper/wrapper_utils.py | 5 +++-- studio/app/common/routers/algolist.py | 5 +++-- studio/app/const.py | 2 ++ studio/app/optinist/core/edit_ROI/edit_ROI.py | 3 ++- 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/studio/app/common/core/experiment/experiment_reader.py b/studio/app/common/core/experiment/experiment_reader.py index ef5749f41..a56a28c5d 100644 --- a/studio/app/common/core/experiment/experiment_reader.py +++ b/studio/app/common/core/experiment/experiment_reader.py @@ -6,6 +6,7 @@ from studio.app.common.core.param.param import ParamChild, ParamParent from studio.app.common.core.param.param_utils import ParamUtils from studio.app.common.core.workflow.workflow import OutputPath +from studio.app.const import FUNC_KEY class ExptConfigReader: @@ -22,7 +23,7 @@ def read(cls, filepath) -> ExptConfig: finished_at=config.get("finished_at"), success=config.get("success", "running"), hasNWB=config["hasNWB"], - function=cls.read_function(config["function"]), + function=cls.read_function(config[FUNC_KEY]), nwb=cls.read_params(config.get("nwb"), "nwb"), snakemake=cls.read_params(config.get("snakemake"), "snakemake"), ) @@ -129,7 +130,7 @@ def rename(cls, filepath, new_name: str) -> ExptConfig: finished_at=config.get("finished_at"), success=config.get("success", "running"), hasNWB=config["hasNWB"], - function=cls.read_function(config["function"]), + function=cls.read_function(config[FUNC_KEY]), nwb=config.get("nwb"), snakemake=config.get("snakemake"), ) diff --git a/studio/app/common/core/rules/runner.py b/studio/app/common/core/rules/runner.py index 54c60d657..fdb7bcb69 100644 --- a/studio/app/common/core/rules/runner.py +++ b/studio/app/common/core/rules/runner.py @@ -13,7 +13,7 @@ from studio.app.common.core.utils.config_handler import ConfigWriter from studio.app.common.core.utils.filepath_creater import join_filepath from studio.app.common.core.utils.pickle_handler import PickleReader, PickleWriter -from studio.app.const import DATE_FORMAT +from studio.app.const import DATE_FORMAT, FUNC_KEY from studio.app.dir_path import DIRPATH from studio.app.optinist.core.nwb.nwb_creater import ( merge_nwbfile, @@ -123,7 +123,7 @@ def save_all_nwb(cls, save_path, all_nwbfile): @classmethod def execute_function(cls, path, params, nwb_params, output_dir, input_info): wrapper = cls.dict2leaf(wrapper_dict, path.split("/")) - wrapper = copy.deepcopy(wrapper["function"]) + wrapper = copy.deepcopy(wrapper[FUNC_KEY]) flatten_params = ParamUtils.get_flatten_params(params) output_info = ( wrapper.set_output_dir(output_dir) diff --git a/studio/app/common/core/wrapper/wrapper_utils.py b/studio/app/common/core/wrapper/wrapper_utils.py index 5f79d5d01..e9bd972c6 100644 --- a/studio/app/common/core/wrapper/wrapper_utils.py +++ b/studio/app/common/core/wrapper/wrapper_utils.py @@ -1,6 +1,7 @@ from typing import Any, Dict from studio.app.common.core.wrapper.wrapper import Wrapper +from studio.app.const import FUNC_KEY from studio.app.wrappers import wrapper_dict @@ -12,9 +13,9 @@ def find_wrapper_by_name( wrapper_dict: Dict[str, Any] = wrapper_dict, ) -> Wrapper: if name in wrapper_dict: - return wrapper_dict[name]["function"] + return wrapper_dict[name][FUNC_KEY] else: for v in wrapper_dict.values(): - if isinstance(v, dict) and "function" not in v: + if isinstance(v, dict) and FUNC_KEY not in v: return cls.find_wrapper_by_name(name, v) return None diff --git a/studio/app/common/routers/algolist.py b/studio/app/common/routers/algolist.py index e57ae6350..968829bfd 100644 --- a/studio/app/common/routers/algolist.py +++ b/studio/app/common/routers/algolist.py @@ -5,6 +5,7 @@ from studio.app.common.core.param.param import Param from studio.app.common.core.wrapper.wrapper import Wrapper from studio.app.common.schemas.algolist import Algo, AlgoList, Arg, Return +from studio.app.const import FUNC_KEY from studio.app.wrappers import wrapper_dict router = APIRouter() @@ -16,12 +17,12 @@ def get_nest_dict(cls, parent_value, parent_key: str) -> AlgoList: algo_dict = {} for key, value in parent_value.items(): algo_dict[key] = {} - if isinstance(value, dict) and "function" not in value: + if isinstance(value, dict) and FUNC_KEY not in value: algo_dict[key]["children"] = cls.get_nest_dict( value, cls._parent_key(parent_key, key) ) else: - wrapper: Wrapper = value["function"] + wrapper: Wrapper = value[FUNC_KEY] algo_dict[key] = Algo( args=cls._args_list(wrapper._INPUT_NODES), diff --git a/studio/app/const.py b/studio/app/const.py index 579a778c2..efc49f8d2 100644 --- a/studio/app/const.py +++ b/studio/app/const.py @@ -13,4 +13,6 @@ class FILETYPE: ACCEPT_CSV_EXT = [".csv"] ACCEPT_HDF5_EXT = [".hdf5", ".nwb", ".HDF5", ".NWB"] +FUNC_KEY = "function" + DATE_FORMAT = "%Y-%m-%d %H:%M:%S" diff --git a/studio/app/optinist/core/edit_ROI/edit_ROI.py b/studio/app/optinist/core/edit_ROI/edit_ROI.py index 03569cdb4..8bebd649c 100644 --- a/studio/app/optinist/core/edit_ROI/edit_ROI.py +++ b/studio/app/optinist/core/edit_ROI/edit_ROI.py @@ -13,6 +13,7 @@ from studio.app.common.core.utils.filepath_finder import find_condaenv_filepath from studio.app.common.core.utils.pickle_handler import PickleReader, PickleWriter from studio.app.common.dataclass.base import BaseData +from studio.app.const import FUNC_KEY from studio.app.dir_path import DIRPATH from studio.app.optinist.core.edit_ROI.wrappers import edit_roi_wrapper_dict from studio.app.optinist.core.nwb.nwb_creater import overwrite_nwb @@ -85,7 +86,7 @@ def excute(cls, config): action = config["action"] params = config["params"] - func = copy.deepcopy(edit_roi_wrapper_dict[algo]["function"][action]) + func = copy.deepcopy(edit_roi_wrapper_dict[algo][FUNC_KEY][action]) output_info = func(node_dirpath, **params) del func gc.collect() From 4ed540119355faa1da0548277a70d56d09827536 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Tue, 7 Nov 2023 12:08:21 +0900 Subject: [PATCH 06/10] use abc --- studio/app/common/core/wrapper/wrapper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/studio/app/common/core/wrapper/wrapper.py b/studio/app/common/core/wrapper/wrapper.py index 0e57fc52f..ec25198bc 100644 --- a/studio/app/common/core/wrapper/wrapper.py +++ b/studio/app/common/core/wrapper/wrapper.py @@ -1,9 +1,10 @@ +from abc import ABC, abstractmethod from typing import List from studio.app.common.core.param.param import Param -class Wrapper: +class Wrapper(ABC): _INPUT_NODES: List[Param] = [] _OUTPUT_NODES: List[Param] = [] _DEFAULT_PARAMS: List[Param] = [] @@ -21,5 +22,6 @@ def set_nwb_params(self, nwb_params): def docval_params(params: List[Param]): return [param.docval_dict() for param in params] + @abstractmethod def func(self, **kwargs): pass From c960eef0f39bb2ce97ea5d266ca7ff8fb8a905c5 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:12:53 +0900 Subject: [PATCH 07/10] add return docs --- studio/app/common/core/wrapper/wrapper.py | 14 ++++++++++++++ studio/app/optinist/wrappers/caiman/cnmf.py | 5 ++++- .../optinist/wrappers/caiman/motion_correction.py | 5 ++++- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/studio/app/common/core/wrapper/wrapper.py b/studio/app/common/core/wrapper/wrapper.py index ec25198bc..e497e8df8 100644 --- a/studio/app/common/core/wrapper/wrapper.py +++ b/studio/app/common/core/wrapper/wrapper.py @@ -22,6 +22,20 @@ def set_nwb_params(self, nwb_params): def docval_params(params: List[Param]): return [param.docval_dict() for param in params] + @staticmethod + def docval_returns(returns: List[Param]): + return { + "returns": ",".join([r.name for r in returns]), + "rtype": ",".join( + [ + ",".join([t.__name__ for t in r.type]) + if isinstance(r.type, List) + else r.type.__name__ + for r in returns + ] + ), + } + @abstractmethod def func(self, **kwargs): pass diff --git a/studio/app/optinist/wrappers/caiman/cnmf.py b/studio/app/optinist/wrappers/caiman/cnmf.py index a810856df..8f54a1ad9 100644 --- a/studio/app/optinist/wrappers/caiman/cnmf.py +++ b/studio/app/optinist/wrappers/caiman/cnmf.py @@ -152,7 +152,10 @@ class CaimanCnmf(Wrapper): ), ] - @docval(*Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS])) + @docval( + *Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS]), + **Wrapper.docval_returns([*_OUTPUT_NODES]), + ) def func(self, **kwargs): """caiman_cnmf diff --git a/studio/app/optinist/wrappers/caiman/motion_correction.py b/studio/app/optinist/wrappers/caiman/motion_correction.py index 29f2b5a0f..51f892f30 100644 --- a/studio/app/optinist/wrappers/caiman/motion_correction.py +++ b/studio/app/optinist/wrappers/caiman/motion_correction.py @@ -38,7 +38,10 @@ class CaimanMc(Wrapper): Param(name="use_cuda", type=bool, default=False), ] - @docval(*Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS])) + @docval( + *Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS]), + **Wrapper.docval_returns(_OUTPUT_NODES), + ) def func(self, **kwargs): """caiman_mc From 38a56ac863d1f408973a2e913e92037be967b299 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:23:48 +0900 Subject: [PATCH 08/10] move internal function to Wrapper --- studio/app/optinist/wrappers/caiman/cnmf.py | 115 ++++++++++---------- 1 file changed, 58 insertions(+), 57 deletions(-) diff --git a/studio/app/optinist/wrappers/caiman/cnmf.py b/studio/app/optinist/wrappers/caiman/cnmf.py index 8f54a1ad9..9e081dbaa 100644 --- a/studio/app/optinist/wrappers/caiman/cnmf.py +++ b/studio/app/optinist/wrappers/caiman/cnmf.py @@ -11,61 +11,6 @@ from studio.app.optinist.dataclass import CaimanCnmfData, FluoData, IscellData, RoiData -def get_roi(A, thr, thr_method, swap_dim, dims): - from scipy.ndimage import binary_fill_holes - from skimage.measure import find_contours - - d, nr = np.shape(A) - - # for each patches - ims = [] - coordinates = [] - for i in range(nr): - pars = dict() - # we compute the cumulative sum of the energy of the Ath component - # that has been ordered from least to highest - patch_data = A.data[A.indptr[i] : A.indptr[i + 1]] - indx = np.argsort(patch_data)[::-1] - - if thr_method == "nrg": - cumEn = np.cumsum(patch_data[indx] ** 2) - if len(cumEn) == 0: - pars = dict( - coordinates=np.array([]), - CoM=np.array([np.NaN, np.NaN]), - neuron_id=i + 1, - ) - coordinates.append(pars) - continue - else: - # we work with normalized values - cumEn /= cumEn[-1] - Bvec = np.ones(d) - # we put it in a similar matrix - Bvec[A.indices[A.indptr[i] : A.indptr[i + 1]][indx]] = cumEn - else: - Bvec = np.zeros(d) - Bvec[A.indices[A.indptr[i] : A.indptr[i + 1]]] = ( - patch_data / patch_data.max() - ) - - if swap_dim: - Bmat = np.reshape(Bvec, dims, order="C") - else: - Bmat = np.reshape(Bvec, dims, order="F") - - r_mask = np.zeros_like(Bmat, dtype="bool") - contour = find_contours(Bmat, thr) - for c in contour: - r_mask[np.round(c[:, 0]).astype("int"), np.round(c[:, 1]).astype("int")] = 1 - - # Fill in the hole created by the contour boundary - r_mask = binary_fill_holes(r_mask) - ims.append(r_mask + (i * r_mask)) - - return ims - - class CaimanCnmf(Wrapper): _INPUT_NODES = [Param(name="images", type=ImageData)] _OUTPUT_NODES = [ @@ -241,14 +186,14 @@ def func(self, **kwargs): ] ).astype(bool) - ims = get_roi(cnm.estimates.A, thr, thr_method, swap_dim, dims) + ims = self.get_roi(cnm.estimates.A, thr, thr_method, swap_dim, dims) ims = np.stack(ims) cell_roi = np.nanmax(ims, axis=0).astype(float) cell_roi[cell_roi == 0] = np.nan cell_roi -= 1 if cnm.estimates.b is not None and cnm.estimates.b.size != 0: - non_cell_roi_ims = get_roi( + non_cell_roi_ims = self.get_roi( scipy.sparse.csc_matrix(cnm.estimates.b), thr, thr_method, @@ -359,3 +304,59 @@ def func(self, **kwargs): } return info + + def get_roi(self, A, thr, thr_method, swap_dim, dims): + from scipy.ndimage import binary_fill_holes + from skimage.measure import find_contours + + d, nr = np.shape(A) + + # for each patches + ims = [] + coordinates = [] + for i in range(nr): + pars = dict() + # we compute the cumulative sum of the energy of the Ath component + # that has been ordered from least to highest + patch_data = A.data[A.indptr[i] : A.indptr[i + 1]] + indx = np.argsort(patch_data)[::-1] + + if thr_method == "nrg": + cumEn = np.cumsum(patch_data[indx] ** 2) + if len(cumEn) == 0: + pars = dict( + coordinates=np.array([]), + CoM=np.array([np.NaN, np.NaN]), + neuron_id=i + 1, + ) + coordinates.append(pars) + continue + else: + # we work with normalized values + cumEn /= cumEn[-1] + Bvec = np.ones(d) + # we put it in a similar matrix + Bvec[A.indices[A.indptr[i] : A.indptr[i + 1]][indx]] = cumEn + else: + Bvec = np.zeros(d) + Bvec[A.indices[A.indptr[i] : A.indptr[i + 1]]] = ( + patch_data / patch_data.max() + ) + + if swap_dim: + Bmat = np.reshape(Bvec, dims, order="C") + else: + Bmat = np.reshape(Bvec, dims, order="F") + + r_mask = np.zeros_like(Bmat, dtype="bool") + contour = find_contours(Bmat, thr) + for c in contour: + r_mask[ + np.round(c[:, 0]).astype("int"), np.round(c[:, 1]).astype("int") + ] = 1 + + # Fill in the hole created by the contour boundary + r_mask = binary_fill_holes(r_mask) + ims.append(r_mask + (i * r_mask)) + + return ims From 0eedb9ccdeaf736209f99c42ac05ef2376dadae3 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Wed, 8 Nov 2023 09:52:11 +0900 Subject: [PATCH 09/10] set wrapper related function as classmethod --- studio/app/optinist/wrappers/caiman/cnmf.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/studio/app/optinist/wrappers/caiman/cnmf.py b/studio/app/optinist/wrappers/caiman/cnmf.py index 9e081dbaa..da6938a41 100644 --- a/studio/app/optinist/wrappers/caiman/cnmf.py +++ b/studio/app/optinist/wrappers/caiman/cnmf.py @@ -305,7 +305,8 @@ def func(self, **kwargs): return info - def get_roi(self, A, thr, thr_method, swap_dim, dims): + @classmethod + def get_roi(cls, A, thr, thr_method, swap_dim, dims): from scipy.ndimage import binary_fill_holes from skimage.measure import find_contours From 718190cdae46d720fcbd43e0098bffe90e9a1a94 Mon Sep 17 00:00:00 2001 From: ReiHashimoto <42664619+ReiHashimoto@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:16:50 +0900 Subject: [PATCH 10/10] rename Wrapper class to AlgoTemplate --- studio/app/common/core/param/param_utils.py | 4 ++-- studio/app/common/core/wrapper/{wrapper.py => algo.py} | 2 +- .../core/wrapper/{wrapper_utils.py => algo_utils.py} | 10 +++++----- studio/app/common/routers/algolist.py | 4 ++-- studio/app/optinist/wrappers/caiman/cnmf.py | 8 ++++---- .../app/optinist/wrappers/caiman/motion_correction.py | 8 ++++---- 6 files changed, 18 insertions(+), 18 deletions(-) rename studio/app/common/core/wrapper/{wrapper.py => algo.py} (97%) rename studio/app/common/core/wrapper/{wrapper_utils.py => algo_utils.py} (69%) diff --git a/studio/app/common/core/param/param_utils.py b/studio/app/common/core/param/param_utils.py index b04435fce..0894ca839 100644 --- a/studio/app/common/core/param/param_utils.py +++ b/studio/app/common/core/param/param_utils.py @@ -2,7 +2,7 @@ from studio.app.common.core.param.param import ParamChild, ParamParent from studio.app.common.core.snakemake.smk import SnakemakeParams -from studio.app.common.core.wrapper.wrapper_utils import WrapperUtils +from studio.app.common.core.wrapper.algo_utils import AlgoUtils from studio.app.optinist.core.nwb.nwb import NWBParams @@ -14,7 +14,7 @@ def get_default_params(cls, name): elif name == "nwb": params = NWBParams.PARAMS else: - wrapper = WrapperUtils.find_wrapper_by_name(name=name) + wrapper = AlgoUtils.find_algo_by_name(name=name) if wrapper is None: return None params = wrapper._DEFAULT_PARAMS diff --git a/studio/app/common/core/wrapper/wrapper.py b/studio/app/common/core/wrapper/algo.py similarity index 97% rename from studio/app/common/core/wrapper/wrapper.py rename to studio/app/common/core/wrapper/algo.py index e497e8df8..c7775446f 100644 --- a/studio/app/common/core/wrapper/wrapper.py +++ b/studio/app/common/core/wrapper/algo.py @@ -4,7 +4,7 @@ from studio.app.common.core.param.param import Param -class Wrapper(ABC): +class AlgoTemplate(ABC): _INPUT_NODES: List[Param] = [] _OUTPUT_NODES: List[Param] = [] _DEFAULT_PARAMS: List[Param] = [] diff --git a/studio/app/common/core/wrapper/wrapper_utils.py b/studio/app/common/core/wrapper/algo_utils.py similarity index 69% rename from studio/app/common/core/wrapper/wrapper_utils.py rename to studio/app/common/core/wrapper/algo_utils.py index e9bd972c6..39be96b25 100644 --- a/studio/app/common/core/wrapper/wrapper_utils.py +++ b/studio/app/common/core/wrapper/algo_utils.py @@ -1,21 +1,21 @@ from typing import Any, Dict -from studio.app.common.core.wrapper.wrapper import Wrapper +from studio.app.common.core.wrapper.algo import AlgoTemplate from studio.app.const import FUNC_KEY from studio.app.wrappers import wrapper_dict -class WrapperUtils: +class AlgoUtils: @classmethod - def find_wrapper_by_name( + def find_algo_by_name( cls, name: str, wrapper_dict: Dict[str, Any] = wrapper_dict, - ) -> Wrapper: + ) -> AlgoTemplate: if name in wrapper_dict: return wrapper_dict[name][FUNC_KEY] else: for v in wrapper_dict.values(): if isinstance(v, dict) and FUNC_KEY not in v: - return cls.find_wrapper_by_name(name, v) + return cls.find_algo_by_name(name, v) return None diff --git a/studio/app/common/routers/algolist.py b/studio/app/common/routers/algolist.py index 968829bfd..4578d7e24 100644 --- a/studio/app/common/routers/algolist.py +++ b/studio/app/common/routers/algolist.py @@ -3,7 +3,7 @@ from fastapi import APIRouter from studio.app.common.core.param.param import Param -from studio.app.common.core.wrapper.wrapper import Wrapper +from studio.app.common.core.wrapper.algo import AlgoTemplate from studio.app.common.schemas.algolist import Algo, AlgoList, Arg, Return from studio.app.const import FUNC_KEY from studio.app.wrappers import wrapper_dict @@ -22,7 +22,7 @@ def get_nest_dict(cls, parent_value, parent_key: str) -> AlgoList: value, cls._parent_key(parent_key, key) ) else: - wrapper: Wrapper = value[FUNC_KEY] + wrapper: AlgoTemplate = value[FUNC_KEY] algo_dict[key] = Algo( args=cls._args_list(wrapper._INPUT_NODES), diff --git a/studio/app/optinist/wrappers/caiman/cnmf.py b/studio/app/optinist/wrappers/caiman/cnmf.py index da6938a41..4f0403b59 100644 --- a/studio/app/optinist/wrappers/caiman/cnmf.py +++ b/studio/app/optinist/wrappers/caiman/cnmf.py @@ -5,13 +5,13 @@ from studio.app.common.core.param.param import Param from studio.app.common.core.utils.filepath_creater import join_filepath -from studio.app.common.core.wrapper.wrapper import Wrapper +from studio.app.common.core.wrapper.algo import AlgoTemplate from studio.app.common.dataclass import ImageData from studio.app.optinist.core.nwb.nwb import NWBDATASET from studio.app.optinist.dataclass import CaimanCnmfData, FluoData, IscellData, RoiData -class CaimanCnmf(Wrapper): +class CaimanCnmf(AlgoTemplate): _INPUT_NODES = [Param(name="images", type=ImageData)] _OUTPUT_NODES = [ Param(name="fluorescence", type=FluoData), @@ -98,8 +98,8 @@ class CaimanCnmf(Wrapper): ] @docval( - *Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS]), - **Wrapper.docval_returns([*_OUTPUT_NODES]), + *AlgoTemplate.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS]), + **AlgoTemplate.docval_returns([*_OUTPUT_NODES]), ) def func(self, **kwargs): """caiman_cnmf diff --git a/studio/app/optinist/wrappers/caiman/motion_correction.py b/studio/app/optinist/wrappers/caiman/motion_correction.py index 51f892f30..73734ced6 100644 --- a/studio/app/optinist/wrappers/caiman/motion_correction.py +++ b/studio/app/optinist/wrappers/caiman/motion_correction.py @@ -7,13 +7,13 @@ create_directory, join_filepath, ) -from studio.app.common.core.wrapper.wrapper import Wrapper +from studio.app.common.core.wrapper.algo import AlgoTemplate from studio.app.common.dataclass import ImageData from studio.app.optinist.core.nwb.nwb import NWBDATASET from studio.app.optinist.dataclass import RoiData -class CaimanMc(Wrapper): +class CaimanMc(AlgoTemplate): _INPUT_NODES = [Param(name="image", type=ImageData)] _OUTPUT_NODES = [Param(name="mc_images", type=ImageData)] _DEFAULT_PARAMS = [ @@ -39,8 +39,8 @@ class CaimanMc(Wrapper): ] @docval( - *Wrapper.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS]), - **Wrapper.docval_returns(_OUTPUT_NODES), + *AlgoTemplate.docval_params([*_INPUT_NODES, *_DEFAULT_PARAMS]), + **AlgoTemplate.docval_returns(_OUTPUT_NODES), ) def func(self, **kwargs): """caiman_mc