Merge pull request #946 from arayabrain/feature/porting_finish_workflow_without_run

Porting finish-workflow-without-run

Author: itutu-tienday

frontend/src/components/Workspace/FlowChart/FlowChartNode/AlgorithmNode.tsx

@@ -43,6 +43,7 @@ import { NodeData, NodeIdProps } from "store/slice/FlowElement/FlowElementType"
 import {
   selectPipelineLatestUid,
   selectPipelineNodeResultMessage,
+  selectPipelineNodeResultOutputPathsExists,
   selectPipelineNodeResultStatus,
   selectPipelineStatus,
 } from "store/slice/Pipeline/PipelineSelectors"
@@ -108,6 +109,10 @@ const AlgorithmNodeImple = memo(function AlgorithmNodeImple({
   const isUpdated = useSelector(selectAlgorithmIsUpdated(nodeId))
   const isParentParamsUpdated = useSelector(isParentNodeUpdatedParams(nodeId))
 
+  const nodeResultOutputPathsExists = useSelector(
+    selectPipelineNodeResultOutputPathsExists(nodeId),
+  )
+
   const updated =
     typeof workflowId !== "undefined" &&
     (isUpdated || ancestorIsUpdated || isParentParamsUpdated)
@@ -118,7 +123,7 @@ const AlgorithmNodeImple = memo(function AlgorithmNodeImple({
         "suite2p_roi",
         "caiman_cnmf",
         "lccd_cell_detection",
-	"vacant_roi",
+        "vacant_roi",
         "caiman_cnmfe",
         "cnmf_multisession",
       ].includes(data.label),
@@ -158,7 +163,10 @@ const AlgorithmNodeImple = memo(function AlgorithmNodeImple({
         <Button
           size="small"
           onClick={onClickOutputButton}
-          disabled={status !== NODE_RESULT_STATUS.SUCCESS}
+          disabled={
+            status !== NODE_RESULT_STATUS.SUCCESS ||
+            !nodeResultOutputPathsExists
+          }
         >
           Output
         </Button>
@@ -167,7 +175,9 @@ const AlgorithmNodeImple = memo(function AlgorithmNodeImple({
             size="small"
             onClick={onClickFilterButton}
             disabled={
-              status !== NODE_RESULT_STATUS.SUCCESS || isParentParamsUpdated
+              status !== NODE_RESULT_STATUS.SUCCESS ||
+              !nodeResultOutputPathsExists ||
+              isParentParamsUpdated
             }
             style={{ backgroundColor: isUpdateFilterParams ? "#ff98004d" : "" }}
           >

frontend/src/store/slice/DisplayData/DisplayDataType.ts

@@ -64,11 +64,12 @@ export type DisplayData = {
 }
 
 export const DATA_TYPE_SET = {
-  TIME_SERIES: "timeSeries",
-  HEAT_MAP: "heatMap",
+  EMPTY: "empty",
   IMAGE: "image",
   CSV: "csv",
   ROI: "roi",
+  TIME_SERIES: "timeSeries",
+  HEAT_MAP: "heatMap",
   SCATTER: "scatter",
   BAR: "bar",
   HDF5: "hdf5",

frontend/src/store/slice/DisplayData/DisplayDataUtils.ts

@@ -8,6 +8,8 @@ import {
  */
 export function toDataType(value: string): DATA_TYPE {
   switch (value) {
+    case "empty":
+      return DATA_TYPE_SET.EMPTY
     case "images":
       return DATA_TYPE_SET.IMAGE
     case "timeseries":

frontend/src/store/slice/Pipeline/PipelineSelectors.ts

@@ -1,3 +1,4 @@
+import { DATA_TYPE_SET } from "store/slice/DisplayData/DisplayDataType"
 import {
   NodeResult,
   NodeResultPending,
@@ -165,13 +166,27 @@ const selectPipelineNodeResultOutputPaths =
         return nodeResult.outputPaths
       }
     }
-    throw new Error(`key error. nodeId:${nodeId}`)
+    return null
+  }
+
+export const selectPipelineNodeResultOutputPathsExists =
+  (nodeId: string) => (state: RootState) => {
+    const outputPaths = selectPipelineNodeResultOutputPaths(nodeId)(state)
+
+    if (outputPaths == null) {
+      return false
+    }
+
+    const firstOutputPath = Object.values(outputPaths).at(0)
+    const outputPathsExists =
+      firstOutputPath != null && firstOutputPath.type !== DATA_TYPE_SET.EMPTY
+    return outputPathsExists
   }
 
 export const selectPipelineNodeResultOutputFilePath =
   (nodeId: string, outputKey: string) => (state: RootState) => {
     const outputPaths = selectPipelineNodeResultOutputPaths(nodeId)(state)
-    if (Object.keys(outputPaths).includes(outputKey)) {
+    if (outputPaths && Object.keys(outputPaths).includes(outputKey)) {
       return outputPaths[outputKey].path
     } else {
       throw new Error(`key error. outputKey:${outputKey}`)
@@ -182,7 +197,7 @@ export const selectOutputFilePathCellRoi =
   (nodeId?: string | null) => (state: RootState) => {
     if (!nodeId) return ""
     const outputPaths = selectPipelineNodeResultOutputPaths(nodeId)(state)
-    if (Object.keys(outputPaths).includes("cell_roi")) {
+    if (outputPaths && Object.keys(outputPaths).includes("cell_roi")) {
       return outputPaths["cell_roi"].path
     }
     return ""
@@ -191,7 +206,7 @@ export const selectOutputFilePathCellRoi =
 export const selectPipelineNodeResultOutputFileDataType =
   (nodeId: string, outputKey: string) => (state: RootState) => {
     const outputPaths = selectPipelineNodeResultOutputPaths(nodeId)(state)
-    if (Object.keys(outputPaths).includes(outputKey)) {
+    if (outputPaths && Object.keys(outputPaths).includes(outputKey)) {
       return outputPaths[outputKey].type
     } else {
       throw new Error(`key error. outputKey:${outputKey}`)

frontend/src/store/slice/VisualizeItem/VisualizeItemSlice.ts

@@ -13,11 +13,12 @@ import {
   setNewDisplayDataPath,
 } from "store/slice/VisualizeItem/VisualizeItemActions"
 import {
-  HeatMapItem,
+  EmptyItem,
   ImageItem,
   CsvItem,
   TimeSeriesItem,
   RoiItem,
+  HeatMapItem,
   ScatterItem,
   VisualaizeItem,
   VISUALIZE_ITEM_TYPE_SET,
@@ -62,6 +63,10 @@ const displayDataCommonInitialValue = {
   saveFileName: "newPlot",
   saveFormat: "png",
 }
+const emptyItemInitialValue: EmptyItem = {
+  ...displayDataCommonInitialValue,
+  dataType: DATA_TYPE_SET.EMPTY,
+}
 const imageItemInitialValue: ImageItem = {
   ...displayDataCommonInitialValue,
   dataType: DATA_TYPE_SET.IMAGE,
@@ -178,6 +183,8 @@ const matlabItemInitialValue: MatlabItem = {
 
 function getDisplayDataItemInitialValue(dataType: DATA_TYPE) {
   switch (dataType) {
+    case DATA_TYPE_SET.EMPTY:
+      return emptyItemInitialValue
     case DATA_TYPE_SET.IMAGE:
       return imageItemInitialValue
     case DATA_TYPE_SET.HEAT_MAP:

frontend/src/store/slice/VisualizeItem/VisualizeItemType.ts

@@ -39,6 +39,7 @@ export type VISUALIZE_ITEM_TYPE =
   (typeof VISUALIZE_ITEM_TYPE_SET)[keyof typeof VISUALIZE_ITEM_TYPE_SET]
 
 export type DisplayDataItem =
+  | EmptyItem
   | ImageItem
   | TimeSeriesItem
   | HeatMapItem
@@ -65,6 +66,10 @@ export interface DisplayDataItemBaseType extends ItemBaseType<"displayData"> {
   saveFormat: string
 }
 
+export interface EmptyItem extends DisplayDataItemBaseType {
+  dataType: typeof DATA_TYPE_SET.EMPTY
+}
+
 export interface ImageItem extends DisplayDataItemBaseType {
   dataType: typeof DATA_TYPE_SET.IMAGE
   activeIndex: number

studio/app/common/core/workflow/workflow.py

@@ -107,6 +107,7 @@ def check_nodetype_from_filetype(file_type: str) -> str:
 
 @dataclass
 class OutputType:
+    EMPTY: str = "empty"
     IMAGE: str = "images"
     TIMESERIES: str = "timeseries"
     HEATMAP: str = "heatmap"

studio/app/common/core/workflow/workflow_runner.py

@@ -1,7 +1,14 @@
 import uuid
 from dataclasses import asdict
+from datetime import datetime
 from typing import Dict, List
 
+from fastapi import BackgroundTasks
+
+from studio.app.common.core.experiment.experiment_reader import ExptConfigReader
+from studio.app.common.core.experiment.experiment_record_services import (
+    ExperimentRecordService,
+)
 from studio.app.common.core.experiment.experiment_writer import ExptConfigWriter
 from studio.app.common.core.rules.runner import Runner
 from studio.app.common.core.snakemake.smk import FlowConfig, Rule, SmkParam
@@ -12,9 +19,17 @@
 from studio.app.common.core.snakemake.snakemake_reader import SmkParamReader
 from studio.app.common.core.snakemake.snakemake_rule import SmkRule
 from studio.app.common.core.snakemake.snakemake_writer import SmkConfigWriter
-from studio.app.common.core.workflow.workflow import NodeType, NodeTypeUtil, RunItem
+from studio.app.common.core.workflow.workflow import (
+    NodeType,
+    NodeTypeUtil,
+    OutputPath,
+    OutputType,
+    RunItem,
+    WorkflowRunStatus,
+)
 from studio.app.common.core.workflow.workflow_params import get_typecheck_params
 from studio.app.common.core.workflow.workflow_writer import WorkflowConfigWriter
+from studio.app.const import DATE_FORMAT
 
 
 class WorkflowRunner:
@@ -47,7 +62,7 @@ def create_workflow_unique_id() -> str:
         new_unique_id = str(uuid.uuid4())[:8]
         return new_unique_id
 
-    def run_workflow(self, background_tasks):
+    def run_workflow(self, background_tasks: BackgroundTasks):
         self.set_smk_config()
 
         snakemake_params: SmkParam = get_typecheck_params(
@@ -68,6 +83,54 @@ def run_workflow(self, background_tasks):
             snakemake_execute, self.workspace_id, self.unique_id, snakemake_params
         )
 
+    def finish_workflow_without_run(
+        self, status: WorkflowRunStatus = WorkflowRunStatus.SUCCESS
+    ):
+        """
+        Saves the settings and finishes the workflow without actually running it.
+        - Function solely for creating experiment record.
+        """
+
+        # Load current configs
+        expt_config = ExptConfigReader.read(self.workspace_id, self.unique_id)
+
+        # Construct update data (ExptConfig.*)
+        update_expt_config = ExptConfigReader.create_empty_experiment_config()
+        now = datetime.now().strftime(DATE_FORMAT)
+        update_expt_config.success = status.value
+        update_expt_config.finished_at = now
+        update_expt_config.data_usage = 0
+
+        # Construct update data (ExptConfig.function)
+        update_expt_config.function = {}
+        for node_id, function in expt_config.function.items():
+            function.success = WorkflowRunStatus.SUCCESS.value
+            function.outputPaths = {
+                "empty": OutputPath(
+                    path="empty",
+                    type=OutputType.EMPTY,
+                    max_index=1,
+                )
+            }
+
+            update_expt_config.function[node_id] = function
+
+        # Prepare data (dict variable) for overwriting the config file
+        update_expt_config_dict = {
+            k: v for k, v in asdict(update_expt_config).items() if v is not None
+        }
+
+        # Overwrite config file
+        ExptConfigWriter(self.workspace_id, self.unique_id).overwrite(
+            update_expt_config_dict
+        )
+
+        # Update experiment database record
+        if ExperimentRecordService.is_available():
+            ExperimentRecordService.regist_record_on_workflow_completed(
+                self.workspace_id, self.unique_id
+            )
+
     def set_smk_config(self):
         rules, last_output = self.rulefile()
 

studio/tests/app/common/core/workflow/test_workflow_runner.py

@@ -0,0 +1,73 @@
+import os
+import shutil
+
+from studio.app.common.core.experiment.experiment_reader import ExptConfigReader
+from studio.app.common.core.workflow.workflow import (
+    Edge,
+    Node,
+    NodeData,
+    RunItem,
+    WorkflowRunStatus,
+)
+from studio.app.common.core.workflow.workflow_runner import WorkflowRunner
+from studio.app.dir_path import DIRPATH
+
+workspace_id = "default"
+unique_id = "workflow_test"
+
+node_data = NodeData(label="a", param={}, path="", type="")
+
+nodeDict = {
+    "test1": Node(
+        id="node_id",
+        type="a",
+        data=node_data,
+        position={"x": 0, "y": 0},
+        style={
+            "border": None,
+            "borderRadius": 0,
+            "height": 100,
+            "padding": 0,
+            "width": 180,
+        },
+    )
+}
+
+edgeDict = {
+    "test2": Edge(
+        id="edge_id",
+        type="a",
+        animated=False,
+        source="",
+        sourceHandle="",
+        target="",
+        targetHandle="",
+        style={},
+    )
+}
+
+
+dirpath = f"{DIRPATH.OUTPUT_DIR}/{workspace_id}/{unique_id}"
+
+
+def test_finish_workflow_without_run():
+    if os.path.exists(dirpath):
+        shutil.rmtree(dirpath)
+
+    runItem = RunItem(
+        name="New Flow",
+        nodeDict=nodeDict,
+        edgeDict=edgeDict,
+        snakemakeParam={},
+        nwbParam={},
+        forceRunList=[],
+    )
+
+    WorkflowRunner(workspace_id, unique_id, runItem).finish_workflow_without_run()
+
+    assert os.path.exists(f"{dirpath}/experiment.yaml")
+    assert os.path.exists(f"{dirpath}/workflow.yaml")
+
+    exp_config = ExptConfigReader.read(workspace_id, unique_id)
+
+    assert exp_config.success == WorkflowRunStatus.SUCCESS.value