SF plotter use structure to aggregate argument data, add support for full plotting spec

Packages/MIES/MIES_Constants.ipf

@@ -2146,6 +2146,7 @@ StrConstant SF_META_TRACETOFRONT       = "/TraceToFront"     // number, boolean,
 StrConstant SF_META_DONOTPLOT          = "/DoNotPlot"        // number, boolean, defaults to false (0)
 StrConstant SF_META_WINDOW_HOOK        = "/WindowHook"       // string
 StrConstant SF_META_FORMULA            = "/Formula"          // string
+StrConstant SF_META_PLOT               = "/Plot"             // number, boolean, defaults to false (0)
 
 /// A color group allows to have matching colors for sweep data with the same channel type/number and sweep.
 /// It is applied before the matching headstage/average colors in #SF_GetTraceColor().
@@ -2207,6 +2208,9 @@ StrConstant SF_DATATYPE_ANAFUNCPARAM        = "AnaFunc"
 
 StrConstant SF_WREF_MARKER     = "\"WREF@\":"
 StrConstant SF_VARIABLE_MARKER = "/SF_IsVariable" // numeric
+
+StrConstant SF_ANNOTATION_NAME = "metadata"
+StrConstant SF_VARNAME_REGEXP  = "[A-Z]{1}[A-Z0-9_]*"
 ///@}
 
 /// @name Constants for SweepFormula Clampmode codes returned by operation selcm()
@@ -2548,6 +2552,10 @@ StrConstant SF_OP_TPINST              = "tpinst"
 StrConstant SF_OP_TPBASE              = "tpbase"
 StrConstant SF_OP_TPFIT               = "tpfit"
 StrConstant SF_OP_EXTRACT             = "extract"
+
+#ifdef AUTOMATED_TESTING
+StrConstant SF_OP_TESTOP = "testop"
+#endif // AUTOMATED_TESTING
 ///@}
 
 StrConstant SF_PROPERTY_TABLE = "Table"

Packages/MIES/MIES_GlobalStringAndVariableAccess.ipf

@@ -824,3 +824,14 @@ Function/S GetSweepFormulaLastRightClickedDisplayWindow()
 
 	return GetSVARAsString(GetSweepFormulaPath(), "LastRightClickedDisplayWindow", initialValue = "")
 End
+
+#ifdef AUTOMATED_TESTING
+/// @brief Returns the full path to the global with the name of the function that is declared in a test
+///        that is called when testop is executed in a formula, used in testing
+Function/S GetSFTestopName(string graph)
+
+	DFREF dfr = SF_GetBrowserDF(graph)
+
+	return GetSVARAsString(dfr, "TestopFunctionName", initialValue = "")
+End
+#endif // AUTOMATED_TESTING

Packages/MIES/MIES_SweepFormula_Executor.ipf

@@ -25,7 +25,11 @@ static Constant SFE_VARIABLE_PREFIX = 36
 /// @param useVariables [optional, default 1], when not set, hint the function that the formula string contains only an expression and no variable definitions
 /// @param line         [optional, default NaN], line number of formula in SF notebook, when set, stores the information for the case of an SFH_ASSERT
 /// @param offset       [optional, default NaN], offset of a formula in SF notebook in characters from the start of the line (x-formulas), when set, stores the information for the case of an SFH_ASSERT
-Function/WAVE SFE_ExecuteFormula(string formula, string graph, [variable singleResult, variable checkExist, variable useVariables, variable line, variable offset])
+/// @param preProcess   [optional, default 1], when set to 0 then the formula is not in any way preprocessed and must not contain any variable definitions.
+///                                            Also the current error information for SFH_ASSERT is kept as is. The current variable storage is used.
+///                                            This allows to internally execute a formula  where a triggered SFH_ASSERT should result in the marking
+///                                            of the "outer" formula in the current SF notebook.
+Function/WAVE SFE_ExecuteFormula(string formula, string graph, [variable singleResult, variable checkExist, variable useVariables, variable line, variable offset, variable preProcess])
 
 	STRUCT SF_ExecutionData exd
 	variable jsonId, srcLocId
@@ -37,12 +41,15 @@ Function/WAVE SFE_ExecuteFormula(string formula, string graph, [variable singleR
 	useVariables = ParamIsDefault(useVariables) ? 1 : !!useVariables
 	line         = ParamIsDefault(line) ? NaN : line
 	offset       = ParamIsDefault(offset) ? NaN : offset
+	preProcess   = ParamIsDefault(preProcess) ? 1 : !!preProcess
 
-	formula = SF_PreprocessInput(formula)
-	if(useVariables)
-		formula = SFE_ExecuteVariableAssignments(graph, formula)
+	if(preProcess)
+		formula = SF_PreprocessInput(formula)
+		if(useVariables)
+			formula = SFE_ExecuteVariableAssignments(graph, formula)
+		endif
+		SFH_StoreAssertInfoParser(line, offset)
 	endif
-	SFH_StoreAssertInfoParser(line, offset)
 	[jsonId, srcLocId] = SFP_ParseFormulaToJSON(formula)
 	exd.jsonId         = jsonId
 	WAVE/Z result = SFE_FormulaExecutor(exd, srcLocId = srcLocId)
@@ -61,12 +68,20 @@ Function/WAVE SFE_ExecuteFormula(string formula, string graph, [variable singleR
 	return out
 End
 
-Function/S SFE_ExecuteVariableAssignments(string graph, string preProcCode)
+/// @brief Executes each variable assignment expression and stores the result in the variable storage
+///
+/// @param graph          SweepBrowser graph
+/// @param preProcCode    preprocessed sweep formula notebook text
+/// @param allowEmptyCode [optional, default 0] when set then the check for empty formula code is disabled, such that
+///                       input that contains only variable expressions can be evaluated
+Function/S SFE_ExecuteVariableAssignments(string graph, string preProcCode, [variable allowEmptyCode])
 
 	STRUCT SF_ExecutionData exd
 	variable i, numAssignments, jsonId, srcLocId, line, offset
 	string code, sfWin, nbText
 
+	allowEmptyCode = ParamisDefault(allowEmptyCode) ? 0 : !!allowEmptyCode
+
 	exd.graph = graph
 
 	WAVE/WAVE varStorage = GetSFVarStorage(graph)
@@ -96,7 +111,7 @@ Function/S SFE_ExecuteVariableAssignments(string graph, string preProcCode)
 		JSON_Release(srcLocId)
 	endfor
 
-	if(IsEmpty(code))
+	if(!allowEmptyCode && IsEmpty(code))
 		if(!StringEndsWith(preProcCode, SF_CHAR_CR))
 			sfWin   = BSP_GetSFFormula(graph)
 			nbText  = GetNotebookText(sfWin, mode = 2)
@@ -532,6 +547,11 @@ Function/WAVE SFE_FormulaExecutor(STRUCT SF_ExecutionData &exd, [variable srcLoc
 		case SF_OP_TABLE:
 			WAVE out = SFO_OperationTable(exdop)
 			break
+#ifdef AUTOMATED_TESTING
+		case SF_OP_TESTOP:
+			WAVE out = SFO_OperationTestop(exdop)
+			break
+#endif // AUTOMATED_TESTING
 		default:
 			SFH_FATAL_ERROR("Undefined Operation", jsonId = exdop.jsonId)
 	endswitch

Packages/MIES/MIES_SweepFormula_Helpers.ipf

@@ -2177,3 +2177,46 @@ Function/WAVE SFH_GetDatasetArrayAsResolvedWaverefs(STRUCT SF_ExecutionData &exd
 
 	return dataFromEachGroup
 End
+
+/// @brief Executes a formula from within an operation with low overhead
+///        - the currently active variable storage is used
+///        - the formula string is not preprocessed
+Function/WAVE SFH_ExecuteFormulaInternal(string graph, string formula)
+
+	STRUCT SF_ExecutionData exd
+	variable jsonId, srcLocId
+
+	exd.graph          = graph
+	[jsonId, srcLocId] = SFP_ParseFormulaToJSON(formula)
+	exd.jsonId         = jsonId
+	WAVE dataRef = SFE_FormulaExecutor(exd, srcLocId = srcLocId)
+
+	JSON_Release(exd.jsonId)
+	JSON_Release(srcLocId)
+
+	WAVE resolved = SF_ResolveDataset(dataRef)
+
+	return resolved
+End
+
+/// @brief Adds a variable to the variable storage. If the variable already exists it is overwritten.
+Function SFH_AddVariableToStorage(string graph, string name, WAVE result)
+
+	variable size, idx
+	string varName
+	string regex = "^(?i)(" + SF_VARNAME_REGEXP + ")$"
+
+	SplitString/E=regex name, varName
+	ASSERT(V_flag == 1, "Invalid SF variable name")
+
+	WAVE/WAVE varStorage = GetSFVarStorage(graph)
+	idx = FindDimLabel(varStorage, ROWS, varName)
+	if(idx == -2)
+		size = DimSize(varStorage, ROWS)
+		Redimension/N=(size + 1) varStorage
+		idx = size
+		SetDimLabel ROWS, size, $varName, varStorage
+	endif
+	JWN_SetNumberInWaveNote(result, SF_VARIABLE_MARKER, 1)
+	varStorage[idx] = result
+End

Packages/MIES/MIES_SweepFormula_Operations.ipf

@@ -2481,3 +2481,24 @@ Function/WAVE SFO_OperationTable(STRUCT SF_ExecutionData &exd)
 
 	return SFH_GetOutputForExecutor(output, exd.graph, SF_OP_TABLE)
 End
+
+#ifdef AUTOMATED_TESTING
+Function/WAVE SFO_OperationTestop_PROTO(STRUCT SF_ExecutionData &exd)
+
+	INFO("TestOp PROTO function was called. A proper implementation function for testop must be registered by setting the SVAR from GetSFTestopName")
+	FAIL()
+End
+
+/// testop(...)
+Function/WAVE SFO_OperationTestop(STRUCT SF_ExecutionData &exd)
+
+	string funcName = ROStr(GetSFTestopName(exd.graph))
+
+	REQUIRE_PROPER_STR(funcName)
+
+	FUNCREF SFO_OperationTestop_PROTO func = $funcName
+	WAVE                              wv   = func(exd)
+
+	return wv
+End
+#endif // AUTOMATED_TESTING

Packages/MIES/MIES_Utilities_Conversions.ipf

@@ -54,24 +54,17 @@ threadsafe Function/S TextWaveToList(WAVE/Z/T txtWave, string rowSep, [string co
 	endif
 
 	ASSERT_TS(IsTextWave(txtWave), "Expected a text wave")
-	ASSERT_TS(!IsEmpty(rowSep), "Expected a non-empty row list separator")
 
 	if(ParamIsDefault(colSep))
 		colSep = ","
-	else
-		ASSERT_TS(!IsEmpty(colSep), "Expected a non-empty column list separator")
 	endif
 
 	if(ParamIsDefault(layerSep))
 		layerSep = ":"
-	else
-		ASSERT_TS(!IsEmpty(layerSep), "Expected a non-empty layer list separator")
 	endif
 
 	if(ParamIsDefault(chunkSep))
 		chunkSep = "/"
-	else
-		ASSERT_TS(!IsEmpty(chunkSep), "Expected a non-empty chunk list separator")
 	endif
 
 	if(ParamIsDefault(maxElements))

Packages/MIES/MIES_WaveDataFolderGetters.ipf

@@ -9382,3 +9382,26 @@ Function/WAVE GetSFPlotProperties()
 
 	return wv
 End
+
+/// @brief Returns a permanent SF wave reference wave for the AND branch of a full plotting specification.
+///        The elements of this wave must be filled with WITH branches from GetFullPlottingWITH.
+///        At the end of the operation code this wave can be returned with
+///        return SFH_GetOutputForExecutor(plotAND, exd.graph, opShort)
+Function/WAVE GetFullPlottingAND(string graph, string opShort, variable size)
+
+	WAVE/WAVE plotAND = SFH_CreateSFRefWave(graph, opShort, size)
+	JWN_SetNumberInWaveNote(plotAND, SF_META_PLOT, 1)
+
+	return plotAND
+End
+
+/// @brief Returns a free wave for the WITH branch of a full plotting specification
+///        At least the FORMULAY element must be filled by the caller
+Function/WAVE GetFullPlottingWITH(variable size)
+
+	Make/FREE/WAVE/N=(size, 2) plotWITH
+	SetDimlabel COLS, 0, FORMULAX, plotWITH
+	SetDimlabel COLS, 1, FORMULAY, plotWITH
+
+	return plotWITH
+End