Merge pull request #2387 from AllenInstitute/feature/2387-sf-enhancements-range-concat

SF: Fix range and add concat

Author: t-b

Packages/MIES/MIES_Constants.ipf

@@ -2109,6 +2109,7 @@ StrConstant SF_DATATYPE_AVG                 = "Average"
 StrConstant SF_DATATYPE_MAX                 = "Max"
 StrConstant SF_DATATYPE_MIN                 = "Min"
 StrConstant SF_DATATYPE_RANGE               = "Range"
+StrConstant SF_DATATYPE_CONCAT              = "Concat"
 StrConstant SF_DATATYPE_EPOCHS              = "Epochs"
 StrConstant SF_DATATYPE_TP                  = "TestPulse"
 StrConstant SF_DATATYPE_TPSS                = "TestPulseMode_SteadyState"

Packages/MIES/MIES_JSONWaveNotes.ipf

@@ -9,29 +9,34 @@
 static Constant JWN_DEFAULT_RELEASE = 1
 
 /// @brief Gets the JSON wave note part as string
-threadsafe Function/S JWN_GetWaveNoteAsString(WAVE wv)
+threadsafe static Function/S JWN_GetWaveNoteAsString(string noteStr)
 
 	variable pos, len
-	string noteStr
 
-	ASSERT_TS(WaveExists(wv), "Missing wave")
-	noteStr = note(wv)
-	pos     = strsearch(noteStr, WAVE_NOTE_JSON_SEPARATOR, 0)
-	len     = strlen(WAVE_NOTE_JSON_SEPARATOR)
+	pos = strsearch(noteStr, WAVE_NOTE_JSON_SEPARATOR, 0)
+	len = strlen(WAVE_NOTE_JSON_SEPARATOR)
 	if(pos >= 0 && strlen(noteStr) > (pos + len))
 		return noteStr[pos + len, Inf]
 	endif
 
 	return WAVE_NOTE_EMPTY_JSON
 End
 
+threadsafe static Function JWN_Parse(string noteStr)
+
+	return JSON_Parse(JWN_GetWaveNoteAsString(noteStr))
+End
+
 /// @brief Gets the JSON wave note part as JSON object
 ///        The caller is responsible to release the returned jsonId after use.
 threadsafe Function JWN_GetWaveNoteAsJSON(WAVE wv)
 
+	string noteStr
+
 	ASSERT_TS(WaveExists(wv), "Missing wave")
+	noteStr = note(wv)
 
-	return JSON_Parse(JWN_GetWaveNoteAsString(wv))
+	return JWN_Parse(noteStr)
 End
 
 /// @brief Set the JSON json document as JSON wave note. Releases json if `release` is true (default).
@@ -176,12 +181,17 @@ End
 /// @returns the value on success. An empty string is returned if it could not be found
 threadsafe Function/S JWN_GetStringFromWaveNote(WAVE wv, string jsonPath)
 
+	return JWN_GetStringFromNote(note(wv), jsonPath)
+End
+
+threadsafe Function/S JWN_GetStringFromNote(string noteStr, string jsonPath)
+
 	variable jsonID
 	string   str
 
 	ASSERT_TS(!IsEmpty(jsonPath), "Empty jsonPath")
 
-	jsonID = JWN_GetWaveNoteAsJSON(wv)
+	jsonID = JWN_Parse(noteStr)
 	str    = JSON_GetString(jsonID, jsonPath, ignoreErr = 1)
 	JSON_Release(jsonID)
 

Packages/MIES/MIES_SweepFormula.ipf

@@ -64,6 +64,7 @@ static StrConstant SF_OP_PLUS                = "+"
 static StrConstant SF_OP_MULT                = "*"
 static StrConstant SF_OP_DIV                 = "~1"
 static StrConstant SF_OP_RANGE               = "range"
+static StrConstant SF_OP_CONCAT              = "concat"
 static StrConstant SF_OP_RANGESHORT          = "…"
 static StrConstant SF_OP_MIN                 = "min"
 static StrConstant SF_OP_MAX                 = "max"
@@ -255,7 +256,7 @@ Function/WAVE SF_GetNamedOperations()
 	                  SF_OP_MERGE, SF_OP_FIT, SF_OP_FITLINE, SF_OP_DATASET, SF_OP_SELECTVIS, SF_OP_SELECTCM, SF_OP_SELECTSTIMSET,       \
 	                  SF_OP_SELECTIVSCCSWEEPQC, SF_OP_SELECTIVSCCSETQC, SF_OP_SELECTRANGE, SF_OP_SELECTEXP, SF_OP_SELECTDEV,            \
 	                  SF_OP_SELECTEXPANDSCI, SF_OP_SELECTEXPANDRAC, SF_OP_SELECTSETCYCLECOUNT, SF_OP_SELECTSETSWEEPCOUNT,               \
-	                  SF_OP_SELECTSCIINDEX, SF_OP_SELECTRACINDEX, SF_OP_ANAFUNCPARAM}
+	                  SF_OP_SELECTSCIINDEX, SF_OP_SELECTRACINDEX, SF_OP_ANAFUNCPARAM, SF_OP_CONCAT}
 
 	return wt
 End
@@ -1044,6 +1045,9 @@ static Function/WAVE SF_FormulaExecutor(string graph, variable jsonID, [string j
 		case SF_OP_RANGESHORT:
 			WAVE out = SF_OperationRange(jsonId, jsonPath, graph)
 			break
+		case SF_OP_CONCAT:
+			WAVE out = SF_OperationConcat(jsonId, jsonPath, graph)
+			break
 		case SF_OP_MIN:
 			WAVE out = SF_OperationMin(jsonId, jsonPath, graph)
 			break
@@ -4023,37 +4027,68 @@ End
 /// range (start[, stop[, step]])
 static Function/WAVE SF_OperationRange(variable jsonId, string jsonPath, string graph)
 
-	variable numArgs
+	variable start, stop, step, stopDefault
 
-	numArgs = SFH_CheckArgumentCount(jsonId, jsonPath, SF_OP_RANGE, 1, maxArgs = 3)
+	SFH_CheckArgumentCount(jsonId, jsonPath, SF_OP_RANGE, 1, maxArgs = 3)
 
-	WAVE arg0 = SFH_ResolveDatasetElementFromJSON(jsonId, jsonpath, graph, SF_OP_RANGE, 0, checkExist = 1)
-	if(numArgs == 1)
-		SFH_ASSERT(IsNumericWave(arg0), "range first argument must be numeric")
-		Make/FREE/D/N=(abs(trunc(arg0[0]))) range
-		MultiThread range = p
-	elseif(numArgs == 2)
-		WAVE arg1 = SFH_ResolveDatasetElementFromJSON(jsonId, jsonpath, graph, SF_OP_RANGE, 1, checkExist = 1)
-		SFH_ASSERT(IsNumericWave(arg1), "range second argument must be numeric")
-		Make/FREE/D/N=(abs(trunc(arg0[0]) - trunc(arg1[0]))) range
-		MultiThread range[] = arg0[0] + p
-		SFH_CleanUpInput(arg1)
-	elseif(numArgs == 3)
-		WAVE arg1 = SFH_ResolveDatasetElementFromJSON(jsonId, jsonpath, graph, SF_OP_RANGE, 1, checkExist = 1)
-		WAVE arg2 = SFH_ResolveDatasetElementFromJSON(jsonId, jsonpath, graph, SF_OP_RANGE, 2, checkExist = 1)
-		SFH_ASSERT(IsNumericWave(arg1), "range second argument must be numeric")
-		SFH_ASSERT(IsNumericWave(arg2), "range third argument must be numeric")
-		Make/FREE/D/N=(ceil(abs((arg0[0] - arg1[0]) / arg2[0]))) range
-		MultiThread range[] = arg0[0] + p * arg2[0]
-		SFH_CleanUpInput(arg1)
-		SFH_CleanUpInput(arg2)
+	start = SFH_GetArgumentAsNumeric(jsonId, jsonpath, graph, SF_OP_RANGE, 0)
+	stop  = SFH_GetArgumentAsNumeric(jsonId, jsonpath, graph, SF_OP_RANGE, 1, defValue = NaN)
+	step  = SFH_GetArgumentAsNumeric(jsonId, jsonpath, graph, SF_OP_RANGE, 2, defValue = 1)
+
+	if(IsNaN(stop))
+		stop        = 0
+		stopDefault = 1
 	endif
 
-	SFH_CleanUpInput(arg0)
+	Make/FREE/D/N=(ceil(abs((start - stop) / step))) range
+
+	if(stopDefault)
+		MultiThread range[] = p * step
+	else
+		MultiThread range[] = start + p * step
+	endif
 
 	return SFH_GetOutputForExecutorSingle(range, graph, SF_OP_RANGE, dataType = SF_DATATYPE_RANGE)
 End
 
+/// concat(array0, array1, array2, ...)
+static Function/WAVE SF_OperationConcat(variable jsonId, string jsonPath, string graph)
+
+	variable numArgs, i, err, majorType, sliceMajorType
+	variable constantDataType
+	string refDataType, dataType, wvNote, errMsg
+
+	numArgs = SFH_CheckArgumentCount(jsonId, jsonPath, SF_OP_CONCAT, 1)
+
+	WAVE result = SFH_GetArgumentAsWave(jsonId, jsonpath, graph, SF_OP_CONCAT, 0, copy = 1, singleResult = 1, wvNote = wvNote)
+	majorType = WaveType(result, 1)
+
+	refDataType      = JWN_GetStringFromNote(wvNote, SF_META_DATATYPE)
+	dataType         = refDataType
+	constantDataType = !IsEmpty(refDataType)
+	Note/K result
+
+	AssertOnAndClearRTError()
+	for(i = 1; i < numArgs; i += 1)
+		WAVE slice = SFH_GetArgumentAsWave(jsonId, jsonpath, graph, SF_OP_CONCAT, i, singleResult = 1, wvNote = wvNote)
+		sliceMajorType = WaveType(slice, 1)
+
+		if(majorType != sliceMajorType)
+			sprintf errMsg, "Concatenate failed as the wave types of the first argument and #%d don't match: %s vs %s", i, WaveTypeToStringSelectorOne(majorType), WaveTypeToStringSelectorOne(sliceMajorType)
+			SFH_ASSERT(0, errMsg)
+		endif
+
+		dataType         = JWN_GetStringFromNote(wvNote, SF_META_DATATYPE)
+		constantDataType = constantDataType && !CmpStr(refDataType, dataType)
+
+		Concatenate/FREE/NP {slice}, result; errMsg = GetRTErrMessage(); err = GetRTError(1)
+		SFH_ASSERT(!err, "Error concatenating waves: " + errMsg)
+	endfor
+
+	dataType = SelectString(constantDataType, SF_DATATYPE_CONCAT, dataType)
+	return SFH_GetOutputForExecutorSingle(result, graph, SF_OP_CONCAT, discardOpStack = 1, dataType = dataType)
+End
+
 static Function/WAVE SF_OperationMin(variable jsonId, string jsonPath, string graph)
 
 	WAVE/WAVE input  = SF_GetNumericVarArgs(jsonId, jsonPath, graph, SF_OP_MIN)

Packages/MIES/MIES_SweepFormula_Helpers.ipf

@@ -235,14 +235,34 @@ End
 /// returned.
 ///
 /// The second argument `birdTypes` is optional, if not present the operation `birdTypes()` is called and its result returned. Alternatively `defWave` can be supplied which is then returned if the argument is not present.
-Function/WAVE SFH_GetArgumentAsWave(variable jsonId, string jsonPath, string graph, string opShort, variable argNum, [string defOp, WAVE/Z defWave, variable singleResult, variable expectedMinorType, variable expectedMajorType, variable copy])
-
-	variable checkExist, numArgs, checkMinorType, checkMajorType
+///
+/// @param jsonId            JSON identifier
+/// @param jsonPath          Location in the AST
+/// @param graph             Databrowser graph
+/// @param opShort           Short name of the operation
+/// @param argNum            Argument index
+/// @param defOp             [optional, defaults to None] SF code to execute in case the argument is not present
+/// @param defWave           [optional] wave to return in case the argument is not present and defOp is not supplied
+/// @param singleResult      [optional, defaults to 0] Return the first wave of the dataset if it has only one entry
+/// @param expectedMinorType [optional, defaults to None] Expected minor wave type, possible values are from WaveType(wv, 0)
+/// @param expectedMajorType [optional, defaults to None] Expected major wave type, possible values are from WaveType(wv, 1)
+/// @param copy              [optional, defaults to 0] If the returned data should be safe for modification (true) or is only read (false)
+/// @param[out] wvNote       [optional, defaults to None] Wave note of the dataset, useful for single result cases where you still need
+///                          to query JSON wave note entries
+Function/WAVE SFH_GetArgumentAsWave(variable jsonId, string jsonPath, string graph, string opShort, variable argNum, [string defOp, WAVE/Z defWave, variable singleResult, variable expectedMinorType, variable expectedMajorType, variable copy, string &wvNote])
+
+	variable checkExist, numArgs, checkMinorType, checkMajorType, result
 	string msg
 
+	if(!ParamIsDefault(wvNote))
+		ASSERT(ParamIsDefault(defOp) && ParamIsDefault(defWave),                        \
+		       "The optional parameters wvNote and defOp/defWave can't be used together")
+	endif
+
 	if(ParamIsDefault(defOp) && ParamIsDefault(defWave))
 		checkExist = 1
 	else
+		ASSERT((ParamIsDefault(defOp) + ParamIsDefault(defWave)) == 1, "Can only supply one of defOp and defWave")
 		checkExist = 0
 	endif
 
@@ -265,6 +285,10 @@ Function/WAVE SFH_GetArgumentAsWave(variable jsonId, string jsonPath, string gra
 			sprintf msg, "Argument #%d of operation %s: Too many input values", argNum, opShort
 			SFH_ASSERT(DimSize(input, ROWS) == 1, msg)
 
+			if(!ParamIsDefault(wvNote))
+				wvNote = note(input)
+			endif
+
 			WAVE/Z data = input[0]
 			SFH_CleanUpInput(input)
 		else
@@ -278,12 +302,18 @@ Function/WAVE SFH_GetArgumentAsWave(variable jsonId, string jsonPath, string gra
 				WAVE/WAVE dataAsRef = data
 				Make/FREE/N=(DimSize(data, ROWS)) types = WaveType(dataAsRef[p])
 			endif
-			sprintf msg, "Argument #%d of operation %s: Expected minor wave type %d", argNum, opShort, expectedMinorType
-			if(expectedMinorType)
+
+			if(expectedMinorType > 0)
 				types[] = !!(types[p] & expectedMinorType)
-				SFH_ASSERT(sum(types) == DimSize(types, ROWS), msg)
+				result  = sum(types) == DimSize(types, ROWS)
 			else
-				SFH_ASSERT(IsConstant(types, expectedMinorType), msg)
+				result = IsConstant(types, expectedMinorType)
+			endif
+
+			if(!result)
+				Make/T/FREE/N=(DimSize(types, ROWS)) typesText = WaveTypeToStringSelectorZero(types[p])
+				sprintf msg, "Argument #%d of operation %s: Expected minor wave type %s but got %s", argNum, opShort, WaveTypeToStringSelectorZero(expectedMinorType), TextWaveToList(typesText, ", ", trailSep = 0)
+				SFH_ASSERT(result, msg)
 			endif
 		endif
 		if(checkMajorType)
@@ -294,8 +324,11 @@ Function/WAVE SFH_GetArgumentAsWave(variable jsonId, string jsonPath, string gra
 				Make/FREE/N=(DimSize(data, ROWS)) types = WaveType(dataAsRef[p], 1)
 			endif
 
-			sprintf msg, "Argument #%d of operation %s: Expected major wave type %d", argNum, opShort, expectedMajorType
-			SFH_ASSERT(IsConstant(types, expectedMajorType), msg)
+			if(!IsConstant(types, expectedMajorType))
+				Make/T/FREE/N=(DimSize(types, ROWS)) typesText = WaveTypeToStringSelectorOne(types[p])
+				sprintf msg, "Argument #%d of operation %s: Expected major wave type %s but got %s", argNum, opShort, WaveTypeToStringSelectorOne(expectedMajorType), TextWaveToList(typesText, ", ", trailSep = 0)
+				SFH_ASSERT(0, msg)
+			endif
 		endif
 
 		return SFH_CopyDataIfRequired(copy, input, data)

Packages/MIES/MIES_Utilities_Strings.ipf

@@ -514,3 +514,66 @@ threadsafe Function/S UpperCaseFirstChar(string str)
 
 	return UpperStr(str[0]) + str[1, len - 1]
 End
+
+/// @brief Human readable name for possible return values of WaveType(wv, 0)
+///
+/// We don't do any error checking if the given type really exists.
+threadsafe Function/S WaveTypeToStringSelectorZero(variable type)
+
+	string result = ""
+	variable trim
+
+	if(type == IGOR_TYPE_TEXT_WREF_DFR)
+		return "non-numeric (text, wave ref, dfref)"
+	endif
+
+	if(type & IGOR_TYPE_COMPLEX)
+		result += "complex "
+	endif
+
+	if(type & IGOR_TYPE_32BIT_FLOAT)
+		result += "32-bit float"
+	elseif(type & IGOR_TYPE_64BIT_FLOAT)
+		result += "64-bit float"
+	elseif(type & IGOR_TYPE_8BIT_INT)
+		result += "8-bit int"
+	elseif(type & IGOR_TYPE_16BIT_INT)
+		result += "16-bit int"
+	elseif(type & IGOR_TYPE_32BIT_INT)
+		result += "32-bit int"
+	elseif(type & IGOR_TYPE_64BIT_INT)
+		result += "64-bit int"
+	else
+		// do nothing here
+		trim = 1
+	endif
+
+	if(type & IGOR_TYPE_UNSIGNED)
+		result += " unsigned"
+	endif
+
+	if(trim)
+		return trimstring(result)
+	endif
+
+	return result
+End
+
+/// @brief Human readable name for possible return values of WaveType(wv, 1)
+threadsafe Function/S WaveTypeToStringSelectorOne(variable type)
+
+	switch(type)
+		case IGOR_TYPE_NULL_WAVE:
+			return "null"
+		case IGOR_TYPE_NUMERIC_WAVE:
+			return "numeric"
+		case IGOR_TYPE_TEXT_WAVE:
+			return "text"
+		case IGOR_TYPE_DFREF_WAVE:
+			return "datafolder reference"
+		case IGOR_TYPE_WAVEREF_WAVE:
+			return "wave reference"
+		default:
+			ASSERT_TS(0, "Unknown constant: " + num2str(type, "%d"))
+	endswitch
+End

Packages/MIES/SweepFormulaHelp.ifn

@@ -1722,13 +1722,13 @@ the Unicode Character 'HORIZONTAL ELLIPSIS' (U+2026). Writing "..." is automatic
 The function generally accepts 1 to 3 arguments. The operation is intended to be
 used with two arguments.
 
-The operation accepts also multiple data waves as first argument. Each data wave content must follow the operation argument order and size in that case.
-For this case the operation is applied on each input data wave independently and returns the same number of data waves.
 The returned data type is `SF_DATATYPE_RANGE`.
 
 .. code-block:: bash
 
    range(1, 5, 0.7) == [1, 1.7, 2.4, 3.1, 3.8, 4.5]
+   range(3) == [0, 1, 2]
+   range(1, 4) == [1, 2, 3]
 
 epochs
 """"""
@@ -1983,6 +1983,26 @@ The operation currently throws away all metadata.
 
    merge(4, 7, 8) == [4, 7, 8]
 
+concat
+""""""
+
+The concat operation allows to concatenate multiple arrays together.
+
+.. code-block:: bash
+
+   concat(array data1, array data2, ...)
+
+data1, data2, ...
+  data waves (numeric and text)
+
+The operation accepts 1 to unlimited arguments. The dimensionality of all input
+waves and their types must match.
+
+.. code-block:: bash
+
+   concat([1, 5, [3, 8]) == [1, 5, 3, 8]
+   concat(["a", "b], ["e", "f"]) == [a, b, e, f]
+
 dataset
 """""""