diff --git a/Packages/Conversion/MIES_MassExperimentProcessing.ipf b/Packages/Conversion/MIES_MassExperimentProcessing.ipf index 176a6aa448..552d0683d5 100644 --- a/Packages/Conversion/MIES_MassExperimentProcessing.ipf +++ b/Packages/Conversion/MIES_MassExperimentProcessing.ipf @@ -16,6 +16,7 @@ /// - Ensure that only MIES is installed and no other Igor Pro packages /// - In the MIES installation folder (All Users: `C:\Program Files\MIES`, User: `C:\Users\$User\Documents\MIES`) /// create an empty file named `UserConfig.txt`. +/// - Execute CreateEmptyFiles() to create required empty files in `User Procedures` /// /// Running: /// - Start Igor Pro @@ -30,8 +31,8 @@ #ifdef MEP_DEBUGGING -static StrConstant INPUT_FOLDER = "C:tim-data:pxp_examples_for_nwb_2:" -static StrConstant OUTPUT_FOLDER = "C:tim-data:output:" +static StrConstant INPUT_FOLDER = "E:tim-data:pxp_examples_for_nwb_2:" +static StrConstant OUTPUT_FOLDER = "E:tim-data:output:" #else @@ -85,8 +86,8 @@ End static Function ProcessCurrentExperiment(STRUCT MultiExperimentProcessPrefs &prefs) - variable jsonID, index - string outputFilePath, inputFile, outputFolder + variable jsonID, index, ref, error + string outputFileTemplate, inputFile, outputFolder, history, path, message, file, regex jsonID = GetJSON(prefs) @@ -97,28 +98,44 @@ static Function ProcessCurrentExperiment(STRUCT MultiExperimentProcessPrefs &pre PathInfo home inputFile = S_path + GetExperimentName() + ".pxp" - outputFilePath = outputFolder + S_path + GetExperimentName() + ".nwb" + outputFileTemplate = outputFolder + S_path + GetExperimentName() - index = JSON_GetVariable(jsonID, "/index") - JSON_AddString(jsonID, "/log/" + num2str(index) + "/from", inputFile) - JSON_AddString(jsonID, "/log/" + num2str(index) + "/to", outputFilePath) + path = "/log/" + num2str(JSON_GetVariable(jsonID, "/index")) + JSON_AddString(jsonID, path + "/from", inputFile) + JSON_AddString(jsonID, path + "/to", outputFileTemplate) - DoWindow/K HistoryCarbonCopy - NewNotebook/V=0/F=0/N=HistoryCarbonCopy + ref = CaptureHistoryStart() + AssertOnAndClearRTError() try - PerformMiesTasks(outputFilePath); AbortOnRTE + PerformMiesTasks(outputFileTemplate); AbortOnRTE catch - ClearRTError() - print "Caught an RTE" - JSON_AddBoolean(jsonID, "/log/" + num2str(index) + "/error", 1) + message = GetRTErrMessage() + error = ClearRTError() + + if(error >= 0) + printf "Encountered lingering RTE of %d (message: %s) after executing PerformMiesTasks.\r", error, message + else + printf "Encountered Abort with V_AbortCode: %d\r", V_AbortCode + endif + + JSON_AddBoolean(jsonID, path + "/error", 1) JSON_SetVariable(jsonID, "/errors", JSON_GetVariable(jsonID, "/errors") + 1) HDF5CloseFile/A/Z 0 - DeleteFile/Z outputFilePath + + regex = "\\Q" + outputFileTemplate + "\\E" + ".*\.nwb$" + WAVE/Z/T files = GetAllFilesRecursivelyFromPath("home", regex = regex) + + if(WaveExists(files)) + for(file : files) + DeleteFile/Z file + endfor + endif endtry - Notebook HistoryCarbonCopy, getData=1 - JSON_AddString(jsonID, "/log/" + num2str(index) + "/output", trimstring(S_Value)) + history = CaptureHistory(ref, 1) + + JSON_AddString(jsonID, path + "/output", trimstring(history)) JSON_SetVariable(jsonID, "/processed", JSON_GetVariable(jsonID, "/processed") + 1) else @@ -130,28 +147,20 @@ static Function ProcessCurrentExperiment(STRUCT MultiExperimentProcessPrefs &pre StoreJSON(prefs, jsonID) End -static Function PerformMiesTasks(string outputFilePath) +static Function PerformMiesTasks(string outputFileTemplate) - string folder, message - variable nwbVersion, error + string folder printf "Free Memory: %g GB\r", GetFreeMemory() - folder = GetFolder(outputFilePath) + folder = GetFolder(outputFileTemplate) if(!FolderExists(folder)) CreateFolderOnDisk(folder) endif - ClearRTError() - - nwbVersion = 2 - NWB_ExportAllData(nwbVersion, overrideFullFilePath = outputFilePath) + NWB_ExportAllData(NWB_VERSION_LATEST, overrideFileTemplate = outputFileTemplate) HDF5CloseFile/A/Z 0 - - message = GetRTErrMessage() - error = GetRTError(1) - ASSERT(error == 0, "Encountered lingering RTE of " + num2str(error) + "(message: " + message + ") after executing NWB_ExportAllData.") End static Function IsAppropriateExperiment() @@ -232,23 +241,32 @@ static Function AfterFileOpenHook(variable refNum, string file, string pathName, ProcessCurrentExperiment(prefs) - // See if there are more experiments to process. - string nextExperimentFullPath = FindNextExperiment(prefs) - if(strlen(nextExperimentFullPath) == 0) - // Process is finished - prefs.processRunning = 0 // Flag process is finished. - Execute/P "NEWEXPERIMENT " // Post command to close this experiment. - print "Multi-experiment process is finished." - else - // Load the next experiment in the designated folder, if any. - PostLoadNextExperiment(nextExperimentFullPath) // Post operation queue commands to load next experiment - endif - - SavePackagePrefs(prefs) + NextFile(prefs) return 0 // Tell Igor to handle file in default fashion. End +Function CreateEmptyFiles() + + string file + string path = SpecialDirPath("Igor Pro User Files", 0, 0, 0) + "User Procedures:" + + Make/T/FREE files = {"MIES_Include.ipf", \ + "TJ_MIES_AnalysisBrowser.ipf", \ + "TJ_MIES_Include.ipf", \ + "UTF_HardwareHelperFunctions.ipf", \ + "UTF_HardwareMain.ipf", \ + "UserAnalysisFunctions.ipf", \ + "tango_Panel.ipf", \ + "tango_loader.ipf", \ + "unit-testing.ipf", \ + "UserConfig.txt"} + + for(file : files) + SaveTextFile("", path + file) + endfor +End + // This function enables our special Igor hooks which skip saving the experiment Function StartMultiExperimentProcess() @@ -260,7 +278,7 @@ End // Allow user to choose the folder containing the experiment files and start the process. Function StartMultiExperimentProcessWrapper() - string message, settingsFile, inputFolder, outputFolder, files + string message, settingsFile, inputFolder, outputFolder variable jsonID STRUCT MultiExperimentProcessPrefs prefs @@ -296,7 +314,7 @@ Function StartMultiExperimentProcessWrapper() outputFolder = S_Path ASSERT(V_flag, "Invalid path") - WAVE/Z files = GetAllFilesRecursivelyFromPath("MultiExperimentInputFolder", regex = "(?i)\.pxp$") + WAVE/Z/T files = GetAllFilesRecursivelyFromPath("MultiExperimentInputFolder", regex = "(?i)\.pxp$") if(WaveExists(files)) Sort/A=2 files, files @@ -312,10 +330,10 @@ Function StartMultiExperimentProcessWrapper() JSON_AddVariable(jsonID, "/processed", 0) JSON_AddVariable(jsonID, "/errors", 0) JSON_AddVariable(jsonID, "/skipped", 0) - JSON_AddVariable(jsonID, "/total", DimSize(inputPXPs, ROWS)) + JSON_AddVariable(jsonID, "/total", DimSize(files, ROWS)) JSON_AddTreeArray(jsonID, "/log") - JSON_AddObjects(jsonID, "/log", objCount = DimSize(inputPXPs, ROWS)) + JSON_AddObjects(jsonID, "/log", objCount = DimSize(files, ROWS)) prefs.settingsFile = outputFolder + "conversion.json" StoreJSON(prefs, jsonID) @@ -331,6 +349,23 @@ Function StartMultiExperimentProcessWrapper() return 0 End +Function NextFile(STRUCT MultiExperimentProcessPrefs &prefs) + + // See if there are more experiments to process. + string nextExperimentFullPath = FindNextExperiment(prefs) + if(strlen(nextExperimentFullPath) == 0) + // Process is finished + prefs.processRunning = 0 // Flag process is finished. + Execute/P "NEWEXPERIMENT " // Post command to close this experiment. + print "Multi-experiment process is finished." + else + // Load the next experiment in the designated folder, if any. + PostLoadNextExperiment(nextExperimentFullPath) // Post operation queue commands to load next experiment + endif + + SavePackagePrefs(prefs) +End + #ifdef MEP_DEBUGGING Function TestMe() @@ -339,6 +374,7 @@ Function TestMe() LoadPackagePrefs(prefs) ProcessCurrentExperiment(prefs) + NextFile(prefs) End #endif // MEP_DEBUGGING diff --git a/Packages/IPNWB b/Packages/IPNWB index 4db1c167ee..623646aba3 160000 --- a/Packages/IPNWB +++ b/Packages/IPNWB @@ -1 +1 @@ -Subproject commit 4db1c167ee4636f5cfd96fe823c697202d7947b4 +Subproject commit 623646aba35189073f4236f8d3104c6907d28e2b diff --git a/Packages/MIES/MIES_AnalysisFunctionHelpers.ipf b/Packages/MIES/MIES_AnalysisFunctionHelpers.ipf index 12b51deaff..f3e681f75e 100644 --- a/Packages/MIES/MIES_AnalysisFunctionHelpers.ipf +++ b/Packages/MIES/MIES_AnalysisFunctionHelpers.ipf @@ -469,19 +469,19 @@ End /// parameters, possibly including the type, as specified by the function /// `$func_GetParams` /// -/// @param func Analysis function `V3` which must be valid and existing -/// @param mode Bit mask values from @ref GetListOfParamsModeFlags -Function/S AFH_GetListOfAnalysisParams(string func, variable mode) +/// @param funcname Analysis function `V3` which must be valid and existing +/// @param mode Bit mask values from @ref GetListOfParamsModeFlags +Function/S AFH_GetListOfAnalysisParams(string funcname, variable mode) string params, re - FUNCREF AFP_PARAM_GETTER_V3 f = $(func + "_GetParams") + FUNCREF AFP_PARAM_GETTER_V3 func = $(funcname + "_GetParams") - if(!FuncRefIsAssigned(FuncRefInfo(f))) // no such getter functions + if(!FuncRefIsAssigned(FuncRefInfo(func))) // no such getter functions return "" endif - params = f() + params = func() ASSERT(strsearch(params, ";", 0) == -1, "Entries must be separated with ,") @@ -501,19 +501,19 @@ End /// @brief Get help string from optional `$func_GetHelp` /// -/// @param func Analysis function `V3` -/// @param name Parameter name -Function/S AFH_GetHelpForAnalysisParameter(string func, string name) +/// @param funcname Analysis function `V3` +/// @param name Parameter name +Function/S AFH_GetHelpForAnalysisParameter(string funcname, string name) - FUNCREF AFP_PARAM_HELP_GETTER_V3 f = $(func + "_GetHelp") + FUNCREF AFP_PARAM_HELP_GETTER_V3 func = $(funcname + "_GetHelp") - if(!FuncRefIsAssigned(FuncRefInfo(f))) + if(!FuncRefIsAssigned(FuncRefInfo(func))) return "" endif AssertOnAndClearRTError() try - return f(name); AbortOnRTE + return func(name); AbortOnRTE catch ClearRTError() // ignoring errors here diff --git a/Packages/MIES/MIES_Async.ipf b/Packages/MIES/MIES_Async.ipf index 843d5282cf..e2ab66b4fd 100644 --- a/Packages/MIES/MIES_Async.ipf +++ b/Packages/MIES/MIES_Async.ipf @@ -124,7 +124,7 @@ threadsafe static Function/DF ASYNC_Run_Worker(DFREF dfr) DFREF dfrOut, dfrTemp SVAR WFunc = dfr:$ASYNC_WORKERFUNC_STR - FUNCREF ASYNC_Worker f = $WFunc + FUNCREF ASYNC_Worker func = $WFunc DFREF dfrInp = dfr:input DFREF dfrAsync = dfr:async @@ -139,7 +139,7 @@ threadsafe static Function/DF ASYNC_Run_Worker(DFREF dfr) dfrOut = $"" AssertOnAndClearRTError() try - dfrOut = f(dfrInp); AbortOnRTE + dfrOut = func(dfrInp); AbortOnRTE catch rtErrMsg = GetRTErrMessage() rtErr = ClearRTError() @@ -243,7 +243,7 @@ Function ASYNC_ThreadReadOut() track[%$workloadClass][%OUTPUTCOUNT] += 1 SVAR RFunc = dfr:$ASYNC_READOUTFUNC_STR - FUNCREF ASYNC_ReadOut f = $RFunc + FUNCREF ASYNC_ReadOut func = $RFunc NVAR rtErr = dfr:$ASYNC_RTERROR_STR SVAR rtErrMsg = dfr:$ASYNC_RTERRORMSG_STR NVAR abortCode = dfr:$ASYNC_ABORTCODE_STR @@ -256,7 +256,7 @@ Function ASYNC_ThreadReadOut() statCnt += 1 AssertOnAndClearRTError() try - f(ar); AbortOnRTE + func(ar); AbortOnRTE catch msg = GetRTErrMessage() ASSERT(!ClearRTError(), "ReadOut function " + RFunc + " encountered an RTE: " + msg) diff --git a/Packages/MIES/MIES_Constants.ipf b/Packages/MIES/MIES_Constants.ipf index a8fb0b502f..1e0eb8d0a4 100644 --- a/Packages/MIES/MIES_Constants.ipf +++ b/Packages/MIES/MIES_Constants.ipf @@ -39,7 +39,7 @@ Constant SWEEP_EPOCH_VERSION = 9 /// - New/Changed layers of entries /// ///@{ -Constant LABNOTEBOOK_VERSION = 81 +Constant LABNOTEBOOK_VERSION = 82 Constant RESULTS_VERSION = 3 ///@} diff --git a/Packages/MIES/MIES_DAEphys.ipf b/Packages/MIES/MIES_DAEphys.ipf index 45dc03cf2d..4f5a939a66 100644 --- a/Packages/MIES/MIES_DAEphys.ipf +++ b/Packages/MIES/MIES_DAEphys.ipf @@ -4601,11 +4601,7 @@ Function DAP_LockDevice(string win) headstage = GetSliderPositionIndex(deviceLocked, "slider_DataAcq_ActiveHeadstage") P_SaveUserSelectedHeadstage(deviceLocked, headstage) - // upgrade all four labnotebook waves in wanna-be atomic way - GetLBNumericalKeys(deviceLocked) - GetLBNumericalValues(deviceLocked) - GetLBTextualKeys(deviceLocked) - GetLBTextualValues(deviceLocked) + UpgradeLabNotebook(deviceLocked) NVAR sessionStartTime = $GetSessionStartTime() sessionStartTime = DateTimeInUTC() diff --git a/Packages/MIES/MIES_DataBrowser.ipf b/Packages/MIES/MIES_DataBrowser.ipf index ada6fc7f69..0f573f8751 100644 --- a/Packages/MIES/MIES_DataBrowser.ipf +++ b/Packages/MIES/MIES_DataBrowser.ipf @@ -326,6 +326,8 @@ static Function/S DB_LockToDevice(string win, string device) for(i = first; i <= last; i += 1) SplitAndUpgradeSweepGlobal(device, i) endfor + + UpgradeLabNotebook(device) endif UpdateSweepPlot(win) @@ -421,10 +423,6 @@ Function DB_UpdateSweepPlot(string win) return NaN endif - // fetch keys waves to trigger a potential labnotebook upgrade - WAVE numericalKeys = DB_GetLBNWave(win, LBN_NUMERICAL_KEYS) - WAVE textualKeys = DB_GetLBNWave(win, LBN_TEXTUAL_KEYS) - WAVE numericalValues = DB_GetLBNWave(win, LBN_NUMERICAL_VALUES) WAVE textualValues = DB_GetLBNWave(win, LBN_TEXTUAL_VALUES) diff --git a/Packages/MIES/MIES_MiesUtilities_Device.ipf b/Packages/MIES/MIES_MiesUtilities_Device.ipf index 8a279681af..9d56985efc 100644 --- a/Packages/MIES/MIES_MiesUtilities_Device.ipf +++ b/Packages/MIES/MIES_MiesUtilities_Device.ipf @@ -14,12 +14,13 @@ Function/S GetAllDevices() variable i, j, numEntries, numDevices - string folder, number, device, folders, subFolders, subFolder + string folder, number, device, folders, subFolders, subFolder, deviceType, deviceNumber string path string list = "" - string devicesFolderPath = GetDAQDevicesFolderAsString() + // ensure that the folder location upgarde in GetDAQDevicesFolder is done first DFREF devicesFolder = GetDAQDevicesFolder() + string devicesFolderPath = GetDAQDevicesFolderAsString() folders = GetListOfObjects(devicesFolder, ".*", typeFlag = COUNTOBJECTS_DATAFOLDER) numEntries = ItemsInList(folders) @@ -50,7 +51,12 @@ Function/S GetAllDevices() else // other hardware has no subfolder device = folder - path = GetDevicePathAsString(device) + + if(ParseDeviceString(device, deviceType, deviceNumber) == 0) + continue + endif + + path = GetDevicePathAsString(device) if(DataFolderExists(path)) DFREF dfr = $path @@ -73,7 +79,7 @@ static Function DeviceHasUserComments(string device) userComment = ROStr(GetUserComment(device)) if(WindowExists(device)) - userCommentDraft = DAG_GetTextualValue(device, "SetVar_DataAcq_Comment") + userCommentDraft = GetSetVariableString(device, "SetVar_DataAcq_Comment") commentNotebook = DAP_GetCommentNotebook(device) if(WindowExists(commentNotebook)) diff --git a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf index e6d24840d8..8df3613b13 100644 --- a/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf +++ b/Packages/MIES/MIES_NeuroDataWithoutBorders.ipf @@ -637,18 +637,25 @@ Function NWB_ExportAllData(variable nwbVersion, [string device, string overrideF values = {num2str(nwbVersion), num2str(writeStoredTestPulses), \ num2str(writeIgorHistory), CompressionModeToString(compressionMode)}) - if(ParamIsDefault(device)) - WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") - if(!DimSize(devicesWithContent, ROWS)) - if(verbose) - print "No devices with acquired content found for NWB export" - ControlWindowToFront() - endif + UpgradeAllDataFolderLocations() - LOG_AddEntry(PACKAGE_MIES, "end") - return 1 + WAVE/T devicesWithContent = ListToTextWave(GetAllDevicesWithContent(contentType = CONTENT_TYPE_ALL), ";") + if(!DimSize(devicesWithContent, ROWS)) + if(verbose) + print "No devices with acquired content found for NWB export" + ControlWindowToFront() endif - else + + LOG_AddEntry(PACKAGE_MIES, "end") + return 1 + endif + + for(device : devicesWithContent) + UpgradeLabNotebook(device) + endfor + + if(!ParamIsDefault(device)) + ASSERT(GetRowIndex(devicesWithContent, str = device) >= 0, "Could not find data for device: " + device) Make/FREE/T devicesWithContent = {device} endif @@ -1243,7 +1250,8 @@ threadsafe static Function NWB_AppendSweepLowLevel(STRUCT NWBAsyncParameters &s) params.samplingRate = ConvertSamplingIntervalToRate(GetSamplingInterval(s.DAQConfigWave, params.channelType)) * KILO_TO_ONE col = AFH_GetDAQDataColumn(s.DAQConfigWave, params.channelNumber, params.channelType) writtenDataColumns[col] = 1 - WAVE params.data = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, params.channelNumber) + WAVE/Z params.data = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, params.channelNumber) + ASSERT_TS(WaveExists(params.data), "Missing AD data for channel " + num2str(adc) + " of headstage " + num2str(i) + ".") NWB_GetTimeSeriesProperties(s.nwbVersion, s.numericalKeys, s.numericalValues, params, tsp) params.groupIndex = IsFinite(params.groupIndex) ? params.groupIndex : GetNextFreeGroupIndex(s.locationID, path) WriteSingleChannel(s.locationID, path, s.nwbVersion, params, tsp, compressionMode = s.compressionMode) @@ -1256,7 +1264,8 @@ threadsafe static Function NWB_AppendSweepLowLevel(STRUCT NWBAsyncParameters &s) params.samplingRate = ConvertSamplingIntervalToRate(GetSamplingInterval(s.DAQConfigWave, params.channelType)) * KILO_TO_ONE col = AFH_GetDAQDataColumn(s.DAQConfigWave, params.channelNumber, params.channelType) writtenDataColumns[col] = 1 - WAVE params.data = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, params.channelNumber) + WAVE/Z params.data = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, params.channelNumber) + ASSERT_TS(WaveExists(params.data), "Missing DA data for channel " + num2str(dac) + " of headstage " + num2str(i) + ".") NWB_GetTimeSeriesProperties(s.nwbVersion, s.numericalKeys, s.numericalValues, params, tsp) params.groupIndex = IsFinite(params.groupIndex) ? params.groupIndex : GetNextFreeGroupIndex(s.locationID, path) WAVE/Z/T params.epochs = EP_FetchEpochs_TS(s.numericalValues, s.textualValues, s.sweep, params.channelNumber, params.channelType) @@ -1301,7 +1310,8 @@ threadsafe static Function NWB_AppendSweepLowLevel(STRUCT NWBAsyncParameters &s) NWB_GetTimeSeriesProperties(s.nwbVersion, s.numericalKeys, s.numericalValues, params, tsp) params.groupIndex = IsFinite(params.groupIndex) ? params.groupIndex : GetNextFreeGroupIndex(s.locationID, path) - WAVE params.data = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, i) + WAVE/Z params.data = GetDAQDataSingleColumnWaveNG(s.numericalValues, s.textualValues, s.sweep, sweepDFR, params.channelType, i) + ASSERT_TS(WaveExists(params.data), "Missing TTL data for channel " + num2str(i) + ".") WAVE/Z/T params.epochs = EP_FetchEpochs_TS(s.numericalValues, s.textualValues, s.sweep, i, params.channelType) s.locationID = WriteSingleChannel(s.locationID, path, s.nwbVersion, params, tsp, compressionMode = s.compressionMode, nwbFilePath = s.nwbFilePath) diff --git a/Packages/MIES/MIES_Utilities_Algorithm.ipf b/Packages/MIES/MIES_Utilities_Algorithm.ipf index 9f9c49379f..6153befd34 100644 --- a/Packages/MIES/MIES_Utilities_Algorithm.ipf +++ b/Packages/MIES/MIES_Utilities_Algorithm.ipf @@ -219,7 +219,7 @@ End /// /// The function's type must be #CALL_FUNCTION_LIST_PROTOTYPE where the return /// type is ignored. -Function CallFunctionForEachListItem(FUNCREF CALL_FUNCTION_LIST_PROTOTYPE f, string list, [string sep]) +Function CallFunctionForEachListItem(FUNCREF CALL_FUNCTION_LIST_PROTOTYPE func, string list, [string sep]) variable i, numEntries string entry @@ -232,14 +232,14 @@ Function CallFunctionForEachListItem(FUNCREF CALL_FUNCTION_LIST_PROTOTYPE f, str for(i = 0; i < numEntries; i += 1) entry = StringFromList(i, list, sep) - f(entry) + func(entry) endfor End /// Compatibility wrapper for threadsafe functions `f` /// /// @see CallFunctionForEachListItem() -threadsafe Function CallFunctionForEachListItem_TS(FUNCREF CALL_FUNCTION_LIST_PROTOTYPE_TS f, string list, [string sep]) +threadsafe Function CallFunctionForEachListItem_TS(FUNCREF CALL_FUNCTION_LIST_PROTOTYPE_TS func, string list, [string sep]) variable i, numEntries string entry @@ -252,7 +252,7 @@ threadsafe Function CallFunctionForEachListItem_TS(FUNCREF CALL_FUNCTION_LIST_PR for(i = 0; i < numEntries; i += 1) entry = StringFromList(i, list, sep) - f(entry) + func(entry) endfor End diff --git a/Packages/MIES/MIES_WaveDataFolderGetters.ipf b/Packages/MIES/MIES_WaveDataFolderGetters.ipf index 3ae72cf33d..a868ae5812 100644 --- a/Packages/MIES/MIES_WaveDataFolderGetters.ipf +++ b/Packages/MIES/MIES_WaveDataFolderGetters.ipf @@ -717,6 +717,14 @@ Function/S GetDeviceDataPathAsString(string device) return GetDevicePathAsString(device) + ":Data" End +/// @brief Upgrade all DFREF locations we have changed in earlier versions +/// +/// UTF_NOINSTRUMENTATION +Function UpgradeAllDataFolderLocations() + + GetDAQDevicesFolder() +End + /// @brief Returns a data folder reference to the mies base folder /// /// UTF_NOINSTRUMENTATION @@ -1372,7 +1380,8 @@ End /// @brief Handle upgrades of the numerical/textual labnotebooks in one step /// -/// This function is idempotent and must stay that way. +/// This function is idempotent in the sense that doing the upgrade steps +/// multiple times must succeed again with the same results. /// /// Supported upgrades: /// - Addition of the third column "TimeStampSinceIgorEpochUTC" @@ -1383,9 +1392,9 @@ End /// - Fix unit and tolerance of "Repeat Sets" /// - Reapplying the dimension labels as the old ones were cut off after 31 bytes /// - Making dimension labels valid liberal object names -/// - Extending the row dimension to 6 for the key waves -/// - Fixing empty column dimension labels in key waves -static Function UpgradeLabNotebook(string device) +/// - Extending the row dimension to 6 for the key waves including setting the dimension labels +/// - Fixing empty column dimension labels in the columns of the value waves +Function UpgradeLabNotebook(string device) variable numCols, i, col, numEntries, sourceCol, timeStampColumn, nextFreeRow string list, key @@ -1434,7 +1443,7 @@ static Function UpgradeLabNotebook(string device) // END IP9 dimension labels // BEGIN UTC timestamps - if(cmpstr(numericalKeys[0][2], "TimeStampSinceIgorEpochUTC")) + if(DimSize(numericalKeys, COLS) < 3 || cmpstr(numericalKeys[0][2], "TimeStampSinceIgorEpochUTC")) numCols = DimSize(numericalKeys, COLS) @@ -1451,7 +1460,7 @@ static Function UpgradeLabNotebook(string device) DEBUGPRINT("Upgraded numerical labnotebook to hold UTC timestamps") endif - if(cmpstr(textualKeys[0][2], "TimeStampSinceIgorEpochUTC")) + if(DimSize(textualKeys, COLS) < 3 || cmpstr(textualKeys[0][2], "TimeStampSinceIgorEpochUTC")) numCols = DimSize(textualKeys, COLS) @@ -1470,7 +1479,7 @@ static Function UpgradeLabNotebook(string device) // END UTC timestamps // BEGIN epoch source type - if(cmpstr(numericalKeys[0][3], "EntrySourceType")) + if(DimSize(numericalKeys, COLS) < 4 || cmpstr(numericalKeys[0][3], "EntrySourceType")) numCols = DimSize(numericalKeys, COLS) @@ -1487,7 +1496,7 @@ static Function UpgradeLabNotebook(string device) DEBUGPRINT("Upgraded numerical labnotebook to hold entry source type column") endif - if(cmpstr(textualKeys[0][3], "EntrySourceType")) + if(DimSize(textualKeys, COLS) < 4 || cmpstr(textualKeys[0][3], "EntrySourceType")) numCols = DimSize(textualKeys, COLS) @@ -1559,7 +1568,7 @@ static Function UpgradeLabNotebook(string device) endif // BEGIN acquisition state - if(cmpstr(numericalKeys[0][4], "AcquisitionState")) + if(DimSize(numericalKeys, COLS) < 5 || cmpstr(numericalKeys[0][4], "AcquisitionState")) numCols = DimSize(numericalKeys, COLS) @@ -1576,7 +1585,7 @@ static Function UpgradeLabNotebook(string device) DEBUGPRINT("Upgraded numerical labnotebook to hold acquisition state column") endif - if(cmpstr(textualKeys[0][4], "AcquisitionState")) + if(DimSize(textualKeys, COLS) < 5 || cmpstr(textualKeys[0][4], "AcquisitionState")) numCols = DimSize(textualKeys, COLS) @@ -1604,7 +1613,7 @@ static Function UpgradeLabNotebook(string device) endif // END extending rows - // BEGIN fix missing column dimension labels in keyWaves + // BEGIN fix missing column dimension labels in columns of values if(WaveVersionIsSmaller(numericalKeys, 74)) numCols = DimSize(numericalValues, COLS) for(i = 0; i < numCols; i += 1) @@ -1622,7 +1631,7 @@ static Function UpgradeLabNotebook(string device) endif endfor endif - // END fix missing column dimension labels in keyWaves + // END fix missing column dimension labels in columns of values // we don't remove the wavenote entry of sweep rollback as we might need to adapt the reading code // in the future to handle labnotebooks with that specially @@ -1637,12 +1646,14 @@ static Function UpgradeLabNotebook(string device) nextFreeRow = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) if(IsNaN(nextFreeRow)) - FindValue/FNAN/RMD=[][timeStampColumn][0]/R numericalValues - if(!(V_row >= 0)) - // labnotebook is completely full - V_row = DimSize(numericalValues, ROWS) + WAVE/Z indizes = FindIndizes(numericalValues, col = timeStampColumn, prop = PROP_NOT | PROP_EMPTY, startLayer = 0, endLayer = 0) + if(!WaveExists(indizes)) + // labnotebook is empty + nextFreeRow = 0 + else + nextFreeRow = WaveMax(indizes) + 1 endif - SetNumberInWaveNote(numericalValues, NOTE_INDEX, V_row) + SetNumberInWaveNote(numericalValues, NOTE_INDEX, nextFreeRow) endif endif @@ -1650,14 +1661,30 @@ static Function UpgradeLabNotebook(string device) nextFreeRow = GetNumberFromWaveNote(textualValues, NOTE_INDEX) if(IsNaN(nextFreeRow)) - FindValue/TEXT=("")/RMD=[][timeStampColumn][0]/R textualValues - if(!(V_row >= 0)) - V_row = DimSize(textualValues, ROWS) + WAVE/Z indizes = FindIndizes(textualValues, col = timeStampColumn, prop = PROP_NOT | PROP_EMPTY, startLayer = 0, endLayer = 0) + if(!WaveExists(indizes)) + // labnotebook is empty + nextFreeRow = 0 + else + nextFreeRow = WaveMax(indizes) + 1 endif - SetNumberInWaveNote(textualValues, NOTE_INDEX, V_row) + SetNumberInWaveNote(textualValues, NOTE_INDEX, nextFreeRow) endif endif // END add note index + + // BEGIN add dimension labels for key waves + if(IsEmpty(GetDimLabel(numericalKeys, ROWS, DimSize(numericalKeys, ROWS) - 1))) + SetLBKeysRowDimensionLabels(numericalKeys) + endif + + if(IsEmpty(GetDimLabel(textualKeys, ROWS, DimSize(textualKeys, ROWS) - 1))) + SetLBKeysRowDimensionLabels(textualKeys) + endif + // END add dimension labels for key waves + + SetWaveVersion(numericalKeys, LABNOTEBOOK_VERSION) + SetWaveVersion(textualkeys, LABNOTEBOOK_VERSION) End static Function/S FixInvalidLabnotebookKey(string name) @@ -1727,7 +1754,6 @@ Function/WAVE GetLBTextualKeys(string device) return wv elseif(WaveExists(wv)) UpgradeLabNotebook(device) - SetWaveVersion(wv, versionOfNewWave) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv @@ -1739,6 +1765,7 @@ Function/WAVE GetLBTextualKeys(string device) wv[0][] = StringFromList(q, LABNOTEBOOK_KEYS_INITIAL) SetLBKeysRowDimensionLabels(wv) + SetDimensionLabels(wv, LABNOTEBOOK_KEYS_INITIAL, COLS) SetWaveVersion(wv, versionOfNewWave) @@ -1781,7 +1808,6 @@ Function/WAVE GetLBNumericalKeys(string device) return wv elseif(WaveExists(wv)) UpgradeLabNotebook(device) - SetWaveVersion(wv, versionOfNewWave) return wv else Make/T/N=(6, INITIAL_KEY_WAVE_COL_COUNT) newDFR:$newName/WAVE=wv @@ -1790,6 +1816,7 @@ Function/WAVE GetLBNumericalKeys(string device) wv = "" SetLBKeysRowDimensionLabels(wv) + SetDimensionLabels(wv, LABNOTEBOOK_KEYS_INITIAL, COLS) WAVE/T desc = GetLBNumericalDescription(forceReload = 1) ASSERT(DimSize(desc, ROWS) == DimSize(wv, ROWS), "Non-matching number of rows") @@ -8840,7 +8867,8 @@ threadsafe Function [WAVE/T sortedKeys, WAVE/D indices] GetLogbookSortedKeys(WAV variable numKeys string keysName, sortedKeysName, sortedKeysIndicesName, cacheKey - WAVE/T keys = GetLogbookKeysFromValues(values) + WAVE/Z/T keys = GetLogbookKeysFromValues(values) + ASSERT_TS(WaveExists(keys), "Keys wave is missing") cacheKey = CA_GenKeyLogbookSortedKeys(keys) DFREF dfrTmp = createDFWithAllParents(GetWavesDataFolder(keys, 1) + LOGBOOK_WAVE_TEMP_FOLDER) diff --git a/Packages/tests/Basic/Basic.pxp b/Packages/tests/Basic/Basic.pxp index 6a9d77e62c..a4a45f08d8 100644 Binary files a/Packages/tests/Basic/Basic.pxp and b/Packages/tests/Basic/Basic.pxp differ diff --git a/Packages/tests/Basic/UTF_Labnotebook.ipf b/Packages/tests/Basic/UTF_Labnotebook.ipf index 2337da1095..9e2fb6b321 100644 --- a/Packages/tests/Basic/UTF_Labnotebook.ipf +++ b/Packages/tests/Basic/UTF_Labnotebook.ipf @@ -48,6 +48,13 @@ static Function/WAVE PrepareLBNTextualValues(WAVE textualValuesSrc) return $LBN_TEXTUAL_VALUES_NAME End +static Function PrepareLabnotebookWaves(DFREF sourceDFR, string device) + + DFREF destDFR = GetLabNotebookFolder() + + DuplicateDataFolder/O=1 sourceDFR, destDFR:$(device) +End + /// GetLastSetting with numeric wave /// @{ Function GetLastSettingEntrySourceTypes() @@ -1417,6 +1424,112 @@ Function LabnotebookUpgradeMissingNoteIndexTextual() CHECK_EQUAL_VAR(idxRedone, 0) End +Function LabnotebookUpgradeMissingNoteIndexNumericalWithHoles() + + variable idx, idxRedone + string device, key, keyTxt + + device = "ITC16USB_0_DEV" + [key, keyTxt] = PrepareLBN_IGNORE(device) + + WAVE/Z numericalValues = GetLBNumericalValues(device) + WAVE/Z/T numericalKeys = GetLBNumericalKeys(device) + + idx = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_GT_VAR(idx, 0) + + Note/K numericalKeys + + MIES_WAVEGETTERS#UpgradeLabNotebook(device) + + idxRedone = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(idxRedone, idx) + + Note/K numericalKeys + Note/K numericalValues + + numericalValues[0][][] = NaN + + MIES_WAVEGETTERS#UpgradeLabNotebook(device) + + idxRedone = GetNumberFromWaveNote(numericalValues, NOTE_INDEX) + CHECK_EQUAL_VAR(idxRedone, idx) +End + +Function LabnotebookUpgradeMissingNoteIndexTextualWithHoles() + + variable idx, idxRedone + string device, key, keyTxt + + device = "ITC16USB_0_DEV" + [key, keyTxt] = PrepareLBN_IGNORE(device) + + WAVE/Z/T textualValues = GetLBTextualValues(device) + WAVE/Z/T textualKeys = GetLBTextualKeys(device) + + idx = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + CHECK_GT_VAR(idx, 0) + + Note/K textualKeys + + MIES_WAVEGETTERS#UpgradeLabNotebook(device) + + idxRedone = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + CHECK_EQUAL_VAR(idxRedone, idx) + + Note/K textualValues + Note/K textualKeys + + textualValues[0][][] = "" + + MIES_WAVEGETTERS#UpgradeLabNotebook(device) + + idxRedone = GetNumberFromWaveNote(textualValues, NOTE_INDEX) + CHECK_EQUAL_VAR(idxRedone, idx) +End + +// Labnotebook waves as created in version 5872e55614 (Modified files: DR_MIES_TangoInteract: changes recommended by Thomas TJ, 2014-09-11) +Function LabnotebookUpgradeWithInitialWaveSizes() + + DFREF dfr = root:Labnotebook_initial_column_sizes + string device = "Dev1" + + PrepareLabnotebookWaves(dfr, device) + UpgradeLabNotebook(device) + CHECK_NO_RTE() +End + +Function LabnotebookUpgradeDoesNotModifyDefaultWaves() + + string device + + device = "ITC16USB_0_DEV" + + WAVE numericalValues = GetLBNumericalValues(device) + Duplicate/FREE numericalValues, numericalValuesRef + + WAVE numericalKeys = GetLBNumericalKeys(device) + Duplicate/FREE numericalKeys, numericalKeysRef + + WAVE textualValues = GetLBTextualValues(device) + Duplicate/FREE textualValues, textualValuesRef + + WAVE textualKeys = GetLBTextualKeys(device) + Duplicate/FREE textualKeys, textualKeysRef + + Note/K numericalValues + Note/K numericalKeys + Note/K textualValues + Note/K textualKeys + + UpgradeLabNotebook(device) + + CHECK_EQUAL_WAVES(numericalValuesRef, numericalValues) + CHECK_EQUAL_WAVES(numericalKeysRef, numericalKeys) + CHECK_EQUAL_WAVES(textualValuesRef, textualValues) + CHECK_EQUAL_WAVES(textualKeysRef, textualKeys) +End + Function EmptyLabnotebookWorks() string device diff --git a/Packages/tests/UTF_All_Includes.ipf b/Packages/tests/UTF_All_Includes.ipf index 2953b13aa1..011512bbdb 100644 --- a/Packages/tests/UTF_All_Includes.ipf +++ b/Packages/tests/UTF_All_Includes.ipf @@ -3,6 +3,7 @@ #pragma rtFunctionErrors = 1 #pragma ModuleName = AllIncludes +#include "::Conversion:MIES_MassExperimentProcessing" #include "UTF_Basic_Includes" #include "UTF_HardwareBasic_Includes" #include "UTF_HardwareAnalysisFunctions_Includes" diff --git a/tools/clean_mies_installation.sh b/tools/clean_mies_installation.sh index 290ce879f4..92b7e55e08 100755 --- a/tools/clean_mies_installation.sh +++ b/tools/clean_mies_installation.sh @@ -127,6 +127,9 @@ do cp -r "$top_level"/Packages/tests "$user_proc" cp -r "$top_level"/Packages/doc/ipf "$user_proc" + # only install to $user_proc as it contains specialized igor hooks + cp -r "$top_level"/Packages/conversion "$user_proc" + if [ "$sourceLoc" = "installer" ] then # move shortcut to the main include file