Skip to content

Commit 949bf95

Browse files
aglowackiaglowacki
authored
Merge pull request #225 from aglowacki/master
fixed bug loading netcdf raw files
2 parents 9e3a225 + 2bd903f commit 949bf95

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

49 files changed

+707
-432
lines changed

‎reference/Scaler_to_PV_map.yaml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ BeamLines:
1212
DS_AMP_NUM: 2idd:A4sens_num.VAL
1313
DS_AMP_UNIT: 2idd:A4sens_unit.VAL
1414
US_IC: 2idd:scaler1_cts1.C
15-
DS_IC: 2idd:scaler1_cts1.B
15+
DS_IC:
16+
- 2idd:scaler1_cts1.B
17+
- 2iddtmm1:Current4:MeanValue_RBV
18+
- 2iddtmm1:Current4
19+
DS_IC_SCALE_FACTOR: 2iddtmm1:CurrentScale4
1620
ELT1: 2iddXMAP:mca1.ELTM
1721
ELT2: 2iddXMAP:mca2.ELTM
1822
ELT3: 2iddXMAP:mca3.ELTM
@@ -33,6 +37,7 @@ BeamLines:
3337
US_FM:
3438
- 2iddtmm1:Current3:MeanValue_RBV
3539
- 2iddtmm1:Current3
40+
US_FM_SCALE_FACTOR: 2iddtmm1:CurrentScale3
3641
TimeNormalizedScalers:
3742
Timing: [2idd:3820:mca1.VAL, 50000000.0]
3843
Scalers:

‎src/core/defines.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -282,6 +282,11 @@ const std::string STR_CHANNEL_NAMES = "Channel_Names";
282282
const std::string STR_CHANNEL_UNITS = "Channel_Units";
283283
const std::string STR_FIT_PARAMETERS_OVERRIDE = "Fit_Parameters_Override";
284284
const std::string STR_FIT_PARAMETERS = "Fit_Parameters";
285+
const std::string STR_BRANCHING_RATIOS = "Brancing_Ratios";
286+
const std::string STR_K_SHELL = "K_Shell";
287+
const std::string STR_L_SHELL = "L_Shell";
288+
const std::string STR_M_SHELL = "M_Shell";
289+
const std::string STR_L_FAMILY = "L_Family";
285290

286291
const std::string STR_QUANTIFICATION = "Quantification";
287292
const std::string STR_CALIBRATION = "Calibration";

‎src/core/main.cpp‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ void set_optimizer(Command_Line_Parser& clp, data_struct::Analysis_Job<T_real>&
179179
// ----------------------------------------------------------------------------
180180

181181
template <typename T_real>
182-
void set_mem_limit(Command_Line_Parser& clp, data_struct::Analysis_Job<T_real>& analysis_job)
182+
void set_mem_limit(Command_Line_Parser& clp, [[maybe_unused]] data_struct::Analysis_Job<T_real>& analysis_job)
183183
{
184184
//check if we should set a ram memory limit
185185
if (clp.option_exists("--mem-limit"))
@@ -448,8 +448,8 @@ int set_dir_and_files(Command_Line_Parser& clp, data_struct::Analysis_Job<T_real
448448
continue;
449449
}
450450
logI<<dataset_dir << DIR_END_CHAR << itr << DIR_END_CHAR << "*.h5"<<"\n";
451-
std::vector<std::string> flist = io::file::File_Scan::inst()->find_all_dataset_files_by_list(dataset_dir + DIR_END_CHAR + itr + DIR_END_CHAR, search_ext_h5_list);
452-
for (const auto& fitr : flist)
451+
std::vector<std::string> flist1 = io::file::File_Scan::inst()->find_all_dataset_files_by_list(dataset_dir + DIR_END_CHAR + itr + DIR_END_CHAR, search_ext_h5_list);
452+
for (const auto& fitr : flist1)
453453
{
454454
//logI<<fitr<<"\n";
455455
analysis_job.dataset_files.push_back(itr + DIR_END_CHAR + fitr);
@@ -957,8 +957,8 @@ int main(int argc, char* argv[])
957957

958958
// get location of where we are running from and use it to find ref files
959959
std::string exe_loc = std::string(argv[0]);
960-
int prog_idx = exe_loc.find("xrf_maps");
961-
if (prog_idx > 0)
960+
size_t prog_idx = exe_loc.find("xrf_maps");
961+
if (prog_idx != std::string::npos)
962962
{
963963
exe_loc = exe_loc.substr(0, prog_idx);
964964
}

‎src/core/process_streaming.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,15 @@ DLL_EXPORT data_struct::Stream_Block<T_real>* proc_spectra_block( data_struct::S
7373
{
7474
stream_block->fitting_blocks[itr.first].fit_counts[el_itr.first] = counts_dict[el_itr.first] / stream_block->spectra->elapsed_livetime();
7575
}
76-
stream_block->fitting_blocks[itr.first].fit_counts[STR_NUM_ITR] = counts_dict[STR_NUM_ITR];
76+
if(counts_dict.count(STR_NUM_ITR) > 0)
77+
{
78+
stream_block->fitting_blocks[itr.first].fit_counts[STR_NUM_ITR] = counts_dict[STR_NUM_ITR];
79+
}
80+
if(counts_dict.count(STR_RESIDUAL) > 0)
81+
{
82+
stream_block->fitting_blocks[itr.first].fit_counts[STR_RESIDUAL] = counts_dict[STR_RESIDUAL];
83+
}
84+
stream_block->fitting_blocks[itr.first].fit_counts[STR_TOTAL_FLUORESCENCE_YIELD] = stream_block->spectra->sum() / stream_block->spectra->elapsed_livetime();
7785
}
7886
return stream_block;
7987
}

‎src/core/process_whole.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ bool optimize_integrated_fit_params(data_struct::Analysis_Job<double> * analysis
127127
ret_val = false;
128128
break;
129129
}
130-
io::file::save_optimized_fit_params(analysis_job->output_dir, save_filename, detector_num, result, &out_fitp, &int_spectra, &(params_override->elements_to_fit));
130+
io::file::save_optimized_fit_params(analysis_job->output_dir, save_filename, (int)detector_num, result, &out_fitp, &int_spectra, &(params_override->elements_to_fit));
131131

132132
delete fit_routine;
133133
}
@@ -142,7 +142,7 @@ void generate_optimal_params(data_struct::Analysis_Job<double>* analysis_job)
142142
{
143143
std::unordered_map<int, data_struct::Fit_Parameters<double>> fit_params_avgs;
144144
std::unordered_map<int, data_struct::Params_Override<double>*> params;
145-
std::unordered_map<int, float> detector_file_cnt;
145+
std::unordered_map<size_t, float> detector_file_cnt;
146146
data_struct::Params_Override<double>* params_override = nullptr;
147147
data_struct::Spectra<double> int_spectra;
148148

@@ -463,7 +463,7 @@ bool perform_quantification(data_struct::Analysis_Job<double>* analysis_job, boo
463463
for(size_t detector_num : analysis_job->detector_num_arr)
464464
{
465465
data_struct::Detector<double>* detector = analysis_job->get_detector(detector_num);
466-
data_struct::Params_Override<double>* override_params = &(detector->fit_params_override_dict);
466+
//data_struct::Params_Override<double>* override_params = &(detector->fit_params_override_dict);
467467

468468

469469
load_and_fit_quatification_datasets(analysis_job, detector_num);
@@ -549,8 +549,8 @@ bool perform_quantification(data_struct::Analysis_Job<double>* analysis_job, boo
549549
}
550550

551551
// check if esrf dataset and remove folder
552-
int didx = dataset_file.find(DIR_END_CHAR);
553-
if (didx > -1)
552+
size_t didx = dataset_file.find(DIR_END_CHAR);
553+
if (didx != std::string::npos)
554554
{
555555
dataset_file = dataset_file.substr(didx + 1);
556556
}
@@ -783,7 +783,7 @@ void optimize_single_roi(data_struct::Analysis_Job<double>& analysis_job,
783783
// int specs loaded from v10 version
784784
std::unordered_map<std::string, data_struct::Spectra<double> > int_specs;
785785

786-
int slen = roi_file_name.size();
786+
size_t slen = roi_file_name.size();
787787
if (slen < 6)
788788
{
789789
logE << "Roi file name too short " << roi_file_name << ". Skipping file.\n";
@@ -804,7 +804,7 @@ void optimize_single_roi(data_struct::Analysis_Job<double>& analysis_job,
804804
// search for detector in list of int specs loaded
805805
for (const auto& spec_itr : int_specs)
806806
{
807-
int slen = spec_itr.first.length();
807+
slen = spec_itr.first.length();
808808
if (slen > 0 && spec_itr.first[slen - 1] == str_detector_num[0])
809809
{
810810
search_filename = spec_itr.first;

‎src/core/process_whole.h‎

Lines changed: 76 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ POSSIBILITY OF SUCH DAMAGE.
6060
#include <limits>
6161
#include <sstream>
6262
#include <fstream>
63+
#include <typeinfo>
6364

6465
#include <stdlib.h>
6566

@@ -119,8 +120,6 @@ DLL_EXPORT bool optimize_integrated_fit_params(data_struct::Analysis_Job<double>
119120

120121
DLL_EXPORT void generate_optimal_params(data_struct::Analysis_Job<double>* analysis_job);
121122

122-
void load_and_fit_quatification_datasets(data_struct::Analysis_Job<double>* analysis_job, size_t detector_num, std::vector<Quantification_Standard<double>>& standard_element_weights, std::unordered_map<size_t, double>& quant_map);
123-
124123
DLL_EXPORT void optimize_single_roi(data_struct::Analysis_Job<double>& analysis_job, std::string roi_file_name, std::map<int, std::map<std::string, data_struct::Fit_Parameters<double>>>& out_roi_fit_params, Callback_Func_Status_Def* status_callback = nullptr);
125124

126125
DLL_EXPORT void optimize_rois(data_struct::Analysis_Job<double>& analysis_job);
@@ -158,6 +157,45 @@ DLL_EXPORT data_struct::Fit_Count_Dict<T_real>* generate_fit_count_dict(const Fi
158157
return element_fit_counts_dict;
159158
}
160159

160+
// ----------------------------------------------------------------------------
161+
162+
template<typename T_real>
163+
DLL_EXPORT data_struct::Fit_Count_Dict<T_real>* generate_fit_count_by_shell_dict(const Fit_Element_Map_Dict<T_real>* elements_to_fit, size_t height, size_t width, bool alloc_iter_count)
164+
{
165+
data_struct::Fit_Count_Dict<T_real>* element_fit_counts_dict = new data_struct::Fit_Count_Dict<T_real>();
166+
for (auto& e_itr : *elements_to_fit)
167+
{
168+
element_fit_counts_dict->emplace(std::pair<std::string, data_struct::ArrayXXr<T_real> >(e_itr.first, data_struct::ArrayXXr<T_real>()));
169+
element_fit_counts_dict->at(e_itr.first).resize(height, width);
170+
for (auto& s_itr : e_itr.second->generate_roi_centers_per_shell())
171+
{
172+
element_fit_counts_dict->emplace(std::pair<std::string, data_struct::ArrayXXr<T_real> >(e_itr.first + "_" + s_itr.first, data_struct::ArrayXXr<T_real>()));
173+
element_fit_counts_dict->at(e_itr.first + "_" + s_itr.first).resize(height, width);
174+
}
175+
}
176+
177+
if (alloc_iter_count)
178+
{
179+
//Allocate memeory to save number of fit iterations
180+
element_fit_counts_dict->emplace(std::pair<std::string, data_struct::ArrayXXr<T_real> >(STR_NUM_ITR, data_struct::ArrayXXr<T_real>()));
181+
element_fit_counts_dict->at(STR_NUM_ITR).resize(height, width);
182+
element_fit_counts_dict->emplace(std::pair<std::string, data_struct::ArrayXXr<T_real> >(STR_RESIDUAL, data_struct::ArrayXXr<T_real>()));
183+
element_fit_counts_dict->at(STR_RESIDUAL).resize(height, width);
184+
}
185+
186+
// TOTAL_FLUORESCENCE_YIELD
187+
element_fit_counts_dict->emplace(std::pair<std::string, data_struct::ArrayXXr<T_real> >(STR_TOTAL_FLUORESCENCE_YIELD, data_struct::ArrayXXr<T_real>()));
188+
element_fit_counts_dict->at(STR_TOTAL_FLUORESCENCE_YIELD).resize(height, width);
189+
190+
//SUM_ELASTIC_INELASTIC
191+
element_fit_counts_dict->emplace(std::pair<std::string, data_struct::ArrayXXr<T_real> >(STR_SUM_ELASTIC_INELASTIC_AMP, data_struct::ArrayXXr<T_real>()));
192+
element_fit_counts_dict->at(STR_SUM_ELASTIC_INELASTIC_AMP).resize(height, width);
193+
194+
195+
return element_fit_counts_dict;
196+
}
197+
198+
161199
// ----------------------------------------------------------------------------
162200

163201
template<typename T_real>
@@ -172,35 +210,26 @@ DLL_EXPORT bool fit_single_spectra(fitting::routines::Base_Fit_Routine<T_real>*
172210
std::unordered_map<std::string, T_real> counts_dict;
173211
fit_routine->fit_spectra(model, spectra, elements_to_fit, counts_dict);
174212
//save count / sec
175-
for (auto& el_itr : *elements_to_fit)
213+
for (auto& itr : *out_fit_counts)
176214
{
177-
(*out_fit_counts)[el_itr.first](i, j) = counts_dict[el_itr.first] / spectra->elapsed_livetime();
215+
if (counts_dict.count(itr.first) > 0)
216+
{
217+
itr.second(i, j) = counts_dict[itr.first] / spectra->elapsed_livetime();
218+
}
219+
//(*out_fit_counts)[el_itr.first](i, j) = counts_dict[el_itr.first] / spectra->elapsed_livetime();
178220
}
179221
(*out_fit_counts)[STR_NUM_ITR](i, j) = counts_dict[STR_NUM_ITR];
180222

181223
(*out_fit_counts)[STR_RESIDUAL](i, j) = counts_dict[STR_RESIDUAL];
182224
// add total fluorescense yield
183225
if (out_fit_counts->count(STR_TOTAL_FLUORESCENCE_YIELD))
184226
{
185-
(*out_fit_counts)[STR_TOTAL_FLUORESCENCE_YIELD](i, j) = spectra->sum();
227+
(*out_fit_counts)[STR_TOTAL_FLUORESCENCE_YIELD](i, j) = spectra->sum() / spectra->elapsed_livetime();
186228
}
187229
// add sum coherent and compton
188230
if (out_fit_counts->count(STR_SUM_ELASTIC_INELASTIC_AMP) > 0 && counts_dict.count(STR_COHERENT_SCT_AMPLITUDE) > 0 && counts_dict.count(STR_COMPTON_AMPLITUDE) > 0)
189231
{
190232
(*out_fit_counts)[STR_SUM_ELASTIC_INELASTIC_AMP](i, j) = counts_dict[STR_COHERENT_SCT_AMPLITUDE] + counts_dict[STR_COMPTON_AMPLITUDE];
191-
// add total fluorescense yield
192-
if (out_fit_counts->count(STR_TOTAL_FLUORESCENCE_YIELD))
193-
{ // (sum - (elastic + inelastic)) / live time
194-
(*out_fit_counts)[STR_TOTAL_FLUORESCENCE_YIELD](i, j) = (spectra->sum() - (*out_fit_counts)[STR_SUM_ELASTIC_INELASTIC_AMP](i, j)) / spectra->elapsed_livetime();
195-
}
196-
}
197-
else
198-
{
199-
// add total fluorescense yield
200-
if (out_fit_counts->count(STR_TOTAL_FLUORESCENCE_YIELD))
201-
{
202-
(*out_fit_counts)[STR_TOTAL_FLUORESCENCE_YIELD](i, j) = spectra->sum() / spectra->elapsed_livetime();
203-
}
204233
}
205234

206235
return true;
@@ -236,7 +265,7 @@ DLL_EXPORT void proc_spectra(data_struct::Spectra_Volume<T_real>* spectra_volume
236265
fitting::models::Range energy_range = data_struct::get_energy_range(spectra_volume->samples_size(), &(detector->fit_params_override_dict.fit_params));
237266

238267
std::chrono::time_point<std::chrono::system_clock> start, end;
239-
268+
240269
for (auto& itr : detector->fit_routines)
241270
{
242271
fitting::routines::Base_Fit_Routine<T_real>* fit_routine = itr.second;
@@ -253,8 +282,17 @@ DLL_EXPORT void proc_spectra(data_struct::Spectra_Volume<T_real>* spectra_volume
253282
//Fit job queue
254283
std::queue<std::future<bool> >* fit_job_queue = new std::queue<std::future<bool> >();
255284

285+
data_struct::Fit_Count_Dict<T_real>* element_fit_count_dict = nullptr;
256286
//Allocate memeory to save fit counts
257-
data_struct::Fit_Count_Dict<T_real>* element_fit_count_dict = generate_fit_count_dict(&override_params->elements_to_fit, spectra_volume->rows(), spectra_volume->cols(), true);
287+
if (typeid(*fit_routine) == typeid(fitting::routines::ROI_Fit_Routine<T_real>) && scan_type == STR_SCAN_TYPE_POLAR_XANES)
288+
{
289+
((fitting::routines::ROI_Fit_Routine<T_real> *)fit_routine)->set_fit_separate_shells(true);
290+
element_fit_count_dict = generate_fit_count_by_shell_dict(&override_params->elements_to_fit, spectra_volume->rows(), spectra_volume->cols(), true);
291+
}
292+
else
293+
{
294+
element_fit_count_dict = generate_fit_count_dict(&override_params->elements_to_fit, spectra_volume->rows(), spectra_volume->cols(), true);
295+
}
258296

259297
for (size_t i = 0; i < spectra_volume->rows(); i++)
260298
{
@@ -280,7 +318,7 @@ DLL_EXPORT void proc_spectra(data_struct::Spectra_Volume<T_real>* spectra_volume
280318
cur_block++;
281319
}
282320

283-
std::chrono::time_point<std::chrono::system_clock> end = std::chrono::system_clock::now();
321+
end = std::chrono::system_clock::now();
284322
std::chrono::duration<double> elapsed_seconds = end - start;
285323
logI << "Fitting [ " << fit_routine->get_name() << " ] elapsed time: " << elapsed_seconds.count() << "s" << "\n";
286324

@@ -302,8 +340,8 @@ DLL_EXPORT void proc_spectra(data_struct::Spectra_Volume<T_real>* spectra_volume
302340
}
303341
// save png
304342
std::string dataset_fullpath = io::file::HDF5_IO::inst()->get_filename();
305-
int sidx = dataset_fullpath.find("img.dat");
306-
if (dataset_fullpath.length() > 0 && sidx > 0)
343+
size_t sidx = dataset_fullpath.find("img.dat");
344+
if (dataset_fullpath.length() > 0 && sidx != std::string::npos)
307345
{
308346
dataset_fullpath.replace(sidx, 7, "output"); // 7 = sizeof("img.dat")
309347
std::string str_path = dataset_fullpath + "_" + fit_routine->get_name() + ".png";
@@ -322,11 +360,10 @@ DLL_EXPORT void proc_spectra(data_struct::Spectra_Volume<T_real>* spectra_volume
322360
if (itr.first == data_struct::Fitting_Routines::GAUSS_MATRIX)
323361
{
324362
fitting::routines::Matrix_Optimized_Fit_Routine<T_real>* matrix_fit = (fitting::routines::Matrix_Optimized_Fit_Routine<T_real>*)fit_routine;
325-
io::file::HDF5_IO::inst()->save_max_10_spectra(fit_routine->get_name(),
326-
matrix_fit->energy_range(),
327-
matrix_fit->max_integrated_spectra(),
328-
matrix_fit->max_10_integrated_spectra(),
329-
matrix_fit->fitted_integrated_background());
363+
io::file::HDF5_IO::inst()->save_max_10_spectra(matrix_fit->energy_range(),
364+
matrix_fit->max_integrated_spectra(),
365+
matrix_fit->max_10_integrated_spectra(),
366+
matrix_fit->fitted_integrated_background());
330367
}
331368

332369
delete fit_job_queue;
@@ -397,6 +434,17 @@ void find_quantifier_scalers(std::unordered_map<std::string, double>& pv_map, Qu
397434
}
398435
}
399436

437+
// itr again to look for scale_factor
438+
for (auto& itr : pv_map)
439+
{
440+
double multiplier = (T_real)1.0;
441+
if(pv_map.count(itr.first+"_SCALE_FACTOR") > 0)
442+
{
443+
multiplier = pv_map.at(itr.first+"_SCALE_FACTOR");
444+
}
445+
itr.second *= multiplier;
446+
}
447+
400448
// add any summded scalers to pv_map
401449
const std::vector<struct Summed_Scaler>* summed_scalers = data_struct::Scaler_Lookup::inst()->get_summed_scaler_list(beamline);
402450
if (summed_scalers != nullptr)

‎src/data_struct/detector.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ void Detector<T_real>::append_element(Fitting_Routines routine, std::string quan
111111
Element_Info<T_real>* element = Element_Info_Map<T_real>::inst()->get_element(name);
112112
if (element != nullptr)
113113
{
114-
Electron_Shell shell = get_shell_by_name(name);
114+
data_struct::Electron_Shell shell = get_shell_by_name(name);
115115
fitting_quant_map.at(routine).update_weight_if_greater(shell, element->number, weight);
116116

117117
if (fitting_quant_map.at(routine).quant_scaler_map.count(quant_scaler) > 0)
@@ -161,11 +161,11 @@ void Detector<T_real>::update_element_quants(Fitting_Routines routine,
161161
if (eq_itr.weight > 0.0)
162162
{
163163
std::string name = eq_itr.name;
164-
if (shell_itr == Electron_Shell::L_SHELL)
164+
if (shell_itr == data_struct::Electron_Shell::L_SHELL)
165165
{
166166
name += "_L";
167167
}
168-
if (shell_itr == Electron_Shell::M_SHELL)
168+
if (shell_itr == data_struct::Electron_Shell::M_SHELL)
169169
{
170170
name += "_M";
171171
}

0 commit comments

Comments
 (0)