Skip to content

Commit 654b784

Browse files
luigiferianiluigiferiani
authored
Parallel summaries (ver228#74)
Get Features Summaries parallelised. Up to n_cores - 1 can be used. Limit is not enforced in GUI, but at execution. No limit applied if Tierpsy detects it's running in a PBS or SLURM HPC (Tierpsy's estimate of n_cores might disagree with actual cores allocated) file_id is not guaranteed to always increase in filename_* files, as file_id is assigned before results files are processed, and results are written on a first-finished first-written basis
1 parent 18aaba9 commit 654b784

5 files changed

Lines changed: 347 additions & 155 deletions

File tree

‎tierpsy/gui/QtDesignerFiles/Summarizer.ui‎

Lines changed: 57 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,18 @@
2727
</property>
2828
</widget>
2929
</item>
30-
<item row="1" column="1">
31-
<widget class="QComboBox" name="p_feature_type"/>
30+
<item row="2" column="1">
31+
<widget class="QComboBox" name="p_summary_type"/>
32+
</item>
33+
<item row="1" column="0">
34+
<widget class="QLabel" name="label_4">
35+
<property name="text">
36+
<string>Features Type</string>
37+
</property>
38+
<property name="alignment">
39+
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
40+
</property>
41+
</widget>
3242
</item>
3343
<item row="0" column="1" colspan="2">
3444
<widget class="QLineEdit" name="p_root_dir"/>
@@ -41,15 +51,8 @@ edited trajectories?</string>
4151
</property>
4252
</widget>
4353
</item>
44-
<item row="1" column="0">
45-
<widget class="QLabel" name="label_4">
46-
<property name="text">
47-
<string>Features Type</string>
48-
</property>
49-
<property name="alignment">
50-
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
51-
</property>
52-
</widget>
54+
<item row="1" column="1">
55+
<widget class="QComboBox" name="p_feature_type"/>
5356
</item>
5457
<item row="0" column="0">
5558
<widget class="QPushButton" name="pushButton_rootdir">
@@ -64,31 +67,28 @@ edited trajectories?</string>
6467
</property>
6568
</widget>
6669
</item>
67-
<item row="2" column="1">
68-
<widget class="QComboBox" name="p_summary_type"/>
70+
<item row="3" column="1">
71+
<widget class="QSpinBox" name="p_n_parallel">
72+
<property name="toolTip">
73+
<string extracomment="Number of files to process in parallel. Limited by the available system resources. Any value &lt;= 1 disables multiprocessing"/>
74+
</property>
75+
</widget>
76+
</item>
77+
<item row="3" column="0">
78+
<widget class="QLabel" name="label_20">
79+
<property name="text">
80+
<string>Number of Processes</string>
81+
</property>
82+
<property name="alignment">
83+
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
84+
</property>
85+
</widget>
6986
</item>
7087
</layout>
7188
</item>
7289
<item>
7390
<widget class="QWidget" name="FoldArgs" native="true">
7491
<layout class="QGridLayout" name="gridLayout">
75-
<item row="1" column="1">
76-
<widget class="QLabel" name="label_2">
77-
<property name="sizePolicy">
78-
<sizepolicy hsizetype="Preferred" vsizetype="Maximum">
79-
<horstretch>0</horstretch>
80-
<verstretch>0</verstretch>
81-
</sizepolicy>
82-
</property>
83-
<property name="text">
84-
<string>Fraction of trajectories
85-
to sample</string>
86-
</property>
87-
<property name="alignment">
88-
<set>Qt::AlignBottom|Qt::AlignHCenter</set>
89-
</property>
90-
</widget>
91-
</item>
9292
<item row="0" column="1">
9393
<widget class="QDoubleSpinBox" name="p_frac_worms_to_keep">
9494
<property name="maximum">
@@ -99,16 +99,6 @@ edited trajectories?</string>
9999
</property>
100100
</widget>
101101
</item>
102-
<item row="0" column="0">
103-
<widget class="QSpinBox" name="p_n_folds"/>
104-
</item>
105-
<item row="0" column="2">
106-
<widget class="QDoubleSpinBox" name="p_time_sample_seconds">
107-
<property name="maximum">
108-
<double>100000000000.000000000000000</double>
109-
</property>
110-
</widget>
111-
</item>
112102
<item row="1" column="0">
113103
<widget class="QLabel" name="label">
114104
<property name="sizePolicy">
@@ -142,6 +132,33 @@ edited trajectories?</string>
142132
</property>
143133
</widget>
144134
</item>
135+
<item row="0" column="2">
136+
<widget class="QDoubleSpinBox" name="p_time_sample_seconds">
137+
<property name="maximum">
138+
<double>100000000000.000000000000000</double>
139+
</property>
140+
</widget>
141+
</item>
142+
<item row="1" column="1">
143+
<widget class="QLabel" name="label_2">
144+
<property name="sizePolicy">
145+
<sizepolicy hsizetype="Preferred" vsizetype="Maximum">
146+
<horstretch>0</horstretch>
147+
<verstretch>0</verstretch>
148+
</sizepolicy>
149+
</property>
150+
<property name="text">
151+
<string>Fraction of trajectories
152+
to sample</string>
153+
</property>
154+
<property name="alignment">
155+
<set>Qt::AlignBottom|Qt::AlignHCenter</set>
156+
</property>
157+
</widget>
158+
</item>
159+
<item row="0" column="0">
160+
<widget class="QSpinBox" name="p_n_folds"/>
161+
</item>
145162
</layout>
146163
</widget>
147164
</item>

‎tierpsy/gui/Summarizer_ui.py‎

Lines changed: 39 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# -*- coding: utf-8 -*-
22

3-
# Form implementation generated from reading ui file 'Summarizer.ui'
3+
# Form implementation generated from reading ui file 'QtDesignerFiles/Summarizer.ui'
44
#
55
# Created by: PyQt5 UI code generator 5.9.2
66
#
@@ -22,19 +22,22 @@ def setupUi(self, Summarizer):
2222
self.label_5.setAlignment(QtCore.Qt.AlignRight|QtCore.Qt.AlignTrailing|QtCore.Qt.AlignVCenter)
2323
self.label_5.setObjectName("label_5")
2424
self.gridLayout_2.addWidget(self.label_5, 2, 0, 1, 1)
25-
self.p_feature_type = QtWidgets.QComboBox(self.centralwidget)
26-
self.p_feature_type.setObjectName("p_feature_type")
27-
self.gridLayout_2.addWidget(self.p_feature_type, 1, 1, 1, 1)
25+
self.p_summary_type = QtWidgets.QComboBox(self.centralwidget)
26+
self.p_summary_type.setObjectName("p_summary_type")
27+
self.gridLayout_2.addWidget(self.p_summary_type, 2, 1, 1, 1)
28+
self.label_4 = QtWidgets.QLabel(self.centralwidget)
29+
self.label_4.setAlignment(QtCore.Qt.AlignRight|QtCore.Qt.AlignTrailing|QtCore.Qt.AlignVCenter)
30+
self.label_4.setObjectName("label_4")
31+
self.gridLayout_2.addWidget(self.label_4, 1, 0, 1, 1)
2832
self.p_root_dir = QtWidgets.QLineEdit(self.centralwidget)
2933
self.p_root_dir.setObjectName("p_root_dir")
3034
self.gridLayout_2.addWidget(self.p_root_dir, 0, 1, 1, 2)
3135
self.p_is_manual_index = QtWidgets.QCheckBox(self.centralwidget)
3236
self.p_is_manual_index.setObjectName("p_is_manual_index")
3337
self.gridLayout_2.addWidget(self.p_is_manual_index, 1, 2, 1, 1)
34-
self.label_4 = QtWidgets.QLabel(self.centralwidget)
35-
self.label_4.setAlignment(QtCore.Qt.AlignRight|QtCore.Qt.AlignTrailing|QtCore.Qt.AlignVCenter)
36-
self.label_4.setObjectName("label_4")
37-
self.gridLayout_2.addWidget(self.label_4, 1, 0, 1, 1)
38+
self.p_feature_type = QtWidgets.QComboBox(self.centralwidget)
39+
self.p_feature_type.setObjectName("p_feature_type")
40+
self.gridLayout_2.addWidget(self.p_feature_type, 1, 1, 1, 1)
3841
self.pushButton_rootdir = QtWidgets.QPushButton(self.centralwidget)
3942
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Maximum, QtWidgets.QSizePolicy.Fixed)
4043
sizePolicy.setHorizontalStretch(0)
@@ -43,35 +46,24 @@ def setupUi(self, Summarizer):
4346
self.pushButton_rootdir.setSizePolicy(sizePolicy)
4447
self.pushButton_rootdir.setObjectName("pushButton_rootdir")
4548
self.gridLayout_2.addWidget(self.pushButton_rootdir, 0, 0, 1, 1)
46-
self.p_summary_type = QtWidgets.QComboBox(self.centralwidget)
47-
self.p_summary_type.setObjectName("p_summary_type")
48-
self.gridLayout_2.addWidget(self.p_summary_type, 2, 1, 1, 1)
49+
self.p_n_parallel = QtWidgets.QSpinBox(self.centralwidget)
50+
self.p_n_parallel.setToolTip("")
51+
self.p_n_parallel.setObjectName("p_n_parallel")
52+
self.gridLayout_2.addWidget(self.p_n_parallel, 3, 1, 1, 1)
53+
self.label_20 = QtWidgets.QLabel(self.centralwidget)
54+
self.label_20.setAlignment(QtCore.Qt.AlignRight|QtCore.Qt.AlignTrailing|QtCore.Qt.AlignVCenter)
55+
self.label_20.setObjectName("label_20")
56+
self.gridLayout_2.addWidget(self.label_20, 3, 0, 1, 1)
4957
self.verticalLayout_3.addLayout(self.gridLayout_2)
5058
self.FoldArgs = QtWidgets.QWidget(self.centralwidget)
5159
self.FoldArgs.setObjectName("FoldArgs")
5260
self.gridLayout = QtWidgets.QGridLayout(self.FoldArgs)
5361
self.gridLayout.setObjectName("gridLayout")
54-
self.label_2 = QtWidgets.QLabel(self.FoldArgs)
55-
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Maximum)
56-
sizePolicy.setHorizontalStretch(0)
57-
sizePolicy.setVerticalStretch(0)
58-
sizePolicy.setHeightForWidth(self.label_2.sizePolicy().hasHeightForWidth())
59-
self.label_2.setSizePolicy(sizePolicy)
60-
self.label_2.setAlignment(QtCore.Qt.AlignBottom|QtCore.Qt.AlignHCenter)
61-
self.label_2.setObjectName("label_2")
62-
self.gridLayout.addWidget(self.label_2, 1, 1, 1, 1)
6362
self.p_frac_worms_to_keep = QtWidgets.QDoubleSpinBox(self.FoldArgs)
6463
self.p_frac_worms_to_keep.setMaximum(1.0)
6564
self.p_frac_worms_to_keep.setSingleStep(0.01)
6665
self.p_frac_worms_to_keep.setObjectName("p_frac_worms_to_keep")
6766
self.gridLayout.addWidget(self.p_frac_worms_to_keep, 0, 1, 1, 1)
68-
self.p_n_folds = QtWidgets.QSpinBox(self.FoldArgs)
69-
self.p_n_folds.setObjectName("p_n_folds")
70-
self.gridLayout.addWidget(self.p_n_folds, 0, 0, 1, 1)
71-
self.p_time_sample_seconds = QtWidgets.QDoubleSpinBox(self.FoldArgs)
72-
self.p_time_sample_seconds.setMaximum(100000000000.0)
73-
self.p_time_sample_seconds.setObjectName("p_time_sample_seconds")
74-
self.gridLayout.addWidget(self.p_time_sample_seconds, 0, 2, 1, 1)
7567
self.label = QtWidgets.QLabel(self.FoldArgs)
7668
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Maximum)
7769
sizePolicy.setHorizontalStretch(0)
@@ -90,6 +82,22 @@ def setupUi(self, Summarizer):
9082
self.label_3.setAlignment(QtCore.Qt.AlignBottom|QtCore.Qt.AlignHCenter)
9183
self.label_3.setObjectName("label_3")
9284
self.gridLayout.addWidget(self.label_3, 1, 2, 1, 1)
85+
self.p_time_sample_seconds = QtWidgets.QDoubleSpinBox(self.FoldArgs)
86+
self.p_time_sample_seconds.setMaximum(100000000000.0)
87+
self.p_time_sample_seconds.setObjectName("p_time_sample_seconds")
88+
self.gridLayout.addWidget(self.p_time_sample_seconds, 0, 2, 1, 1)
89+
self.label_2 = QtWidgets.QLabel(self.FoldArgs)
90+
sizePolicy = QtWidgets.QSizePolicy(QtWidgets.QSizePolicy.Preferred, QtWidgets.QSizePolicy.Maximum)
91+
sizePolicy.setHorizontalStretch(0)
92+
sizePolicy.setVerticalStretch(0)
93+
sizePolicy.setHeightForWidth(self.label_2.sizePolicy().hasHeightForWidth())
94+
self.label_2.setSizePolicy(sizePolicy)
95+
self.label_2.setAlignment(QtCore.Qt.AlignBottom|QtCore.Qt.AlignHCenter)
96+
self.label_2.setObjectName("label_2")
97+
self.gridLayout.addWidget(self.label_2, 1, 1, 1, 1)
98+
self.p_n_folds = QtWidgets.QSpinBox(self.FoldArgs)
99+
self.p_n_folds.setObjectName("p_n_folds")
100+
self.gridLayout.addWidget(self.p_n_folds, 0, 0, 1, 1)
93101
self.verticalLayout_3.addWidget(self.FoldArgs)
94102
self.TimeWindows = QtWidgets.QWidget(self.centralwidget)
95103
self.TimeWindows.setObjectName("TimeWindows")
@@ -298,14 +306,15 @@ def retranslateUi(self, Summarizer):
298306
_translate = QtCore.QCoreApplication.translate
299307
Summarizer.setWindowTitle(_translate("Summarizer", "MainWindow"))
300308
self.label_5.setText(_translate("Summarizer", "Summary Type"))
309+
self.label_4.setText(_translate("Summarizer", "Features Type"))
301310
self.p_is_manual_index.setText(_translate("Summarizer", "Use manually \n"
302311
"edited trajectories?"))
303-
self.label_4.setText(_translate("Summarizer", "Features Type"))
304312
self.pushButton_rootdir.setText(_translate("Summarizer", "Root Directory"))
305-
self.label_2.setText(_translate("Summarizer", "Fraction of trajectories\n"
306-
" to sample"))
313+
self.label_20.setText(_translate("Summarizer", "Number of Processes"))
307314
self.label.setText(_translate("Summarizer", "Number of Folds"))
308315
self.label_3.setText(_translate("Summarizer", "Time (seconds)\n"
316+
" to sample"))
317+
self.label_2.setText(_translate("Summarizer", "Fraction of trajectories\n"
309318
" to sample"))
310319
self.label_6.setText(_translate("Summarizer", "Time windows (optional)"))
311320
self.p_time_windows.setText(_translate("Summarizer", "0:end"))

‎tierpsy/helper/params/docs_summarizer_param.py‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,16 @@
44

55
from .helper import repack_dflt_list
66

7+
try:
8+
from os import sched_getaffinity
9+
_max_n_processes = max(1, len(sched_getaffinity(0))-1)
10+
except (AttributeError, ImportError):
11+
from multiprocessing import cpu_count
12+
_max_n_processes = max(1, cpu_count()-1)
13+
except:
14+
_max_n_processes = 1
15+
16+
717
summarizer_valid_options = {
818
'feature_type':['openworm','tierpsy'],
919
'summary_type' : ['plate', 'trajectory', 'plate_augmented'],
@@ -31,6 +41,16 @@
3141
is going to be a data augmentation by randomingly sampling over a subset of the plate trajectories.
3242
'''
3343
),
44+
('n_parallel',
45+
1,
46+
'''
47+
Number of files to be processed in parallel. Not limited in the GUI,
48+
but in reality limited by the resources available. Capped to one less
49+
than the number of cores available, unless running on PBS or SLURM HPC,
50+
in which case no upper cap is set as python does not know the number of
51+
cores allocated to it
52+
'''
53+
),
3454
('is_manual_index',
3555
False,
3656
'Set to true to calculate data using manually edited trajectories used Tierpsy Viewer.'
@@ -39,12 +59,12 @@
3959
'0:end',
4060
'''
4161
Define time windows to get feature summaries from the parts of the video included in each window.
42-
Each window must be defined by the start_time and the end_time connected by ':',
43-
for example 'start_time:end_time'.
44-
Different windows must be separated by ',', for example 'start_time_1:end_time_1, start_time_2:end_time_2'.
45-
A sequence of equally sized windows can be defined using the format 'start_time:end_time:step'.
62+
Each window must be defined by the start_time and the end_time connected by ':',
63+
for example 'start_time:end_time'.
64+
Different windows must be separated by ',', for example 'start_time_1:end_time_1, start_time_2:end_time_2'.
65+
A sequence of equally sized windows can be defined using the format 'start_time:end_time:step'.
4666
A single window can be comprised of several time intervals using the format
47-
'start_time1:end_time1 + start_time2:end_time2'.
67+
'start_time1:end_time1 + start_time2:end_time2'.
4868
ATTENTION: the start_time is included in the window, but the end_time is not included.
4969
'''
5070
),

0 commit comments

Comments
 (0)