Skip to content

Commit 27eb84f

Browse files
kifkif
authored
Merge pull request #80 from kif/BM29
Ubuntu 24.04 upgrade on BM29
2 parents 4de22d6 + 6c75d4e commit 27eb84f

File tree

11 files changed

+97
-59
lines changed

11 files changed

+97
-59
lines changed

Diff for: ‎build-deb.sh

+10-8
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
#!/bin/sh
2-
rm -rf deb_dist
3-
export PYBUILD_DISABLE_python2=test
4-
export PYBUILD_DISABLE_python3=test
5-
python3 setup.py --command-packages=stdeb.command bdist_deb
6-
if [ -z $1 ]
7-
then
8-
sudo dpkg -i deb_dist/python3-*.deb
9-
fi
2+
export PATH=$PATH:/opt/bliss/conda/venv/dahu/bin
3+
rm -rf deb_dist/
4+
/usr/bin/python3 -m pip wheel .
5+
wheel2deb --output-dir deb_dist --exclude numpy*
6+
cd deb_dist/python3-dahu*_amd64
7+
dpkg-buildpackage -r -uc -us
8+
cd ..
9+
sudo dpkg -i python3-dahu*.deb
10+
cd ..
11+

Diff for: ‎plugins/bm29/__init__.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,16 @@
44
55
* bm29.IntegrateMultiframe
66
* bm29.SubtractBuffer
7+
* bm29.hplc
78
"""
89

910
__authors__ = ["Jérôme Kieffer"]
1011
__contact__ = "[email protected]"
1112
__license__ = "MIT"
1213
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
13-
__date__ = "12/10/2020"
14+
__date__ = "03/12/2024"
1415
__status__ = "development"
15-
__version__ = "0.1.0"
16+
__version__ = "0.2.0"
1617

1718
from dahu.factory import register
1819
from .integrate import IntegrateMultiframe

Diff for: ‎plugins/bm29/common.py

+3-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
__contact__ = "[email protected]"
1212
__license__ = "MIT"
1313
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
14-
__date__ = "29/10/2020"
14+
__date__ = "03/12/2024"
1515
__status__ = "development"
1616
version = "0.0.2"
1717

@@ -143,6 +143,8 @@ def get_equivalent_frames(proba, absolute=0.1, relative=0.2):
143143
res = []
144144
sizes = []
145145
size = len(proba)
146+
if size<2:
147+
return (0,1)
146148
ext_diag = numpy.zeros(size + 1, dtype=numpy.int16)
147149
delta = numpy.zeros(size + 1, dtype=numpy.int16)
148150
ext_diag[1:-1] = numpy.diagonal(proba, 1) >= relative

Diff for: ‎plugins/bm29/hplc.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
__contact__ = "[email protected]"
1111
__license__ = "MIT"
1212
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
13-
__date__ = "16/09/2022"
13+
__date__ = "03/12/2024"
1414
__status__ = "development"
1515
__version__ = "0.2.0"
1616

@@ -117,7 +117,7 @@ def build_background(I, std=None, keep=0.3):
117117
"""
118118
U, S, V = numpy.linalg.svd(I.T, full_matrices=False)
119119
bg1 = numpy.median(V[0]) * S[0] * U[:, 0]
120-
Pscore = [freesas.cormap.measure_longest(numpy.ascontiguousarray(bg1 - i, dtype=numpy.float)) for i in I]
120+
Pscore = [freesas.cormap.measure_longest(numpy.ascontiguousarray(bg1 - i, dtype=numpy.float64)) for i in I]
121121
orderd = numpy.argsort(Pscore)
122122
nkeep = int(math.ceil(keep * I.shape[0]))
123123
to_keep = numpy.sort(orderd[:nkeep])

Diff for: ‎plugins/bm29/integrate.py

+9-3
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
__contact__ = "[email protected]"
1212
__license__ = "MIT"
1313
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
14-
__date__ = "03/10/2022"
14+
__date__ = "03/12/2024"
1515
__status__ = "development"
1616
__version__ = "0.3.0"
1717

@@ -151,8 +151,9 @@ def setup(self, kwargs=None):
151151
lst = list(os.path.splitext(self.input_file))
152152
lst.insert(1, "-integrate")
153153
dirname, basename = os.path.split("".join(lst))
154+
dirname = dirname.replace("RAW_DATA", "PROCESSED_DATA")
154155
dirname = os.path.dirname(dirname)
155-
dirname = os.path.join(dirname, "processed")
156+
# dirname = os.path.join(dirname, "processed")
156157
dirname = os.path.join(dirname, "integrate")
157158
self.output_file = os.path.join(dirname, basename)
158159
if not os.path.isdir(dirname):
@@ -578,7 +579,12 @@ def process1_integration(self, data):
578579
if self.ispyb.url:
579580
self.to_pyarch[idx] = res
580581
idx += 1
581-
return IntegrationResult(res.radial, intensity, sigma)
582+
if idx == 0:
583+
self.log_error(f"No frame iterated over in process1_integration! len(frames): {len(data)} len(monitor): {len(self.monitor_values)}", do_raise=False)
584+
radial = numpy.zeros(self.npt, dtype=numpy.float32)
585+
else:
586+
radial = res.radial
587+
return IntegrationResult(radial, intensity, sigma)
582588

583589
def process2_cormap(self, curves, fidelity_abs, fidelity_rel):
584590
"Take the integrated data as input, returns a CormapResult namedtuple"

Diff for: ‎plugins/bm29/ispyb.py

+35-18
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111
__contact__ = "[email protected]"
1212
__license__ = "MIT"
1313
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
14-
__date__ = "03/11/2022"
14+
__date__ = "10/12/2024"
1515
__status__ = "development"
16-
version = "0.2.1"
16+
version = "0.2.3"
1717

1818
import logging
1919
logger = logging.getLogger("bm29.ispyb")
@@ -36,7 +36,7 @@
3636

3737
import matplotlib.pyplot
3838
matplotlib.use("Agg")
39-
from freesas.collections import RG_RESULT, RT_RESULT, StatsResult
39+
from freesas.containers import RG_RESULT, RT_RESULT, StatsResult
4040
from freesas.plot import kratky_plot, guinier_plot, scatter_plot, density_plot, hplc_plot
4141

4242

@@ -78,7 +78,7 @@ def __init__(self, url, login=None, passwd=None, gallery=None, pyarch=None,
7878
self.gallery = os.path.abspath(gallery)
7979
if not os.path.isdir(self.gallery):
8080
try:
81-
os.makedirs(self.gallery)
81+
os.makedirs(self.gallery)
8282
except Exception as err:
8383
logger.warning(f"Unable to create dir {self.gallery}. {type(err)}: {err}")
8484
else:
@@ -105,22 +105,39 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
105105
:param path: directory name where processed data are staying
106106
:param raw: directory name of the raw data (not the processed ones)
107107
:param data: dict with all data sent to ISpyB
108+
108109
"""
109110
tmp = self.gallery.strip("/").split("/")
110111
idx_process = [i for i,j in enumerate(tmp) if j.lower().startswith("process")][-1]
111-
assert idx_process>5
112-
if proposal is None:
113-
proposal = tmp[idx_process-5]
114-
if beamline is None:
115-
beamline = tmp[idx_process-4]
116-
if sample is None:
117-
sample = tmp[idx_process-2]
118-
if dataset is None:
119-
dataset = tmp[idx_process+1]
120-
if path is None:
121-
path = os.path.dirname(self.gallery)
122-
if raw is None:
123-
raw = os.path.abspath(self.gallery[:self.gallery.lower().index("process")])
112+
if tmp[idx_process] == "process":
113+
assert idx_process>5
114+
if proposal is None:
115+
proposal = tmp[idx_process-5]
116+
if beamline is None:
117+
beamline = tmp[idx_process-4]
118+
if sample is None:
119+
sample = tmp[idx_process-2]
120+
if dataset is None:
121+
dataset = tmp[idx_process+1]
122+
if path is None:
123+
path = os.path.dirname(self.gallery)
124+
if raw is None:
125+
raw = os.path.abspath(self.gallery[:self.gallery.lower().index("process")])
126+
elif tmp[idx_process] == "PROCESSED_DATA":
127+
if proposal is None:
128+
proposal = tmp[idx_process-3]
129+
if beamline is None:
130+
beamline = tmp[idx_process-2]
131+
if sample is None:
132+
sample = tmp[idx_process+1]
133+
if dataset is None:
134+
dataset = tmp[idx_process+2]
135+
if path is None:
136+
path = os.path.dirname(self.gallery)
137+
if raw is None:
138+
raw = os.path.dirname(os.path.dirname(os.path.abspath(self.gallery.replace("PROCESSED_DATA", "RAW_DATA"))))
139+
else:
140+
logger.error("Unrecognized path layout")
124141

125142
metadata = {"definition": "SAXS",
126143
"Sample_name": sample}
@@ -157,7 +174,7 @@ def send_icat(self, proposal=None, beamline=None, sample=None, dataset=None, pat
157174
if volume:
158175
metadata["SAXS_porod_volume"] = str(volume)
159176
#Other metadata one may collect ...
160-
metadata["SAXS_experimentType"]= data.get("experiment_type", "")
177+
metadata["SAXS_experiment_type"]= data.get("experiment_type", "UNKNOWN")
161178
metadata["datasetName"] = dataset
162179
icat_client = IcatClient(metadata_urls=["bcu-mq-01.esrf.fr:61613", "bcu-mq-02.esrf.fr:61613"])
163180
kwargs = {"beamline":beamline,

Diff for: ‎plugins/bm29/requirements.txt

+6-2
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ pyFAI
66
h5py
77
hdf5plugin
88
freesas
9-
suds-jurko
10-
memcache
9+
suds
10+
python-memcached
1111
pyicat-plus
12+
urllib3
13+
siphash24
14+
pyopencl
15+
stompest

Diff for: ‎plugins/bm29/subtracte.py

+10-6
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111
__contact__ = "[email protected]"
1212
__license__ = "MIT"
1313
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
14-
__date__ = "03/11/2022"
14+
__date__ = "03/12/2024"
1515
__status__ = "development"
16-
__version__ = "0.2.1"
16+
__version__ = "0.2.1"
1717

1818
import os
1919
import json
@@ -166,8 +166,7 @@ def process(self):
166166
self.send_to_ispyb()
167167
except Exception as err2:
168168
import traceback
169-
self.log_warning("Processing failed and unable to send remaining data to ISPyB: %s %s\n%s" %
170-
(type(err2), err2, "\n".join(traceback.format_exc(limit=10))))
169+
self.log_warning(f"Processing failed and unable to send remaining data to ISPyB: {type(err2)} {err2}\n{traceback.format_exc(limit=10)}")
171170
raise(err)
172171
else:
173172
self.send_to_ispyb()
@@ -700,7 +699,12 @@ def read_nexus(filename):
700699
npt = len(q)
701700
unit = pyFAI.units.to_unit(axis + "_" + nxdata_grp[axis].attrs["units"])
702701
integration_grp = nxdata_grp.parent
703-
poni = str(integration_grp["configuration/file_name"][()]).strip()
702+
poni = integration_grp["configuration/file_name"][()]
703+
if isinstance(poni, bytes):
704+
poni = poni.decode()
705+
else:
706+
poni = str(poni)
707+
poni = poni.strip()
704708
if not os.path.exists(poni):
705709
poni = str(integration_grp["configuration/data"][()]).strip()
706710
polarization = integration_grp["configuration/polarization_factor"][()]
@@ -732,7 +736,7 @@ def send_to_ispyb(self):
732736
if self.ispyb.url and parse_url(self.ispyb.url).host:
733737
ispyb = IspybConnector(*self.ispyb)
734738
ispyb.send_subtracted(self.to_pyarch)
735-
self.to_pyarch["experiment_type"]="sample-changer"
739+
self.to_pyarch["experiment_type"]="sampleChanger"
736740
self.to_pyarch["sample"] = self.sample_juice.sample
737741
ispyb.send_icat(data=self.to_pyarch)
738742
else:

Diff for: ‎src/dahu/plugin.py

+5-4
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
__contact__ = "[email protected]"
1313
__license__ = "MIT"
1414
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
15-
__date__ = "01/09/2020"
15+
__date__ = "25/11/2024"
1616
__status__ = "production"
1717

1818
import os
@@ -127,10 +127,11 @@ def wait_for(self, job_id):
127127
:return: the job object
128128
"""
129129
from .job import Job
130-
assert isinstance(job_id, int)
130+
if not isinstance(job_id, int):
131+
self.log_warning(f"Unable to synchronize on {job_id} job, invalid type: {type(job_id)}, expected an int !")
131132
TIMEOUT = getattr(self, 'TIMEOUT', 10.0) #default timeout to 10s
132133
if job_id>Job._id_class:
133-
self.log_warning("Not synchronizing job, invalid id: %s" % job_id)
134+
self.log_warning(f"Not synchronizing job, invalid id: {job_id}")
134135
else:
135136
status = Job.synchronize_job(job_id, TIMEOUT)
136137
abort_time = time.time() + TIMEOUT
@@ -142,7 +143,7 @@ def wait_for(self, job_id):
142143
self.log_error("Timeout while waiting other job to finish")
143144
break
144145
if status != Job.STATE_SUCCESS:
145-
self.log_error("Other job ended in %s: aborting myself" % status)
146+
self.log_error(f"Other job {job_id} ended in {status}: aborting myself")
146147
return Job.getJobFromId(job_id)
147148

148149

Diff for: ‎src/dahu/test/utilstest.py

+13-12
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
__contact__ = "[email protected]"
2828
__license__ = "MIT"
2929
__copyright__ = "European Synchrotron Radiation Facility, Grenoble, France"
30-
__date__ = "22/02/2024"
30+
__date__ = "04/12/2024"
3131

3232
PACKAGE = "dahu"
3333
DATA_KEY = "DAHU_DATA"
@@ -132,11 +132,12 @@ def timeoutDuringDownload(cls, imagename=None):
132132
"""
133133
if imagename is None:
134134
imagename = "2252/testimages.tar.bz2 unzip it "
135-
raise RuntimeError("Could not automatically \
136-
download test images!\n \ If you are behind a firewall, \
137-
please set both environment variable http_proxy and https_proxy.\
138-
This even works under windows ! \n \
139-
Otherwise please try to download the images manually from \n %s/%s and put it in in test/testimages." % (cls.url_base, imagename))
135+
raise RuntimeError(f"""Could not automatically download test images!
136+
If you are behind a firewall, please set both environment variable http_proxy and https_proxy.
137+
This even works under windows !
138+
Otherwise please try to download the images manually from:
139+
{cls.url_base}/{imagename}
140+
and put it in in test/testimages.""")
140141

141142
@classmethod
142143
def getimage(cls, imagename):
@@ -193,11 +194,11 @@ def getimage(cls, imagename):
193194
data to disk at %s" % cls.image_home)
194195

195196
if not os.path.isfile(fullimagename):
196-
raise RuntimeError("Could not automatically \
197-
download test images %s!\n \ If you are behind a firewall, \
198-
please set both environment variable http_proxy and https_proxy.\
199-
This even works under windows ! \n \
200-
Otherwise please try to download the images manually from \n%s/%s" % (imagename, cls.url_base, imagename))
197+
raise RuntimeError(f"""Could not automatically download test images {imagename}!
198+
If you are behind a firewall, please set both environment variable http_proxy and https_proxy.
199+
This even works under windows !
200+
Otherwise, please try to download the images manually from:
201+
{cls.url_base}/{imagename}""")
201202

202203
return fullimagename
203204

@@ -275,7 +276,7 @@ def script_path(cls, script):
275276

276277

277278
def Rwp(obt, ref, comment="Rwp"):
278-
""" ___________________________
279+
r""" ___________________________
279280
Calculate \/ 4 ( obt - ref)²
280281
V Sum( --------------- )
281282
(obt + ref)²

Diff for: ‎version.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@
5959
"final": 15}
6060

6161
MAJOR = 2024
62-
MINOR = 2
62+
MINOR = 12
6363
MICRO = 0
6464
RELEV = "final" # <16
6565
SERIAL = 0 # <16

0 commit comments

Comments
 (0)