Skip to content

Commit 20bd67e

Browse files
dagewadagewa
authored
Use image files from dials-data image_examples, not from dials_regression (#801)
* Fix test_imageset.py::test_format to use dials-data * Turns out bzip2-compressed RAXIS images could not be read, but that could be got around by copying image reading code from iotbx into get_raw_data, ensuring the file is only opened by the compression-aware Format class open_file method. * Fix FormatRAXISII too * Fixes for test_experiment_list.py * test_datablock.py no longer contains any tests. So delete it * Fix two more tests that read particular files from image_examples * Fix test_FormatCBFMiniPilatusDLS12M.py::test_bad_pixel_mask * Fix test_imageset.py::test_multi_panel * Fix test_cbf_mini_as_file.py::test_cbf_writer * Fxi test_FormatPYunspecified.py::test_static_mask * Fix test_FormatSMVADSC.py * fix test_xds.py::test_to_xds_multi_panel_i23 * Move test_regression_images.py::test_detectorbase to use dials-data * fix test_regression_images.py::test_read_image and test_regression_images.py::test_format_class_API_assumptions * Remove test_regression_images.py::test_compressed_images as this is adequately tested now anyway by the compressed images on dials-data
1 parent d44e229 commit 20bd67e

15 files changed

+217
-432
lines changed

‎newsfragments/801.bugfix‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
- Change dxtbx tests to use publicly-available data on dials-data
2+
- ``FormatRAXIS``: Allow the possibility of reading compressed files

‎src/dxtbx/format/FormatRAXIS.py‎

Lines changed: 114 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -64,14 +64,84 @@
6464

6565
import calendar
6666
import datetime
67+
import io
6768
import math
6869
import struct
6970

71+
from iotbx.detectors import ReadRAXIS
7072
from iotbx.detectors.raxis import RAXISImage
7173

7274
from dxtbx import IncorrectFormatError
7375
from dxtbx.format.Format import Format
7476

77+
header_struct = [
78+
("Device", 10, "s"),
79+
("Version", 10, "s"),
80+
("Crystal", 20, "s"),
81+
("CrystalSystem", 12, "s"),
82+
(None, 24),
83+
("SpaceGroup", 12, "s"),
84+
("mosaic1", 4, "!f"),
85+
("memo", 80, "s"),
86+
("reserve1", 84, "s"),
87+
("date", 12, "s"),
88+
("operatorname", 20, "s"),
89+
("target", 4, "s"),
90+
("wavelength", 4, "!f"),
91+
("monotype", 20, "s"),
92+
("mono2theta", 4, "!f"),
93+
("collimator", 20, "s"),
94+
("filter", 4, "s"),
95+
("distance", 4, "!f"),
96+
("Kv", 4, "!f"),
97+
("mA", 4, "!f"),
98+
("focus", 12, "s"),
99+
("Xmemo", 80, "s"),
100+
("cyl", 4, "!i"),
101+
(None, 60),
102+
("Spindle", 4, "s"), # Crystal mount axis closest to spindle axis
103+
("Xray_axis", 4, "s"), # Crystal mount axis closest to beam axis
104+
("phidatum", 4, "!f"),
105+
("phistart", 4, "!f"),
106+
("phiend", 4, "!f"),
107+
("noscillations", 4, "!i"),
108+
("minutes", 4, "!f"), # Exposure time in minutes?
109+
("beampixels_x", 4, "!f"),
110+
("beampixels_y", 4, "!f"), # Direct beam position in pixels
111+
("omega", 4, "!f"),
112+
("chi", 4, "!f"),
113+
("twotheta", 4, "!f"),
114+
("Mu", 4, "!f"), # Spindle inclination angle?
115+
("ScanTemplate", 204, "s"), # This space is now used for storing the scan
116+
# templates information
117+
("nFast", 4, "!i"),
118+
("nSlow", 4, "!i"), # Number of fast, slow pixels
119+
("sizeFast", 4, "!f"),
120+
("sizeSlow", 4, "!f"), # Size of fast, slow direction in mm
121+
("record_length", 4, "!i"), # Record length in bytes
122+
("number_records", 4, "!i"), # number of records
123+
("Read_start", 4, "!i"), # For partial reads, 1st read line
124+
("IP_num", 4, "!i"), # Which imaging plate 1, 2 ?
125+
("Ratio", 4, "!f"), # Output ratio for high value pixels
126+
("Fading_start", 4, "!f"), # Fading time to start of read
127+
("Fading_end", 4, "!f"), # Fading time to end of read
128+
("computer", 10, "s"), # Type of computer "IRIS", "VAX", "SUN", etc
129+
("plate_type", 10, "s"), # Type of IP
130+
("Dr", 4, "!i"),
131+
("Dx", 4, "!i"),
132+
("Dz", 4, "!i"), # IP scanning codes??
133+
("PixShiftOdd", 4, "!f"), # Pixel shift to odd lines
134+
("IntRatioOdd", 4, "!f"), # Intensity ratio to odd lines
135+
("MagicNum", 4, "!i"), # Magic number to indicate next values are legit
136+
("NumGonAxes", 4, "!i"), # Number of goniometer axes
137+
("a5x3fGonVecs", 60, "!fffffffffffffff"), # Goniometer axis vectors
138+
("a5fGonStart", 20, "!fffff"), # Start angles for each of 5 axes
139+
("a5fGonEnd", 20, "!fffff"), # End angles for each of 5 axes
140+
("a5fGonOffset", 20, "!fffff"), # Offset values for each of 5 axes
141+
("ScanAxisNum", 4, "!i"), # Which axis is the scan axis?
142+
("AxesNames", 40, "s"), # Names of the axes (space or comma separated?)'''
143+
]
144+
75145

76146
class RAXISHelper:
77147
def __init__(self, image_file, **kwargs):
@@ -83,8 +153,22 @@ def __init__(self, image_file, **kwargs):
83153
def _start(self):
84154
with Format.open_file(self._image_file) as fh:
85155
self._header_bytes = fh.read(1024)
86-
87-
if self._header_bytes[812:822].strip() in (b"SGI", b"IRIS"):
156+
self.head = {}
157+
stream = io.BytesIO(self._header_bytes)
158+
for item in header_struct:
159+
if item[0] is None:
160+
stream.read(item[1])
161+
elif item[2] == "s":
162+
self.head[item[0]] = stream.read(item[1])[0 : item[1]]
163+
elif len(item[2]) > 2:
164+
rawdata = stream.read(item[1])
165+
assert len(rawdata) == struct.calcsize(item[2])
166+
self.head[item[0]] = struct.unpack(item[2], rawdata)
167+
else:
168+
rawdata = stream.read(item[1])
169+
assert len(rawdata) == struct.calcsize(item[2])
170+
self.head[item[0]] = struct.unpack(item[2], rawdata)[0]
171+
if self.head["computer"].strip() in (b"SGI", b"IRIS"):
88172
self._f = ">f"
89173
self._i = ">i"
90174
else:
@@ -98,8 +182,34 @@ def detectorbase_start(self):
98182
def get_raw_data(self):
99183
"""Get the pixel intensities (i.e. read the image and return as a flex array."""
100184
self.detectorbase_start()
101-
# super() resolves to the other (true) parent class
102-
return super().get_raw_data()
185+
186+
dim0 = self.head["nFast"] # number of fast pixels
187+
to_read = self.head["record_length"]
188+
read_lines = self.head["number_records"]
189+
190+
with Format.open_file(self._image_file) as fh:
191+
fh.seek(to_read)
192+
raw_data = fh.read(to_read * read_lines)
193+
194+
# For a normal image, there should be no padding per line
195+
# Each line might be padded, so figure this out
196+
bytes_per_line = dim0 * 2
197+
if bytes_per_line < to_read:
198+
# Remove all padding bytes
199+
raw_data = b"".join(
200+
raw_data[record * to_read : record * to_read + bytes_per_line]
201+
for record in range(read_lines)
202+
)
203+
204+
raw_data = ReadRAXIS(
205+
raw_data,
206+
self.head["record_length"] // self.head["nFast"],
207+
self.head["nSlow"] * self.detectorbase.bin,
208+
self.head["nFast"] * self.detectorbase.bin,
209+
self.detectorbase.endian_swap_required(),
210+
)
211+
212+
return raw_data
103213

104214
def _detector_helper(self):
105215
"""Returns image header values as a dictionary."""

‎src/dxtbx/format/FormatRAXISII.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ def detectorbase_start(self):
2121
pass # override with an empty function
2222

2323
def _start(self):
24+
super()._start()
2425
self.detectorbase = NonSquareRAXISImage(self._image_file)
2526
self.detectorbase.readHeader()
2627

‎tests/format/test_FormatCBFMiniPilatusDLS12M.py‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from __future__ import annotations
22

33
import calendar
4-
import os
54

65
import pytest
76

@@ -49,18 +48,16 @@
4948
),
5049
),
5150
)
52-
def test_bad_pixel_mask(
53-
timestamp, multi_panel, masked_pixel_count, dials_regression, mocker
54-
):
51+
def test_bad_pixel_mask(timestamp, multi_panel, masked_pixel_count, dials_data, mocker):
5552
if timestamp is not None:
5653
# Fool the format class into masking out bad modules
5754
mocked_timestamp = mocker.patch(
5855
"dxtbx.format.FormatCBFMiniPilatusDLS12M.get_pilatus_timestamp"
5956
)
6057
mocked_timestamp.return_value = timestamp
6158

62-
image_path = os.path.join(
63-
dials_regression, "image_examples", "DLS_I23", "germ_13KeV_0001.cbf"
59+
image_path = str(
60+
dials_data("image_examples", pathlib=True) / "DLS_I23-germ_13KeV_0001.cbf.bz2"
6461
)
6562
experiments = ExperimentListFactory.from_filenames(
6663
[image_path], format_kwargs={"multi_panel": multi_panel}

‎tests/format/test_FormatPYunspecified.py‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,13 @@
11
from __future__ import annotations
22

3-
import os
4-
53
from dxtbx.format.FormatPYunspecifiedStill import FormatPYunspecifiedStill
64
from dxtbx.model.experiment_list import ExperimentListFactory
75

86

9-
def test_static_mask(dials_regression):
10-
filename = os.path.join(
11-
dials_regression,
12-
"image_examples/LCLS_CXI/shot-s00-2011-12-02T21_07Z29.723_00569.pickle",
7+
def test_static_mask(dials_data):
8+
data_dir = dials_data("image_examples", pathlib=True)
9+
filename = str(
10+
data_dir / "LCLS_CXI-shot-s00-2011-12-02T21_07Z29.723_00569.pickle",
1311
)
1412
assert FormatPYunspecifiedStill.understand(filename)
1513

‎tests/format/test_FormatSMVADSC.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,15 @@
11
from __future__ import annotations
22

3-
import os
3+
import bz2
44

55
import dxtbx
66

77

8-
def test_noninteger_pedestal(dials_regression, tmpdir):
9-
filename = os.path.join(dials_regression, "image_examples/APS_14BMC/q315_1_001.img")
8+
def test_noninteger_pedestal(dials_data, tmpdir):
9+
data_dir = dials_data("image_examples", pathlib=True)
10+
filename = str(data_dir / "APS_14BMC-q315_1_001.img.bz2")
1011
# Read this file in as data
11-
with open(filename, "rb") as f:
12+
with bz2.open(filename, "rb") as f:
1213
data = f.read()
1314

1415
# Write out with an inserted header item of a noninteger pedestal

‎tests/format/test_cbf_mini_as_file.py‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
from __future__ import annotations
22

3-
import os
4-
53
import pytest
64

75
from dxtbx.format.FormatCBFMini import FormatCBFMini
@@ -17,12 +15,13 @@
1715
@pytest.mark.parametrize(
1816
"image_file",
1917
[
20-
("image_examples", "ALS_831", "q315r_lyso_001.img"),
21-
("image_examples", "DLS_I02", "X4_wide_M1S4_1_0001.cbf"),
18+
("ALS_831-q315r_lyso_001.img.bz2"),
19+
("DLS_I02-X4_wide_M1S4_1_0001.cbf.bz2"),
2220
],
2321
)
24-
def test_cbf_writer(image_file, dials_regression, tmp_path):
25-
filename = os.path.join(dials_regression, *image_file)
22+
def test_cbf_writer(image_file, dials_data, tmp_path):
23+
data_dir = dials_data("image_examples", pathlib=True)
24+
filename = str(data_dir / image_file)
2625
imageset = ExperimentListFactory.from_filenames([filename])[0].imageset
2726
output_file = tmp_path / "image_0001.cbf"
2827

‎tests/imagelist.py‎

Lines changed: 14 additions & 86 deletions
Original file line numberDiff line numberDiff line change
@@ -1,88 +1,16 @@
11
from __future__ import annotations
22

3-
4-
def id_conversion(path):
5-
return path.replace("/", "-")
6-
7-
8-
smv_images = [
9-
"image_examples/ALS_501/als501_q4_1_001.img",
10-
"image_examples/SPring8_BL26B1_SaturnA200/A200_000001.img",
11-
"image_examples/SPring8_BL26B1_SaturnA200/A200_000002.img",
12-
"image_examples/APS_19ID/q315_unbinned_a.0001.img",
13-
"image_examples/ALS_821/q210_lyso_1_101.img",
14-
"image_examples/MLFSOM_simulation/fake_00001.img",
15-
"image_examples/ALS_831/q315r_lyso_001.img",
16-
"image_examples/DESY_ID141/q210_2_001.img",
17-
"image_examples/SSRL_bl91/q315_1_001.img",
18-
"image_examples/APS_24IDC/q315_1_001.img",
19-
"image_examples/ALS_1231/q315r_lyso_1_001.img",
20-
"image_examples/SRS_142/q4_1_001.img",
21-
"image_examples/ALS_422/lyso_041013a_1_001.img",
22-
"image_examples/APS_17ID/q210_1_001.img",
23-
"image_examples/saturn/lyso_00001.img",
24-
]
25-
smv_image_ids = list(map(id_conversion, smv_images))
26-
27-
tiff_images = [
28-
"image_examples/SPring8_BL32XU_MX225HS/ds_000045.img",
29-
"image_examples/SPring8_BL32XU_MX225HS/ds_000001.img",
30-
"image_examples/SPring8_BL44XU_MX300HE/bl44xu_lys_000002.img",
31-
"image_examples/SPring8_BL44XU_MX300HE/bl44xu_lys_000001.img",
32-
"image_examples/SPring8_BL32XU/rayonix225hs_0001.img",
33-
"image_examples/SPring8_BL32XU/rayonix225_0001.img",
34-
"image_examples/SLS_X06SA/mar225_2_001.img",
35-
"image_examples/CLS1_08ID1/mar225_2_E0_0001.img",
36-
"image_examples/SPring8_BL38B1_MX225HE/bl38b1_001.img",
37-
"image_examples/SPring8_BL38B1_MX225HE/bl38b1_090.img",
38-
"image_examples/SRS_101/mar225_001.img",
39-
"image_examples/SPring8_BL12B2_MX225HE/lys001_000091.img",
40-
"image_examples/SPring8_BL12B2_MX225HE/lys001_000001.img",
41-
"image_examples/SPring8_BL26B2_MX225/2sec_Al200um_000001.img",
42-
"image_examples/SPring8_BL26B2_MX225/2sec_Al200um_000090.img",
43-
]
44-
tiff_image_ids = list(map(id_conversion, tiff_images))
45-
46-
cbf_images = [
47-
"image_examples/ESRF_ID29/trypsin_1_0001.cbf",
48-
"image_examples/xia2/merge2cbf_averaged_0001.cbf",
49-
"image_examples/dials-190/whatev1_01_00002.cbf",
50-
"image_examples/dials-190/whatev1_02_00001.cbf",
51-
"image_examples/dials-190/whatev1_02_00002.cbf",
52-
"image_examples/dials-190/whatev1_03_00001.cbf",
53-
"image_examples/dials-190/whatev1_01_00001.cbf",
54-
"image_examples/dials-190/whatev1_03_00002.cbf",
55-
"image_examples/APS_24IDE_test/thaum-12_1_0005.cbf",
56-
"image_examples/APS_24IDE_test/thaum-12_1_0004.cbf",
57-
"image_examples/APS_24IDE_test/thaum-12_1_0006.cbf",
58-
"image_examples/APS_24IDE_test/thaum-12_1_0008.cbf",
59-
"image_examples/APS_24IDE_test/thaum-12_1_0009.cbf",
60-
"image_examples/APS_24IDE_test/thaum-12_1_0010.cbf",
61-
"image_examples/APS_24IDE_test/thaum-12_1_0007.cbf",
62-
"image_examples/APS_24IDE_test/thaum-12_1_0002.cbf",
63-
"image_examples/APS_24IDE_test/thaum-12_1_0001.cbf",
64-
"image_examples/APS_24IDE_test/thaum-12_1_0003.cbf",
65-
"image_examples/APS_24IDC/pilatus_1_0001.cbf",
66-
"image_examples/SLS_Eiger_16M_as_CBF/insu_with_bs_labelit_0901.cbf",
67-
"image_examples/SLS_Eiger_16M_as_CBF/insu_with_bs_labelit_0001.cbf",
68-
"image_examples/SPring8_BL41XU_PILATUS3_6M/data1_000001.cbf",
69-
"image_examples/SPring8_BL41XU_PILATUS3_6M/data1_000901.cbf",
70-
"image_examples/DLS_I02/X4_wide_M1S4_1_0001.cbf",
71-
"image_examples/DLS_I23/I23_P12M_alpha_0001.cbf",
72-
"image_examples/DLS_I23/germ_13KeV_0001.cbf",
73-
"image_examples/SLS_X06SA/pilatus6m_1_00001.cbf",
74-
"image_examples/SPring8_ADSC_SN916/Xtal17-2phi_3_015.cbf",
75-
"image_examples/DLS_I19/I19_P300k_00001.cbf",
76-
"image_examples/ED_From_TIFF/170112330001.cbf",
77-
]
78-
cbf_image_ids = list(map(id_conversion, cbf_images))
79-
80-
cbf_multitile_images = [
81-
"stills_test_data/hit-20111202210224984.cbf",
82-
"stills_test_data/hit-s00-20140306002935980.cbf",
83-
"stills_test_data/hit-s00-20140306002857363.cbf",
84-
]
85-
cbf_multitile_image_ids = list(map(id_conversion, cbf_multitile_images))
86-
87-
hdf5_images = ["image_examples/putative_imgCIF_HDF5_mapping/minicbf.h5"]
88-
hdf5_image_ids = list(map(id_conversion, hdf5_images))
3+
import os
4+
5+
import dials_data.download
6+
7+
df = dials_data.download.DataFetcher()
8+
data_dir = df("image_examples", pathlib=True)
9+
image_examples = []
10+
for f in data_dir.iterdir():
11+
ext = os.path.splitext(f)[1]
12+
if ext == ".h5" and "_master." not in str(f):
13+
continue
14+
if ext in (".expt", ".json"):
15+
continue
16+
image_examples.append(str(f))

0 commit comments

Comments
 (0)