Skip to content

Commit cacd0c1

Browse files
CadairCadair
authored
Update sample data to use productID (#595)
* Update sample data to use productID * Still regenerate old sample data to keep it updated * Add changelogs * Fix doctest * Update figure test hashes * Always write tests for deprecated behaviour * Due to problems uploading the VISP sample data
1 parent 75558a4 commit cacd0c1

19 files changed

Lines changed: 96 additions & 58 deletions

‎.readthedocs.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ build:
1212
pre_build:
1313
# Note $DKIST_SAMPLE_DIR is set by RTD to ~/dkist_data
1414
- python -c "from dkist.data.sample import download_all_sample_data; download_all_sample_data()"
15-
- python -c "from dkist.data.sample import VISP_BKPLX; print(VISP_BKPLX)"
15+
- python -c "from dkist.data.sample import VISP_L1_KMUPT; print(VISP_L1_KMUPT)"
1616

1717
conda:
1818
environment: .rtd-environment.yml

‎changelog/595.feature.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Sample data has been updated to use newly reprocessed datasets where available.

‎changelog/595.removal.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The ``dkist.data.sample.VBI_AJQWW`` and ``dkist.data.sample.VISP_BKPLX`` datasets have been renamed ``dkist.data.sample.VBI_L1_NZJTB`` and ``dkist.data.sample.VISP_L1_KMUPT`` respectively.
2+
These names use the new Product ID which does not change when the data is reprocessed.
3+
(Note that the VBI dataset has been reprocessed so the Dataset ID is also different.)

‎dkist/data/_sample.py‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,12 @@
1111

1212
VISP_HEADER = fits.Header.fromtextfile(Path(__file__).parent / "VISP_HEADER.hdr")
1313
_SAMPLE_DATASETS = {
14-
"VISP_BKPLX": ("https://g-a36282.cd214.a567.data.globus.org/user_tools_tutorial_data/", "BKPLX_stokesI.tar"),
15-
"VBI_AJQWW": ("https://g-a36282.cd214.a567.data.globus.org/user_tools_tutorial_data/", "AJQWW_single_mosaic.tar"),
14+
"VISP_L1_KMUPT": ("https://g-a36282.cd214.a567.data.globus.org/user_tools_tutorial_data/", "BKPLX_stokesI.tar"),
15+
"VBI_L1_NZJTB": ("https://g-a36282.cd214.a567.data.globus.org/user_tools_tutorial_data/", "YCDRFH_single_mosaic.tar"),
16+
}
17+
_DEPRECATED_NAMES = {
18+
"VISP_BKPLX": "VISP_L1_KMUPT",
19+
"VBI_AJQWW": "VBI_L1_NZJTB",
1620
}
1721

1822

‎dkist/data/sample.py‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,13 @@
11
"""
22
This module provides some (partial) sample datasets.
33
"""
4+
import warnings as _warnings
45

5-
from ._sample import _SAMPLE_DATASETS, VISP_HEADER, _get_sample_datasets
6+
from dkist.utils.exceptions import DKISTDeprecationWarning as _DKISTDeprecationWarning
67

7-
__all__ = ["download_all_sample_data", *sorted(_SAMPLE_DATASETS.keys()), "VISP_HEADER"] # noqa: PLE0604
8+
from ._sample import _DEPRECATED_NAMES, _SAMPLE_DATASETS, VISP_HEADER, _get_sample_datasets
9+
10+
__all__ = ["download_all_sample_data", *sorted(_SAMPLE_DATASETS.keys()), *sorted(_DEPRECATED_NAMES.keys()), "VISP_HEADER"] # noqa: PLE0604
811

912

1013
# See PEP 562 (https://peps.python.org/pep-0562/) for module-level __dir__()
@@ -17,6 +20,14 @@ def __getattr__(name):
1720
if name in _SAMPLE_DATASETS:
1821
return _get_sample_datasets(name)[0]
1922

23+
if name in _DEPRECATED_NAMES:
24+
new_name = _DEPRECATED_NAMES[name]
25+
_warnings.warn(
26+
f"The sample data name {name} is deprecated and has been replaced by {new_name}.",
27+
_DKISTDeprecationWarning,
28+
)
29+
return _get_sample_datasets(new_name)[0]
30+
2031
raise AttributeError(f"module '{__name__}' has no attribute '{name}'")
2132

2233

‎dkist/data/tests/test_sample.py‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44

55
import pytest
66

7+
from dkist.utils.exceptions import DKISTDeprecationWarning
8+
79

810
@pytest.fixture
911
def tmp_sample_dir(tmp_path):
@@ -16,11 +18,11 @@ def tmp_sample_dir(tmp_path):
1618
def test_module_dir():
1719
import dkist.data.sample # noqa: PLC0415
1820

19-
assert "VBI_AJQWW" in dir(dkist.data.sample)
20-
assert "VISP_BKPLX" in dir(dkist.data.sample)
21+
assert "VBI_L1_NZJTB" in dir(dkist.data.sample)
22+
assert "VISP_L1_KMUPT" in dir(dkist.data.sample)
2123

2224

23-
@pytest.mark.parametrize("attrname", ["VBI_AJQWW", "VISP_BKPLX"])
25+
@pytest.mark.parametrize("attrname", ["VBI_L1_NZJTB", "VISP_L1_KMUPT"])
2426
def test_module_getattr(mocker, attrname):
2527
mock = mocker.patch("dkist.data.sample._get_sample_datasets")
2628
import dkist.data.sample # noqa: PLC0415
@@ -30,11 +32,23 @@ def test_module_getattr(mocker, attrname):
3032
mock.assert_has_calls([call(attrname), call().__getitem__(0)])
3133

3234

35+
@pytest.mark.parametrize("attrname", ["VBI_AJQWW", "VISP_BKPLX"])
36+
def test_module_getattr_deprecated(mocker, attrname):
37+
mock = mocker.patch("dkist.data.sample._get_sample_datasets")
38+
import dkist.data.sample # noqa: PLC0415
39+
from dkist.data._sample import _DEPRECATED_NAMES # noqa: PLC0415
40+
41+
with pytest.warns(DKISTDeprecationWarning, match=attrname):
42+
getattr(dkist.data.sample, attrname)
43+
44+
mock.assert_has_calls([call(_DEPRECATED_NAMES[attrname]), call().__getitem__(0)])
45+
46+
3347
@pytest.mark.skipif(platform.system() == "Windows", reason="Internet not properly disabled on Windows")
3448
@pytest.mark.internet_off
3549
def test_fail(tmp_sample_dir):
3650
"""
3751
No remote data means this test should fail.
3852
"""
3953
with pytest.raises(RuntimeError, match="1 sample data files failed"):
40-
from dkist.data.sample import VISP_BKPLX # noqa: F401 PLC0415
54+
from dkist.data.sample import VISP_L1_KMUPT # noqa: F401 PLC0415

‎dkist/dataset/loader.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,13 @@ def load_dataset(target, *, ignore_version_mismatch=False):
6161
6262
>>> dkist.load_dataset(Path("/path/to/ABCDE")) # doctest: +SKIP
6363
64-
>>> from dkist.data.sample import VISP_BKPLX # doctest: +REMOTE_DATA
65-
>>> print(dkist.load_dataset(VISP_BKPLX)) # doctest: +REMOTE_DATA
64+
>>> from dkist.data.sample import VISP_L1_KMUPT # doctest: +REMOTE_DATA
65+
>>> print(dkist.load_dataset(VISP_L1_KMUPT)) # doctest: +REMOTE_DATA
6666
This VISP Dataset consists of 1700 frames.
67-
Files are stored in ...VISP_BKPLX
67+
Files are stored in ...VISP_L1_KMUPT
6868
<BLANKLINE>
6969
This calibration has Dataset ID BKPLX.
70-
The unique identifier for the input observe frames (Product ID) is (no ProductID).
70+
The unique identifier for the input observe frames (Product ID) is ...
7171
<BLANKLINE>
7272
This Dataset has 4 pixel and 5 world dimensions.
7373
<BLANKLINE>

‎dkist/dataset/tests/test_tiled_dataset.py‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,8 @@ def test_tiled_dataset_invalid_construction(dataset, dataset_4d):
107107
@pytest.mark.remote_data
108108
@pytest.mark.parametrize("share_zscale", [True, False], ids=["share_zscale", "indpendent_zscale"])
109109
def test_tileddataset_plot(share_zscale):
110-
from dkist.data.sample import VBI_AJQWW # noqa: PLC0415
111-
ori_ds = load_dataset(VBI_AJQWW)
110+
from dkist.data.sample import VBI_L1_NZJTB # noqa: PLC0415
111+
ori_ds = load_dataset(VBI_L1_NZJTB)
112112

113113
newtiles = []
114114
for tile in ori_ds.flat:
@@ -132,8 +132,8 @@ def test_tileddataset_plot(share_zscale):
132132
@figure_test
133133
@pytest.mark.remote_data
134134
def test_masked_tileddataset_plot():
135-
from dkist.data.sample import VBI_AJQWW # noqa: PLC0415
136-
ds = load_dataset(VBI_AJQWW)
135+
from dkist.data.sample import VBI_L1_NZJTB # noqa: PLC0415
136+
ds = load_dataset(VBI_L1_NZJTB)
137137
ds._data.mask[:2, 0] = True
138138
ds._data.mask[0, 1] = True
139139

@@ -148,8 +148,8 @@ def test_masked_tileddataset_plot():
148148
def test_tileddataset_plot_limit_swapping(swap_tile_limits):
149149
# Also test that row/column sizes are correct
150150

151-
from dkist.data.sample import VBI_AJQWW # noqa: PLC0415
152-
ori_ds = load_dataset(VBI_AJQWW)
151+
from dkist.data.sample import VBI_L1_NZJTB # noqa: PLC0415
152+
ori_ds = load_dataset(VBI_L1_NZJTB)
153153

154154
# Swap WCS to make the `swap_tile_limits` option more natural
155155
for tile in ori_ds.flat:
@@ -191,8 +191,8 @@ def test_tileddataset_plot_limit_swapping(swap_tile_limits):
191191

192192
@pytest.mark.remote_data
193193
def test_tileddataset_plot_non2d_sliceindex():
194-
from dkist.data.sample import VBI_AJQWW # noqa: PLC0415
195-
ds = load_dataset(VBI_AJQWW)
194+
from dkist.data.sample import VBI_L1_NZJTB # noqa: PLC0415
195+
ds = load_dataset(VBI_L1_NZJTB)
196196

197197
newtiles = []
198198
for tile in ds.flat:

‎dkist/dataset/tiled_dataset.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -369,8 +369,8 @@ def slice_tiles(self):
369369
.. code-block:: python
370370
371371
>>> from dkist import load_dataset
372-
>>> from dkist.data.sample import VBI_AJQWW # doctest: +REMOTE_DATA
373-
>>> ds = load_dataset(VBI_AJQWW) # doctest: +REMOTE_DATA
372+
>>> from dkist.data.sample import VBI_L1_NZJTB # doctest: +REMOTE_DATA
373+
>>> ds = load_dataset(VBI_L1_NZJTB) # doctest: +REMOTE_DATA
374374
>>> ds.slice_tiles[0, 10:-10] # doctest: +REMOTE_DATA
375375
<dkist.dataset.tiled_dataset.TiledDataset object at ...>
376376
This VBI Dataset is an array of (3, 3) Dataset objects and

‎dkist/tests/figure_hashes_mpl_391_ft_261_astropy_611_animators_111_ndcube_222.json‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,11 @@
66
"dkist.dataset.tests.test_plotting.test_2d_plot[aslice1]": "cbb84fbae51d8238803f8f0d6820c575f024fe54b1656f1b181dc4ec645e9ff9",
77
"dkist.dataset.tests.test_plotting.test_2d_plot[aslice2]": "132c5615832daff457dacb4cb770498f1fbb4460a5b90b5d4d01d224c70eeb28",
88
"dkist.dataset.tests.test_plotting.test_2d_plot2": "409b5a10ad8ccf005331261505e63ce8febdc38eb8b5a34f8863e567e3cccb9c",
9-
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[share_zscale]": "bc1561c95587a12245dc201eb25487f53b8759f9faf361089308ff95cc21fab5",
10-
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[indpendent_zscale]": "b0a77981f035b9ac5ec8f231a54986f3e1973cb95a68f333d6c7fcba6bb8fc4c",
11-
"dkist.dataset.tests.test_tiled_dataset.test_masked_tileddataset_plot": "7c822bd267a690953a0013535afb33ac9fec17cbb1602a6f90d17ac954825b8e",
12-
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[x]": "56548cd44d6b68c7586fd1b37b16dea46cc6b50ce37aa6d19367a92bc8402555",
13-
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[y]": "debd24cf94d88a4526974ce4e1cf073715b2f1d7da8b558f16fd0f97047e477d",
14-
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[xy]": "3e130694abf6e1a2fc31f9d4c2bdd65d73c87c58645430ebd590bf95f0aed27d",
15-
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[None]": "6aeac69c6b5f1376e8e0334590250c7127fe847031642f05cf82d3cf1b22661f"
9+
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[share_zscale]": "f9990cf9dd604d773f9c5f4a1281f879332cc00796cede8c14fba5c4c458c845",
10+
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot[indpendent_zscale]": "95ef2ace83d982ca33c866549fa7351bbaa1272921303db9927111c79b0397e5",
11+
"dkist.dataset.tests.test_tiled_dataset.test_masked_tileddataset_plot": "8a748fc8e884b4a0e6c4f29ae3becc0125758fa7c02e15108105fce957d46aa8",
12+
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[x]": "d3e09a30154f7aa27e074b5c29201acb9a2810d5180984159c8f9a83fa929b28",
13+
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[y]": "4896113fad41a6e0a89a0fac7a1801b3e18a7361c36df121ecfe76683c0fbeb6",
14+
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[xy]": "146f600f468454ed20a41f72bdba2b9096ed1be99e4a45b7755506222439036f",
15+
"dkist.dataset.tests.test_tiled_dataset.test_tileddataset_plot_limit_swapping[None]": "d041f555e8294022922eab516ecfb106f2eca66211f4922c95b06503c11a7dc2"
1616
}

0 commit comments

Comments
 (0)