Merge branch 'main' into prepare-v1.18

Author: SolarDrew

.github/workflows/codspeed-walltime.yml

@@ -54,7 +54,7 @@ jobs:
       - name: Install requirements
         run: python -m pip install .[tests] pytest-codspeed
       - name: Run benchmarks
-        uses: CodspeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4.11.1
+        uses: CodspeedHQ/action@9d332c4d90b43981c3e55ae8e38e68709996240f # v4.17.0
         with:
           mode: "walltime"
           token: ${{ secrets.CODSPEED_TOKEN }}  # zizmor: ignore[secrets-outside-env] We want this to run everywhere so it always needs the token

.github/workflows/codspeed.yml

@@ -30,7 +30,7 @@ jobs:
           python-version: '3.14'
       - run: python -m pip install .[tests] pytest-codspeed
       - name: Run benchmarks
-        uses: CodspeedHQ/action@281164b0f014a4e7badd2c02cecad9b595b70537 # v4.11.1
+        uses: CodspeedHQ/action@9d332c4d90b43981c3e55ae8e38e68709996240f # v4.17.0
         with:
           mode: "simulation"
           token: ${{ secrets.CODSPEED_TOKEN }}  # zizmor: ignore[secrets-outside-env] We want this to run everywhere so it always needs the token

changelog/723.feature.rst

@@ -0,0 +1 @@
+Add an example notebook for retrieving DKIST dataset metadata and inspecting quality and processing metadata.

changelog/730.trivial.rst

@@ -0,0 +1 @@
+Fix some minor breakages to allow release

dkist/data/tests/test_sample.py

@@ -1,9 +1,11 @@
 import os
 import platform
 from unittest.mock import call
+from urllib.request import urlopen
 
 import pytest
 
+from dkist.data._sample import _BASE_URL
 from dkist.utils.exceptions import DKISTDeprecationWarning
 
 
@@ -50,5 +52,14 @@ def test_fail(tmp_sample_dir):
     """
     No remote data means this test should fail.
     """
+    # Sometimes this test runs and it can still access the internet
+    # even though internet_off should prevent this.  We skip the test
+    # if we can access the internet.
+    try:
+        urlopen(_BASE_URL)
+        pytest.skip("Internet not disabled")
+    except Exception:  # noqa BLE001
+        pass
+
     with pytest.raises(RuntimeError, match="1 sample data files failed"):
         from dkist.data.sample import VISP_L1_KMUPT  # noqa: F401 PLC0415

dkist/io/asdf/resources/schemas/dataset-1.3.0.yaml

@@ -52,15 +52,21 @@ properties:
 
       parameters:
         description: A copy of the parameters used to generate this dataset.
-        type: object
+        anyOf:
+        - type: object
+        - type: array
 
       observation_input_frames:
         description: A copy of the observation input frames used to generate this dataset.
-        type: object
+        anyOf:
+        - type: object
+        - type: array
 
       calibration_input_frames:
         description: A copy of the calibration input frames used to generate this dataset.
-        type: object
+        anyOf:
+        - type: object
+        - type: array
 
       recipe_run_config:
         description: A copy of the recipe configurations used to generate this dataset.

docs/examples/dataset_retrieval_and_metadata.md

@@ -0,0 +1,94 @@
+---
+jupytext:
+  formats: md:myst
+  text_representation:
+    extension: .md
+    format_name: myst
+    format_version: 0.13
+    jupytext_version: 1.17.2
+kernelspec:
+  display_name: Python 3 (ipykernel)
+  language: python
+  name: python3
+---
+
+(dkist:examples:dataset-retrieval-and-metadata)=
+# Retrieving a Dataset and Inspecting Metadata
+
+In this example we will search for a DKIST dataset by dataset ID, download its metadata ASDF file, and inspect some of the metadata stored in it.
+
+```{code-cell} ipython3
+from sunpy.net import Fido, attrs as a
+
+import dkist
+import dkist.net
+```
+
+Here we use the VISP dataset [ALQRZ](https://dkist.data.nso.edu/datasetview/ALQRZ).
+To inspect a different dataset, edit only `DATASET_ID`.
+
+```{code-cell} ipython3
+DATASET_ID = "ALQRZ"
+```
+
+First we search for the dataset.
+The `Status("any")` attribute means the search will return this dataset ID irrespective of its processing status.
+
+```{code-cell} ipython3
+search_results = Fido.search(a.dkist.Dataset(DATASET_ID), a.dkist.Status("any"))
+search_results
+```
+
+The search result points to the metadata ASDF file.
+We can fetch that file directly with `Fido.fetch`.
+
+```{code-cell} ipython3
+asdf_file = Fido.fetch(search_results)
+asdf_file
+```
+
+Now we can load the ASDF file as a `dkist.Dataset`.
+This describes the dataset and its file references without downloading all of the FITS files.
+
+```{code-cell} ipython3
+dataset = dkist.load_dataset(asdf_file)
+dataset
+```
+
+The dataset inventory contains searchable metadata from the DKIST Data Center.
+Here we display the entries related to data quality.
+
+```{code-cell} ipython3
+{
+    key: value
+    for key, value in dataset.meta["inventory"].items()
+    if key.lower().startswith("quality")
+}
+```
+
+Each calibrated dataset also has a quality report.
+This method downloads the report PDF and returns a `parfive.Results` object with the local filename.
+
+```{code-cell} ipython3
+quality_report = dataset.files.quality_report()
+quality_report
+```
+
+The ASDF file can also contain more detailed metadata about how the dataset was generated.
+The following fields describe the input parameters, the observe and calibration frames, and the recipe configuration.
+
+```{code-cell} ipython3
+dataset.meta["parameters"]
+```
+
+```{code-cell} ipython3
+dataset.meta["observation_input_frames"]
+```
+
+```{code-cell} ipython3
+dataset.meta["calibration_input_frames"]
+```
+
+```{code-cell} ipython3
+dataset.meta["recipe_run_config"]
+```

docs/examples/index.md

@@ -8,6 +8,7 @@ Each example is a self-contained tutorial working through one task.
 
 reproject_vbi_mosaic
 vbi_extents
+dataset_retrieval_and_metadata
 example_cryo_plots
 rebinning_and_dask
 ```

pytest.ini

@@ -65,6 +65,9 @@ filterwarnings =
     # See https://github.com/numpy/numpy/issues/15748#issuecomment-598584838
     ignore:numpy.ufunc size changed:RuntimeWarning
     ignore:numpy.ndarray size changed:RuntimeWarning
+    # Warnings introduced in astropy modelling by NumPy 2.5, these can be removed when astropy fix it
+    ignore:.*Setting the shape on a NumPy array has been deprecated in NumPy 2.5:DeprecationWarning
+    ignore:.*Setting the dtype on a NumPy array has been deprecated in NumPy 2.5:DeprecationWarning
     # This isn't really an issue it's an annoyance, it's triggered by printing
     # an astropy table inside a test, which is generally only done when
     # debugging.