diff --git a/.github/scripts/publish_fornax.py b/.github/scripts/publish_fornax.py new file mode 100644 index 000000000..bf05567c4 --- /dev/null +++ b/.github/scripts/publish_fornax.py @@ -0,0 +1,65 @@ +""" +Script to handle Fornax branch deployment with CI +""" +import glob +from pathlib import Path +import shutil +import yaml + +FORNAX_NOTEBOOK_BASEPATH = "mast_notebooks" +FORNAX_MANIFEST = "fornax-manifest.yml" +FORNAX_REQUIREMENTS = "requirements_mast_tutorials.txt" +FORNAX_METADATA_YAML = "notebook_metadata.yml" + + +def main(): + """ + Do Fornax branch formatting. + """ + + with open(FORNAX_MANIFEST, "r", encoding="utf-8") as f: + manifest = yaml.safe_load(f) + + # Fornax-preferred renaming: + Path(FORNAX_NOTEBOOK_BASEPATH).mkdir(parents=True, exist_ok=True) + + # Copy & rename requirements: + shutil.copy( + manifest["global-requirements"][0]["requirements-file"], + f"{FORNAX_NOTEBOOK_BASEPATH}/{FORNAX_REQUIREMENTS}" + ) + + # Copy notebooks over & create reformatted metadata dict: + nbs_metadata = [] + for nb in manifest["notebooks"]: + # Copy files: + orig_path = '/'.join(nb['file'].split('/')[:-1]) + new_path = orig_path.replace("notebooks/", f"{FORNAX_NOTEBOOK_BASEPATH}/") + shutil.copytree(orig_path, new_path, dirs_exist_ok=True) + + # Reformatted metadata: + metadict = {} + for key in ["title", "file", "section", "subsection", "description"]: + val = nb.get(key, None) + if val is not None: + metadict[key] = val + + # Update filename: + metadict[key] = metadict[key].replace("notebooks/", f"{FORNAX_NOTEBOOK_BASEPATH}/") + nbs_metadata.append(metadict) + + # Remove the individual file reqiurements.txt files: + for file in glob.glob( + f"{FORNAX_NOTEBOOK_BASEPATH}/**/requirements.txt", recursive=True + ): + # print(file) + Path(file).unlink(missing_ok=True) + + # Write out a stripped down yml file with metadata: + # Title, section, subsection, file + with open(f"{FORNAX_METADATA_YAML}", 'w', encoding="utf-8") as f: + yaml.dump(nbs_metadata, f, default_flow_style=False, sort_keys=False) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/fornax_branch_deploy.yml b/.github/workflows/fornax_branch_deploy.yml new file mode 100644 index 000000000..647b2ab7e --- /dev/null +++ b/.github/workflows/fornax_branch_deploy.yml @@ -0,0 +1,38 @@ +name: Deploy MAST notebooks to Fornax branch + +on: + push: + branches: [ main, fornax-manifest ] + workflow_dispatch: + +jobs: + deploy_notebooks: + name: Deploy deploy_to_fornax branch + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install dependencies + run: pip install pyaml + + - name: Deploy to deploy_to_fornax branch + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + gith=$(git rev-parse HEAD) + git checkout --orphan deploy_to_fornax + git rm --cached -r . + # Do Fornax preferred renaming + python .github/scripts/publish_fornax.py + git add mast_notebooks + git commit -m "Deploy for fornax for commit ${gith}" + git add notebook_metadata.yml + git commit -m "Adding notebook metadata" + git push origin deploy_to_fornax --force diff --git a/fornax-manifest.yml b/fornax-manifest.yml new file mode 100644 index 000000000..aed9bf906 --- /dev/null +++ b/fornax-manifest.yml @@ -0,0 +1,332 @@ +notebooks: + - title: Accessing Cloud-Hosted Data with Astroquery + file: notebooks/multi_mission/cloud_data_access/cloud_data_access.ipynb + section: Multi-Mission Tools + description: Learn how to discover and access cloud-hosted data from cloud-hosted repositories using the astroquery.mast module. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/multi_mission/cloud_data_access/requirements.txt + keywords: [Astroquery, Observations, Cloud] + + - title: Surveying dust structure via GALEX MIS + file: notebooks/GALEX/mis_mosaic/mis_mosaic.ipynb + section: GALEX + description: Create a composite UV mosaic of a high-latitute cloud, using cloud data from GALEX. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/GALEX/mis_mosaic/requirements.txt + keywords: [mosaic, dust, GALEX] + + - title: "Hubble Source Catalog API Notebook: SMC Color-Magnitude Diagram" + file: notebooks/HSC/HSCV3_SMC_API/hscv3_smc_api.ipynb + section: HST + description: Learn how to query the Hubble Source Catalog to make a color-magnitude diagram in the SMC. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/HSC/HSCV3_SMC_API/requirements.txt + keywords: [ ] + + - title: MAST Table Access Protocol Hubble Source Catalog Demo + file: notebooks/HSC/HSC_TAP/HSC_TAP.ipynb + section: HST + description: Demonstration of how to use astroquery to access the Hubble Source Catalog (HSC) via a Virtual Observatory standard Table Access Protocol (TAP) service at MAST, and work with the resultant data. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/HSC/HSC_TAP/requirements.txt + keywords: [ ] + + - title: Hubble Source Catalog SWEEPS Proper Motion (API Version) + file: notebooks/HSC/SWEEPS_HSCV3P1_API/sweeps_hscv3p1_api.ipynb + section: HST + description: Learn how to query the Hubble Source Catalog (HSC) for proper motions of over 400,000 stars in the Sagittarius Window Eclipsing Extrasolar Planet Search (SWEEPS) HST field. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/HSC/SWEEPS_HSCV3P1_API/requirements.txt + keywords: [ ] + + - title: Hubble Catalog of Variables Notebook (API version) + file: notebooks/HSC/HCV_API/HCV_API_demo.ipynb + section: HST + description: Demonstration of how to access the Hubble Catalog of Variables, containing faint variable objects from the Hubble Source Catalog (HSC) v3. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/HSC/HCV_API/requirements.txt + keywords: [ ] + + - title: Exploring UV Extinction Curves + file: notebooks/IUE/exploring_UV_extinction_curves/exploring_UV_extinction_curves.ipynb + section: IUE + description: Learn how to query, retrieve, and work with UV spectra taken by the International Ultraviolet Explorer (IUE). + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/IUE/exploring_UV_extinction_curves/requirements.txt + keywords: [ ] + + - title: "Astroquery: Exploring Metadata from the James Webb Space Telescope" + file: notebooks/JWST/MAST_metadata_search/MAST_metadata_search.ipynb + section: JWST + description: Learn how to use astroquery.mast to perform metadata queries and filtering of JWST observations, based on coordinates, object name, or non-positional criteria. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/JWST/MAST_metadata_search/requirements.txt + keywords: [Tutorial, JWST, Astroquery, MastMissions] + + ## KEPLER: Introductory Notebooks + - title: Using Kepler Data to Plot a Light Curve + file: notebooks/Kepler/plotting_lightcurves/plotting_lightcurves.ipynb + section: Kepler + subsection: Introductory Notebooks + description: This notebook tutorial demonstrates the process of loading and extracting information from Kepler light curve FITS files to plot a light curve and display the photometric aperture. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/plotting_lightcurves/requirements.txt + keywords: [ ] + + - title: Plotting Images from Kepler Target Pixel Files + file: notebooks/Kepler/plotting_images_from_tpf/plotting_images_from_tpf.ipynb + section: Kepler + subsection: Introductory Notebooks + description: This notebook tutorial demonstrates the loading and extracting of information from Kepler Target Pixel Files to plot images that show the pixels used to create data found in Kepler light curve files. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/plotting_images_from_tpf/requirements.txt + keywords: [ ] + + - title: Read and Plot A Kepler Data Validation Timeseries File + file: notebooks/Kepler/plotting_dvts/plotting_dvts.ipynb + section: Kepler + subsection: Introductory Notebooks + description: This notebook tutorial demonstrates how to load and plot the flux timeseries contained in a Kepler data validation timeseries (dvt) file. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/plotting_dvts/requirements.txt + keywords: [ ] + + - title: Plotting a Catalog over a Kepler Full Frame Image File + file: notebooks/Kepler/plotting_catalog_over_FFI/plotting_catalog_over_FFI.ipynb + section: Kepler + subsection: Introductory Notebooks + description: This tutorial demonstrates how to access the WCS (World Coordinate System) from a full frame image file and use this data to plot a catalog of objects over the FFI. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/plotting_catalog_over_FFI/requirements.txt + keywords: [ ] + + ## KEPLER: Identifying Periodic Signals + - title: notebooks/Kepler/identifying_transiting_planet_signals/identifying_transiting_planet_signals.ipynb + file: notebooks/Kepler/identifying_transiting_planet_signals/identifying_transiting_planet_signals.ipynb + section: Kepler + subsection: Identifying Periodic Signals + description: Use the Box Least Squares (BLS) method to identify transiting planet signals in light curves. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/identifying_transiting_planet_signals/requirements.txt + keywords: [ ] + + - title: notebooks/Kepler/measuring_a_rotation_period/measuring_a_rotation_period.ipynb + file: notebooks/Kepler/measuring_a_rotation_period/measuring_a_rotation_period.ipynb + section: Kepler + subsection: Identifying Periodic Signals + description: Not every oscillating signal is a planet; a key factor in stellar brightness is the star itself! This Notebook focuses on the impact of stellar rotation on a lightcurve, and how to create a model for this rotation. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/measuring_a_rotation_period/requirements.txt + keywords: [ ] + + - title: Visualizing Periodic Signals Using a River Plot + file: notebooks/Kepler/visualizing_periodic_signals_using_a_river_plot/visualizing_periodic_signals_using_a_river_plot.ipynb + section: Kepler + subsection: Identifying Periodic Signals + description: A river plot is a method of visualizing a periodic signal that changes over time. In this lesson, we'll use a multi-planetary system to look at a planet with a non-constant period. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/visualizing_periodic_signals_using_a_river_plot/requirements.txt + keywords: [ ] + + - title: Verifying the Location of a Signal in Kepler Pixel Data + file: notebooks/Kepler/verifying_the_location_of_a_signal/verifying_the_location_of_a_signal.ipynb + section: Kepler + subsection: Identifying Periodic Signals + description: Learn to decontaminate data, even in a crowded field, by performing a pixel-level analysis of the data. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/verifying_the_location_of_a_signal/requirements.txt + keywords: [ ] + + ## KEPLER: Periodograms & Astroseismology + - title: Creating Periodograms + file: notebooks/Kepler/creating_periodograms/creating_periodograms.ipynb + section: Kepler + subsection: Periodograms & Astroseismology + description: A periodogram is a tool for finding the "strongest" frequencies in a signal. This might be useful when trying to find the orbital period of an exoplanet in noisy data. In this notebook, you’ll learn about the mathematics behind periodograms and how to identify significant peaks. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/creating_periodograms/requirements.txt + keywords: [ ] + + - title: How to Understand and Manipulate the Periodogram of an Oscillating Star + file: notebooks/Kepler/how_to_understand_and_manipulate_the_periodogram_of_an_oscillating_star/how_to_understand_and_manipulate_the_periodogram_of_an_oscillating_star.ipynb + section: Kepler + subsection: Periodograms & Astroseismology + description: This notebooks focuses on the signals that come from instrinsic variable stars; namely, δ Scuti stars and solar-like oscillators. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/how_to_understand_and_manipulate_the_periodogram_of_an_oscillating_star/requirements.txt + keywords: [ ] + + - title: How to Estimate a Star's Mass and Radius Using Asteroseismology + file: notebooks/Kepler/how_to_estimate_a_stars_mass_and_radius_using_asteroseismology/how_to_estimate_a_stars_mass_and_radius_using_asteroseismology.ipynb + section: Kepler + subsection: Periodograms & Astroseismology + description: Buidling on work in the above notebook, this exercise teaches you how to estimate the mass and radius of a solar-like oscillator. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/Kepler/how_to_estimate_a_stars_mass_and_radius_using_asteroseismology/requirements.txt + keywords: [ ] + + - title: Removing Instrumental Noise from K2 and TESS Light Curves Using Pixel Level Decorrelation (PLD) + file: notebooks/K2/removing_instrumental_noise_using_pld/removing_instrumental_noise_using_pld.ipynb + section: K2 + description: Tutorial demonstrating how to use Lightkurve's PLDCorrector tool to remove instrumental noise from K2 and TESS light curves. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/K2/removing_instrumental_noise_using_pld/requirements.txt + keywords: [ ] + + - title: Working with FIMS-SPEAR Hyperspectral HEALPix Maps + file: notebooks/MCCM/FIMS-SPEAR/hyperspectral_healpix_maps/hyperspectral_healpix_maps.ipynb + section: MAST Community-Contributed Missions (MCCM) + description: Learn how to retrieve, extract, manipulate, and plot data from FIMS-SPEAR, plotting hyperspectral HEALPix maps in various projections and extract spectra from spatial regions of interest. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/MCCM/FIMS-SPEAR/hyperspectral_healpix_maps/requirements.txt + keywords: [ ] + + - title: Mapping Galaxy Properties with MaNGA + HST + file: notebooks/SDSS/MaNGA_HST_tutorial/MaNGA_HST_tutorial.ipynb + section: Sloan Digital Sky Survey (SDSS) + description: Tutorial for how to search and download MaNGA data at MAST using astroquery.mast. Combines H-alpha emission maps from MaNGA and images from HST to make galaxy maps for for interacting galaxy pair Mrk-848B. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/SDSS/MaNGA_HST_tutorial/requirements.txt + keywords: [Tutorial, SDSS, MaNGA, HST, galaxies] + + - title: Characterizing Variable Stars with APOGEE + TESS + file: notebooks/SDSS/APOGEE_TESS_tutorial/APOGEE_TESS_tutorial.ipynb + section: Sloan Digital Sky Survey (SDSS) + description: Tutorial for how to search and download APOGEE data at MAST using astroquery.mast. Combines infrared spectra from APOGEE and corresponding lightcurves from TESS to characterize different types of variable stars. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/SDSS/APOGEE_TESS_tutorial/requirements.txt + keywords: [Tutorial, SDSS, APOGEE, TESS, stars] + + - title: Exploring High-Redshift Quasars with eBOSS and JWST + file: notebooks/SDSS/eBOSS_JWST_tutorial/eBOSS_JWST_tutorial.ipynb + section: Sloan Digital Sky Survey (SDSS) + description: Tutorial for how to search and download eBOSS data at MAST using astroquery.mast. Characterizing emission lines in high-redshift quasars using optical spectra from eBOSS and infrared spectra from JWST. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/SDSS/eBOSS_JWST_tutorial/requirements.txt + keywords: [Tutorial, SDSS, eBOSS, JWST, QSO, quasar] + + - title: Calculating the Hubble Constant with eBOSS + file: notebooks/SDSS/eBOSS_HST_hubbles_law/hubbles_law.ipynb + section: Sloan Digital Sky Survey (SDSS) + description: Tutorial for how to search and download eBOSS data at MAST using astroquery.mast. Calculating the Hubble constant using spectral data from eBOSS as well as creating a color image of Abell 2199 using HST. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/SDSS/eBOSS_HST_hubbles_law/requirements.txt + keywords: [Tutorial, eBOSS, HST, SDSS, galaxies] + + - title: Creating a Color-Magnitude Diagram for Messier 13 + file: notebooks/SDSS/SDSS_LEGACY_IMAGING_cmd/cmd_diagram.ipynb + section: Sloan Digital Sky Survey (SDSS) + description: Tutorial for how to search and download SDSS Legacy Imaging Survey data and Gaia data from MAST. Calculating the age of M13 with Color-Magnitude Diagrams using parallax from Gaia and stellar isochrones from the PARSEC database. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/SDSS/SDSS_LEGACY_IMAGING_cmd/requirements.txt + keywords: [notebooks/SDSS/SDSS_LEGACY_IMAGING_cmd/cmd_diagram.ipynb] + + - title: Galaxy Morphology with SDSS and Astrocut + file: notebooks/SDSS/SDSS_LEGACY_IMAGING_galaxy_morphology/galaxy_morphology.ipynb + section: Sloan Digital Sky Survey (SDSS) + description: Tutorial for how to search and download SDSS Legacy Imaging Survey data from MAST and use the Astrocut Python package. Creating a Hubble Tuning Fork Diagram with cutout images to understand galaxy morphology. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/SDSS/SDSS_LEGACY_IMAGING_galaxy_morphology/requirements.txt + keywords: [notebooks/SDSS/SDSS_LEGACY_IMAGING_galaxy_morphology/galaxy_morphology.ipynb] + + - title: Exploring Stellar Spectra with SDSS SEGUE + file: notebooks/SDSS/SDSS_SEGUE_spectra/sdss_stellar_spectra.ipynb + section: Sloan Digital Sky Survey (SDSS) + description: Tutorial for how to search for and download SDSS Legacy Spectra data from MAST, plotting spectra for different classifications of stars from the SEGUE survey. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/SDSS/SDSS_SEGUE_spectra/requirements.txt + keywords: [notebooks/SDSS/SDSS_SEGUE_spectra/sdss_stellar_spectra.ipynb] + + - title: MAST Table Access Protocol PanSTARRS 1 DR2 Demo + file: notebooks/PanSTARRS/PS1_DR2_TAP/PS1_DR2_TAP.ipynb + section: PanSTARRS + description: This tutorial demonstrates how to access PanSTARRS 1 Data Release 2 via a Virtual Observatory standard Table Access Protocol (TAP) service at MAST. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/PanSTARRS/PS1_DR2_TAP/requirements.txt + keywords: [ ] + + - title: Read and Display a TESS Full Frame Image + file: notebooks/TESS/beginner_how_to_use_ffi/beginner_how_to_use_ffi.ipynb + section: TESS + description: A Full Frame Image (FFI) is created when TESS reads out the entire frame of all four of its cameras. Learn to plot an FFI. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/TESS/beginner_how_to_use_ffi/requirements.txt + keywords: [ ] + + - title: Read and Plot A TESS Light Curve File + file: notebooks/TESS/beginner_how_to_use_lc/beginner_how_to_use_lc.ipynb + section: TESS + description: MAST stores light curves that were created using TESS data. Learn how to read and plot them. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/TESS/beginner_how_to_use_lc/requirements.txt + keywords: [ ] + + - title: Search The TESS Input Catalog Centered On HD 209458 + file: notebooks/TESS/beginner_tic_search_hd209458/beginner_tic_search_hd209458.ipynb + section: TESS + description: The TESS Input Catalog is a collection of every bright, optically persistent object in the sky. Learn how to search it for a target of interest. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/TESS/beginner_tic_search_hd209458/requirements.txt + keywords: [ ] + + - title: "Intermediate: Finding Flares and Variable Stars in TASOC Light Curves" + file: notebooks/TESS/interm_tasoc_lc/interm_tasoc_lc.ipynb + section: TESS + description: Access TESS time series data from TASOC light curves while exploring a flaring star. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/TESS/interm_tasoc_lc/requirements.txt + keywords: [ ] + + - title: Finding the Rotation Curve of an Asteroid or Comet with TESScut and Lightkurve + file: notebooks/TESS/asteroid_rotation/asteroid_rotation_soutions.ipynb + section: TESS + description: Use TESS to create a light curve for the rotating asteroid Eleonora. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/TESS/asteroid_rotation/requirements.txt + keywords: [ ] + + - title: "Intermediate: Overlay a Cutout of the TESS FFIs with DSS imaging" + file: notebooks/TESS/interm_tesscut_dss_overlay/interm_tesscut_dss_overlay.ipynb + section: TESS + description: Create a cutout and light curve from an FFI, then overlay an image from DSS. + fornax-env: mast-fornax + extra-dependencies: + requirements-file: notebooks/TESS/interm_tesscut_dss_overlay/requirements.txt + keywords: [ ] + +global-requirements: + - requirements-file: requirements_fornax_mast.txt \ No newline at end of file diff --git a/requirements_fornax_mast.txt b/requirements_fornax_mast.txt new file mode 100644 index 000000000..5bde587a4 --- /dev/null +++ b/requirements_fornax_mast.txt @@ -0,0 +1,20 @@ +astrocut >= 1.2.0 +astropy >= 7.2.0 +astroquery @ git+https://github.com/astropy/astroquery.git +bokeh >= 3.9.0 +boto3 >= 1.43.0 +fastkde >= 2.1.5 +healpy >= 1.19.0 +ipywidgets >= 8.1.8 +lightkurve >= 2.6.0 +matplotlib >= 3.10.9 +numpy >= 2.4.6 +pandas >= 2.3.3 +photutils >= 3.0.0 +Pillow >= 12.2.0 +pyvo >= 1.8.1 +reproject >= 0.19.0 +requests >= 2.34.2 +scipy >= 1.17.1 +specutils >= 2.3.0 +s3fs >= 2026.4.0 \ No newline at end of file