Merge pull request #71 from nasa/cleanup

Tidying up the Repo

Author: danielfromearth

.git-hooks/README.md

@@ -72,6 +72,7 @@ python .git-hooks/check-unused-dependencies.py --include-optional
 
 # Exclude archived notebooks
 python .git-hooks/check-unused-dependencies.py --exclude-archived
+```
 
 ## Common Options
 

.git-hooks/check-notebook-imports.py

@@ -7,6 +7,7 @@
 
 import ast
 import sys
+import warnings
 
 try:
     import tomllib  # Python 3.11+
@@ -27,8 +28,21 @@
 )
 
 
+# Packages that are intentionally allowed but cannot or should not be declared
+# in pyproject.toml (local helper modules, platform-specific, proprietary, etc.)
+ALLOWLIST = {
+    # Local helper modules shipped alongside notebooks
+    "local_sageiii_plot",
+    "tolnet_v01",
+    # Proprietary / platform-specific (cannot be pip-installed)
+    "arcpy",
+    # Installed via !pip in notebook cells rather than as a project dependency
+    "gnss_lib_py",
+}
+
+
 def load_allowed_packages() -> Set[str]:
-    """Load package names from pyproject.toml."""
+    """Load package names from pyproject.toml plus the static allowlist."""
     pyproject_path = Path("pyproject.toml")
 
     if not pyproject_path.exists():
@@ -58,6 +72,9 @@ def load_allowed_packages() -> Set[str]:
     # Standard library (already lowercase, but be explicit)
     packages.update(get_stdlib_modules())
 
+    # Static allowlist for packages that can't be declared as dependencies
+    packages.update(ALLOWLIST)
+
     return packages
 
 
@@ -85,7 +102,10 @@ def extract_imports_from_code(code: str) -> Set[str]:
     imports = set()
 
     try:
-        tree = ast.parse(code)
+        # Suppress SyntaxWarnings from invalid escape sequences in notebook code
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", SyntaxWarning)
+            tree = ast.parse(code)
         for node in ast.walk(tree):
             if isinstance(node, ast.Import):
                 for alias in node.names:

.git-hooks/notebook_utils.py

@@ -261,6 +261,7 @@ def get_notebooks_from_args(args: argparse.Namespace) -> tuple[List[Path], str]:
 
         if args.exclude_archived:
             exclude_patterns.append("archived")
+            exclude_patterns.append("_archive")
 
         if args.exclude_additional_drafts:
             exclude_patterns.append("additional_drafts")
@@ -272,6 +273,7 @@ def get_notebooks_from_args(args: argparse.Namespace) -> tuple[List[Path], str]:
         exclusions = []
         if args.exclude_archived:
             exclusions.append("archived")
+            exclusions.append("_archive")
         if args.exclude_additional_drafts:
             exclusions.append("drafts")
         if exclusions:

.github/workflows/validate-pr.yml

@@ -0,0 +1,71 @@
+name: "Validate Pull Request"
+
+on:
+  pull_request:
+    branches: [main]
+
+permissions:
+  contents: read
+
+jobs:
+  lint:
+    name: Lint & format check
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - name: Ruff lint
+        run: uvx ruff check .
+
+      - name: Ruff format check
+        run: uvx ruff format --check .
+
+  notebooks:
+    name: Validate notebook structure
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+
+      - run: uv sync --frozen
+
+      - name: Check notebook headers
+        run: >
+          uv run python .git-hooks/check-notebook-headers.py
+          --all-notebooks --exclude-archived --exclude-additional-drafts
+
+      - name: Check notebook imports (advisory)
+        run: >
+          uv run python .git-hooks/check-notebook-imports.py
+          --all-notebooks --exclude-archived --exclude-additional-drafts --warn-only
+
+  render:
+    name: Quarto render (no deploy)
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: quarto-dev/quarto-actions/setup@v2
+
+      - uses: conda-incubator/setup-miniconda@v3
+        with:
+          activate-environment: quarto-env
+          environment-file: environment.yml
+
+      - name: Render site
+        uses: quarto-dev/quarto-actions/render@v2
+
+      - name: Upload rendered site
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: site-preview
+          path: _site/
+          retention-days: 5

.gitignore

@@ -139,3 +139,25 @@ dmypy.json
 .DS_Store
 
 index_files/*
+
+# Data or notebook files at the top-level in the repository
+*.nc
+*.h5
+*.hdf
+*.csv
+*.json
+*.txt
+*.zip
+*.tar.gz
+*.ipynb
+
+# Data and binary files in subdirectories (already blocked at top-level)
+**/*.nc
+**/*.h5
+**/*.hdf
+**/*.zip
+**/*.tar.gz
+**/*.gif
+
+# Keep small reference images but block large ones by convention
+# (rely on PR review for image files)

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ci:
   autoupdate_schedule: "monthly" # Like dependabot
   autoupdate_commit_msg: "chore: update pre-commit hooks"
-  autoupdate_branch: "develop"
+  autoupdate_branch: "main"
   autofix_prs: false # Comment "pre-commit.ci autofix" on a PR to trigger
 
 default_language_version:

POWER/index.qmd

@@ -1,5 +1,5 @@
 ---
-title: "POWER Prediction Of Worldwide Energy Resources) Project resources"
+title: "POWER (Prediction Of Worldwide Energy Resources) Project resources"
 date: last-modified
 author: "Atmospheric Science Data Center (ASDC)"
 ---

README.md

@@ -1,14 +1,68 @@
 # Data Tutorials and User Services for the ASDC
 
-This GitHub page serves as a comprehensive resource for end users
-seeking guidance and instructions on
-working with data from select missions that are archived and distributed by
-the Atmospheric Science Data Center (ASDC).
-Its aim is to enhance the knowledge and proficiency of data users,
-enabling more effective usage of the valuable resources made available by ASDC.
-Through this transfer of expertise from ASDC to users, we hope to
-foster a collaborative environment that drives innovation and progress in space-related endeavors.
-
-**View the rendered page [here](https://nasa.github.io/ASDC_Data_and_User_Services/).**
-
-*This site is under active, open development. Stay tuned for more and ever-evolving content!* 
+How-to guides, tutorials, and scripts for working with data from missions
+archived and distributed by NASA's
+[Atmospheric Science Data Center (ASDC)](https://www.earthdata.nasa.gov/centers/asdc-daac).
+The site covers CALIPSO, CERES, DSCOVR, MAIA, MISR, MOPITT, POWER, PREFIRE,
+SAGE III-ISS, STAQS, TEMPO, and TOLNet.
+
+**Browse the rendered site: <https://nasa.github.io/ASDC_Data_and_User_Services/>**
+
+*This site is under active, open development. Stay tuned for more and ever-evolving content!*
+
+## Getting started (contributors)
+
+You need [Python 3.10+](https://www.python.org/),
+[uv](https://docs.astral.sh/uv/), and
+[Quarto](https://quarto.org/docs/get-started/).
+
+```bash
+git clone https://github.com/nasa/ASDC_Data_and_User_Services.git
+cd ASDC_Data_and_User_Services
+
+uv sync              # install Python dependencies
+pre-commit install   # enable local quality checks on commit
+quarto preview       # live-preview the site at localhost
+```
+
+See the [Contributing Guide](https://nasa.github.io/ASDC_Data_and_User_Services/contributing.html)
+for notebook style and content guidelines.
+
+### Environment files
+
+This repo has two environment definitions. They serve different purposes:
+
+| File | Managed by | Purpose |
+|------|------------|---------|
+| `pyproject.toml` / `uv.lock` | `uv` | **Local development.** All packages needed to run every notebook. Use `uv sync` to install. |
+| `environment.yml` | conda (CI only) | **Site builds.** Minimal Jupyter kernel so Quarto can render pre-executed notebooks. You should not need this locally. |
+
+Quarto is configured with `freeze: true`, which means it never re-executes
+notebook code during a build. It converts the already-rendered `.ipynb` outputs
+straight to HTML. This keeps CI fast and avoids the need for data access
+credentials in the build environment.
+
+### Pre-commit hooks
+
+The pre-commit configuration runs on every commit:
+
+- **gitleaks** — secret detection
+- **ruff** — Python linting and formatting
+- **markdown-link-check** — broken link detection
+- **check-notebook-headers** — validates required sections (summary, prerequisites, author)
+- **check-notebook-imports** — cross-references imports against `pyproject.toml` (warn-only)
+
+See [`.git-hooks/README.md`](.git-hooks/README.md) for details on the custom
+notebook hooks.
+
+## Data files
+
+Notebooks should fetch data programmatically at runtime (e.g., via
+`earthaccess` or `harmony-py`) rather than committing data files to the
+repository. Large files such as `.nc`, `.hdf`, and `.h5` are not tracked
+by git and should stay that way.
+
+## Questions or issues?
+
+Open an [issue](https://github.com/nasa/ASDC_Data_and_User_Services/issues) or
+reach out via the [Earthdata Forum](https://forum.earthdata.nasa.gov/).