Merge branch 'main' into dependabot/pip/sphinx-gte-7.2-and-lt-10

luigilcsilva · web-flow · commit be0e3394d400 · 2026-01-07T12:35:28.000-03:00
diff --git a/.github/workflows/asv-nightly.yml b/.github/workflows/asv-nightly.yml
@@ -51,7 +51,7 @@ jobs:
         echo "yesterday=$(date -d yesterday +'%Y-%m-%d')" >> $GITHUB_OUTPUT
         echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT
     - name: Use last nightly commit hash from cache
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: ${{env.WORKING_DIR}}
         key: nightly-results-${{steps.nightly-dates.outputs.yesterday}}
@@ -66,7 +66,7 @@ jobs:
         fi
         echo $CURRENT_HASH > $HASH_FILE
     - name: Update last nightly hash in cache
-      uses: actions/cache@v4
+      uses: actions/cache@v5
       with:
         path: ${{env.WORKING_DIR}}
         key: nightly-results-${{steps.nightly-dates.outputs.today}}
diff --git a/.github/workflows/asv-pr.yml b/.github/workflows/asv-pr.yml
@@ -64,7 +64,7 @@ jobs:
       env:
         STEP_URL: ${{steps.jobs.outputs.html_url}}#step:10:1
     - name: Upload artifacts (PR number and benchmarks output)
-      uses: actions/upload-artifact@v5
+      uses: actions/upload-artifact@v6
       with:
         name: benchmark-artifacts
         path: ${{env.ARTIFACTS_DIR}}
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -13,7 +13,7 @@ jobs:
     permissions:
       contents: write
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v6
       - uses: actions/setup-python@v5
         with:
           python-version: "3.11"
diff --git a/.github/workflows/publish-benchmarks-pr.yml b/.github/workflows/publish-benchmarks-pr.yml
@@ -27,7 +27,7 @@ jobs:
         echo "Conclusion: ${{ github.event.workflow_run.conclusion }}"
         echo "Event: ${{ github.event.workflow_run.event }}"
     - name: Download artifact
-      uses: dawidd6/action-download-artifact@v11
+      uses: dawidd6/action-download-artifact@v12
       with:
         name: benchmark-artifacts
         run_id: ${{ github.event.workflow_run.id }}
diff --git a/.github/workflows/smoke-test.yml b/.github/workflows/smoke-test.yml
@@ -23,13 +23,13 @@ jobs:
         python-version: ['3.11', '3.12', '3.13']
 
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install uv
-      uses: astral-sh/setup-uv@v5
+      uses: astral-sh/setup-uv@v7
     - name: Install dependencies
       run: |
         sudo apt-get update
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,11 @@
 # Changelog
 
+## 0.1.1
+
+- Fix `score_density_hybrid` stage-1 de-duplication for LSDB catalogs by deriving unique IDs from pixel metadata and partition context.
+- Add tests for unique ID generation in Dask and LSDB paths.
+- Pin `sphinx-rtd-theme>=3.0,<4` to avoid Sphinx 7+ theme incompatibility; update docs for mag_global hist_peak clipping.
+
 ## 0.1.0
 
 - First publishable release of `hipscatalog-gen`.
diff --git a/README.md b/README.md
@@ -139,6 +139,7 @@ Each run generates a HiPS-compliant directory structure under output.out_dir:
 ## Mode Summary
 
 - **mag_global**: magnitude-complete slices across all depths.
+- mag_global hist_peak default bounds: when `adaptive_range=hist_peak` and `mag_min`/`mag_max` are not provided, the histogram range clips the global min/max to [-2, 40] (mag_min clipped to >= -2; mag_max from the peak within [-2, min(global_max, 40)]).
 - **score_global**: score-based slices across all depths.
 - **score_density_hybrid**: density-driven tiles for depths 1–3, then score slices for deeper levels.
 - Ordering and ties: `order_desc` controls ascending/descending (default ascending); optional `tie_column` breaks ties before falling back to RA/DEC.
diff --git a/docs/index.rst b/docs/index.rst
@@ -9,6 +9,7 @@ Overview
 - Three selection modes (``algorithm.selection_mode``):
 
   - ``mag_global``: magnitude-complete selection (see ``algorithm.mag_global.*``).
+  - ``mag_global`` hist_peak defaults: when no ``mag_min``/``mag_max`` are provided, the histogram range clips global min/max to [-2, 40] (min clipped to >= -2; max from the peak within [-2, min(global_max, 40)]).
   - ``score_global``: selection driven by an arbitrary score/expression (see ``algorithm.score_global.*``).
   - ``score_density_hybrid``: density-driven depths 1–3 with score-driven remainder (see ``algorithm.score_density_hybrid.*``).
 
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -7,4 +7,4 @@ nbsphinx
 sphinx
 sphinx-autoapi
 sphinx-copybutton
-sphinx-rtd-theme
+sphinx-rtd-theme>=3.0,<4
diff --git a/pyproject.toml b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "hipscatalog-gen"
-version = "0.1.0"
+version = "0.1.1"
 license = "MIT"
 license-files = ["LICENSE"]
 readme = { file = "README.md", content-type = "text/markdown" }
@@ -65,7 +65,7 @@ dev = [
 ]
 docs = [
   "sphinx>=7.2,<10",
-  "sphinx-rtd-theme>=2.0,<4",
+  "sphinx-rtd-theme>=3.0,<4",
   "myst-parser>=2.0,<5",
   "sphinx-autodoc-typehints>=2.0,<4",
 ]
diff --git a/src/hipscatalog_gen/score_density_hybrid/pipeline.py b/src/hipscatalog_gen/score_density_hybrid/pipeline.py
@@ -3,11 +3,20 @@
 from __future__ import annotations
 
 import time
-from typing import Any, Dict, Iterable, List
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List
 
 import numpy as np
 import pandas as pd
 
+if TYPE_CHECKING:
+    from lsdb.catalog import Catalog as LsdbCatalog
+else:  # Optional LSDB import for HATS catalogs.
+    LsdbCatalog: type[Any] | None
+    try:
+        from lsdb.catalog import Catalog as LsdbCatalog
+    except Exception:  # pragma: no cover - optional dependency
+        LsdbCatalog = None
+
 from ..io.output import build_header_line_from_keep
 from ..pipeline.common import maybe_persist_ddf, write_tiles_with_allsky
 from ..pipeline.params import ScoreDensityHybridParams
@@ -277,18 +286,71 @@ def prepare_score_density_hybrid(
     meta_with_id = meta_sel.copy()
     meta_with_id["__sdh_id__"] = pd.Series([], dtype="int64")
 
-    def _attach_id_safe(pdf: pd.DataFrame, partition_info=None) -> pd.DataFrame:
+    def _attach_id_partition(pdf: pd.DataFrame, partition_info=None, **kwargs) -> pd.DataFrame:
         """Attach __sdh_id__ deterministically using partition number."""
-        part_no = int(partition_info["number"]) if partition_info and "number" in partition_info else 0
+        if not isinstance(partition_info, dict) or "number" not in partition_info:
+            if pdf.empty:
+                pdf["__sdh_id__"] = pd.Series([], dtype="int64")
+                return pdf[meta_with_id.columns]
+            raise RuntimeError("score_density_hybrid: missing partition_info; cannot build unique IDs.")
+        part_no = int(partition_info["number"])
         base = np.int64(part_no) << 32
         seq = np.arange(len(pdf), dtype="int64")
         pdf = pdf.copy()
         pdf["__sdh_id__"] = base + seq
         return pdf[meta_with_id.columns]
 
-    ddf_sel = ddf_sel.map_partitions(
-        _attach_id_safe, meta=meta_with_id, partition_info=True, enforce_metadata=True
-    )
+    def _parse_pixel_order_pixel(pixel_obj) -> tuple[int | None, int | None]:
+        """Extract (order, pixel) from an LSDB pixel object or its string form."""
+        order = getattr(pixel_obj, "order", None)
+        pix = getattr(pixel_obj, "pixel", None)
+        if order is not None and pix is not None:
+            return int(order), int(pix)
+        try:
+            text = str(pixel_obj)
+            parts = text.replace(",", "").split()
+            if "Order:" in parts and "Pixel:" in parts:
+                order = int(parts[parts.index("Order:") + 1])
+                pix = int(parts[parts.index("Pixel:") + 1])
+                return order, pix
+        except Exception:
+            return None, None
+        return None, None
+
+    def _attach_id_pixel(pdf: pd.DataFrame, pixel=None, **kwargs) -> pd.DataFrame:
+        """Attach __sdh_id__ using (order, pixel, row_index)."""
+        if pdf.empty:
+            pdf["__sdh_id__"] = pd.Series([], dtype="int64")
+            return pdf[meta_with_id.columns]
+        order, pix = _parse_pixel_order_pixel(pixel)
+        if order is None or pix is None:
+            if pdf.empty:
+                pdf["__sdh_id__"] = pd.Series([], dtype="int64")
+                return pdf[meta_with_id.columns]
+            raise RuntimeError("score_density_hybrid: missing LSDB pixel metadata; cannot build unique IDs.")
+        base = (np.int64(order) << np.int64(58)) | (np.int64(pix) << np.int64(32))
+        seq = np.arange(len(pdf), dtype="int64")
+        pdf = pdf.copy()
+        pdf["__sdh_id__"] = base + seq
+        return pdf[meta_with_id.columns]
+
+    is_lsdb = LsdbCatalog is not None and isinstance(ddf, LsdbCatalog)
+    if is_lsdb:
+        log_fn(
+            "[score_density_hybrid] Using LSDB pixel-based IDs for stage-1 de-duplication.",
+            always=True,
+        )
+        ddf_sel = ddf_sel.map_partitions(
+            _attach_id_pixel, meta=meta_with_id, include_pixel=True, enforce_metadata=True
+        )
+    else:
+        log_fn(
+            "[score_density_hybrid] Using partition_info-based IDs for stage-1 de-duplication.",
+            always=True,
+        )
+        ddf_sel = ddf_sel.map_partitions(
+            _attach_id_partition, meta=meta_with_id, partition_info=True, enforce_metadata=True
+        )
 
     should_persist = persist_ddfs or (not avoid_computes)
     reason = "cluster.persist_ddfs=True" if persist_ddfs else "avoid_computes_wherever_possible=False"
diff --git a/tests/hipscatalog_gen/score_density_hybrid/test_score_density_hybrid_pipeline.py b/tests/hipscatalog_gen/score_density_hybrid/test_score_density_hybrid_pipeline.py
@@ -182,6 +182,101 @@ def fake_maybe_persist(ddf_like, should_persist, diag_ctx, log_fn, **kwargs):
     assert captured["diag_label"] == "dask_sdh_persist_filtered"
 
 
+def test_prepare_ids_unique_dask(diag_ctx, log_capture):
+    """Dask path attaches unique ids across partitions."""
+    _, log_fn = log_capture
+    pdf = pd.DataFrame(
+        {
+            "__score__": [0.1, 0.2, 0.3, 0.4],
+            "RA": [0.0, 1.0, 2.0, 3.0],
+            "DEC": [0.0, 1.0, 2.0, 3.0],
+        }
+    )
+    ddf = dd.from_pandas(pdf, npartitions=2)
+    cfg = _cfg()
+    params = SimpleNamespace(score_min=0.0, score_max=1.0, sentinel=None)
+
+    ddf_sel = sdh_pipeline.prepare_score_density_hybrid(
+        ddf,
+        cfg,
+        diag_ctx,
+        log_fn,
+        params,
+        persist_ddfs=False,
+        avoid_computes=True,
+    )
+    result = ddf_sel.compute()
+    assert result["__sdh_id__"].nunique() == len(result)
+    dropped = sdh_pipeline._drop_selected_ids(result, ids=result["__sdh_id__"].iloc[:2])
+    assert len(dropped) == len(result) - 2
+    assert not set(result["__sdh_id__"].iloc[:2]).intersection(set(dropped["__sdh_id__"]))
+
+
+def test_prepare_ids_unique_lsdb(monkeypatch, diag_ctx, log_capture):
+    """LSDB path attaches unique ids using pixel metadata."""
+    _, log_fn = log_capture
+
+    class FakePixel:
+        def __init__(self, order: int, pixel: int) -> None:
+            self.order = order
+            self.pixel = pixel
+
+        def __str__(self) -> str:
+            return f"Order: {self.order}, Pixel: {self.pixel}"
+
+    class FakeLsdbCatalog:
+        """Minimal LSDB-like catalog with include_pixel support."""
+
+        def __init__(self, parts: list[pd.DataFrame], pixels: list[FakePixel]) -> None:
+            self._parts = parts
+            self._pixels = pixels
+            self._ddf = None
+
+        def map_partitions(self, func, *args, meta=None, include_pixel=False, **kwargs):
+            out_parts = []
+            for pdf, pix in zip(self._parts, self._pixels, strict=True):
+                if include_pixel:
+                    out = func(pdf.copy(), pix, *args, **kwargs)
+                else:
+                    out = func(pdf.copy(), *args, **kwargs)
+                out_parts.append(out)
+            new = FakeLsdbCatalog(out_parts, self._pixels)
+            return new
+
+        def head(self, n=5):
+            return self.compute().head(n)
+
+        def to_delayed(self):
+            return [p for p in self._parts]
+
+        def compute(self):
+            return pd.concat(self._parts, ignore_index=True)
+
+    monkeypatch.setattr(sdh_pipeline, "LsdbCatalog", FakeLsdbCatalog)
+
+    parts = [
+        pd.DataFrame({"__score__": [0.1, 0.2], "RA": [0.0, 1.0], "DEC": [0.0, 1.0]}),
+        pd.DataFrame({"__score__": [0.3, 0.4], "RA": [2.0, 3.0], "DEC": [2.0, 3.0]}),
+    ]
+    pixels = [FakePixel(order=3, pixel=2), FakePixel(order=3, pixel=536)]
+    cat = FakeLsdbCatalog(parts, pixels)
+
+    cfg = _cfg()
+    params = SimpleNamespace(score_min=0.0, score_max=1.0, sentinel=None)
+
+    ddf_sel = sdh_pipeline.prepare_score_density_hybrid(
+        cat,
+        cfg,
+        diag_ctx,
+        log_fn,
+        params,
+        persist_ddfs=False,
+        avoid_computes=True,
+    )
+    result = ddf_sel.compute()
+    assert result["__sdh_id__"].nunique() == len(result)
+
+
 def test_attach_unique_id_and_drop_selected_ids():
     """Direct helper coverage for id attachment and dropping selected ids."""
     pdf = pd.DataFrame({"a": [1, 2, 3]})
