fix(dpa-adapt): resolve type-map validation, cache identity, label loading, and fparam_dim validation issues

- Split try/except in _resolve_type_maps so unsupported-element errors propagate
  instead of being silently swallowed as missing-atom-names
- Make read_data_type_map_union skip all-Type_* placeholder names, consistent
  with _read_data_type_map, so MFT does not reject valid raw-index data
- Add set.*/{key}.npy direct fallback to load_dataset for custom label files
  (e.g. homo.npy, bandgap.npy) not loaded into dpdata.System.data
- Replace first/last-64 sampling in _system_fingerprint with full-array hashing
  so descriptor cache keys correctly invalidate when structures change
- Validate fparam_dim as non-negative int in DPAFineTuner.__init__, matching
  DPATrainer and MFTFineTuner
- Add scikit-learn to the test extra so DPA-ADAPT tests can run in all CI paths

Author: zirenjin

dpa_adapt/data/dataset.py

@@ -64,6 +64,10 @@ def load_dataset(
         *every* candidate was skipped, in which case a ``DPADataError``
         is raised (fail-fast for training workflows).
     """
+    from dpa_adapt.data.loader import (
+        _get_source,
+    )
+
     systems = load_data(data)
 
     resolved_key = _resolve_label_key(label_key)
@@ -76,6 +80,25 @@ def load_dataset(
         # ``data`` dict; label_key (after alias resolution) presence is the litmus test.
         if resolved_key in system.data:
             validated.append(system)
+            continue
+
+        # Fallback: check set.*/{key}.npy directly (same logic as
+        # _load_labels() in finetuner.py).  Custom labels such as
+        # "homo.npy", "bandgap.npy" under set.*/ are not generally
+        # loaded into dpdata.System.data, so this direct check prevents
+        # valid datasets from being incorrectly skipped.
+        source = _get_source(system)
+        if source is not None:
+            source_path = Path(source)
+            set_dirs = sorted(source_path.glob("set.*"))
+            for sd in set_dirs:
+                if (sd / f"{resolved_key}.npy").exists():
+                    validated.append(system)
+                    break
+            else:
+                # None of the set.* dirs had the label file.
+                identifier = getattr(system, "_dpa_source", f"system[{i}]")
+                skipped.append(f"{identifier} (missing {resolved_key!r})")
         else:
             identifier = getattr(system, "_dpa_source", f"system[{i}]")
             skipped.append(f"{identifier} (missing {resolved_key!r})")

dpa_adapt/data/desc_cache.py

@@ -47,44 +47,46 @@ def _cache_dir() -> Path:
 
 
 # ---------------------------------------------------------------------------
-# lightweight system fingerprint (O(1) on array size, O(n) on atom count)
+# system fingerprint (O(n) over the full descriptor-relevant arrays)
 # ---------------------------------------------------------------------------
 
 
-def _system_fingerprint(system: dpdata.System) -> str:
-    """Return a short hex fingerprint for a dpdata System.
+def _hash_array(h: "hashlib._Hash", arr: np.ndarray) -> None:
+    """Fold an array's shape, dtype, and full byte content into *h*.
+
+    The contiguous buffer is fed to :meth:`hashlib._Hash.update` directly via
+    the buffer protocol, so no large intermediate ``bytes`` copy is made.
+    """
+    arr = np.ascontiguousarray(arr)
+    h.update(str(arr.shape).encode())
+    h.update(str(arr.dtype).encode())
+    h.update(arr)
 
-    Uses only metadata and a tiny sample of coordinate data so it is fast
-    even for large (10⁵+ frame) systems.  Collisions are possible in
-    principle but vanishingly unlikely in practice given the combination of
-    shape, dtype, atom_types, and first/last bytes.
+
+def _system_fingerprint(system: dpdata.System) -> str:
+    """Return a hex fingerprint for a dpdata System.
+
+    Hashes the *full* contents of the descriptor-relevant arrays — ``coords``,
+    ``cells`` and ``atom_types`` — together with ``atom_names``.  Sampling
+    only the first/last few entries (as an earlier version did) let any change
+    in the middle of a long trajectory keep the same key, so the cache could
+    return descriptors extracted from a different structure.  Hashing every
+    element costs O(total array size), but that is negligible next to the
+    descriptor extraction the cache guards, and it makes the key collision-safe
+    for changed systems.
     """
     d = system.data
-    coords = np.asarray(d["coords"])
-    atom_types = np.asarray(d["atom_types"])
 
     h = hashlib.sha1()
-    # structural identity
-    h.update(str(coords.shape).encode())
-    h.update(str(coords.dtype).encode())
-    h.update(atom_types.tobytes())
+    # atom-type identity
+    _hash_array(h, np.asarray(d["atom_types"]))
     # atom_names (if present)
     names = d.get("atom_names", [])
     h.update("|".join(str(n) for n in names).encode())
-    # first / last 64 bytes of coords (captures actual content without
-    # hashing the entire array)
-    if coords.size > 0:
-        flat = coords.ravel()
-        h.update(flat[: min(64, len(flat))].tobytes())
-        h.update(flat[-min(64, len(flat)) :].tobytes())
-    # same for cells, if present
+    # full geometry
+    _hash_array(h, np.asarray(d["coords"]))
     if "cells" in d:
-        cells = np.asarray(d["cells"])
-        h.update(str(cells.shape).encode())
-        if cells.size > 0:
-            fc = cells.ravel()
-            h.update(fc[: min(64, len(fc))].tobytes())
-            h.update(fc[-min(64, len(fc)) :].tobytes())
+        _hash_array(h, np.asarray(d["cells"]))
     return h.hexdigest()[:16]
 
 

dpa_adapt/data/type_map.py

@@ -95,6 +95,13 @@ def read_data_type_map_union(systems: list) -> list[str]:
     elems: set[str] = set()
     for sys in systems:
         names = sys.data.get("atom_names", [])
+        # dpdata generates "Type_0", "Type_1", ... when no type_map.raw was
+        # present.  Treat an all-placeholder type map as "no real atom_names"
+        # so that callers allow raw atom indices instead of rejecting valid
+        # data as unsupported elements (consistent with _read_data_type_map
+        # in finetuner.py).
+        if names and all(str(n).startswith("Type_") for n in names):
+            continue
         for name in names:
             if name:
                 elems.add(str(name))

dpa_adapt/finetuner.py

@@ -898,6 +898,10 @@ def __init__(
                 f"strategy must be one of {sorted(self._VALID_STRATEGIES)}; "
                 f"got {strategy!r}"
             )
+        if not isinstance(fparam_dim, int) or fparam_dim < 0:
+            raise ValueError(
+                f"fparam_dim must be a non-negative int; got {fparam_dim!r}."
+            )
 
         self.strategy = strategy
 
@@ -1099,9 +1103,10 @@ def _resolve_type_maps(self, train_data: str | list[str]) -> list[str]:
 
         try:
             elements = read_data_type_map_union(systems)
-            validate_type_map_subset(elements, tm, label="train data")
         except ValueError:
             pass  # no atom_names — deepmd uses raw atom indices
+        else:
+            validate_type_map_subset(elements, tm, label="train data")
 
         return tm
 

pyproject.toml

@@ -91,6 +91,8 @@ test = [
     "pytest-split",
     "pytest-timeout",
     "dpgui",
+    # DPA-ADAPT tests import sklearn via dpa_adapt.cv at module load time.
+    "scikit-learn",
     # to support Array API 2024.12
     'array-api-strict>=2.2;python_version>="3.9"',
 ]