Merge pull request #18 from zirenjin/master

fix convert() api and glob format

Author: zhaiwenxi

doc/dpa_adapt/README.md

@@ -54,7 +54,7 @@ model.fit(train_data="/data/qm9", aux_data="/data/spice2")
 ## Data preparation
 
 DPA-ADAPT trains on `deepmd/npy` data. Use `dpa-adapt data convert` (or the Python
-`auto_convert` helper) to route common inputs into the right conversion pipeline:
+`convert` helper) to route common inputs into the right conversion pipeline:
 
 - **SMILES CSV**: a `.csv` file with a `SMILES`/`smiles` column. RDKit generates 3D
   conformers, or existing `.mol`/`.sdf`/`.xyz`/`.pdb` files can be supplied with
@@ -67,19 +67,19 @@ DPA-ADAPT trains on `deepmd/npy` data. Use `dpa-adapt data convert` (or the Pyth
   inputs.
 
 ```python
-from dpa_adapt import auto_convert
+from dpa_adapt import convert
 
 # Structure file / trajectory → dpdata → deepmd/npy
-auto_convert("POSCAR", "./npy")
-auto_convert("OUTCAR", "./npy", fmt="vasp/outcar")
-auto_convert("traj.extxyz", "./npy", fmt="extxyz")
+convert("POSCAR", "./npy")
+convert("OUTCAR", "./npy", fmt="vasp/outcar")
+convert("traj.extxyz", "./npy", fmt="extxyz")
 
 # Glob patterns: one match is converted as one system; multiple matches are batched.
-auto_convert("calcs/**/OUTCAR", "./npy_root", fmt="vasp/outcar")
+convert("calcs/**/OUTCAR", "./npy_root", fmt="vasp/outcar")
 
 # CSV with a SMILES column → RDKit 3D conformers → deepmd/npy.
 # property_col names the input target column and output label name.
-auto_convert(
+convert(
     "molecules.csv",
     "./npy",
     fmt="smiles",          # optional when a SMILES/smiles column is present
@@ -89,7 +89,7 @@ auto_convert(
 )
 
 # CSV + pre-generated molecular structures: skip RDKit conformer generation.
-auto_convert(
+convert(
     "molecules.csv",
     "./npy",
     fmt="smiles",
@@ -103,7 +103,7 @@ auto_convert(
 # CSV: header required; defaults are formula_col="formula" and property_col="Property".
 # e.g.  formula,Property
 #       Ni0.65Gd0.15Fe0.10Co0.05Yb0.05O2H1,291.9
-auto_convert(
+convert(
     "compositions.csv",
     "./npy",
     fmt="formula",
@@ -134,10 +134,10 @@ dpa-adapt data convert --input "calcs/**/OUTCAR" --output ./npy_root --fmt vasp/
 Lower-level helpers:
 
 ```python
-from dpa_adapt import convert, batch_convert, attach_labels, check_data
+from dpa_adapt import convert, attach_labels, check_data
 
 convert("OUTCAR", "./npy", fmt="vasp/outcar")
-batch_convert("calcs/**/OUTCAR", "./npy_root", fmt="vasp/outcar")
+convert("calcs/**/OUTCAR", "./npy_root", fmt="vasp/outcar")
 attach_labels(system, head="bandgap", values=np.array([1.0, 2.0, 3.0]))
 check_data("/data/system")  # → list[Issue]
 ```
@@ -216,11 +216,9 @@ from dpa_adapt import (
     extract_descriptors,  # standalone descriptor extraction
     cross_validate,  # leak-proof cross-validation
     train_test_split,  # formula-grouped splitting
-    auto_convert,  # format-sniffing data conversion
+    convert,  # format-sniffing data conversion
     smiles_to_npy,  # CSV+SMILES → deepmd/npy
     formula_to_npy,  # composition formula CSV + POSCAR → deepmd/npy
-    convert,  # structure file → deepmd/npy
-    batch_convert,  # glob-based batch conversion
     check_data,  # data sanity checks
     attach_labels,  # inject label arrays
     load_dataset,  # label-filtered data loading

doc/dpa_adapt/input_formats.md

@@ -6,7 +6,8 @@
 > **Optional short alias:** `dpaad`
 > **Display name:** DPA-ADAPT — Atomistic DPA Adaptation for Property Tasks
 
-`dpa-adapt data convert` auto-detects the input type and routes it to the correct pipeline:
+`dpa-adapt data convert` and the Python `dpa_adapt.convert()` helper
+auto-detect the input type and route it to the correct pipeline:
 **SMILES table** → RDKit 3D conformer generation,
 **formula table** → random doping from a POSCAR template,
 **structure files** → dpdata (auto-detect or explicit `--fmt`).
@@ -45,19 +46,30 @@ dpaad data convert --input data.csv --output ./npy --fmt smiles \
     --split-seed 42 --conformer-seed 43
 ```
 
-## 2. Formula Tables (CSV + POSCAR Template)
+## 2. Formula Tables (CSV/TXT + POSCAR Template)
 
-**Trigger:** `--fmt formula`. Reads a CSV of elemental composition formulas
-(e.g. `Ni0.65Gd0.15O2H1`) and a template POSCAR, then generates doped structures
-by randomly substituting atoms on the host-element sublattice.
+**Trigger:** `--fmt formula`. Reads a table of elemental composition formulas
+(e.g. `Ni0.65Gd0.15O2H1`) and a template POSCAR, then generates doped
+structures by randomly substituting atoms on the host-element sublattice.
+
+Formula input supports two table styles:
+
+- Headered CSV/TSV: comma- or tab-delimited with named columns, such as
+  `formula,Property`.
+- Headered delimited text: comma, tab, semicolon, or pipe (`|`) delimiters
+  with named columns.
+- Headerless delimited or whitespace rows: use integer column indices, such as
+  `Ni0.65Gd0.15Fe0.10Co0.05Yb0.05O2H1    291.9` or
+  `Ni0.65Gd0.15Fe0.10Co0.05Yb0.05O2H1|291.9`.
 
 | Parameter | Default | Description |
 |-----------|---------|-------------|
 | `--poscar` | *(required)* | Template POSCAR file for the host lattice |
-| `--formula-col` | `formula` | Input CSV column name to read composition formulas from |
+| `--formula-col` | `formula` | Input table column to read composition formulas from; use a column name for headered files or a 0-based index for headerless whitespace files |
 | `--base-element` | auto | Host element to substitute. Inferred as the most frequent non-O/H element in the template if omitted. |
 | `--sets` | `1` | Number of random structures generated per formula row |
-| `--property-col` | `Property` | Input CSV column name to read target values from; also used as the output label name |
+| `--property-col` | `Property` | Input table column to read target values from; use a column name for headered files or a 0-based index for headerless whitespace files |
+| `--property-name` | value of `--property-col` | Output label name written as `set.*/{property_name}.npy` |
 | `--seed` | `42` | Random seed for selecting substituted host-atom sites |
 
 ```bash
@@ -68,6 +80,15 @@ dpa-adapt data convert --input compositions.csv --output ./npy --fmt formula \
 dpaad data convert --input compositions.csv --output ./npy --fmt formula \
     --poscar template.POSCAR --sets 3 \
     --formula-col formula --property-col bandgap
+
+# Headerless whitespace-delimited TXT: formula in column 0, target in column 1
+dpa-adapt data convert --input 20260514.txt --output ./npy --fmt formula \
+    --poscar template.POSCAR --formula-col 0 --property-col 1 \
+    --property-name overpotential
+
+# Headerless pipe-delimited TXT works the same way
+dpa-adapt data convert --input compositions.txt --output ./npy --fmt formula \
+    --poscar template.POSCAR --formula-col 0 --property-col 1
 ```
 
 ## 3. Structure Files via dpdata
@@ -155,46 +176,21 @@ dpaad data convert --input traj.extxyz --output ./npy --fmt extxyz
 
 ### Glob patterns
 
-When `--input` contains wildcards (`*`, `?`, `[`):
+When `--input` contains wildcards (`*`, `?`, `[`), conversion uses mirrored
+batch output:
 
-- **1 match** → treated as a single file (output directly into `--output`).
-- **N > 1 matches** → each match is converted into a numbered subdirectory
-  `{output}/sys_{i:04d}/` (zero-indexed, sorted).
+- **1 or more matches** → each matched file is converted into an output
+  directory that mirrors its path relative to the non-wildcard prefix.
 - **0 matches** → `FileNotFoundError`.
+- A `manifest.json` is written into the output root, recording converted and
+  skipped files.
 
 ```bash
-# Single match (only one OUTCAR found)
-dpa-adapt data convert --input "run*/OUTCAR" --output ./npy
-dpaad data convert --input "run*/OUTCAR" --output ./npy
-
-# Multi-match: outputs sys_0000/, sys_0001/, …
+# Glob output mirrors the input tree under ./npy_root
 dpa-adapt data convert --input "calcs/**/OUTCAR" --output ./npy_root --fmt vasp/outcar
 dpaad data convert --input "calcs/**/OUTCAR" --output ./npy_root --fmt vasp/outcar
 ```
 
-## 4. Batch Mode
-
-**Trigger:** `--input` with glob wildcards and N > 1 matches. Uses
-`batch_convert()` internally.
-
-Key behaviors:
-
-- Output directory tree mirrors the input tree structure (relative to the
-  non-wildcard prefix of the glob pattern).
-- A `manifest.json` is written into the output root, recording every
-  converted and skipped file.
-- When `--strict` is set, the first conversion error fails immediately.
-  Without it (default), errors are skipped and logged.
-
-```bash
-# Batch convert all OUTCAR files; each lands in a mirrored subdirectory
-dpa-adapt data convert --input "scan/**/OUTCAR" --output ./all_npy --fmt vasp/outcar
-dpaad data convert --input "scan/**/OUTCAR" --output ./all_npy --fmt vasp/outcar
-
-# Strict mode — abort on first failure
-dpa-adapt data convert --input "scan/**/OUTCAR" --output ./all_npy --fmt vasp/outcar --strict
-dpaad data convert --input "scan/**/OUTCAR" --output ./all_npy --fmt vasp/outcar --strict
-
-# Check the manifest
-cat ./all_npy/manifest.json
-```
+For example, `calcs/run1/OUTCAR` is written as `npy_root/run1/OUTCAR/`.
+When `--strict` is set, the first conversion error fails immediately. Without
+it, errors are skipped and logged in the manifest.

dpa_adapt/__init__.py

@@ -18,8 +18,6 @@
     "MFTFineTuner",
     "SmilesDataResult",
     "attach_labels",
-    "auto_convert",
-    "batch_convert",
     "check_data",
     "convert",
     "cross_validate",
@@ -37,8 +35,6 @@
     "train_test_split": (".cv", "train_test_split"),
     "SmilesDataResult": (".data", "SmilesDataResult"),
     "attach_labels": (".data", "attach_labels"),
-    "auto_convert": (".data", "auto_convert"),
-    "batch_convert": (".data", "batch_convert"),
     "check_data": (".data", "check_data"),
     "convert": (".data", "convert"),
     "formula_to_npy": (".data", "formula_to_npy"),

dpa_adapt/cli.py

@@ -246,32 +246,13 @@ def _cmd_evaluate(args: argparse.Namespace) -> int:
 def _cmd_data_convert(args: argparse.Namespace) -> int:
 
     type_map = _maybe_split_list(args.type_map)
-    input_val = args.input
 
-    # Detect glob patterns — batch mode.
-    if any(ch in input_val for ch in "*?["):
-        from dpa_adapt import (
-            batch_convert,
-        )
-
-        outputs = batch_convert(
-            glob_pattern=input_val,
-            output_dir=args.output,
-            fmt=args.fmt or "auto",
-            type_map=type_map,
-            validate=args.validate,
-            strict=args.strict,
-        )
-        _LOG.info("Wrote %d deepmd/npy dirs under %s", len(outputs), args.output)
-        return 0
-
-    # Single-file mode.
-    from dpa_adapt.data.convert import (
-        auto_convert,
+    from dpa_adapt import (
+        convert,
     )
 
-    result = auto_convert(
-        input_path=input_val,
+    result = convert(
+        input_path=args.input,
         output_dir=args.output,
         fmt=args.fmt,
         type_map=type_map,
@@ -301,6 +282,9 @@ def _cmd_data_convert(args: argparse.Namespace) -> int:
         print(f"Failed rows  : {len(result['failed_rows'])}")
         print(f"Skipped zero : {result['skipped_zero']}")
         print(f"Skipped overlap: {result['skipped_overlap']}")
+    elif result["method"] == "batch_dpdata":
+        print(f"Output dirs  : {len(result['output_dirs'])}")
+        print(f"Manifest     : {result['manifest']}")
     else:
         _LOG.info("Wrote deepmd/npy → %s", result["output_dir"])
     return 0

dpa_adapt/data/__init__.py

@@ -11,8 +11,6 @@
     "Issue",
     "SmilesDataResult",
     "attach_labels",
-    "auto_convert",
-    "batch_convert",
     "check_data",
     "convert",
     "formula_to_npy",
@@ -32,10 +30,8 @@
     "read_checkpoint_type_map": (".type_map", "read_checkpoint_type_map"),
     "read_data_type_map_union": (".type_map", "read_data_type_map_union"),
     "validate_type_map_subset": (".type_map", "validate_type_map_subset"),
-    "auto_convert": (".convert", "auto_convert"),
     "convert": (".convert", "convert"),
     "attach_labels": (".convert", "attach_labels"),
-    "batch_convert": (".convert", "batch_convert"),
     "formula_to_npy": (".formula", "formula_to_npy"),
     "check_data": (".validate", "check_data"),
     "Issue": (".validate", "Issue"),