chg: Move HuggingFace card templates to vulntrain/cards/

Move dataset and model card templates from datasets/ and trainers/ into
a dedicated cards/ directory. Add dynamic dataset card generation for
multi-source datasets with per-source breakdown table.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: cedricbonhomme

vulntrain/datasets/dataset_card_cnvd.md vulntrain/cards/dataset_card_cnvd.mdvulntrain/datasets/dataset_card_cnvd.md renamed to vulntrain/cards/dataset_card_cnvd.md

@@ -0,0 +1,78 @@
+---
+task_categories:
+- text-classification
+license: cc-by-4.0
+library_name: datasets
+tags:
+- vulnerability
+- cybersecurity
+- security
+- cve
+- cvss
+---
+
+# vulnerability-scores
+
+This dataset comprises **{total_entries:,}** real-world vulnerabilities used to train and evaluate VLAI, a transformer-based model designed to predict software vulnerability severity levels directly from text descriptions, enabling faster and more consistent triage.
+
+The dataset is presented in the paper [VLAI: A RoBERTa-Based Model for Automated Vulnerability Severity Classification](https://huggingface.co/papers/2507.03607).
+
+Project page: [https://vulnerability.circl.lu](https://vulnerability.circl.lu)
+Associated code: [https://github.com/vulnerability-lookup/ML-Gateway](https://github.com/vulnerability-lookup/ML-Gateway)
+
+## Sources
+
+{source_table}
+
+Extracted from the database of [Vulnerability-Lookup](https://vulnerability.circl.lu).
+Dumps of the data are available [here](https://vulnerability.circl.lu/dumps/).
+
+## Splits
+
+| Split | Examples |
+|-------|----------|
+| train | {train_examples:,} |
+| test  | {test_examples:,} |
+
+## Fields
+
+| Field | Type | Description |
+|-------|------|-------------|
+| `id` | string | Vulnerability identifier (e.g., CVE-2024-1234, GHSA-xxxx, PYSEC-2024-xxx) |
+| `title` | string | Vulnerability title |
+| `description` | string | Vulnerability description in English |
+| `cpes` | list[string] | Common Platform Enumeration identifiers |
+| `cvss_v4_0` | float | CVSS v4.0 score |
+| `cvss_v3_1` | float | CVSS v3.1 score |
+| `cvss_v3_0` | float | CVSS v3.0 score |
+| `cvss_v2_0` | float | CVSS v2.0 score |
+| `patch_commit_url` | string | URL to the patch commit on GitHub, if available |
+| `source` | string | Data source identifier |
+
+## Usage
+
+```python
+import json
+from datasets import load_dataset
+
+dataset = load_dataset("{repo_id}")
+
+vulnerabilities = ["CVE-2012-2339", "RHSA-2023:5964", "GHSA-7chm-34j8-4f22", "PYSEC-2024-225"]
+
+filtered_entries = dataset.filter(lambda elem: elem["id"] in vulnerabilities)
+
+for entry in filtered_entries["train"]:
+    print(json.dumps(entry, indent=4))
+```
+
+## Related models
+
+- [CIRCL/vulnerability-severity-classification-roberta-base](https://huggingface.co/CIRCL/vulnerability-severity-classification-roberta-base) — RoBERTa severity classifier
+- [CIRCL/vulnerability-severity-classification-distilbert-base-uncased](https://huggingface.co/CIRCL/vulnerability-severity-classification-distilbert-base-uncased) — DistilBERT severity classifier
+
+## References
+
+- [Vulnerability-Lookup](https://vulnerability.circl.lu) — the vulnerability data source
+- [VulnTrain](https://github.com/vulnerability-lookup/VulnTrain) — training pipeline
+- [ML-Gateway](https://github.com/vulnerability-lookup/ML-Gateway) — inference API
+- [VLAI paper](https://arxiv.org/abs/2507.03607) — Bonhomme, C., Dulaunoy, A. (2025)

vulntrain/cards/dataset_card_vulnerability_scores.md

@@ -1,5 +1,6 @@
 import argparse
 import json
+from collections import Counter
 from pathlib import Path
 from typing import Any, Generator
 
@@ -227,6 +228,58 @@ def extract_patch_commit_url(self, vuln: dict[str, Any]) -> str | None:
         return None
 
 
+SOURCE_LABELS = {
+    "cvelistv5": "CVE Program (enriched with vulnrichment and Fraunhofer FKIE)",
+    "github": "GitHub Security Advisories",
+    "pysec": "PySec advisories",
+    "csaf_redhat": "CSAF Red Hat",
+    "csaf_cisco": "CSAF Cisco",
+    "csaf_cisa": "CSAF CISA",
+    "cnvd": "China National Vulnerability Database (CNVD)",
+}
+
+
+def _generate_dataset_card(
+    sources: list[str],
+    vulns: list[dict[str, Any]],
+    dataset_dict: DatasetDict,
+    repo_id: str,
+) -> str | None:
+    """Generate a dataset card from a template, populated with actual stats."""
+    # Single-source: use static template if available
+    if len(sources) == 1:
+        static_card = Path(__file__).parent.parent / "cards" / f"dataset_card_{sources[0]}.md"
+        if static_card.exists():
+            return static_card.read_text()
+        return None
+
+    # Multi-source: use the vulnerability-scores template
+    template_path = (
+        Path(__file__).parent.parent / "cards" / "dataset_card_vulnerability_scores.md"
+    )
+    if not template_path.exists():
+        return None
+
+    total = len(vulns)
+    source_counts = Counter(v.get("source", "unknown") for v in vulns)
+
+    # Build source table sorted by count descending
+    rows = ["| Source | Label | Entries | Share |", "|--------|-------|---------|-------|"]
+    for src, count in source_counts.most_common():
+        label = SOURCE_LABELS.get(src, src)
+        pct = 100 * count / total if total else 0
+        rows.append(f"| `{src}` | {label} | {count:,} | {pct:.1f}% |")
+
+    template = template_path.read_text()
+    return template.format(
+        repo_id=repo_id,
+        total_entries=total,
+        source_table="\n".join(rows),
+        train_examples=len(dataset_dict["train"]),
+        test_examples=len(dataset_dict["test"]),
+    )
+
+
 def main():
     parser = argparse.ArgumentParser(description="Dataset generation.")
     parser.add_argument(
@@ -270,22 +323,26 @@ def main():
         else:
             dataset_dict.push_to_hub(args.repo_id)
 
-        # Push dataset card if available for this source
-        source_key = sources[0] if len(sources) == 1 else None
-        if source_key:
-            card_path = Path(__file__).parent / f"dataset_card_{source_key}.md"
-            if card_path.exists():
-                from huggingface_hub import HfApi
-
-                api = HfApi()
-                api.upload_file(
-                    path_or_fileobj=str(card_path),
-                    path_in_repo="README.md",
-                    repo_id=args.repo_id,
-                    repo_type="dataset",
-                    commit_message="Update dataset card",
-                )
-                print(f"Dataset card pushed to {args.repo_id}")
+        # Push dataset card
+        card_content = _generate_dataset_card(
+            sources, vulns, dataset_dict, args.repo_id
+        )
+        if card_content:
+            from huggingface_hub import HfApi
+
+            card_path = Path("dataset_card_generated.md")
+            card_path.write_text(card_content)
+
+            api = HfApi()
+            api.upload_file(
+                path_or_fileobj=str(card_path),
+                path_in_repo="README.md",
+                repo_id=args.repo_id,
+                repo_type="dataset",
+                commit_message="Update dataset card",
+            )
+            card_path.unlink()
+            print(f"Dataset card pushed to {args.repo_id}")
 
 
 if __name__ == "__main__":

…ain/trainers/model_card_cnvd_severity.md …ntrain/cards/model_card_cnvd_severity.mdvulntrain/trainers/model_card_cnvd_severity.md renamed to vulntrain/cards/model_card_cnvd_severity.md vulntrain/trainers/model_card_cnvd_severity.md renamed to vulntrain/cards/model_card_cnvd_severity.md

@@ -358,7 +358,7 @@ def tokenize_function(examples):
         template_vars[f"{label_name}_support"] = f"{count:,}"
         template_vars[f"{label_name}_pct"] = 100 * count / test_samples if test_samples else 0
 
-    model_card_template = Path(__file__).parent / "model_card_cnvd_severity.md"
+    model_card_template = Path(__file__).parent.parent / "cards" / "model_card_cnvd_severity.md"
     if model_card_template.exists():
         from huggingface_hub import HfApi