Dataverse

.gitignore

@@ -91,4 +91,7 @@ ENV/
 
 # HashStore settings
 # Allow hashstore/ and its immediate subfolders
-hashstore/*
+hashstore/*
+
+# Our dataverse ignore rules
+.venv/

metadig/checks.py

@@ -255,7 +255,15 @@ def run_check(
     sysmeta_check_vars = get_sysmeta_vars(metadata_sysmeta_path)
     identifier = sysmeta_check_vars.get("identifier")
     auth_mn_node = sysmeta_check_vars.get("authoritative_member_node")
-    data_pids = get_data_pids(identifier, auth_mn_node)
+
+    # Try to get data PIDs from DataONE, but don't fail if network is unavailable
+    try:
+        data_pids = get_data_pids(identifier, auth_mn_node)
+    except Exception:
+        # If we can't reach DataONE (network issues, SSL errors, etc.),
+        # just use an empty list. Most metadata checks don't need data PIDs.
+        data_pids = []
+
     # TODO: Potential Optimization Point
     # `variables-congruent` is the only check at this time that requires it, and seems
     # like we're adding a lot of overhead for just one check.

requirements.txt

@@ -0,0 +1,47 @@
+anyio==4.11.0
+build==1.3.0
+CacheControl==0.14.3
+certifi==2025.10.5
+cffi==2.0.0
+charset-normalizer==3.4.4
+cleo==2.1.0
+crashtest==0.4.1
+distlib==0.4.0
+dulwich==0.24.7
+fastjsonschema==2.21.2
+filelock==3.20.0
+findpython==0.7.0
+h11==0.16.0
+httpcore==1.0.9
+httpx==0.28.1
+idna==3.11
+installer==0.7.0
+jaraco.classes==3.4.0
+jaraco.context==6.0.1
+jaraco.functools==4.3.0
+keyring==25.6.0
+lxml==6.0.2
+more-itertools==10.8.0
+msgpack==1.1.2
+packaging==25.0
+pbs-installer==2025.10.14
+pkginfo==1.12.1.2
+platformdirs==4.5.0
+poetry==2.2.1
+poetry-core==2.2.1
+pycparser==2.23
+pyproject_hooks==1.2.0
+python-dotenv==1.2.1
+RapidFuzz==3.14.1
+requests==2.32.5
+requests-toolbelt==1.0.0
+setuptools==80.9.0
+shellingham==1.5.4
+sniffio==1.3.1
+tomlkit==0.13.3
+trove-classifiers==2025.9.11.17
+typing_extensions==4.15.0
+urllib3==2.5.0
+virtualenv==20.35.3
+xattr==1.3.0
+zstandard==0.25.0

scripts/dataverse_to_metadig.py

@@ -0,0 +1,172 @@
+#!/usr/bin/env python3
+"""
+Fetch metadata from a Dataverse dataset and prepare inputs for MetaDIG-py.
+
+Example:
+  poetry run python scripts/dataverse_to_metadig.py \
+    --base-url https://dataverse.grit.ucsb.edu \
+    --persistent-id doi:10.5072/FK2/IRJI89 \
+    --api-key $DATAVERSE_KEY \
+    --metadata-out /tmp/dataset.xml \
+    --sysmeta-out /tmp/sysmeta.xml \
+    --authoritative-member-node urn:node:KNB
+
+Then run:
+  poetry run metadigpy -runcheck \
+    -store_path=$(pwd)/hashstore \
+    -check_xml=$(pwd)/tests/testdata/checks/resource.license.present-2.0.0.xml \
+    -metadata_doc=/tmp/dataset.xml \
+    -sysmeta_doc=/tmp/sysmeta.xml
+"""
+
+import argparse
+import json
+import os
+import sys
+import urllib.parse
+import urllib.request
+import ssl
+from datetime import datetime, timezone
+
+
+def http_get(url: str, headers: dict | None = None, timeout: int = 60, insecure: bool = False) -> bytes:
+    req = urllib.request.Request(url, headers=headers or {})
+    context = ssl._create_unverified_context() if insecure else None
+    with urllib.request.urlopen(req, timeout=timeout, context=context) as resp:
+        return resp.read()
+
+
+def export_eml(base_url: str, persistent_id: str, api_key: str | None, out_path: str, insecure: bool = False) -> None:
+    export_url = (
+        f"{base_url.rstrip('/')}/api/datasets/export?"
+        f"exporter=eml&persistentId={urllib.parse.quote(persistent_id)}"
+    )
+    headers = {"X-Dataverse-key": api_key} if api_key else {}
+    data = http_get(export_url, headers=headers, insecure=insecure)
+    with open(out_path, "wb") as f:
+        f.write(data)
+
+
+def fetch_dataset_json(base_url: str, persistent_id: str, api_key: str | None, insecure: bool = False) -> dict:
+    url = (
+        f"{base_url.rstrip('/')}/api/v1/datasets/:persistentId/?persistentId="
+        f"{urllib.parse.quote(persistent_id)}"
+    )
+    headers = {"X-Dataverse-key": api_key} if api_key else {}
+    raw = http_get(url, headers=headers, insecure=insecure)
+    return json.loads(raw.decode("utf-8"))
+
+
+def pick_rights_holder(dataset_json: dict, fallback: str | None) -> str:
+    if fallback:
+        return fallback
+    try:
+        latest = dataset_json["data"]["latestVersion"]
+        # Prefer a contact name if available
+        for field in latest.get("metadataBlocks", {}).get("citation", {}).get("fields", []):
+            if field.get("typeName") == "datasetContact":
+                vals = field.get("value", [])
+                if vals and isinstance(vals, list):
+                    contact = vals[0]
+                    name = contact.get("datasetContactName", {}).get("value")
+                    if name:
+                        return name
+        # Fallback to depositor
+        for field in latest.get("metadataBlocks", {}).get("citation", {}).get("fields", []):
+            if field.get("typeName") == "depositor":
+                val = field.get("value")
+                if isinstance(val, str) and val:
+                    return val
+    except Exception:
+        pass
+    return "public"
+
+
+def write_sysmeta_stub(
+    identifier: str,
+    authoritative_member_node: str,
+    rights_holder: str,
+    file_name: str,
+    out_path: str,
+    format_id: str = "https://eml.ecoinformatics.org/eml-2.2.0",
+) -> None:
+    now = datetime.now(timezone.utc).isoformat()
+    xml = f'''<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<ns3:systemMetadata xmlns:ns2="http://ns.dataone.org/service/types/v1" xmlns:ns3="http://ns.dataone.org/service/types/v2.0">
+    <serialVersion>1</serialVersion>
+    <identifier>{identifier}</identifier>
+    <formatId>{format_id}</formatId>
+    <size>0</size>
+    <checksum algorithm="MD5">00000000000000000000000000000000</checksum>
+    <submitter>{rights_holder}</submitter>
+    <rightsHolder>{rights_holder}</rightsHolder>
+    <accessPolicy>
+        <allow>
+            <subject>public</subject>
+            <permission>read</permission>
+        </allow>
+    </accessPolicy>
+    <replicationPolicy replicationAllowed="false" numberReplicas="0"/>
+    <archived>false</archived>
+    <dateUploaded>{now}</dateUploaded>
+    <dateSysMetadataModified>{now}</dateSysMetadataModified>
+    <originMemberNode>{authoritative_member_node}</originMemberNode>
+    <authoritativeMemberNode>{authoritative_member_node}</authoritativeMemberNode>
+    <fileName>{file_name}</fileName>
+</ns3:systemMetadata>
+'''
+    with open(out_path, "w", encoding="utf-8") as f:
+        f.write(xml)
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description="Prepare MetaDIG-py inputs from Dataverse")
+    parser.add_argument("--base-url", required=True, help="Dataverse base URL")
+    parser.add_argument("--persistent-id", required=True, help="Dataset persistentId")
+    parser.add_argument("--api-key", help="Dataverse API key for private/restricted datasets")
+    parser.add_argument("--metadata-out", required=True, help="Path to write exported EML")
+    parser.add_argument("--sysmeta-out", required=True, help="Path to write sysmeta stub XML")
+    parser.add_argument(
+        "--authoritative-member-node",
+        default="urn:node:KNB",
+        help="DataONE member node ID to embed in stub (default: urn:node:KNB)",
+    )
+    parser.add_argument("--rights-holder", help="Override rightsHolder; else inferred from dataset")
+    parser.add_argument(
+        "--format-id",
+        default="https://eml.ecoinformatics.org/eml-2.2.0",
+        help="FormatId for metadata (default: EML 2.2.0)",
+    )
+    parser.add_argument(
+        "--insecure",
+        action="store_true",
+        help="Disable SSL certificate verification for Dataverse requests (use only if you trust the server)",
+    )
+    args = parser.parse_args(argv)
+
+    # 1. Export EML
+    export_eml(args.base_url, args.persistent_id, args.api_key, args.metadata_out, insecure=args.insecure)
+
+    # 2. Fetch dataset JSON to infer rights holder if not provided
+    ds_json = fetch_dataset_json(args.base_url, args.persistent_id, args.api_key, insecure=args.insecure)
+    rights_holder = pick_rights_holder(ds_json, args.rights_holder)
+
+    # 3. Create sysmeta stub
+    file_name = os.path.basename(args.metadata_out) or "metadata.xml"
+    write_sysmeta_stub(
+        identifier=args.persistent_id,
+        authoritative_member_node=args.authoritative_member_node,
+        rights_holder=rights_holder,
+        file_name=file_name,
+        out_path=args.sysmeta_out,
+        format_id=args.format_id,
+    )
+
+    print("Prepared:")
+    print(f"  metadata: {args.metadata_out}")
+    print(f"  sysmeta : {args.sysmeta_out}")
+    print("Next: run metadigpy -runcheck or -runsuite with these paths.")
+
+
+if __name__ == "__main__":
+    main()