Python: Generate model with required fields also

.github/workflows/build-java.yml

@@ -5,14 +5,14 @@ on:
     branches:
       - main
     paths:
-    - "generated/java/**"
-    - ".github/workflows/build-java.yml"
+      - "generated/java/**"
+      - ".github/workflows/build-java.yml"
   pull_request:
     branches:
       - main
     paths:
-    - "generated/java/**"
-    - ".github/workflows/build-java.yml"
+      - "generated/java/**"
+      - ".github/workflows/build-java.yml"
 
 jobs:
   build:
@@ -39,14 +39,13 @@ jobs:
           token_format: access_token
 
       - name: Cache Maven packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
           restore-keys: ${{ runner.os }}-m2
 
       - name: Build with Maven and deploy to Artifact Registry
         run: |
-            pushd generated/java/datadoc-model
-            ./mvnw --batch-mode -P ssb deploy
-
+          pushd generated/java/datadoc-model
+          ./mvnw --batch-mode -P ssb deploy

.github/workflows/generate-python.yml

@@ -25,7 +25,7 @@ jobs:
           private-key: ${{ secrets.DAPLA_BOT_PRIVATE_KEY }}
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
           token: ${{ steps.generate_token.outputs.token }}
@@ -35,35 +35,19 @@ jobs:
           git config user.name "dapla-bot[bot]"
           git config user.email "143391972+dapla-bot[bot]@users.noreply.github.com"
 
-      - uses: actions/setup-python@v2
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
         with:
-          python-version: 3.11.5
+          version: "0.7.2"
+          enable-cache: true
 
-      - name: Install pipx
-        run: |
-          python -m pip install --upgrade pipx
-          python -m pipx ensurepath
-
-      - name: Install datamodel-codegen
-        run: |
-          python -m pipx install datamodel-code-generator==0.25.2
+      - name: Generate All-optional Python models
+        run: >
+          uv run --no-project --with datamodel-code-generator -- ./bin/generate_python.sh --output=generated/python/datadoc_model/datadoc_model/all_optional/model.py --force-optional
 
-      - name: Generate Python
+      - name: Generate Required Python models
         run: >
-          datamodel-codegen
-          --input-file-type jsonschema
-          --input src/metadata-container-json-schema.json
-          --output-model-type pydantic_v2.BaseModel
-          --base-class "datadoc_model.datadoc_base_model.DatadocBaseModel"
-          --use-default
-          --use-title-as-name
-          --use-one-literal-as-default
-          --force-optional
-          --use-subclass-enum
-          --use-standard-collections
-          --use-double-quotes
-          --target-python-version 3.10
-          --output generated/python/datadoc_model/datadoc_model/model.py
+          uv run --no-project --with datamodel-code-generator -- ./bin/generate_python.sh --output=generated/python/datadoc_model/datadoc_model/required/model.py
 
       - name: Check for modified files
         id: git-check

.github/workflows/release-java.yml

@@ -41,7 +41,7 @@ jobs:
           token_format: access_token
 
       - name: Cache Maven packages
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.m2
           key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}

.github/workflows/test-python.yml

@@ -21,12 +21,12 @@ jobs:
       run:
         working-directory: ./generated/python/datadoc_model
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v1
         with:
           python-version: "3.11"
       - uses: Gr1N/setup-poetry@v8
-      - uses: actions/cache@v2
+      - uses: actions/cache@v4
         with:
           path: ~/.cache/pypoetry/virtualenvs
           key: ${{ runner.os }}-poetry-${{ hashFiles('poetry.lock') }}

bin/generate_python.sh

@@ -0,0 +1,29 @@
+#! /usr/bin/env bash
+
+while [ "$#" -gt 0 ]; do
+  case "$1" in
+    --output=*) OUTPUT="${1#*=}"; shift 1;;
+    --output) echo "$1 requires an argument" >&2; exit 1;;
+
+    --force-optional) FORCE_OPTIONAL=true; shift 1;;
+
+    -*) echo "$LOG_PREFIX unknown option: $1" >&2; exit 1;;
+    # Skip positional arguments
+    *) shift 1;;
+  esac
+done
+
+datamodel-codegen \
+    --input-file-type jsonschema \
+    --input src/metadata-container-json-schema.json \
+    --output-model-type pydantic_v2.BaseModel \
+    --base-class "datadoc_model.datadoc_base_model.DatadocBaseModel" \
+    --use-default \
+    --use-title-as-name \
+    --use-one-literal-as-default \
+    --use-subclass-enum \
+    --use-standard-collections \
+    --use-double-quotes \
+    --target-python-version 3.10 \
+    --output "$OUTPUT" \
+    ${FORCE_OPTIONAL:+"--force-optional"}

generated/python/datadoc_model/datadoc_model/__init__.py

@@ -0,0 +1,7 @@
+from . import (
+    all_optional,
+    model,  # For backwards compatibility
+    required,
+)
+
+__all__ = ["model", "all_optional", "required"]

generated/python/datadoc_model/datadoc_model/all_optional/__init__.py

@@ -0,0 +1,4 @@
+"""Module where all model fields are forced to be optional.
+
+This is useful for persisting and validating data from users as it is entered via an interface.
+"""