validate-and-process.yml

name: Validate and process Building Blocks
on:
  workflow_call:
    inputs:
      register_file:
        type: string
        description: JSON register file for output
        default: build/register.json
      items_dir:
        type: string
        description: Root directory where building blocks (subdirectories with bblock.json) are located
        default: _sources
      generated_docs_path:
        type: string
        description: Output directory for documentation
        default: build/generateddocs
      base_url:
        type: string
        description: Base URL to use for asset linkage
      fail_on_error:
        type: string
        description: Whether to fail on errors
        default: 'true'
      annotated_path:
        type: string
        description: Output path for annotated schemas
        default: build/annotated
      clean:
        type: string
        description: Whether to delete all output files and directories before generating new ones
        default: 'true'
      config_file:
        type: string
        description: Building blocks submodule config file
        default: 'bblocks-config.yaml'
      test_outputs_path:
        type: string
        description: Directory for test output resources
        default: build/tests
      github_base_url:
        type: string
        description: Base URL for linking to GitHub content
        default: https://github.com/${{github.repository}}/blob/${{github.ref_name}}/
      ref:
        type: string
        description: Repository ref for checkout
        default: ${{ github.ref_name }}
      skip-build:
        type: boolean
        description: Whether to skip the BB build part of the process
        default: false
      skip-pages:
        type: boolean
        description: Whether to skip the GH pages build part of the process
        default: false
      deploy_viewer:
        type: boolean
        description: Whether to use the new building blocks viewer
        default: true
      viewer_path:
        type: string
        description: Path where to deploy the viewer
        default: '.'
      viewer_show_imported:
        type: number
        description: |
          Level up to which the deployed building blocks viewer will show imported building
          blocks as well as local ones. 0 means "only local", and a negative number means
          "all imports".
          Deprecated: set viewer.show-imported-depth in bblocks-config.yaml instead.
          If that key is present, it takes precedence over this input.
        default: 0
      fallback_rainbow_instances:
        type: string
        description: |
          JSON array of Rainbow proxy URLs for linked data semantic lookups.
          Each URL should accept a ?uri= query parameter.
        default: '["https://defs.opengis.net/prez-backend/object","https://defs-hosted.opengis.net/prez-hosted-b/object"]'
      fallback_sparql_endpoints:
        type: string
        description: |
          JSON array of SPARQL endpoint URLs used as a fallback for linked data lookups.
        default: '[]'
      skip_permissions:
        type: string
        description: Skip interactive permission prompts for transforms and plugins
        default: 'true'
      before_postprocess:
        type: string
        description: |
          Shell code to run before running the bblocks postprocessing (e.g., for setting up custom settings)
        default: ''
      before_pages:
        type: string
        description: |
          Shell code to run before pushing to GH pages (e.g., for cleanup)
        default: ''
    secrets:
      sparql_username:
        description: SPARQL Graph Store Protocol user name for push authentication
        required: false
      sparql_password:
        description: SPARQL Graph Store Protocol password for push authentication
        required: false
      gh_pat:
        description: GitHub Personal Access Token to use when cloning repositories
        required: false


permissions:
  contents: write
  pages: write
  id-token: write

concurrency:
  group: bblocks
  cancel-in-progress: true

jobs:
  validate-and-process:
    runs-on: ubuntu-latest

    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}

    outputs:
      sparql_config: ${{ steps.deploy-viewer.outputs.sparql_config }}
      register_url: ${{ steps.get-pages-url.outputs.result }}/${{ inputs.register_file }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          submodules: recursive
          ref: ${{ inputs.ref }}
      - name: Update submodules
        run: git submodule update --recursive --remote
      - name: Get Pages Base URL
        id: get-pages-url
        uses: actions/github-script@v6
        with:
          result-encoding: string
          script: |
            let base_url;
            if ("${{ inputs.base_url }}"?.trim()) {
              base_url = "${{ inputs.base_url }}";
            } else {
              const { owner, repo } = context.repo;
              try {
                const response = await github.rest.repos.getPages({
                  owner,
                  repo
                });
                base_url = response.data.html_url;
              } catch {
                core.warning('No base_url provided, and the GitHub Pages URL could not be retrieved. Did you forget to enable GitHub Pages?');
                base_url = `https://raw.githubusercontent.com/${owner}/${repo}/${context.ref_name}`;
              }
            }
            // Remove trailing slash
            const parsed = new URL(base_url);
            base_url = base_url.replace(/\/$/, '');
            core.exportVariable('PAGES_BASE_URL', base_url);
            core.exportVariable('PAGES_PATH', parsed.pathname);
            return base_url;
      - name: Process GitHub repository mappings
        run: |
          export RANDOM_DIR="_repos_$(openssl rand -hex 8)"
          mkdir "$RANDOM_DIR"
          echo '*' > "${RANDOM_DIR}/.gitignore" 
          
          if [ -f "bblocks-github.yaml" ]; then
            INPUT_YML="bblocks-github.yaml"
          elif [ -f "bblocks-github.yml" ]; then
            INPUT_YML="bblocks-github.yml"
          fi
          
          if [ -z "$INPUT_YML" ]; then
            echo "No GitHub repository mappings found"
            exit 0
          fi
          
          MAPPINGS_YML="bblocks-config-local.yml"
          echo 'url-mappings:' > $MAPPINGS_YML
          
          if [ -z "${{ secrets.GH_PAT }}" ]; then
            GH_AUTH=
            echo "*WARNING* Cloning repository without a PAT"
          else
            GH_AUTH="x-access-token:${{ secrets.GH_PAT }}@"
            echo "Using a GitHub PAT"
          fi
          export GH_AUTH
          yq -r '."repository-mappings" | to_entries[] |
            "\(strenv(RANDOM_DIR))/\(.value | sub(\"[^A-Za-z0-9]\", \"_\"))" as $SUBDIR |
            "
            if [ ! -d \"\($SUBDIR)\" ]; then
              git clone \"https://\(strenv(GH_AUTH))github.com/\(.value).git\" \"\($SUBDIR)\"
            fi
            echo \"  \\\"\(.key)\\\": \\\"\($SUBDIR)\\\"\" >> bblocks-config-local.yml
            "
          ' < $INPUT_YML > _script.sh
          cat _script.sh
          bash _script.sh
          
          
          echo "Repositories cloned and mapped in $RANDOM_DIR — output saved to $MAPPINGS_YML"

      - name: Restore bblocks sandbox cache
        uses: actions/cache/restore@v4
        with:
          path: .bblocks-sandbox
          key: bblocks-sandbox-${{ runner.os }}-${{ github.run_id }}
          restore-keys: |
            bblocks-sandbox-${{ runner.os }}-
      - name: Before postprocess
        if: ${{inputs.before_postprocess}}
        run: ${{inputs.before_postprocess}}
      - name: OGC BB postprocess
        if: ${{ !inputs.skip-build }}
        uses: opengeospatial/bblocks-postprocess/full@v1
        with:
          register_file: ${{ inputs.register_file }}
          items_dir: ${{ inputs.items_dir }}
          generated_docs_path: ${{ inputs.generated_docs_path }}
          base_url: ${{ steps.get-pages-url.outputs.result }}
          fail_on_error: ${{ inputs.fail_on_error }}
          annotated_path: ${{ inputs.annotated_path }}
          clean: ${{ inputs.clean }}
          config_file: ${{ inputs.config_file }}
          test_outputs_path: ${{ inputs.test_outputs_path }}
          github_base_url: ${{ inputs.github_base_url }}
          deploy_viewer: ${{ inputs.deploy_viewer && 'true' || 'false' }}
          viewer_path: ${{ inputs.viewer_path }}
          skip_permissions: ${{ inputs.skip_permissions }}
          sparql_username: ${{ secrets.sparql_username }}
          sparql_password: ${{ secrets.sparql_password }}
      - name: Fix permissions, compress large _visited_properties.tsv, git pull
        run: |
          sudo chown -R $UID:$(id -g) . || true
          sudo find "${{ inputs.annotated_path }}" -size +10M -name _visited_properties.tsv \
            -exec gzip \{\} \; || true
          git pull
      - name: Add & Commit
        if: ${{ !endsWith(github.repository, '/bblock-template') && !inputs.skip-build }}
        uses: EndBug/add-and-commit@v9
        with:
          message: Building blocks postprocessing
      - name: Save bblocks sandbox cache
        if: ${{ !inputs.skip-build }}
        uses: actions/cache/save@v4
        with:
          path: .bblocks-sandbox
          key: bblocks-sandbox-${{ runner.os }}-${{ github.run_id }}
      - name: Remove build artifacts from pages
        run: |
          sudo find "${{ inputs.annotated_path }}" -name _visited_properties.tsv -delete || true
          sudo find "${{ inputs.annotated_path }}" -name _visited_properties.tsv.gz -delete || true
          rm -rf .bblocks-sandbox
          # Remove LLM tool configuration files
          rm -rf .claude CLAUDE.md AGENTS.md GEMINI.md .cursor .cursorrules .windsurfrules .ruler .continue
          find . -maxdepth 1 -name '.aider*' -exec rm -rf {} + || true
      - name: Deploy Building Blocks viewer
        id: deploy-viewer
        if: ${{ inputs.deploy_viewer }}
        run: |
          OLD_WD="$(pwd)"
          REGISTER_FILE="$(realpath "${{ inputs.register_file }}")"
          if [ -n "${{ inputs.viewer_path }}" ]; then
            mkdir -p "${{ inputs.viewer_path }}"
            cd "${{ inputs.viewer_path }}"
          fi
          
          # If viewer path is '.' or './', clear it.
          # Otherwise, remove any leading './'
          BASE_URL="${{ inputs.viewer_path }}"
          if [[ "${BASE_URL}" =~ ^\./?$ ]]; then
            BASE_URL=
          else
            BASE_URL=${BASE_URL#./}
          fi
          # Add trailing slash to BASE_URL if not empty
          if [[ "${BASE_URL}" =~ [^/]$ ]]; then
            BASE_URL="${BASE_URL}/"
          fi
          
          cat << EOF > index.html
          <!DOCTYPE html>
          <html lang="en">
          <head>
            <meta charset="UTF-8" />
            <link rel="icon" href="https://ogcincubator.github.io/bblocks-viewer/favicon.ico" />
            <meta name="viewport" content="width=device-width, initial-scale=1.0" />
            <title>OGC Location Building Blocks</title>
            <script src="${{ steps.get-pages-url.outputs.result }}/${BASE_URL}config.js"></script>
            <script type="module" crossorigin src="https://ogcincubator.github.io/bblocks-viewer/assets/index.js"></script>
            <link rel="stylesheet" href="https://ogcincubator.github.io/bblocks-viewer/assets/index.css">
          </head>
          <body>
            <div id="app"></div>
          </body>
          </html>
          EOF
          
          BBLOCKS_RAINBOW='${{ inputs.fallback_rainbow_instances }}'
          BBLOCKS_RAINBOW="${BBLOCKS_RAINBOW:-[]}"
          BBLOCKS_SPARQL='${{ inputs.fallback_sparql_endpoints }}'
          BBLOCKS_SPARQL="${BBLOCKS_SPARQL:-[]}"

          # Prefer viewer.show-imported-depth from bblocks-config.yaml if present
          SHOW_IMPORTED=$(yq '.viewer."show-imported-depth" // null' "${OLD_WD}/${{ inputs.config_file }}" 2>/dev/null)
          if [ "$SHOW_IMPORTED" = "null" ] || [ -z "$SHOW_IMPORTED" ]; then
            SHOW_IMPORTED='${{ inputs.viewer_show_imported }}'
          fi

          cat << EOF > config.js
          window.bblocksRegister = '${{ steps.get-pages-url.outputs.result }}/${{ inputs.register_file }}';
          window.bblocksViewer = {
            title: $(jq '.name // "${{ github.event.repository.name }}"' < "$REGISTER_FILE"),
            showImported: ${SHOW_IMPORTED},
            baseUrl: '${PAGES_PATH}${BASE_URL}',
            bblocksFallbackRainbowInstances: ${BBLOCKS_RAINBOW},
            bblocksFallbackSparqlEndpoints: ${BBLOCKS_SPARQL},
          };
          EOF
          
          if [ -f GIT_INFO ]; then
            TEMPFILE="$(mktemp)"
            jq -s '.[0].tooling."bblocks-viewer" = .[1] | .[0]' "$REGISTER_FILE" GIT_INFO > "$TEMPFILE"
            mv "$TEMPFILE" "$REGISTER_FILE"
            rm -f "$TEMPFILE"
          fi
          
          cd "$OLD_WD"
          
          if [ ! -f "404.html" ]; then
            cp "${{ inputs.viewer_path }}/index.html" 404.html
          fi
          
          {
            echo 'sparql_config<<eirue3biu8Cai6cioraaJie8theechi7ahgheiM0dojooho0unii2deech4abiaX'
            yq -o=json '.sparql' '${{ inputs.config_file }}'
            echo 'eirue3biu8Cai6cioraaJie8theechi7ahgheiM0dojooho0unii2deech4abiaX'
          } >> "$GITHUB_OUTPUT"
          
      - name: Before pages
        if: ${{inputs.before_pages}}
        run: ${{inputs.before_pages}}
      - name: Setup Pages
        if: ${{ github.event.repository.has_pages && !inputs.skip-pages }}
        uses: actions/configure-pages@v3
      - name: Upload artifacts
        if: ${{ github.event.repository.has_pages && !inputs.skip-pages }}
        uses: actions/upload-pages-artifact@v3
        with:
          path: .
      - name: Deploy to GH Pages
        if: ${{ github.event.repository.has_pages && !inputs.skip-pages }}
        id: deployment
        uses: actions/deploy-pages@v4
  upload-to-triplestore:
    needs: validate-and-process
    if: ${{ github.event.repository.has_pages && !inputs.skip-pages && needs.validate-and-process.outputs.register_url }}
    uses: ./.github/workflows/upload-to-triplestore.yml
    with:
      sparql_config: ${{ needs.validate-and-process.outputs.sparql_config }}
      register_url: ${{ needs.validate-and-process.outputs.register_url }}
    secrets:
      sparql_username: ${{ secrets.sparql_username }}
      sparql_password: ${{ secrets.sparql_password }}