Import MCP Records #121
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Import MCP Records | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| image_version: | |
| required: true | |
| type: string | |
| default: v1.1.0 | |
| description: "dirctl version to use" | |
| import_config: | |
| description: "JSON config file path or inline JSON content" | |
| type: string | |
| default: ".github/workflows/scripts/import-records.json" | |
| dry_run: | |
| description: "Dry run mode - no records will be imported" | |
| type: boolean | |
| default: true | |
| rate_limit: | |
| description: "LLM API requests per minute to avoid rate limiting errors" | |
| type: number | |
| default: 2 | |
| sign: | |
| description: "Sign records with OIDC after importing" | |
| type: boolean | |
| default: true | |
| force: | |
| description: "Force push even if record already exists" | |
| type: boolean | |
| default: false | |
| scanner_enabled: | |
| description: "Run security scanner on each record" | |
| type: boolean | |
| default: true | |
| server_address: | |
| description: "Directory server address" | |
| type: string | |
| default: "prod.gateway.ads.outshift.io:443" | |
| schedule: | |
| - cron: "0 0 * * *" | |
| jobs: | |
| setup-matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| # Resolved inputs with proper fallback for non-dispatch triggers (github.event.inputs.* returns strings, avoiding boolean falsy issues) | |
| dry_run: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run || 'false' }} | |
| rate_limit: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.rate_limit || '2' }} | |
| server_address: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.server_address || 'prod.gateway.ads.outshift.io:443' }} | |
| sign: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.sign || 'true' }} | |
| force: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.force || 'false' }} | |
| scanner_enabled: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.scanner_enabled || 'false' }} | |
| env: | |
| IMPORT_CONFIG: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.import_config || '.github/workflows/scripts/import-records.json' }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Set up matrix from config | |
| id: set-matrix | |
| run: | | |
| # Determine if IMPORT_CONFIG is a file path or direct JSON content | |
| if [ -f "$IMPORT_CONFIG" ]; then | |
| echo "Loading config from file: $IMPORT_CONFIG" | |
| CONFIG_CONTENT=$(cat "$IMPORT_CONFIG") | |
| else | |
| echo "Loading config from input string" | |
| CONFIG_CONTENT="$IMPORT_CONFIG" | |
| fi | |
| # Validate JSON is an array | |
| LENGTH=$(echo "$CONFIG_CONTENT" | jq 'length') | |
| if [ -z "$LENGTH" ] || [ "$LENGTH" = "null" ]; then | |
| echo "Error: Invalid JSON configuration or not an array" | |
| exit 1 | |
| fi | |
| # Add index to each entry for display purposes | |
| MATRIX=$(echo "$CONFIG_CONTENT" | jq -c '[to_entries | .[] | .value + {index: .key}]') | |
| echo "matrix=$MATRIX" >> $GITHUB_OUTPUT | |
| echo "Generated matrix with $LENGTH entries" | |
| import-mcp: | |
| needs: setup-matrix | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write # Required for OIDC signing with Sigstore | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| entry: ${{ fromJson(needs.setup-matrix.outputs.matrix) }} | |
| env: | |
| DRY_RUN: ${{ needs.setup-matrix.outputs.dry_run }} | |
| RATE_LIMIT: ${{ needs.setup-matrix.outputs.rate_limit }} | |
| SERVER_ADDRESS: ${{ needs.setup-matrix.outputs.server_address }} | |
| SIGN: ${{ needs.setup-matrix.outputs.sign }} | |
| FORCE: ${{ needs.setup-matrix.outputs.force }} | |
| SCANNER_ENABLED: ${{ needs.setup-matrix.outputs.scanner_enabled }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 | |
| - name: Setup dirctl | |
| uses: ./.github/actions/setup-dirctl | |
| with: | |
| version: ${{ inputs.image_version }} | |
| - name: Install Task | |
| uses: go-task/setup-task@70f2430ad412f838533de8c0515c749ffb2b8bd3 # v1.1.0 | |
| with: | |
| version: 3.49.1 | |
| - name: Install mcp-scanner | |
| if: ${{ env.SCANNER_ENABLED == 'true' }} | |
| run: task importer:deps:mcp-scanner | |
| - name: Create Enricher Config | |
| run: | | |
| cat <<'EOF' > mcphost_ci.json | |
| { | |
| "mcpServers": { | |
| "dir-mcp-server": { | |
| "command": "dirctl", | |
| "args": [ | |
| "mcp", | |
| "serve" | |
| ], | |
| "env": { | |
| "OASF_API_VALIDATION_SCHEMA_URL": "https://schema.oasf.outshift.com" | |
| }, | |
| "allowedTools": [ | |
| "agntcy_oasf_get_schema", | |
| "agntcy_oasf_get_schema_skills", | |
| "agntcy_oasf_get_schema_domains" | |
| ] | |
| } | |
| }, | |
| "model": "azure:gpt-4o", | |
| "max-steps": 10 | |
| } | |
| EOF | |
| - name: Import MCP Records | |
| id: import | |
| env: | |
| AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} | |
| AZURE_OPENAI_BASE_URL: ${{ secrets.AZURE_OPENAI_ENDPOINT }} | |
| AZURE_OPENAI_DEPLOYMENT: "gpt-4o" | |
| AZURE_OPENAI_API_VERSION: "2024-10-21" | |
| DIRECTORY_CLIENT_AUTH_MODE: "github" | |
| DIRECTORY_CLIENT_SERVER_ADDRESS: ${{ env.SERVER_ADDRESS }} | |
| DIRECTORY_CLIENT_GITHUB_TOKEN: ${{ secrets.DIRECTORY_MCP_BOT_PAT }} | |
| DIRCTL_PATH: dirctl | |
| run: | | |
| echo "--- Processing Entry $((${{ matrix.entry.index }} + 1)) ---" | |
| CIDS_FILE="imported-cids-${{ matrix.entry.index }}.txt" | |
| CMD=("${DIRCTL_PATH}" import --type=mcp --url=https://registry.modelcontextprotocol.io/v0.1 --enrich-config=mcphost_ci.json --enrich-rate-limit="${{ env.RATE_LIMIT }}") | |
| if [[ "${{ env.DRY_RUN }}" == "true" ]]; then | |
| CMD+=("--dry-run") | |
| else | |
| # Output CIDs for deferred signing (only in non-dry-run mode) | |
| CMD+=("--output-cids=$CIDS_FILE") | |
| fi | |
| if [[ "${{ env.FORCE }}" == "true" ]]; then | |
| CMD+=("--force") | |
| fi | |
| if [[ "${{ env.SCANNER_ENABLED }}" == "true" ]]; then | |
| CMD+=("--scanner-enabled") | |
| fi | |
| if [ -n "${{ matrix.entry.search }}" ]; then | |
| CMD+=(--filter="search=${{ matrix.entry.search }}") | |
| fi | |
| if [ -n "${{ matrix.entry.version }}" ]; then | |
| CMD+=(--filter="version=${{ matrix.entry.version }}") | |
| fi | |
| if [ -n "${{ matrix.entry.updated_since }}" ]; then | |
| CMD+=(--filter="updated_since=${{ matrix.entry.updated_since }}") | |
| fi | |
| echo "Running: ${CMD[*]}" | |
| set +e | |
| OUTPUT=$("${CMD[@]}" 2>&1) | |
| EXIT=$? | |
| set -e | |
| echo "$OUTPUT" | |
| if [ "$EXIT" -ne 0 ]; then | |
| echo "--- Import failed with exit code $EXIT ---" | |
| exit "$EXIT" | |
| fi | |
| # Print jsonl output (dry-run mode) | |
| JSONL_FILE=$(echo "$OUTPUT" | grep 'Records saved to:' | sed 's/.*Records saved to: //') | |
| if [ -n "$JSONL_FILE" ] && [ -f "$JSONL_FILE" ]; then | |
| echo "--- JSONL Output ($JSONL_FILE) ---" | |
| cat "$JSONL_FILE" | |
| fi | |
| # Output CIDs file path for signing step | |
| echo "cids_file=$CIDS_FILE" >> $GITHUB_OUTPUT | |
| - name: Fetch Sigstore OIDC token | |
| if: ${{ env.SIGN == 'true' && env.DRY_RUN == 'false' }} | |
| id: oidc-sigstore | |
| uses: ./.github/actions/fetch-oidc-token | |
| with: | |
| audience: sigstore | |
| # Sign records with fresh OIDC tokens after import completes | |
| # This avoids token expiration issues during long-running imports | |
| - name: Sign Imported Records | |
| if: ${{ env.SIGN == 'true' && env.DRY_RUN == 'false' }} | |
| env: | |
| DIRECTORY_CLIENT_AUTH_MODE: "github" | |
| DIRECTORY_CLIENT_SERVER_ADDRESS: ${{ env.SERVER_ADDRESS }} | |
| DIRECTORY_CLIENT_GITHUB_TOKEN: ${{ secrets.DIRECTORY_MCP_BOT_PAT }} | |
| SIGSTORE_OIDC_TOKEN: ${{ steps.oidc-sigstore.outputs.token }} | |
| DIRCTL_PATH: dirctl | |
| run: | | |
| CIDS_FILE="${{ steps.import.outputs.cids_file }}" | |
| if [ ! -f "$CIDS_FILE" ]; then | |
| echo "No CIDs file found - no records were imported" | |
| exit 0 | |
| fi | |
| CID_COUNT=$(wc -l < "$CIDS_FILE" | tr -d ' ') | |
| if [ "$CID_COUNT" -eq 0 ]; then | |
| echo "No CIDs to sign" | |
| exit 0 | |
| fi | |
| echo "Signing $CID_COUNT records..." | |
| SIGNED=0 | |
| FAILED=0 | |
| while IFS= read -r CID || [ -n "$CID" ]; do | |
| [ -z "$CID" ] && continue | |
| echo "Signing CID: $CID" | |
| if "${DIRCTL_PATH}" sign "$CID" \ | |
| --oidc-token="${SIGSTORE_OIDC_TOKEN}" \ | |
| --oidc-provider-url="https://token.actions.githubusercontent.com" \ | |
| --oidc-client-id="https://github.com/${{ github.repository }}/.github/workflows/import-records.yaml@${{ github.ref }}"; then | |
| SIGNED=$((SIGNED + 1)) | |
| else | |
| echo "Failed to sign CID: $CID" | |
| FAILED=$((FAILED + 1)) | |
| fi | |
| done < "$CIDS_FILE" | |
| echo "=== Signing Summary ===" | |
| echo "Signed: $SIGNED" | |
| echo "Failed: $FAILED" | |
| if [ "$FAILED" -gt 0 ]; then | |
| exit 1 | |
| fi |