[DEV-2583] Add optimized GitBook sync workflow and metadata synchronization script

.changeset/yellow-ears-stay.md

@@ -0,0 +1,5 @@
+---
+"gitbook-docs": patch
+---
+
+Optimize GitBook sync workflow to reduce API calls and improve performance

.github/workflows/sync_gitbook_docs_optimized.yaml

@@ -0,0 +1,283 @@
+name: Sync GitBook Docs (Optimized)
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'The environment used as target'
+        type: choice
+        required: true
+        default: dev
+        options:
+          - dev
+          - uat
+          - prod
+      call_reindex:
+        description: 'Trigger reindex workflow after docs sync'
+        type: boolean
+        required: true
+        default: false
+      metadata_type:
+        description: 'Type of metadata to generate'
+        type: choice
+        required: true
+        default: all
+        options:
+          - all
+          - guides
+          - solutions
+          - release_notes
+      generate_metadata_only:
+        description: 'Generate metadata only without syncing docs'
+        type: boolean
+        required: true
+        default: false
+      incremental_mode:
+        description: 'Run sync in incremental mode by checking only file size'
+        type: boolean
+        required: true
+        default: true
+      inivalidate_opennext_cache:
+        description: 'Invalidate opennext cloufront cache'
+        type: boolean
+        required: true
+        default: false
+
+  schedule:
+    - cron: '0 23 * * *'  # Run daily at midnight UTC
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  manual_sync_gitbook_docs:
+    name: Sync gitbook Docs to S3 (manual on ${{ inputs.environment }})
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: ubuntu-latest
+    outputs:
+      environment: ${{ inputs.environment }}
+    environment: ${{ inputs.environment }}
+    env:
+      ENV_SHORT: ${{ fromJSON('{"dev":"d","uat":"u","prod":"p"}')[inputs.environment] }}
+
+    steps:
+      - name: Checkout current repository
+        uses: actions/checkout@v4
+
+      - name: Checkout devportal-docs repo
+        if: inputs.generate_metadata_only == false
+        uses: actions/checkout@v4
+        with:
+          repository: pagopa/devportal-docs
+          ref: docs/from-gitbook
+          path: devportal-docs
+
+      - name: Setup Node.JS
+        uses: ./.github/actions/setup-node
+
+      - name: Cache npm dependencies
+        id: cache-npm
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-
+
+      - name: Install dependencies
+        run: |
+          npm config set cache ~/.npm
+          npm ci
+
+      - name: Compile packages
+        run: npm run compile
+
+      - name: Generate URL metadata
+        if: inputs.generate_metadata_only == false
+        env:
+          ENVIRONMENT: ${{ inputs.environment }}
+          DOCUMENTATION_PATH: ${{ vars.DOCUMENTATION_PATH || '../../devportal-docs/docs' }}
+          URL_PARSING_METADATA_JSON_PATH: ${{ vars.URL_PARSING_METADATA_JSON_PATH || '../../url-parsing-metadata.json' }}
+          STRAPI_ENDPOINT: ${{ vars.STRAPI_ENDPOINT }}
+          STRAPI_API_TOKEN: ${{ secrets.STRAPI_API_TOKEN }}
+          S3_BUCKET_NAME: devportal-${{ env.ENV_SHORT }}-website-static-content
+          S3_PATH_TO_GITBOOK_DOCS: ${{ vars.S3_PATH_TO_GITBOOK_DOCS || 'devportal-docs/docs' }}
+          NEXT_PUBLIC_COGNITO_REGION: ${{ vars.NEXT_PUBLIC_COGNITO_REGION || 'eu-south-1' }}
+          GENERATE_URL_METADATA: 'true'
+          GENERATE_SITEMAP_METADATA: 'false'
+          SAVE_STRAPI_RESPONSES: 'false'
+          METADATA_TYPE: 'all'
+        run: npm run sync-all-metadata -w gitbook-docs
+
+      - name: Replace urls and include tags in docs
+        if: inputs.generate_metadata_only == false
+        env:
+          ENVIRONMENT: ${{ inputs.environment }}
+          DOCUMENTATION_PATH: ${{ vars.DOCUMENTATION_PATH || '../../devportal-docs/docs' }}
+          URL_PARSING_METADATA_JSON_PATH: ${{ vars.URL_PARSING_METADATA_JSON_PATH || '../../url-parsing-metadata.json' }}
+        run: npm run parse-docs -w gitbook-docs
+
+      - name: Configure AWS Credentials
+        uses: ./.github/actions/configure-aws-credentials
+        with:
+          aws_region: eu-south-1
+          role_to_assume: ${{ secrets.DEPLOY_IAM_ROLE }}
+
+      - name: Sync docs folder to S3
+        if: inputs.generate_metadata_only == false
+        working-directory: ./devportal-docs
+        run: |
+          aws s3 sync docs s3://devportal-${{ env.ENV_SHORT }}-website-static-content/devportal-docs/docs --delete ${{ inputs.incremental_mode == true && ' --size-only' || '' }}
+
+      - name: Generate metadata
+        env:
+          ENVIRONMENT: ${{ inputs.environment }}
+          S3_BUCKET_NAME: devportal-${{ env.ENV_SHORT }}-website-static-content
+          S3_DOC_EXTRACTION_BUCKET_NAME: ${{ vars.S3_DOC_EXTRACTION_BUCKET_NAME }}
+          S3_PATH_TO_GITBOOK_DOCS: ${{ vars.S3_PATH_TO_GITBOOK_DOCS || 'devportal-docs/docs' }}
+          STRAPI_ENDPOINT: ${{ vars.STRAPI_ENDPOINT }}
+          STRAPI_API_TOKEN: ${{ secrets.STRAPI_API_TOKEN }}
+          FETCH_FROM_STRAPI: ${{ vars.FETCH_FROM_STRAPI || 'true' }}
+          NEXT_PUBLIC_COGNITO_REGION: ${{ vars.NEXT_PUBLIC_COGNITO_REGION || 'eu-south-1' }}
+          METADATA_TYPE: ${{ inputs.metadata_type }}
+          GENERATE_URL_METADATA: 'false'
+          GENERATE_SITEMAP_METADATA: 'true'
+          SAVE_STRAPI_RESPONSES: 'true'
+        run: npm run sync-all-metadata -w gitbook-docs
+
+      - name: Invalidate CloudFront asset bucket cache
+        run: |
+          aws cloudfront create-invalidation \
+            --distribution-id ${{ vars.ASSET_BUCKET_CLOUDFRONT_DISTRIBUTION_ID }} \
+            --paths "/*"
+
+      - name: Invalidate CloudFront opennext cache
+        if: inputs.inivalidate_opennext_cache == true
+        run: |
+          aws cloudfront create-invalidation \
+            --distribution-id ${{ vars.OPENNEXT_CLOUDFRONT_DISTRIBUTION_ID }} \
+            --paths "/*"
+
+      - name: Trigger reindex workflow
+        if: inputs.call_reindex
+        run: |
+          curl -X POST \
+          -H "Authorization: token ${{ secrets.TAG_GITHUB_PAT }}" \
+          -H "Accept: application/vnd.github+json" \
+          https://api.github.com/repos/${{ github.repository }}/actions/workflows/141828151/dispatches \
+          -d '{"ref":"main","inputs":{"environment":"${{ inputs.environment }}"}}'
+
+      - name: Summary
+        if: always()
+        run: |
+          echo "## Sync Summary" >> $GITHUB_STEP_SUMMARY
+          echo "- Environment: ${{ inputs.environment }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Metadata Type: ${{ inputs.metadata_type }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Generate Metadata Only: ${{ inputs.generate_metadata_only }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Incremental Mode: ${{ inputs.incremental_mode }}" >> $GITHUB_STEP_SUMMARY
+          echo "- Cache Invalidation: Asset bucket (always), OpenNext (${{ inputs.inivalidate_opennext_cache }})" >> $GITHUB_STEP_SUMMARY
+          echo "- Reindex Triggered: ${{ inputs.call_reindex }}" >> $GITHUB_STEP_SUMMARY
+
+  scheduled_sync_gitbook_docs:
+    name: Sync gitbook Docs to S3 (scheduled on prod)
+    if: github.event_name == 'schedule'
+    runs-on: ubuntu-latest
+    outputs:
+      environment: 'prod'
+    environment: 'prod'
+
+    steps:
+      - name: Checkout current repository
+        uses: actions/checkout@v4
+
+      - name: Checkout devportal-docs repo
+        uses: actions/checkout@v4
+        with:
+          repository: pagopa/devportal-docs
+          ref: docs/from-gitbook
+          path: devportal-docs
+
+      - name: Setup Node.JS
+        uses: ./.github/actions/setup-node
+
+      - name: Cache npm dependencies
+        id: cache-npm
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-npm-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-npm-
+
+      - name: Install dependencies
+        run: |
+          npm config set cache ~/.npm
+          npm ci
+
+      - name: Compile packages
+        run: npm run compile
+
+      - name: Generate URL metadata
+        env:
+          ENVIRONMENT: 'prod'
+          DOCUMENTATION_PATH: ${{ vars.DOCUMENTATION_PATH || '../../devportal-docs/docs' }}
+          URL_PARSING_METADATA_JSON_PATH: ${{ vars.URL_PARSING_METADATA_JSON_PATH || '../../url-parsing-metadata.json' }}
+          STRAPI_ENDPOINT: ${{ vars.STRAPI_ENDPOINT }}
+          STRAPI_API_TOKEN: ${{ secrets.STRAPI_API_TOKEN }}
+          S3_BUCKET_NAME: devportal-p-website-static-content
+          S3_PATH_TO_GITBOOK_DOCS: ${{ vars.S3_PATH_TO_GITBOOK_DOCS || 'devportal-docs/docs' }}
+          NEXT_PUBLIC_COGNITO_REGION: ${{ vars.NEXT_PUBLIC_COGNITO_REGION || 'eu-south-1' }}
+          GENERATE_URL_METADATA: 'true'
+          GENERATE_SITEMAP_METADATA: 'false'
+          SAVE_STRAPI_RESPONSES: 'false'
+          METADATA_TYPE: 'all'
+        run: npm run sync-all-metadata -w gitbook-docs
+
+      - name: Replace urls and include tags in docs
+        env:
+          ENVIRONMENT: 'prod'
+          DOCUMENTATION_PATH: ${{ vars.DOCUMENTATION_PATH || '../../devportal-docs/docs' }}
+          URL_PARSING_METADATA_JSON_PATH: ${{ vars.URL_PARSING_METADATA_JSON_PATH || '../../url-parsing-metadata.json' }}
+        run: npm run parse-docs -w gitbook-docs
+
+      - name: Configure AWS Credentials
+        uses: ./.github/actions/configure-aws-credentials
+        with:
+          aws_region: eu-south-1
+          role_to_assume: ${{ secrets.DEPLOY_IAM_ROLE }}
+
+      - name: Sync docs folder to S3
+        working-directory: ./devportal-docs
+        run: |
+          aws s3 sync docs s3://devportal-p-website-static-content/devportal-docs/docs --delete --size-only
+
+      - name: Generate metadata
+        env:
+          ENVIRONMENT: 'prod'
+          S3_BUCKET_NAME: devportal-p-website-static-content
+          S3_DOC_EXTRACTION_BUCKET_NAME: ${{ vars.S3_DOC_EXTRACTION_BUCKET_NAME }}
+          S3_PATH_TO_GITBOOK_DOCS: ${{ vars.S3_PATH_TO_GITBOOK_DOCS || 'devportal-docs/docs' }}
+          STRAPI_ENDPOINT: ${{ vars.STRAPI_ENDPOINT }}
+          STRAPI_API_TOKEN: ${{ secrets.STRAPI_API_TOKEN }}
+          FETCH_FROM_STRAPI: ${{ vars.FETCH_FROM_STRAPI || 'true' }}
+          NEXT_PUBLIC_COGNITO_REGION: ${{ vars.NEXT_PUBLIC_COGNITO_REGION || 'eu-south-1' }}
+          METADATA_TYPE: 'all'
+          GENERATE_URL_METADATA: 'false'
+          GENERATE_SITEMAP_METADATA: 'true'
+          SAVE_STRAPI_RESPONSES: 'true'
+        run: npm run sync-all-metadata -w gitbook-docs
+
+      - name: Invalidate CloudFront asset bucket cache
+        run: |
+          aws cloudfront create-invalidation \
+            --distribution-id ${{ vars.ASSET_BUCKET_CLOUDFRONT_DISTRIBUTION_ID }} \
+            --paths "/*"
+
+      - name: Trigger reindex workflow
+        run: |
+          curl -X POST \
+          -H "Authorization: token ${{ secrets.TAG_GITHUB_PAT }}" \
+          -H "Accept: application/vnd.github+json" \
+          https://api.github.com/repos/${{ github.repository }}/actions/workflows/141828151/dispatches \
+          -d '{"ref":"main","inputs":{"environment":"prod"}}'

packages/gitbook-docs/package.json

@@ -25,7 +25,8 @@
     "generate-release-notes-metadata": "ts-node src/scripts/generateReleaseNotesMetadata.ts",
     "fetch-soap-api-from-strapi": "ts-node src/scripts/generateSoapApiRepositoriesList.ts",
     "generate-url-parsing-metadata": "ts-node src/scripts/generateUrlParsingMetadata.ts",
-    "parse-docs": "ts-node src/scripts/parseDocUrlsAndIncludes.ts"
+    "parse-docs": "ts-node src/scripts/parseDocUrlsAndIncludes.ts",
+    "sync-all-metadata": "ts-node src/scripts/syncAllMetadata.ts"
   },
   "dependencies": {
     "@markdoc/markdoc": "0.3.0",