[CAI-612] Update sync worflow

.github/workflows/sync_gitbook_docs.yaml

@@ -94,6 +94,16 @@ jobs:
       - name: Compile packages
         run: npm run compile
 
+      - name: Generate Strapi Response Files
+        env:
+          ENVIRONMENT: ${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'dev' }}
+          S3_BUCKET_NAME: devportal-${{ env.ENV_SHORT }}-website-static-content
+          STRAPI_ENDPOINT: ${{ vars.STRAPI_ENDPOINT }}
+          STRAPI_API_TOKEN: ${{ secrets.STRAPI_API_TOKEN }}
+          FETCH_FROM_STRAPI: ${{ vars.FETCH_FROM_STRAPI || 'true' }}
+          NEXT_PUBLIC_COGNITO_REGION: ${{ vars.NEXT_PUBLIC_COGNITO_REGION || 'eu-south-1' }}
+        run: npm run generate-strapi-response-files -w gitbook-docs
+
       - name: Generate UrlReplace metadata
         if: inputs.generate_metadata_only == false
         env:
@@ -224,6 +234,16 @@ jobs:
       - name: Compile packages
         run: npm run compile
 
+      - name: Generate Strapi Response Files
+        env:
+          ENVIRONMENT: ${{ github.event_name == 'workflow_dispatch' && inputs.environment || 'dev' }}
+          S3_BUCKET_NAME: devportal-${{ env.ENV_SHORT }}-website-static-content
+          STRAPI_ENDPOINT: ${{ vars.STRAPI_ENDPOINT }}
+          STRAPI_API_TOKEN: ${{ secrets.STRAPI_API_TOKEN }}
+          FETCH_FROM_STRAPI: ${{ vars.FETCH_FROM_STRAPI || 'true' }}
+          NEXT_PUBLIC_COGNITO_REGION: ${{ vars.NEXT_PUBLIC_COGNITO_REGION || 'eu-south-1' }}
+        run: npm run generate-strapi-response-files -w gitbook-docs
+
       - name: Generate UrlReplace metadata
         env:
           ENVIRONMENT: 'prod'

packages/gitbook-docs/package.json

@@ -25,6 +25,7 @@
     "generate-release-notes-metadata": "ts-node src/scripts/generateReleaseNotesMetadata.ts",
     "fetch-soap-api-from-strapi": "ts-node src/scripts/generateSoapApiRepositoriesList.ts",
     "generate-url-parsing-metadata": "ts-node src/scripts/generateUrlParsingMetadata.ts",
+    "generate-strapi-response-files": "ts-node src/scripts/generateStrapiResponseFiles.ts",
     "parse-docs": "ts-node src/scripts/parseDocUrlsAndIncludes.ts"
   },
   "dependencies": {

packages/gitbook-docs/src/helpers/guidesMetadataHelper.ts

@@ -1,51 +1,3 @@
-export interface StrapiGuide {
-  readonly id: number;
-  readonly attributes: {
-    readonly slug: string;
-    readonly title: string;
-    readonly product?: {
-      readonly data?: {
-        readonly attributes?: {
-          readonly slug: string;
-        };
-      };
-    };
-    readonly versions: readonly {
-      readonly id: number;
-      readonly main: boolean;
-      readonly version: string;
-      readonly dirName: string;
-    }[];
-  };
-}
-
-export interface StrapiReleaseNote {
-  readonly id: number;
-  readonly attributes: {
-    readonly slug: string;
-    readonly title: string;
-    readonly product?: {
-      readonly data?: {
-        readonly attributes?: {
-          readonly slug: string;
-        };
-      };
-    };
-    readonly dirName: string;
-    readonly landingFile: string;
-  };
-}
-
-export interface StrapiSolution {
-  readonly id: number;
-  readonly attributes: {
-    readonly slug: string;
-    readonly title: string;
-    readonly landingUseCaseFile: string;
-    readonly dirName: string;
-  };
-}
-
 export enum MetadataType {
   Guide = 0,
   ReleaseNote = 1,

packages/gitbook-docs/src/helpers/strapiTypes.ts

@@ -0,0 +1,92 @@
+export interface StrapiGuide {
+  readonly id: number;
+  readonly attributes: {
+    readonly slug: string;
+    readonly title: string;
+    readonly product?: {
+      readonly data?: {
+        readonly attributes?: {
+          readonly slug: string;
+        };
+      };
+    };
+    readonly versions: readonly {
+      readonly id: number;
+      readonly main: boolean;
+      readonly version: string;
+      readonly dirName: string;
+    }[];
+  };
+}
+
+export interface StrapiReleaseNote {
+  readonly id: number;
+  readonly attributes: {
+    readonly slug: string;
+    readonly title: string;
+    readonly product?: {
+      readonly data?: {
+        readonly attributes?: {
+          readonly slug: string;
+        };
+      };
+    };
+    readonly dirName: string;
+    readonly landingFile: string;
+  };
+}
+
+export interface StrapiSolution {
+  readonly id: number;
+  readonly attributes: {
+    readonly slug: string;
+    readonly title: string;
+    readonly landingUseCaseFile: string;
+    readonly dirName: string;
+  };
+}
+
+export interface StrapiProduct {
+  readonly attributes: {
+    readonly name: string;
+    readonly shortName: string;
+    readonly slug: string;
+  };
+}
+
+export interface StrapiApiData {
+  readonly id: number;
+  readonly attributes: {
+    readonly title: string;
+    readonly description?: string;
+    readonly icon: {
+      readonly data?: {
+        readonly attributes: {
+          readonly name: string;
+          readonly ext: string;
+          readonly mime: string;
+          readonly size: number;
+          readonly url: string;
+          readonly alternativeText?: string;
+          readonly caption?: string;
+          readonly height?: number;
+          readonly width?: number;
+        };
+      };
+    };
+    readonly apiRestDetail?: {
+      readonly slug: string;
+      readonly specUrls: readonly {
+        readonly id: number;
+        readonly name?: string;
+        readonly url: string;
+        readonly hideTryIt: boolean;
+      }[];
+    };
+    readonly apiSoapDetail?: {
+      readonly slug: string;
+      readonly repositoryUrl: string;
+      readonly dirName: string;
+    };
+  };
+}

packages/gitbook-docs/src/scripts/generateGuidesMetadata.ts

@@ -16,7 +16,8 @@ import {
   fetchFromStrapi,
   getResponseFromStrapi,
 } from '../helpers/fetchFromStrapi';
-import { StrapiGuide, MetadataInfo } from '../helpers/guidesMetadataHelper';
+import { StrapiGuide } from '../helpers/strapiTypes';
+import { MetadataInfo } from '../helpers/guidesMetadataHelper';
 import { sitePathFromS3Path } from '../helpers/sitePathFromS3Path';
 import {
   getSyncedGuideListPagesResponseJsonPath,

packages/gitbook-docs/src/scripts/generateReleaseNotesMetadata.ts

@@ -14,7 +14,7 @@ import {
 import { extractTitleFromMarkdown } from '../helpers/extractTitle.helper';
 import { fetchFromStrapi } from '../helpers/fetchFromStrapi';
 import { sitePathFromS3Path } from '../helpers/sitePathFromS3Path';
-import { StrapiReleaseNote } from '../helpers/guidesMetadataHelper';
+import { StrapiReleaseNote } from '../helpers/strapiTypes';
 import { getSyncedReleaseNotesResponseJsonPath } from '../syncedResponses';
 
 // Load environment variables from .env file

packages/gitbook-docs/src/scripts/generateSolutionsMetadata.ts

@@ -17,7 +17,7 @@ import {
   getResponseFromStrapi,
 } from '../helpers/fetchFromStrapi';
 import { sitePathFromS3Path } from '../helpers/sitePathFromS3Path';
-import { StrapiSolution } from '../helpers/guidesMetadataHelper';
+import { StrapiSolution } from '../helpers/strapiTypes';
 import {
   getSyncedSolutionListPagesResponseJsonPath,
   getSyncedSolutionsResponseJsonPath,

packages/gitbook-docs/src/scripts/generateStrapiResponseFile.ts

@@ -0,0 +1,82 @@
+/* eslint-disable functional/prefer-readonly-type */
+/* eslint-disable functional/no-loop-statements */
+/* eslint-disable functional/no-expression-statements */
+/* eslint-disable functional/immutable-data */
+/* eslint-disable functional/no-try-statements */
+import dotenv from 'dotenv';
+import { fetchFromStrapi } from '../helpers/fetchFromStrapi';
+import { makeS3Client, writeSitemapJson } from '../helpers/s3Bucket.helper';
+import { StrapiApiData, StrapiProduct } from '../helpers/strapiTypes';
+
+dotenv.config();
+
+const SITEMAP_URL =
+  process.env.SITEMAP_URL || 'https://developer.pagopa.it/sitemap.xml';
+const S3_SITEMAP_PATH = process.env.S3_SITEMAP_PATH || 'sitemap.xml';
+const S3_PRODUCTS_METADATA_JSON_PATH =
+  process.env.S3_PRODUCTS_METADATA_JSON_PATH || 'synced-products-response.json';
+const S3_APIS_DATA_METADATA_JSON_PATH =
+  process.env.S3_APIS_DATA_METADATA_JSON_PATH ||
+  'synced-apis-data-response.json';
+const S3_BUCKET_NAME = process.env.S3_BUCKET_NAME;
+
+const s3Client = makeS3Client();
+async function main() {
+  console.log('Recovering products and apis-data from Strapi CMS...');
+  // eslint-disable-next-line functional/no-let
+  let strapiProducts;
+  try {
+    const { data } = await fetchFromStrapi<StrapiProduct>('api/products');
+    strapiProducts = data;
+  } catch (error) {
+    console.error('Error fetching Products from Strapi:', error);
+    process.exit(1);
+  }
+  console.log(`Fetched ${strapiProducts.length} products from Strapi`);
+
+  // eslint-disable-next-line functional/no-let
+  let strapiApisData;
+  try {
+    const { data } = await fetchFromStrapi<StrapiApiData>(
+      'api/apis-data?populate[product]=*&populate[apiRestDetail][populate][specUrls]=*'
+    );
+    strapiApisData = data;
+  } catch (error) {
+    console.error('Error fetching Products from Strapi:', error);
+    process.exit(1);
+  }
+  console.log(`Fetched ${strapiApisData.length} apis-data from Strapi`);
+
+  // eslint-disable-next-line functional/no-let
+  let siteMap;
+  try {
+    const response = await fetch(SITEMAP_URL);
+    siteMap = await response.text();
+  } catch (error) {
+    console.error('Error fetching sitemap.xml:', error);
+    process.exit(1);
+  }
+
+  await writeSitemapJson(
+    siteMap,
+    S3_SITEMAP_PATH,
+    `${S3_BUCKET_NAME}`,
+    s3Client
+  );
+
+  await writeSitemapJson(
+    strapiProducts,
+    S3_PRODUCTS_METADATA_JSON_PATH,
+    `${S3_BUCKET_NAME}`,
+    s3Client
+  );
+
+  await writeSitemapJson(
+    strapiApisData,
+    S3_APIS_DATA_METADATA_JSON_PATH,
+    `${S3_BUCKET_NAME}`,
+    s3Client
+  );
+}
+
+main();

packages/gitbook-docs/src/scripts/generateUrlParsingMetadata.ts

@@ -8,13 +8,12 @@ import { readdir, writeFile } from 'fs/promises';
 import * as fs from 'fs';
 import path from 'path';
 import { fetchFromStrapi } from '../helpers/fetchFromStrapi';
+import { MetadataInfo, MetadataType } from '../helpers/guidesMetadataHelper';
 import {
-  MetadataInfo,
-  MetadataType,
   StrapiGuide,
   StrapiReleaseNote,
   StrapiSolution,
-} from '../helpers/guidesMetadataHelper';
+} from '../helpers/strapiTypes';
 import { sitePathFromLocalPath } from '../helpers/sitePathFromLocalPath';
 import { DOCUMENTATION_PATH } from '../helpers/documentationParsing.helper';
 // Load environment variables from .env file