feat: set a warning message on forbidden pages instead of blocking (#436)

* feat: set a warning message on forbidden pages instead of blocking update

* send error to mattermost

* fix: delete 403 files

* remove color from release

Author: Viczei

.github/workflows/release.yml

@@ -7,7 +7,7 @@ on:
   workflow_dispatch:
 
 permissions:
-  id-token: write  # Required for OIDC token generation
+  id-token: write # Required for OIDC token generation
   contents: write
 
 jobs:
@@ -62,6 +62,21 @@ jobs:
       - name: Check Refs
         run: yarn checkRefs
 
+      - name: Check for 403 errors
+        id: check_403
+        run: |
+          if [ -f "./forbidden-urls.json" ]; then
+            echo "has_403=true" >> $GITHUB_OUTPUT
+            FORBIDDEN_CONTENT=$(cat ./forbidden-urls.json)
+            # Escape newlines and other special characters for GitHub Actions
+            FORBIDDEN_CONTENT="${FORBIDDEN_CONTENT//'%'/'%25'}"
+            FORBIDDEN_CONTENT="${FORBIDDEN_CONTENT//$'\n'/'%0A'}"
+            FORBIDDEN_CONTENT="${FORBIDDEN_CONTENT//$'\r'/'%0D'}"
+            echo "forbidden_urls=$FORBIDDEN_CONTENT" >> $GITHUB_OUTPUT
+          else
+            echo "has_403=false" >> $GITHUB_OUTPUT
+          fi
+
       - name: Get metadata
         id: metadata
         shell: bash
@@ -115,10 +130,24 @@ jobs:
           GITHUB_TOKEN: ${{ steps.token.outputs.token }}
           NPM_TOKEN: ${{ secrets.SOCIALGROOVYBOT_NPM_TOKEN }}
 
+      - uses: mattermost/action-mattermost-notify@master
+        if: steps.check_403.outputs.has_403 == 'true'
+        with:
+          MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }}
+          TEXT: |
+            ⚠️ **Avertissement** : Certaines pages ont retourné une erreur 403 (Forbidden) lors du scraping et ont été ignorées.
+
+            ```json
+            ${{ steps.check_403.outputs.forbidden_urls }}
+            ```
+
+            Le processus de mise à jour a continué normalement.
+            [Les logs complets sont disponibles ici](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+
       - uses: mattermost/action-mattermost-notify@master
         if: failure()
         with:
           MATTERMOST_WEBHOOK_URL: ${{ secrets.MATTERMOST_WEBHOOK_URL }}
           TEXT: |
-            La mise à jour du dépôt fiches-travail-data a échoué.
+            ❌ La mise à jour du dépôt fiches-travail-data a échoué.
             [Les logs sont disponibles ici](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})

data/fiches-travail.json

@@ -500,10 +500,6 @@
       "id": "article373014",
       "url": "https://travail-emploi.gouv.fr/les-titres-professionnels"
     },
-    {
-      "id": "article373015",
-      "url": "https://travail-emploi.gouv.fr/le-fonds-national-de-lemploi-formation-fne-formation"
-    },
     {
       "id": "article373029",
       "url": "https://travail-emploi.gouv.fr/lallocation-de-solidarite-specifique-ass"

local.data.json

@@ -15,11 +15,17 @@ scrapUrl.mockImplementation((id, url) => {
   if (url === "url.fail") {
     const error = new Error(`xxx ${url} fail`);
     error.url = url;
-    return Promise.reject(url);
+    return Promise.reject(error);
+  }
+  if (url === "url.forbidden") {
+    const error = new Error(`HTTP Error: 403 - ${url} - Forbidden`);
+    error.url = url;
+    error.isForbidden = true;
+    return Promise.reject(error);
   }
 });
 
-test("scrap should throw if some scrapped pages failed", async () => {
+test("scrap should throw if some scrapped pages failed with non-403 errors", async () => {
   await expect(
     scrap([
       { id: "url.a", url: "url.sections" },
@@ -28,6 +34,27 @@ test("scrap should throw if some scrapped pages failed", async () => {
   ).rejects.toThrowError(/fetching pages fail/);
 });
 
+test("scrap should not throw if some scrapped pages failed with 403 errors", async () => {
+  // Mock console.warn to verify it's called
+  const consoleWarnSpy = jest
+    .spyOn(console, "warn")
+    .mockImplementation(jest.fn);
+
+  const result = await scrap([
+    { id: "url.a", url: "url.sections" },
+    { id: "noid", url: "url.forbidden" },
+  ]);
+
+  // Verify that console.warn was called
+  expect(consoleWarnSpy).toHaveBeenCalled();
+  // Verify that we still get the successful results
+  expect(result).toHaveLength(1);
+  expect(result[0].pubId).toBe("url.a");
+
+  // Restore the original console.warn
+  consoleWarnSpy.mockRestore();
+});
+
 test("scrap should throw if some scrapped pages have same id", async () => {
   await expect(
     scrap([

src/fetch-data/__tests__/index.test.js

@@ -31,6 +31,16 @@ got.mockImplementation((url) => {
     error.name = "HTTPError";
     return Promise.reject(error);
   }
+  if (url.startsWith("url.forbidden")) {
+    const error = new HTTPError();
+    error.response = {
+      statusCode: 403,
+    };
+    error.options = { url: { href: "url.forbidden" } };
+    error.message = "Forbidden";
+    error.name = "HTTPError";
+    return Promise.reject(error);
+  }
   if (url.startsWith("url.parse.fail")) {
     const error = new ParseError();
     error.message = "parse fail";
@@ -89,4 +99,15 @@ describe("scrapUrl", () => {
       /Parsing Error/
     );
   });
+
+  test("scrapUrl should set isForbidden property for 403 errors", async () => {
+    process.env.TOKEN_MT = "TOKEN";
+    try {
+      await scrapUrl("id", "url.forbidden");
+      expect(true).toBe(false); // This will fail the test if no error is thrown
+    } catch (error) {
+      expect(error.message).toMatch(/HTTP Error: 403/);
+      expect(error.isForbidden).toBe(true);
+    }
+  });
 });

src/fetch-data/__tests__/scrapUrl.test.js

@@ -22,10 +22,40 @@ export async function scrap(urls) {
 
   const failedPromise = results.filter(({ status }) => status === "rejected");
 
-  if (failedPromise.length > 0) {
+  // Separate 403 errors from other errors
+  const forbiddenErrors = failedPromise.filter(
+    ({ reason }) => reason.isForbidden
+  );
+  const otherErrors = failedPromise.filter(({ reason }) => !reason.isForbidden);
+
+  // Log 403 errors as warnings and save them to a file for GitHub Actions
+  if (forbiddenErrors.length > 0) {
+    const forbiddenUrls = forbiddenErrors.map(({ reason }) => reason.url);
+    console.warn(
+      "WARNING: The following pages returned 403 Forbidden and were skipped:",
+      forbiddenUrls
+    );
+
+    // Write forbidden URLs to a file for GitHub Actions to use
+    fs.writeFileSync(
+      path.join(__dirname, "../../forbidden-urls.json"),
+      JSON.stringify(
+        {
+          urls: forbiddenUrls,
+          timestamp: new Date().toISOString(),
+          count: forbiddenUrls.length,
+        },
+        null,
+        2
+      )
+    );
+  }
+
+  // Only fail if there are non-403 errors
+  if (otherErrors.length > 0) {
     console.error(
       "scrap fail",
-      failedPromise.map(({ reason }) => reason)
+      otherErrors.map(({ reason }) => reason)
     );
     throw new Error("Error - fetching pages fail. Some pages are missing");
   }

src/fetch-data/index.js

@@ -34,6 +34,10 @@ export async function scrapUrl(id, url) {
       err = new Error(
         `HTTP Error: ${error.response.statusCode} - ${url} - ${error.message}`
       );
+      // Add a property to identify 403 errors specifically
+      if (error.response.statusCode === 403) {
+        err.isForbidden = true;
+      }
     } else {
       err = new Error(error.message);
     }