[WIP] New test for finding internal and external incorrect links

.github/workflows/test.yaml

@@ -8,9 +8,7 @@ env:
 
 jobs:
   blueberry:
-
     runs-on: ubuntu-latest
-
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
@@ -77,6 +75,38 @@ jobs:
       env:
         GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
 
+  links:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.10.x'
+        architecture: 'x64'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install python-frontmatter
+    - name: Run tests
+      run: python ./ci/links.py ${{ steps.files.outputs.all }}
+
+  links-new:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.10.x'
+        architecture: 'x64'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install python-frontmatter
+    - name: Run tests
+      run: python ./ci/links-new.py ${{ steps.files.outputs.all }}
+
   docs404:
     runs-on: ubuntu-latest
     steps:
@@ -96,7 +126,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.10.x'
+        python-version: '3.11.x'
         architecture: 'x64'
     - name: Cache pip
       uses: actions/cache@v2
@@ -114,7 +144,7 @@ jobs:
     - name: Use Node.js
       uses: actions/setup-node@v1
       with:
-        node-version: '12.x'
+        node-version: '18.x'
     - name: Install dependencies (Node)
       run: npm install
     - name: Set up Hugo

ci/docs404.py

@@ -32,6 +32,11 @@ def __init__(self, *args, **kwargs):
     start_urls = ['http://localhost:1313/docs']
     handle_httpstatus_list = [404]
 
+    custom_settings = {
+      'CONCURRENT_REQUESTS': 300,
+      'CONCURRENT_REQUESTS_PER_DOMAIN': 300
+    }
+
     rules = (
         Rule(LinkExtractor(allow=r'/docs/', deny=r'/docs/contribute'),
              callback='parse_item', follow=True),
@@ -76,4 +81,4 @@ def parse_item(self, response):
         print('404 response in HTML - see logs')
         sys.exit(1)
     else:
-        print('\n\nScraper did not find any 404 links')
+        print('\n\nScraper did not find any 404 links')