add case insensitive redirects for all _materials

Author: camriddell

.github/workflows/fetch-remark-metadata.yml .github/workflows/site-preprocess.yml

@@ -1,4 +1,4 @@
-name: Copy REMARK metadata
+name: Preprocessing to build website
 on:
   push:
     branches:
@@ -22,8 +22,10 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           python -m pip install pyyaml==6.0.0
-      - name: copy metadata
+      - name: copy REMARK metadata
         run: python scripts/populate_materials.py
+      - name: case insensitive redirects
+        run: python scripts/caseinsensitive_redirects.py
       - name: update gh-pages branch
         run: |
           git config --global user.name github-actions

_layouts/material.html

@@ -349,6 +349,8 @@ <h2 class="heading">Metadata</h2>
                 and item != 'title'
                 and item != 'slug'
                 and item != 'ext'
+                and item != 'redirect_from'
+                and item != 'redirect_to'
                 %}
                 {% if page[item] != '' %}
                 <tr>

scripts/caseinsensitive_redirects.py

@@ -0,0 +1,53 @@
+from pathlib import Path
+from itertools import product
+from io import StringIO
+from re import fullmatch, split as re_split
+
+from yaml import safe_load, dump as yaml_dump
+
+def parse_yaml_header(f):
+    metadata = StringIO()
+    # advance to metadata section
+    for line in f:
+        if line.strip() == '---':
+            break
+    # copy metadata section
+    for line in f:
+        if line.strip() == '---':
+            break
+        metadata.write(line)
+    metadata.seek(0)
+    return safe_load(metadata), f
+
+def generate_case_combinations(name):
+    # Find all uppercase letters and their positions
+    pattern = '[A-Z]+'
+    parts = re_split(f'({pattern})', name)
+
+    # Generate case variants for matched groups
+    case_options = [
+        [part.lower(), part.upper()] if fullmatch(pattern, part) else [part]
+        for part in parts
+    ]
+
+    # Create all combinations and join them back
+    yield from (''.join(combo) for combo in product(*case_options))
+
+if __name__ == '__main__':
+    repo_root = Path(__file__).parents[1]
+
+    for path in repo_root.glob('_materials/*.md'):
+        with open(path) as f:
+            metadata, f = parse_yaml_header(f)
+            metadata.setdefault('redirect_from', [])
+            metadata['redirect_from'] += [
+                n for n in generate_case_combinations(path.stem)
+                if n != path.stem
+            ]
+            body = f.read()
+
+        with open(path, 'w') as f:
+            f.write('---\n')
+            yaml_dump(metadata, f, default_flow_style=False)
+            f.write('---\n')
+            f.write(body)

scripts/populate_materials.py

@@ -0,0 +1,65 @@
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from io import StringIO
+from itertools import product
+from re import finditer
+from subprocess import run
+from tempfile import TemporaryDirectory
+
+from yaml import safe_load, dump as yaml_dump
+
+def parse_yaml_header(f):
+    metadata = StringIO()
+    # advance to metadata section
+    for line in f:
+        if line.strip() == '---':
+            break
+    # copy metadata section
+    for line in f:
+        if line.strip() == '---':
+            break
+        metadata.write(line)
+    metadata.seek(0)
+    return safe_load(metadata), f
+
+repo_root = Path(__file__).parents[1]
+
+if __name__ == '__main__':
+    with TemporaryDirectory() as d:
+        tmpdir = Path(d)
+        run(['git', 'clone', '--filter=tree:0', 'https://github.com/econ-ark/REMARK'], cwd=tmpdir)
+
+        remotes = {}
+        for yml_path in tmpdir.glob('REMARK/REMARKs/*yml'):
+            with open(yml_path) as f:
+                metadata = safe_load(f)
+            remotes[yml_path.stem] = metadata['remote']
+
+        futures = {}
+        with ThreadPoolExecutor(4) as pool:
+            for name, uri in remotes.items():
+                futures[name] = pool.submit(
+                    lambda uri: run(['git', 'clone', '--sparse', uri], cwd=tmpdir),
+                    uri=uri
+                )
+
+        for name, result in futures.items():
+            result.result()
+            cff = tmpdir / name / 'CITATION.cff'
+            if not cff.exists():
+                continue
+            with open(cff) as f:
+                remark_data = safe_load(f)
+
+            if (tmpdir / name / 'REMARK.md').exists():
+                with open(tmpdir / name / 'REMARK.md') as f:
+                    mdata, f = parse_yaml_header(f)
+                    body = f.read()
+                    remark_data.update(mdata)
+
+            with open(repo_root / '_materials' / f'{name}.md', 'w') as f:
+                f.write('---\n')
+                yaml_dump(remark_data, f, default_flow_style=False)
+                f.write('---\n')
+                f.write(body)
+