Track number of Sphinx build warnings

.github/workflows/update.yml

@@ -1,6 +1,6 @@
 on:
   schedule:
-    - cron: '*/30 * * * *'
+    - cron: '17 * * * *'
   push:
     branches:
       - 'main'
@@ -20,7 +20,7 @@ jobs:
       - uses: actions/checkout@v4
       - run: sudo apt-get install -y gettext
       - run: uv run generate.py  # generates "index.html"
-      - run: mkdir -p build && cp index.* style.css build
+      - run: mkdir -p build && cp index.* style.css warnings* build
       - name: Deploy 🚀
         if: github.event_name != 'pull_request'
         uses: JamesIves/github-pages-deploy-action@v4

.gitignore

@@ -1 +1,2 @@
 index.html
+warnings-*.txt

build_warnings.py

@@ -0,0 +1,37 @@
+from pathlib import Path
+from re import findall
+from shutil import copyfile
+
+import sphinx.cmd.build
+
+
+def number(clones_dir: str, repo: str, language_code: str) -> int:
+    language_part, *locale = language_code.split('-')
+    if locale:
+        lang_with_locale = f'{language_part}_{locale[0].upper()}'
+    else:
+        lang_with_locale = language_part
+    locale_dir = Path(clones_dir, f'cpython/Doc/locales/{lang_with_locale}/LC_MESSAGES')
+    locale_dir.mkdir(parents=True)
+    for po_file in Path(clones_dir, repo).rglob('*.po'):
+        relative_path = po_file.relative_to(Path(clones_dir, repo))
+        target_file = locale_dir / relative_path
+        target_file.parent.mkdir(parents=True, exist_ok=True)
+        copyfile(po_file, target_file)
+    sphinx.cmd.build.main(
+        (
+            '--builder',
+            'html',
+            '--jobs',
+            'auto',
+            '--define',
+            f'language={language_code}',
+            '--verbose',
+            '--warning-file',
+            warning_file := f'{clones_dir}/warnings-{language_code}.txt',
+            f'{clones_dir}/cpython/Doc',  # sourcedir
+            f'./sphinxbuild/{language_code}',  # outputdir
+        )
+    )
+    copyfile(warning_file, f'warnings-{language_code}.txt')
+    return len(findall('ERROR|WARNING', Path(warning_file).read_text()))

generate.py

@@ -6,6 +6,8 @@
 #     "jinja2",
 #     "requests",
 #     "docutils",
+#     "sphinx",
+#     "python-docs-theme",
 # ]
 # ///
 import json
@@ -23,11 +25,12 @@
 from jinja2 import Template
 from urllib3 import PoolManager
 
-import contribute
+import build_warnings
 import build_status
+import contribute
 from visitors import get_number_of_visitors
 from completion import branches_from_devguide, get_completion, TranslatorsData
-from repositories import get_languages_and_repos, Language
+from repositories import Language, get_languages_and_repos
 
 generation_time = datetime.now(timezone.utc)
 
@@ -48,15 +51,13 @@ def get_completion_progress() -> Iterator['LanguageProjectData']:
         )
         subprocess.run(['make', '-C', cpython_dir / 'Doc', 'venv'], check=True)
         subprocess.run(['make', '-C', cpython_dir / 'Doc', 'gettext'], check=True)
-        languages_built = dict(build_status.get_languages(http := PoolManager()))
-
-        with concurrent.futures.ThreadPoolExecutor() as executor:
+        languages_built = dict(build_status.get_languages(PoolManager()))
+        with concurrent.futures.ProcessPoolExecutor() as executor:
             return executor.map(
                 get_project_data,
                 *zip(*get_languages_and_repos(devguide_dir)),
                 itertools.repeat(languages_built),
                 itertools.repeat(clones_dir),
-                itertools.repeat(http),
             )
 
 
@@ -65,16 +66,21 @@ def get_project_data(
     repo: str | None,
     languages_built: dict[str, bool],
     clones_dir: str,
-    http: PoolManager,
 ) -> 'LanguageProjectData':
     built = language.code in languages_built
     if repo:
         completion, translators_data, branch = get_completion(clones_dir, repo)
-        visitors_num = get_number_of_visitors(language.code, http) if built else 0
+        visitors_num = (
+            get_number_of_visitors(language.code, PoolManager()) if built else 0
+        )
+        warnings = (
+            build_warnings.number(clones_dir, repo, language.code) if completion else 0
+        )
     else:
         completion = 0.0
         translators_data = TranslatorsData(0, False)
         visitors_num = 0
+        warnings = 0
         branch = None
     return LanguageProjectData(
         language,
@@ -83,6 +89,7 @@ def get_project_data(
         completion,
         translators_data,
         visitors_num,
+        warnings,
         built,
         in_switcher=languages_built.get(language.code),
         uses_platform=language.code in contribute.pulling_from_transifex,
@@ -98,6 +105,7 @@ class LanguageProjectData:
     completion: float
     translators: TranslatorsData
     visitors: int
+    warnings: int
     built: bool
     in_switcher: bool | None
     uses_platform: bool

style.css

@@ -39,7 +39,7 @@ th {
 .progress-bar.low + .progress-bar-outer-label {
   display: inline-block;
 }
-td[data-label="visitors"], td[data-label="translators"] {
+td[data-label="visitors"], td[data-label="translators"], td[data-label="warnings"] {
   text-align: right;
 }
 td[data-label="completion"] {

template.html.jinja

@@ -3,6 +3,7 @@
   <title>Python Docs Translation Dashboard</title>
   <link rel="stylesheet" href="style.css">
   <meta charset="UTF-8">
+  <base target="_blank">
 </head>
 <body>
 <h1>Python Docs Translation Dashboard</h1>
@@ -12,8 +13,9 @@
   <th>language</th>
   <th>contribute</th>
   <th>build</th>
-  <th>visitors*</th>
+  <th>visitors¹</th>
   <th>translators</th>
+  <th>warnings²</th>
   <th>completion</th>
 </tr>
 </thead>
@@ -22,31 +24,34 @@
 <tr>
   <td data-label="language">{{ project.language.name }} ({{ project.language.code }})</td>
   <td data-label="contribute">
-    {% if project.contribution_link %}<a href="{{ project.contribution_link }}" target="_blank">{% endif %}
+    {% if project.contribution_link %}<a href="{{ project.contribution_link }}">{% endif %}
     {% if project.uses_platform %}platform{% else %}repository{% endif %}
     {% if project.contribution_link %}</a>{% endif %}
   </td>
   <td data-label="build">
     {% if project.built %}
-      <a href="https://docs.python.org/{{ project.language.code }}/" target="_blank">✓{% if project.in_switcher %} in switcher{% endif %}</a>
+      <a href="https://docs.python.org/{{ project.language.code }}/">✓{% if project.in_switcher %} in switcher{% endif %}</a>
     {% else %}
       ✗
     {% endif %}
   </td>
   <td data-label="visitors">
     {% if project.built %}
-      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))" target="_blank">
+      <a href="https://plausible.io/docs.python.org?filters=((contains,page,(/{{ project.language.code }}/)))">
         {{ '{:,}'.format(project.visitors) }}
       </a>
     {% else %}
       {{ '{:,}'.format(project.visitors) }}
     {% endif %}
   </td>
   <td data-label="translators">
-    {% if project.translators.link %}<a href="{{ project.translators.link }}" target="_blank">{% endif %}
+    {% if project.translators.link %}<a href="{{ project.translators.link }}">{% endif %}
     {{ project.translators.number }}
     {% if project.translators.link %}</a>{% endif %}
   </td>
+  <td data-label="warnings">
+    {% if project.completion %}<a href="warnings-{{ project.language.code }}.txt">{{ project.warnings }}</a>{% else %}{{ project.warnings }}{% endif %}
+  </td>
   <td data-label="completion">
     <div class="progress-bar" style="width: {{ project.completion }}%;">{{ "{:.2f}".format(project.completion) }}%</div>
     <div class="progress-bar-outer-label">{{ "{:.2f}".format(project.completion) }}%</div>
@@ -55,8 +60,9 @@
 {% endfor %}
 </tbody>
 </table>
-<p>* sum of <a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies" target="_blank">daily unique visitors</a> since 8 June 2024</p>
-<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/" target="_blank">Python Developer’s Guide</a>.</p>
+<p>¹ sum of <a href="https://plausible.io/data-policy#how-we-count-unique-users-without-cookies">daily unique visitors</a> since 8 June 2024</p>
+<p>² number of Sphinx build process warnings</p>
+<p>For more information about translations, see the <a href="https://devguide.python.org/documentation/translating/">Python Developer’s Guide</a>.</p>
 <p>Last updated at {{ generation_time.strftime('%A, %-d %B %Y, %-H:%M:%S %Z') }} (in {{ duration // 60 }}:{{ "{:02}".format(duration % 60) }} minutes).</p>
 </body>
 <script>