Bump dependencies (#6312)

Author: Jackenmen

.github/workflows/run_pip_compile.yaml

@@ -20,12 +20,27 @@ jobs:
       - name: Set up Python 3.8.
         uses: actions/setup-python@v4
         with:
-          python-version: '3.8'
+          python-version: |
+            3.11
+            3.10
+            3.9
+            3.8
 
-      - name: Install dependencies
+      - name: Install dependencies on Linux/macOS
+        if: matrix.os != 'windows-latest'
+        run: |
+          python3.11 -m pip install -U pip pip-tools
+          python3.10 -m pip install -U pip pip-tools
+          python3.9 -m pip install -U pip pip-tools
+          python3.8 -m pip install -U pip pip-tools
+
+      - name: Install dependencies on Windows
+        if: matrix.os == 'windows-latest'
         run: |
-          python -m pip install -U pip
-          python -m pip install -U pip-tools
+          py -3.11 -m pip install -U pip pip-tools
+          py -3.10 -m pip install -U pip pip-tools
+          py -3.9 -m pip install -U pip pip-tools
+          py -3.8 -m pip install -U pip pip-tools
 
       - name: Generate requirements files.
         id: compile_requirements

.github/workflows/scripts/compile_requirements.py

@@ -1,19 +1,26 @@
 import os
+import re
 import shutil
 import subprocess
 import sys
 from pathlib import Path
 
 
+EXCLUDE_STEM_RE = re.compile(r".*-3\.(?!8-)(\d+)-extra-(doc|style)")
 GITHUB_OUTPUT = os.environ["GITHUB_OUTPUT"]
 REQUIREMENTS_FOLDER = Path(__file__).parents[3].absolute() / "requirements"
 os.chdir(REQUIREMENTS_FOLDER)
 
 
-def pip_compile(name: str) -> None:
+def pip_compile(version: str, name: str) -> None:
+    stem = f"{sys.platform}-{version}-{name}"
+    if EXCLUDE_STEM_RE.fullmatch(stem):
+        return
+
+    executable = ("py", f"-{version}") if sys.platform == "win32" else (f"python{version}",)
     subprocess.check_call(
         (
-            sys.executable,
+            *executable,
             "-m",
             "piptools",
             "compile",
@@ -22,15 +29,17 @@ def pip_compile(name: str) -> None:
             "--verbose",
             f"{name}.in",
             "--output-file",
-            f"{sys.platform}-{name}.txt",
+            f"{stem}.txt",
         )
     )
 
 
-pip_compile("base")
-shutil.copyfile(f"{sys.platform}-base.txt", "base.txt")
-for file in REQUIREMENTS_FOLDER.glob("extra-*.in"):
-    pip_compile(file.stem)
+for minor in range(8, 11 + 1):
+    version = f"3.{minor}"
+    pip_compile(version, "base")
+    shutil.copyfile(f"{sys.platform}-{version}-base.txt", "base.txt")
+    for file in REQUIREMENTS_FOLDER.glob("extra-*.in"):
+        pip_compile(version, file.stem)
 
 with open(GITHUB_OUTPUT, "a", encoding="utf-8") as fp:
     fp.write(f"sys_platform={sys.platform}\n")

.github/workflows/scripts/merge_requirements.py

@@ -1,6 +1,8 @@
+from __future__ import annotations
+
 import os
 from pathlib import Path
-from typing import List, TextIO
+from typing import Dict, Iterable, List, TextIO, Tuple
 
 from packaging.markers import Marker
 from packaging.requirements import Requirement
@@ -15,6 +17,12 @@ def __init__(self, requirement_string: str) -> None:
         self.req = Requirement(requirement_string)
         self.comments = set()
 
+    def __hash__(self) -> int:
+        return hash(self.req)
+
+    def __eq__(self, other: RequirementData) -> bool:
+        return self.req == other.req
+
     @property
     def name(self) -> str:
         return self.req.name
@@ -49,30 +57,47 @@ def get_requirements(fp: TextIO) -> List[RequirementData]:
     return requirements
 
 
+def iter_envs(envs: Iterable[str]) -> Iterable[Tuple[str, str]]:
+    for env_name in envs:
+        platform, python_version = env_name.split("-", maxsplit=1)
+        yield (platform, python_version)
+
+
 names = ["base"]
 names.extend(file.stem for file in REQUIREMENTS_FOLDER.glob("extra-*.in"))
-base_requirements = []
+base_requirements: List[RequirementData] = []
 
 for name in names:
-    # {req_name: {sys_platform: RequirementData}
-    input_data = {}
+    # {req_data: {sys_platform: RequirementData}
+    input_data: Dict[RequirementData, Dict[str, RequirementData]] = {}
+    all_envs = set()
     all_platforms = set()
+    all_python_versions = set()
     for file in REQUIREMENTS_FOLDER.glob(f"*-{name}.txt"):
-        platform_name = file.stem.split("-", maxsplit=1)[0]
+        platform_name, python_version, _ = file.stem.split("-", maxsplit=2)
+        env_name = f"{platform_name}-{python_version}"
+        all_envs.add(env_name)
         all_platforms.add(platform_name)
+        all_python_versions.add(python_version)
         with file.open(encoding="utf-8") as fp:
             requirements = get_requirements(fp)
 
         for req in requirements:
-            platforms = input_data.setdefault(req.name, {})
-            platforms[platform_name] = req
+            envs = input_data.setdefault(req, {})
+            envs[env_name] = req
 
     output = base_requirements if name == "base" else []
-    for req_name, platforms in input_data.items():
-        req = next(iter(platforms.values()))
-        for other_req in platforms.values():
-            if req.req != other_req.req:
-                raise RuntimeError(f"Incompatible requirements for {req_name}.")
+    for req, envs in input_data.items():
+        # {platform: [python_versions...]}
+        python_versions_per_platform: Dict[str, List[str]] = {}
+        # {python_version: [platforms...]}
+        platforms_per_python_version: Dict[str, List[str]] = {}
+        platforms = python_versions_per_platform.keys()
+        python_versions = platforms_per_python_version.keys()
+        for env_name, other_req in envs.items():
+            platform_name, python_version = env_name.split("-", maxsplit=1)
+            python_versions_per_platform.setdefault(platform_name, []).append(python_version)
+            platforms_per_python_version.setdefault(python_version, []).append(platform_name)
 
             req.comments.update(other_req.comments)
 
@@ -84,30 +109,74 @@ def get_requirements(fp: TextIO) -> List[RequirementData]:
             old_req_marker = req.marker
             req.marker = base_req.marker = None
             if base_req.req != req.req:
-                raise RuntimeError(f"Incompatible requirements for {req_name}.")
+                raise RuntimeError(f"Incompatible requirements for {req.name}.")
 
             base_req.marker = old_base_marker
             req.marker = old_req_marker
             if base_req.marker is None or base_req.marker == req.marker:
                 continue
 
-        if len(platforms) == len(all_platforms):
+        if len(envs) == len(all_envs):
             output.append(req)
             continue
-        elif len(platforms) < len(all_platforms - platforms.keys()):
-            platform_marker = " or ".join(
-                f"sys_platform == '{platform}'" for platform in platforms
+
+        # At this point I'm wondering why I didn't just go for
+        # a more generic boolean algebra simplification (sympy.simplify_logic())...
+        if (
+            len(set(map(frozenset, python_versions_per_platform.values()))) == 1
+            or len(set(map(frozenset, platforms_per_python_version.values()))) == 1
+        ):
+            # Either all platforms have the same Python version set
+            # or all Python versions have the same platform set.
+            # We can generate markers for platform (platform_marker) and Python
+            # (python_version_marker) version sets separately and then simply require
+            # that both markers are fulfilled at the same time (env_marker).
+
+            python_version_marker = (
+                # Requirement present on less Python versions than not.
+                " or ".join(
+                    f"python_version == '{python_version}'" for python_version in python_versions
+                )
+                if len(python_versions) < len(all_python_versions - python_versions)
+                # Requirement present on more Python versions than not
+                # This may generate an empty string when Python version is irrelevant.
+                else " and ".join(
+                    f"python_version != '{python_version}'"
+                    for python_version in all_python_versions - python_versions
+                )
             )
+
+            platform_marker = (
+                # Requirement present on less platforms than not.
+                " or ".join(f"sys_platform == '{platform}'" for platform in platforms)
+                if len(platforms) < len(all_platforms - platforms)
+                # Requirement present on more platforms than not
+                # This may generate an empty string when platform is irrelevant.
+                else " and ".join(
+                    f"sys_platform != '{platform}'" for platform in all_platforms - platforms
+                )
+            )
+
+            if python_version_marker and platform_marker:
+                env_marker = f"({python_version_marker}) and ({platform_marker})"
+            else:
+                env_marker = python_version_marker or platform_marker
         else:
-            platform_marker = " and ".join(
-                f"sys_platform != '{platform}'" for platform in all_platforms - platforms.keys()
+            # Fallback to generic case.
+            env_marker = (
+                # Requirement present on less envs than not.
+                " or ".join(
+                    f"(sys_platform == '{platform}' and python_version == '{python_version}')"
+                    for platform, python_version in iter_envs(envs)
+                )
+                if len(envs) < len(all_envs - envs.keys())
+                else " and ".join(
+                    f"(sys_platform != '{platform}' and python_version != '{python_version}')"
+                    for platform, python_version in iter_envs(all_envs - envs.keys())
+                )
             )
 
-        new_marker = (
-            f"({req.marker}) and ({platform_marker})"
-            if req.marker is not None
-            else platform_marker
-        )
+        new_marker = f"({req.marker}) and ({env_marker})" if req.marker is not None else env_marker
         req.marker = Marker(new_marker)
         if base_req is not None and base_req.marker == req.marker:
             continue

requirements/base.in

@@ -1,4 +1,3 @@
-aiodns
 aiohttp
 aiohttp-json-rpc
 apsw

requirements/base.txt

@@ -1,6 +1,4 @@
-aiodns==3.0.0
-    # via -r base.in
-aiohttp==3.8.5
+aiohttp==3.9.3
     # via
     #   -r base.in
     #   aiohttp-json-rpc
@@ -10,20 +8,14 @@ aiohttp-json-rpc==0.13.3
     # via -r base.in
 aiosignal==1.3.1
     # via aiohttp
-apsw==3.43.1.0
+apsw==3.45.2.0
     # via -r base.in
-async-timeout==4.0.3
-    # via aiohttp
-attrs==23.1.0
+attrs==23.2.0
     # via aiohttp
-babel==2.12.1
+babel==2.14.0
     # via -r base.in
 brotli==1.1.0
     # via -r base.in
-cffi==1.15.1
-    # via pycares
-charset-normalizer==3.2.0
-    # via aiohttp
 click==8.1.7
     # via -r base.in
 contextlib2==21.6.0
@@ -32,71 +24,69 @@ discord-py==2.3.2
     # via
     #   -r base.in
     #   red-lavalink
-frozenlist==1.4.0
+frozenlist==1.4.1
     # via
     #   aiohttp
     #   aiosignal
-idna==3.4
+idna==3.6
     # via yarl
-importlib-metadata==6.8.0
-    # via markdown
-markdown==3.4.4
+markdown==3.6
     # via -r base.in
 markdown-it-py==3.0.0
     # via rich
 mdurl==0.1.2
     # via markdown-it-py
-multidict==6.0.4
+multidict==6.0.5
     # via
     #   aiohttp
     #   yarl
-orjson==3.9.7
+orjson==3.9.15
     # via -r base.in
-packaging==23.1
+packaging==24.0
     # via -r base.in
-platformdirs==3.10.0
+platformdirs==4.2.0
     # via -r base.in
-psutil==5.9.5
+psutil==5.9.8
     # via -r base.in
-pycares==4.3.0
-    # via aiodns
-pycparser==2.21
-    # via cffi
-pygments==2.16.1
+pygments==2.17.2
     # via rich
-python-dateutil==2.8.2
+python-dateutil==2.9.0.post0
     # via -r base.in
-pytz==2023.3.post1
-    # via babel
 pyyaml==6.0.1
     # via -r base.in
-rapidfuzz==3.3.0
+rapidfuzz==3.6.2
     # via -r base.in
 red-commons==1.0.0
     # via
     #   -r base.in
     #   red-lavalink
 red-lavalink==0.11.0
     # via -r base.in
-rich==13.5.2
+rich==13.7.1
     # via -r base.in
 schema==0.7.5
     # via -r base.in
 six==1.16.0
     # via python-dateutil
-typing-extensions==4.7.1
+typing-extensions==4.10.0
     # via
     #   -r base.in
     #   rich
-yarl==1.9.2
+yarl==1.9.4
     # via
     #   -r base.in
     #   aiohttp
-zipp==3.16.2
-    # via importlib-metadata
+async-timeout==4.0.3; python_version != "3.11"
+    # via aiohttp
 colorama==0.4.6; sys_platform == "win32"
     # via click
-distro==1.8.0; sys_platform == "linux" and sys_platform == "linux"
+distro==1.9.0; sys_platform == "linux" and sys_platform == "linux"
     # via -r base.in
-uvloop==0.17.0; (sys_platform != "win32" and platform_python_implementation == "CPython") and sys_platform != "win32"
+importlib-metadata==7.1.0; python_version != "3.10" and python_version != "3.11"
+    # via markdown
+pytz==2024.1; python_version == "3.8"
+    # via babel
+uvloop==0.19.0; (sys_platform != "win32" and platform_python_implementation == "CPython") and sys_platform != "win32"
     # via -r base.in
+zipp==3.18.1; python_version != "3.10" and python_version != "3.11"
+    # via importlib-metadata