[DNM] 🧪 Instrument C-extensions to collect coverage

.github/workflows/ci-cd.yml

@@ -355,6 +355,12 @@
       run: >-
         make clean
       shell: bash
+    - name: Pre-install ccache for gcov  [FIXME]
+      if: steps.build_type.outputs.build_type == 'source'
+      run: >-
+        sudo apt update -y
+        && sudo apt install -y ccache
+      shell: bash
     - name: Self-install (from ${{ steps.build_type.outputs.build_type }})
       env:
         MULTIDICT_NO_EXTENSIONS: ${{ matrix.no-extensions }}
@@ -365,12 +371,56 @@
           && '.'
           || steps.wheel-file.outputs.path
         }}'
+    - name: Experimental display of multidict files  [FIXME]
+      run: pip show multidict --files
     - name: Run unittests
       run: >-
         python -Im pytest tests -v
         --cov-report xml
         --junitxml=.test-results/pytest/test.xml
         --${{ matrix.no-extensions == 'Y' && 'no-' || '' }}c-extensions
+    - name: Experimentally find gcda/gcno [FIXME]
+      if: >-
+        !cancelled()
+        && runner.os == 'Linux'
+      run: |
+        find . -name '*.gc*'
+    - name: Experimentally find gcda/gcno in site-packages [FIXME]
+      if: >-
+        !cancelled()
+        && runner.os == 'Linux'
+      run: |
+        find "$(python -Ic 'import importlib.resources; print(str(importlib.resources.files("multidict")))')" -name '*.gc*'
+    - name: Experimentally install Gcovr [FIXME]
+      if: >-
+        !cancelled()
+        && runner.os == 'Linux'
+      run: |
+        pip install gcovr
+    - name: Experimentally make a coverage dir [FIXME]
+      if: >-
+        !cancelled()
+        && runner.os == 'Linux'
+      run: |
+        mkdir -pv coverage
+    - name: Experimentally run Gcovr [FIXME]
+      if: >-
+        !cancelled()
+        && runner.os == 'Linux'
+      run: |
+        gcovr --verbose
+    - name: See Gcovr leftovers at home [FIXME]
+      if: >-
+        !cancelled()
+        && runner.os == 'Linux'
+      run: |
+        find ~/ -type f -name '*.gcov.json.gz'
+    - name: See Gcovr leftovers in workdir [FIXME]
+      if: >-
+        !cancelled()
+        && runner.os == 'Linux'
+      run: |
+        find ../.. -type f -name '*.gcov.json.gz'
     - name: Produce markdown test summary from JUnit
       if: >-
         !cancelled()

MANIFEST.in

@@ -1,4 +1,5 @@
 include .coveragerc
+include gcovr.cfg
 include pyproject.toml
 include pytest.ini
 include LICENSE

gcovr.cfg

@@ -0,0 +1,10 @@
+filter = multidict/
+
+html-details = coverage/index.html
+
+print-summary = yes
+
+search-path = multidict/__tracing-data__/multidict/_multidict.gcda
+search-path = multidict/__tracing-data__/multidict/_multidict.gcno
+
+xml = coverage/coverage.xml

pyproject.toml

@@ -13,4 +13,7 @@ enable = ["cpython-freethreading"]
 [tool.cibuildwheel.linux]
 # Re-enable 32-bit builds (disabled by default in cibuildwheel 3.0)
 archs = ["auto", "auto32"]
-before-all = "yum install -y libffi-devel || apk add --upgrade libffi-dev || apt-get install libffi-dev"
+before-all = "yum install -y ccache libffi-devel || apk add --upgrade ccache libffi-dev || apt-get install ccache libffi-dev"
+
+[tool.cibuildwheel.linux.environment]
+MULTIDICT_DEBUG_BUILD = "1"

setup.py

@@ -1,7 +1,11 @@
+import functools
+import json
 import os
+import pathlib
 import platform
 import sys
 
+from distutils import log
 from setuptools import Extension, setup
 
 NO_EXTENSIONS = bool(os.environ.get("MULTIDICT_NO_EXTENSIONS"))
@@ -11,8 +15,27 @@
     NO_EXTENSIONS = True
 
 CFLAGS = ["-O0", "-g3", "-UNDEBUG"] if DEBUG_BUILD else ["-O3", "-DNDEBUG"]
+# https://gcc.gnu.org/onlinedocs/gcc/Gcov-Data-Files.html
+# `-ftest-coverage` -> `.gcno` is in the same place as `.o` -> `{self.build_temp}/multidict`
+# `-fprofile-dir` -> change the location of the `.gcda` file
+#
+# https://gcc.gnu.org/onlinedocs/gcc/Invoking-Gcov.html
+# `-fkeep-inline-functions`
+# `-fkeep-static-functions`
+# `-fprofile-prefix-map=old=new`
+# `--coverage` is an alias for `-fprofile-arcs -ftest-coverage` when compiling and `-lgcov` when linking; `-coverage` seems to be its synonym.
+# `-fprofile-abs-path` makes `gcovr` more robust — https://gcovr.com/en/stable/guide/compiling.html#compiler-options
+if DEBUG_BUILD:
+    CFLAGS.extend([
+        '--coverage',
+        '-fkeep-inline-functions',
+        '-fkeep-static-functions',
+        '-fprofile-abs-path',
+    ])
 
-if platform.system() != "Windows":
+LDFLAGS = ['--coverage'] if DEBUG_BUILD else []
+
+if platform.system() != "Windows" and False:
     CFLAGS.extend(
         [
             "-std=c11",
@@ -25,20 +48,201 @@
         ]
     )
 
+if DEBUG_BUILD:
+    # https://gcovr.com/en/stable/cookbook.html#how-to-collect-coverage-for-c-extensions-in-python
+    os.environ['CC'] = 'ccache gcc'  # no distutils/setuptools equivalent
+
 extensions = [
     Extension(
         "multidict._multidict",
         ["multidict/_multidict.c"],
         extra_compile_args=CFLAGS,
+        extra_link_args=LDFLAGS,
     ),
 ]
 
 
+from setuptools.command.build_ext import build_ext
+
+
+class TraceableBinaryExtensionCmd(build_ext):
+    def _ext_mod_path_for(self, ext) -> list[str]:
+        if not DEBUG_BUILD:
+            raise LookupError
+
+        fullname = self.get_ext_fullname(ext.name)
+        return fullname.split('.')
+
+    def _ext_tracing_file_for(self, ext) -> pathlib.Path:
+        if not DEBUG_BUILD:
+            raise LookupError
+
+        modpath = self._ext_mod_path_for(ext)
+
+        # NOTE: `.o` file is unnecessary for gcovr to function
+        return pathlib.Path(*modpath).with_suffix('.gcno')
+
+    def _ext_tracing_data_dir_for(self, ext) -> tuple[str]:
+        if not DEBUG_BUILD:
+            raise LookupError
+
+        modpath = self._ext_mod_path_for(ext)
+        package = '.'.join(modpath[:-1])
+        build_py = self.get_finalized_command('build_py')  # ??
+        package_dir = pathlib.Path(build_py.get_package_dir(package))
+        return package_dir / '__tracing-data__'
+
+    @functools.cached_property
+    def _ext_tracing_data_dir_map(self) -> dict[str, tuple[str]]:
+        extensions_with_tracing_data = self.extensions if DEBUG_BUILD else ()
+
+        return {
+            ext.name: self._ext_tracing_data_dir_for(ext)
+            for ext in extensions_with_tracing_data
+        }
+
+    def _extra_ext_data_files_for(self, ext) -> tuple[str]:
+        if not DEBUG_BUILD:
+            return ()
+
+        tracing_data_in_package_dir = self._ext_tracing_data_dir_map[ext.name]
+        tracing_data_file_in_tmp_dir = self._ext_tracing_file_for(ext)
+        tracing_data_file_in_package_dir = tracing_data_in_package_dir / tracing_data_file_in_tmp_dir
+        build_meta_json_path = tracing_data_in_package_dir / 'build-metadata.json'
+
+        return (build_meta_json_path, tracing_data_file_in_package_dir)
+
+    @functools.cached_property
+    def _extra_ext_data_files_map(self) -> dict[str, tuple[str]]:
+        extensions_with_tracing_data = self.extensions if DEBUG_BUILD else ()
+
+        return {
+            ext.name: self._extra_ext_data_files_for(ext)
+            for ext in extensions_with_tracing_data
+        }
+
+    @functools.cached_property
+    def _extra_wheel_data_files(self) -> tuple[pathlib.Path]:
+        extensions_with_tracing_data = self.extensions if DEBUG_BUILD else ()
+
+        return tuple(
+            relative_path
+            for ext in extensions_with_tracing_data
+            for relative_path in self._extra_ext_data_files_map[ext.name]
+        )
+
+    def get_outputs(self) -> list[str]:
+        """Return absolute file paths to be included in the wheel."""
+        base_outputs = super().get_outputs()
+
+        build_dir_path = pathlib.Path(self.build_lib)
+        tracing_outputs = (
+            build_dir_path / relative_path
+            for relative_path in self._extra_wheel_data_files
+        )
+
+        # NOTE: Files returned here end up in wheels and then `site-packages/`.
+        # NOTE: Editable installs rely on the tracing files to be copied into
+        # NOTE: the source checkout, which is happening in `run()`.
+        return [*base_outputs, *tracing_outputs]
+
+    def build_extension(self, ext):
+        super().build_extension(ext)
+
+        if not DEBUG_BUILD:
+            log.info('Not a debug build. Skipping tracing data...')
+            return
+
+        log.info('Copying tracing data into the build directory')
+        tracing_data_file_in_tmp_dir = self._ext_tracing_file_for(ext)
+        tracing_data_in_package_dir = self._ext_tracing_data_dir_map[ext.name]
+        tracing_data_file_in_package_dir = tracing_data_in_package_dir / tracing_data_file_in_tmp_dir
+
+        tracing_data_in_build_dir = pathlib.Path(self.build_lib) / tracing_data_in_package_dir
+
+        tracing_data_file_in_build_dir_absolute = tracing_data_in_build_dir / tracing_data_file_in_tmp_dir
+
+        tracing_data_file_in_build_dir_absolute.parent.mkdir(exist_ok=True, parents=True)
+        # NOTE: `gcc` writes `.gcno` files next to `.o` which is the temporary
+        # NOTE: directory for us (`build_temp`). This copies it over into the
+        # NOTE: regular build directory (`build_lib`) producing the layout we
+        # NOTE: expect in wheels / site-packages / source checkout. It's later
+        # NOTE: copied over to those places along with the shared libraries.
+        self.copy_file(
+            pathlib.Path(self.build_temp) / tracing_data_file_in_tmp_dir,
+            tracing_data_file_in_build_dir_absolute,
+            level=self.verbose,
+        )
+
+        # NOTE: `gcc` writes an absolute path to `.gcno` files into the shared
+        # NOTE: library at build time. It may point to an arbitrary temporary
+        # NOTE: directory that we don't care for. When pytest imports this
+        # NOTE: file, the C-extension attempts writing a `.gcda` file in the
+        # NOTE: same directory. We want it to be written in the source checkout
+        # NOTE: next to the tracing file. For this, we record information about
+        # NOTE: the desired location relative to the project root, bundle it
+        # NOTE: in the wheel and the tests will read from it, and set the
+        # NOTE: respective environment variables so the coverage data ends up
+        # NOTE: where we expect it to. `gcovr` will also work better with
+        # NOTE: predictable data locations.
+        # NOTE:
+        # NOTE: The contributors won't have to set the env vars manually
+        # NOTE: $ GCOV_PREFIX=multidict/__tracing-data__/ \
+        # NOTE:   GCOV_PREFIX_STRIP=2 \
+        # NOTE:     python -Im pytest
+        build_meta_path = tracing_data_in_build_dir / 'build-metadata.json'
+        tmp_path_length = len(pathlib.Path(self.build_temp).resolve().parents)
+        build_meta_path.write_text(
+            json.dumps(
+                {
+                    'GCOV_PREFIX': str(tracing_data_in_package_dir),
+                    'GCOV_PREFIX_STRIP': str(tmp_path_length),
+                },
+            ),
+            encoding='utf-8',
+        )
+
+    def run(self):
+        super().run()
+
+        if not self.inplace:
+            log.info('Not an editable install. Skipping tracing data...')
+
+        if not DEBUG_BUILD:
+            log.info('Not a debug build. Skipping tracing data...')
+            return
+
+        log.info(
+            'Editable install in debug mode. Copying tracing data in-tree...',
+        )
+
+        # NOTE: Editable installs usually expect data in-tree. This handles
+        # NOTE: the cases of `python setup.py build_ext --inplace` and
+        # NOTE: `pip install -e .`
+        # NOTE: Normal wheel builds include files returned by `get_outputs()`.
+
+        build_dir_path = pathlib.Path(self.build_lib)
+        for relative_path in self._extra_wheel_data_files:
+            relative_path.parent.mkdir(exist_ok=True, parents=True)
+            self.copy_file(
+                build_dir_path / relative_path,
+                relative_path,
+                level=self.verbose,
+            )
+
+        for ext in self.extensions:
+            tracing_data_in_pkg_dir = self._ext_tracing_data_dir_map[ext.name]
+            (tracing_data_in_pkg_dir / '.gitignore').write_text('*\n')
+
+
 if not NO_EXTENSIONS:
     print("*********************")
     print("* Accelerated build *")
     print("*********************")
-    setup(ext_modules=extensions)
+    setup(
+        cmdclass={'build_ext': TraceableBinaryExtensionCmd},
+        ext_modules=extensions,
+    )
 else:
     print("*********************")
     print("* Pure Python build *")