Merge branch 'dev' of github.com:OpenCyphal/pycyphal into dev

Author: songmeo

noxfile.py

@@ -121,7 +121,7 @@ def test(session):
         "mypy   ~= 1.15.0",
         "pylint == 3.3.7",
     )
-    session.run("mypy", *map(str, src_dirs), str(compiled_dir))
+    session.run("mypy", *map(str, src_dirs))
     session.run("pylint", *map(str, src_dirs), env={"PYTHONPATH": str(compiled_dir)})
 
     # Publish coverage statistics. This also has to be run from the test session to access the coverage files.

pycyphal/dsdl/_compiler.py

@@ -15,7 +15,7 @@
 import nunavut
 import nunavut.lang
 import nunavut.jinja
-
+from pycyphal.dsdl._lockfile import Locker
 
 _AnyPath = Union[str, pathlib.Path]
 
@@ -171,26 +171,12 @@ def compile(  # pylint: disable=redefined-builtin
         (root_namespace_name,) = set(map(lambda x: x.root_namespace, composite_types))  # type: ignore
         _logger.info("Read %d definitions from root namespace %r", len(composite_types), root_namespace_name)
 
-        # Generate code
-        assert isinstance(output_directory, pathlib.Path)
         root_ns = nunavut.build_namespace_tree(
             types=composite_types,
             root_namespace_dir=str(root_namespace_directory),
             output_dir=str(output_directory),
             language_context=language_context,
         )
-        code_generator = nunavut.jinja.DSDLCodeGenerator(
-            namespace=root_ns,
-            generate_namespace_types=nunavut.YesNoDefault.YES,
-            followlinks=True,
-        )
-        code_generator.generate_all()
-        _logger.info(
-            "Generated %d types from the root namespace %r in %.1f seconds",
-            len(composite_types),
-            root_namespace_name,
-            time.monotonic() - started_at,
-        )
     else:
         root_ns = nunavut.build_namespace_tree(
             types=[],
@@ -199,28 +185,35 @@ def compile(  # pylint: disable=redefined-builtin
             language_context=language_context,
         )
 
-    support_generator = nunavut.jinja.SupportGenerator(
-        namespace=root_ns,
-    )
-    support_generator.generate_all()
-
-    # A minor UX improvement; see https://github.com/OpenCyphal/pycyphal/issues/115
-    for p in sys.path:
-        if pathlib.Path(p).resolve() == pathlib.Path(output_directory):
-            break
-    else:
-        if os.name == "nt":
-            quick_fix = f'Quick fix: `$env:PYTHONPATH += ";{output_directory.resolve()}"`'
-        elif os.name == "posix":
-            quick_fix = f'Quick fix: `export PYTHONPATH="{output_directory.resolve()}"`'
-        else:
-            quick_fix = "Quick fix is not available for this OS."
-        _logger.info(
-            "Generated package is stored in %r, which is not in Python module search path list. "
-            "The package will fail to import unless you add the destination directory to sys.path or PYTHONPATH. %s",
-            str(output_directory),
-            quick_fix,
-        )
+    if root_namespace_name is not None:
+        with Locker(
+            root_namespace_name=root_namespace_name,
+            output_directory=output_directory,
+        ) as lockfile:
+            if lockfile:
+                assert isinstance(output_directory, pathlib.Path)
+                code_generator = nunavut.jinja.DSDLCodeGenerator(
+                    namespace=root_ns,
+                    generate_namespace_types=nunavut.YesNoDefault.YES,
+                    followlinks=True,
+                )
+                code_generator.generate_all()
+                _logger.info(
+                    "Generated %d types from the root namespace %r in %.1f seconds",
+                    len(composite_types),
+                    root_namespace_name,
+                    time.monotonic() - started_at,
+                )
+
+    with Locker(
+        root_namespace_name="_support_",
+        output_directory=output_directory,
+    ) as support_lockfile:
+        if support_lockfile:
+            support_generator = nunavut.jinja.SupportGenerator(
+                namespace=root_ns,
+            )
+            support_generator.generate_all()
 
     return GeneratedPackageInfo(
         path=pathlib.Path(output_directory) / pathlib.Path(root_namespace_name),

pycyphal/dsdl/_lockfile.py

@@ -0,0 +1,62 @@
+# Copyright (c) 2025 OpenCyphal
+# This software is distributed under the terms of the MIT License.
+# Author: Huong Pham <huong.pham@zubax.com>
+
+import logging
+import pathlib
+import time
+from io import TextIOWrapper
+from pathlib import Path
+from types import TracebackType
+
+_logger = logging.getLogger(__name__)
+
+
+class Locker:
+
+    def __init__(self, output_directory: pathlib.Path, root_namespace_name: str) -> None:
+        self._output_directory = output_directory
+        self._root_namespace_name = root_namespace_name
+        self._lockfile: TextIOWrapper | None = None
+
+    @property
+    def _lockfile_path(self) -> Path:
+        return self._output_directory / f"{self._root_namespace_name}.lock"
+
+    def __enter__(self) -> bool:
+        return self.create()
+
+    def __exit__(
+        self, exc_type: type[BaseException] | None, exc_val: BaseException | None, exc_tb: TracebackType | None
+    ) -> None:
+        if self._lockfile is not None:
+            self.remove()
+
+    def create(self) -> bool:
+        """
+        True means compilation needs to proceed.
+        False means another process already compiled the namespace so we just waited for the lockfile to disappear before returning.
+        """
+        try:
+            pathlib.Path(self._output_directory).mkdir(parents=True, exist_ok=True)
+            self._lockfile = open(self._lockfile_path, "x")
+            _logger.debug("Created lockfile %s", self._lockfile_path)
+            return True
+        except FileExistsError:
+            pass
+        while pathlib.Path(self._lockfile_path).exists():
+            _logger.debug("Waiting for lockfile %s", self._lockfile_path)
+            time.sleep(1)
+
+        _logger.debug("Done waiting %s", self._lockfile_path)
+
+        return False
+
+    def remove(self) -> None:
+        """
+        Invoking remove before creating lockfile is not allowed.
+        """
+        assert self._lockfile is not None
+        self._lockfile.close()
+        pathlib.Path(self._lockfile_path).unlink()
+        _logger.debug("Removed lockfile %s", self._lockfile_path)

setup.cfg

@@ -149,6 +149,10 @@ ignore_missing_imports = True
 ignore_errors = True
 ignore_missing_imports = True
 
+[mypy-test_dsdl_namespace.*]
+ignore_errors = True
+ignore_missing_imports = True
+
 [mypy-numpy]
 ignore_errors = True
 ignore_missing_imports = True

tests/dsdl/_compiler.py

@@ -2,15 +2,16 @@
 # This software is distributed under the terms of the MIT License.
 # Author: Pavel Kirienko <pavel@opencyphal.org>
 
+import random
 import sys
-import typing
-import logging
+import threading
+import time
 import pathlib
 import tempfile
 import pytest
 import pycyphal.dsdl
 from pycyphal.dsdl import remove_import_hooks, add_import_hook
-
+from pycyphal.dsdl._lockfile import Locker
 from .conftest import DEMO_DIR
 
 
@@ -20,28 +21,6 @@ def _unittest_bad_usage() -> None:
         pycyphal.dsdl.compile("irrelevant", "irrelevant")  # type: ignore
 
 
-def _unittest_module_import_path_usage_suggestion(caplog: typing.Any) -> None:
-    caplog.set_level(logging.INFO)
-    output_directory = tempfile.TemporaryDirectory()  # pylint: disable=consider-using-with
-    output_directory_name = pathlib.Path(output_directory.name).resolve()
-    caplog.clear()
-    pycyphal.dsdl.compile(DEMO_DIR / "public_regulated_data_types" / "uavcan", output_directory=output_directory.name)
-    logs = caplog.record_tuples
-    print("Captured log entries:", logs, sep="\n")
-    for e in logs:
-        if "dsdl" in e[0] and str(output_directory_name) in e[2]:
-            assert e[1] == logging.INFO
-            assert " path" in e[2]
-            assert "Path(" not in e[2]  # Ensure decent formatting
-            break
-    else:
-        assert False
-    try:
-        output_directory.cleanup()  # This may fail on Windows with Python 3.7, we don't care.
-    except PermissionError:  # pragma: no cover
-        pass
-
-
 def _unittest_remove_import_hooks() -> None:
     from pycyphal.dsdl._import_hook import DsdlMetaFinder
 
@@ -64,3 +43,28 @@ def _unittest_remove_import_hooks() -> None:
 def _unittest_issue_133() -> None:
     with pytest.raises(ValueError, match=".*output directory.*"):
         pycyphal.dsdl.compile(pathlib.Path.cwd() / "irrelevant")
+
+
+def _unittest_lockfile_cant_be_recreated() -> None:
+    output_directory = pathlib.Path(tempfile.gettempdir())
+    root_namespace_name = str(random.getrandbits(64))
+
+    lockfile1 = Locker(output_directory, root_namespace_name)
+    lockfile2 = Locker(output_directory, root_namespace_name)
+
+    assert lockfile1.create() is True
+
+    def remove_lockfile1() -> None:
+        time.sleep(5)
+        lockfile1.remove()
+
+    threading.Thread(target=remove_lockfile1).start()
+    assert lockfile2.create() is False
+
+
+def _unittest_lockfile_is_removed() -> None:
+    output_directory = pathlib.Path(tempfile.gettempdir())
+
+    pycyphal.dsdl.compile(DEMO_DIR / "public_regulated_data_types" / "uavcan", output_directory=output_directory.name)
+
+    assert pathlib.Path.exists(output_directory / "uavcan.lock") is False