Skip to content

Commit 87b4d5f

Browse files
OwenKephartsmackesey
authored andcommitted
Fix pex build for pyproject.tomls with tool.uv.sources defined (#33080)
## Summary & Motivation When parsing out local dependencies, we were not properly handling uv sources. This PR handles both standard single-project setups (where the tool.uv.sources will just contain the relative path directly), as well as uv workspaces (where tool.uv.sources will just contain `workspace = true`, which requires us to scan upwards to find the workspace config file. ## How I Tested These Changes ## Changelog > Insert changelog entry or delete this section.
1 parent 8563514 commit 87b4d5f

2 files changed

Lines changed: 210 additions & 0 deletions

File tree

python_modules/libraries/dagster-cloud-cli/dagster_cloud_cli/core/pex_builder/deps.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -388,6 +388,51 @@ def get_setup_py_deps(code_directory: str, python_interpreter: str) -> list[str]
388388
return lines
389389

390390

391+
def _read_pyproject_toml(directory: str) -> Optional[dict]:
392+
"""Read and parse pyproject.toml from a directory. Returns None if not found."""
393+
pyproject_path = os.path.join(directory, "pyproject.toml")
394+
if not os.path.exists(pyproject_path):
395+
return None
396+
with open(pyproject_path, "rb") as f:
397+
return tomllib.load(f)
398+
399+
400+
def _find_uv_workspace_root(start_dir: str) -> Optional[tuple[str, dict]]:
401+
"""Walk up directories to find a uv workspace root.
402+
403+
Returns (workspace_root_path, workspace_config) or None if not found.
404+
"""
405+
current = os.path.abspath(start_dir)
406+
while True:
407+
data = _read_pyproject_toml(current)
408+
if data:
409+
workspace_config = data.get("tool", {}).get("uv", {}).get("workspace", {})
410+
if workspace_config:
411+
return current, workspace_config
412+
parent = os.path.dirname(current)
413+
if parent == current:
414+
return None
415+
current = parent
416+
417+
418+
def _resolve_uv_workspace_dep(dep_name: str, code_directory: str) -> Optional[str]:
419+
"""Find the relative path to a uv workspace member by package name.
420+
421+
Returns a relative path like "../shared-lib" or None if not found.
422+
Assumes the directory name matches the package name.
423+
"""
424+
result = _find_uv_workspace_root(code_directory)
425+
if not result:
426+
return None
427+
428+
workspace_root, _ = result
429+
candidate = os.path.join(workspace_root, dep_name)
430+
if os.path.isdir(candidate):
431+
return os.path.relpath(candidate, code_directory)
432+
433+
return None
434+
435+
391436
def get_pyproject_toml_deps(code_directory: str) -> list[str]:
392437
pyproject_path = os.path.join(code_directory, "pyproject.toml")
393438
if not os.path.exists(pyproject_path):
@@ -442,6 +487,24 @@ def get_pyproject_toml_deps(code_directory: str) -> list[str]:
442487
else:
443488
lines.append(dep_name)
444489

490+
# Handle [tool.uv.sources] - resolve workspace and path dependencies to local paths
491+
uv_sources = pyproject_data.get("tool", {}).get("uv", {}).get("sources", {})
492+
if uv_sources:
493+
resolved_lines = []
494+
for line in lines:
495+
source_config = uv_sources.get(line)
496+
if source_config:
497+
if source_config.get("workspace"):
498+
resolved_path = _resolve_uv_workspace_dep(line, code_directory)
499+
if resolved_path:
500+
resolved_lines.append(resolved_path)
501+
continue
502+
elif "path" in source_config:
503+
resolved_lines.append(source_config["path"])
504+
continue
505+
resolved_lines.append(line)
506+
return resolved_lines
507+
445508
return lines
446509

447510

python_modules/libraries/dagster-cloud-cli/dagster_cloud_cli_tests/test_pyproject_toml_support.py

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -548,6 +548,153 @@ def test_get_deps_requirements_multi_location_scenario(tmp_path):
548548
)
549549

550550

551+
def test_get_deps_requirements_uv_sources_path(tmp_path):
552+
"""Test get_deps_requirements handles [tool.uv.sources] with path (no workspace)."""
553+
python_version = version.Version(f"{sys.version_info.major}.{sys.version_info.minor}")
554+
555+
# Create main package
556+
app_dir = tmp_path / "my-app"
557+
app_dir.mkdir()
558+
(app_dir / "my_app").mkdir()
559+
(app_dir / "my_app" / "__init__.py").write_text("")
560+
(app_dir / "pyproject.toml").write_text("""
561+
[project]
562+
name = "my-app"
563+
version = "0.1.0"
564+
dependencies = [
565+
"dagster>=1.0.0",
566+
"dagster-cloud>=1.0.0",
567+
"local-utils",
568+
]
569+
570+
[tool.uv.sources]
571+
local-utils = { path = "../local-utils" }
572+
573+
[build-system]
574+
requires = ["hatchling"]
575+
build-backend = "hatchling.build"
576+
""")
577+
578+
# Create local-utils as a sibling package (not a workspace member)
579+
local_utils_dir = tmp_path / "local-utils"
580+
local_utils_dir.mkdir()
581+
(local_utils_dir / "local_utils").mkdir()
582+
(local_utils_dir / "local_utils" / "__init__.py").write_text("")
583+
(local_utils_dir / "pyproject.toml").write_text("""
584+
[project]
585+
name = "local-utils"
586+
version = "0.1.0"
587+
dependencies = ["pydantic>=2.0.0"]
588+
589+
[build-system]
590+
requires = ["hatchling"]
591+
build-backend = "hatchling.build"
592+
""")
593+
594+
# Get deps from the app directory
595+
local_packages, deps_requirements = deps.get_deps_requirements(str(app_dir), python_version)
596+
597+
deps_lines = [
598+
line.strip()
599+
for line in deps_requirements.requirements_txt.strip().split("\n")
600+
if line.strip()
601+
]
602+
603+
# External deps should be in requirements
604+
assert any("dagster>=" in dep for dep in deps_lines), f"dagster not found in {deps_lines}"
605+
606+
# local-utils should be detected as a local package via path source
607+
assert str(local_utils_dir) in local_packages.local_package_paths, (
608+
f"local-utils should be in local_package_paths: {local_packages.local_package_paths}"
609+
)
610+
assert not any("local-utils" in dep for dep in deps_lines), (
611+
f"local-utils should NOT be in deps (it's a local package): {deps_lines}"
612+
)
613+
614+
615+
def test_get_deps_requirements_uv_workspace(tmp_path):
616+
"""Test get_deps_requirements correctly handles uv workspace local dependencies.
617+
618+
When a uv workspace member has dependencies on other workspace members
619+
(via [tool.uv.sources] with workspace = true), those should be detected
620+
as local packages and included in local_package_paths.
621+
"""
622+
python_version = version.Version(f"{sys.version_info.major}.{sys.version_info.minor}")
623+
624+
# Create workspace root
625+
workspace_root = tmp_path
626+
(workspace_root / "pyproject.toml").write_text("""
627+
[project]
628+
name = "my-workspace"
629+
version = "0.1.0"
630+
631+
[tool.uv.workspace]
632+
members = ["my-app", "shared-lib"]
633+
""")
634+
635+
# Create main app package
636+
app_dir = workspace_root / "my-app"
637+
app_dir.mkdir()
638+
(app_dir / "my_app").mkdir()
639+
(app_dir / "my_app" / "__init__.py").write_text("")
640+
(app_dir / "pyproject.toml").write_text("""
641+
[project]
642+
name = "my-app"
643+
version = "0.1.0"
644+
dependencies = [
645+
"dagster>=1.0.0",
646+
"dagster-cloud>=1.0.0",
647+
"requests>=2.25.0",
648+
"shared-lib",
649+
]
650+
651+
[tool.uv.sources]
652+
shared-lib = { workspace = true }
653+
654+
[build-system]
655+
requires = ["hatchling"]
656+
build-backend = "hatchling.build"
657+
""")
658+
659+
# Create shared-lib workspace member
660+
shared_lib_dir = workspace_root / "shared-lib"
661+
shared_lib_dir.mkdir()
662+
(shared_lib_dir / "shared_lib").mkdir()
663+
(shared_lib_dir / "shared_lib" / "__init__.py").write_text("")
664+
(shared_lib_dir / "pyproject.toml").write_text("""
665+
[project]
666+
name = "shared-lib"
667+
version = "0.1.0"
668+
dependencies = ["pydantic>=2.0.0"]
669+
670+
[build-system]
671+
requires = ["hatchling"]
672+
build-backend = "hatchling.build"
673+
""")
674+
675+
# Get deps from the app directory
676+
local_packages, deps_requirements = deps.get_deps_requirements(str(app_dir), python_version)
677+
678+
deps_lines = [
679+
line.strip()
680+
for line in deps_requirements.requirements_txt.strip().split("\n")
681+
if line.strip()
682+
]
683+
684+
# External deps should be in requirements
685+
assert any("dagster>=" in dep for dep in deps_lines), f"dagster not found in {deps_lines}"
686+
assert any("requests" in dep for dep in deps_lines), f"requests not found in {deps_lines}"
687+
688+
# shared-lib is a workspace dependency - it should be detected as a local package
689+
# and NOT appear in the requirements (it would fail pip install)
690+
assert str(shared_lib_dir) in local_packages.local_package_paths, (
691+
f"shared-lib should be in local_package_paths: {local_packages.local_package_paths}"
692+
)
693+
assert not any("shared-lib" in dep for dep in deps_lines), (
694+
f"shared-lib should NOT be in deps (it's a local package): {deps_lines}"
695+
)
696+
697+
551698
def test_get_pyproject_deps_requirements_multi_location_scenario() -> None:
552699
"""Test scenario that mimics multi-location build with nested packages.
553700

0 commit comments

Comments
 (0)