[dg] Rename PackageObject* -> PluginObject*

Author: smackesey

docs/docs/guides/labs/dg/configuring-dg.md

@@ -0,0 +1,36 @@
+---
+title: 'Configuring dg'
+sidebar_position: 500
+---
+
+import Preview from '@site/docs/partials/\_Preview.md';
+
+<Preview />
+
+`dg` is primarily configured using 
+
+
+
+## Default `dg` behavior (`uv run`)
+
+`dg` itself is installed globally and always executes in an isolated environment. However, many `dg` commands need to spawn subprocesses in environments scoped to specific projects. For example, when you run `dg list component-type` in a project directory, `dg` only lists the component types available in that project.
+
+When using `dg` with default settings, the Python subprocess is spawned after detecting a virtual environment in the project directory. This means that, rather than finding the first `python` executable on your shell `PATH`, it will instead use the Python executable from a locally detected virtual environment even if that virtual environment is not currently activated.
+
+By default, `dg` uses a special `uv` command called `uv run` for environment detection and command execution. `uv run` resolves a local virtual environment (`.venv` in an ancestor directory) before running a command, and additionally makes sure that the environment is up to date with the project's specified dependencies before running the command.
+
+`dg`'s use of `uv run` means you don't need to manage global shell state by activating and deactivating virtual environments during development. Instead, `dg` expects (and will scaffold for you) a `.venv` directory in the root of each project, and will use that virtual environment when executing commands (such as `dg scaffold defs` or `dg list component-type`) against that project.
+
+## Disabling use of `uv run`
+
+You can disable `dg`'s use of `uv run` by passing the `--no-use-dg-managed-environment` flag to `dg` commands. This will cause `dg` to still try to resolve local virtual environments by looking up the ancestor tree for `.venv`, but instead launch Python processes directly rather than going through `uv run`.
+
+If you want to opt out of virtual environment detection entirely and just use your ambient Python environment (system Python or an activated virtual env), you should also (a) set `--no-require-local-venv`, and (b) ensure there are no `.venv` directories in
+the ancestor tree of your project.
+
+:::info
+
+Disabling `uv` integration with `--no-use-dg-managed-environment` will
+currently disable the `dg cache`. This will make some operations noticeably slower.
+
+:::

python_modules/dagster/dagster/components/cli/list.py

@@ -2,7 +2,7 @@
 from typing import Literal, Optional, Union
 
 import click
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 from dagster_shared.serdes.objects.definition_metadata import (
     DgAssetCheckMetadata,
     DgAssetMetadata,
@@ -160,7 +160,7 @@ def list_definitions_command(
 
 def _load_library_objects(
     entry_points: bool, extra_modules: tuple[str, ...]
-) -> dict[PackageObjectKey, object]:
+) -> dict[PluginObjectKey, object]:
     objects = {}
     if entry_points:
         objects.update(discover_entry_point_package_objects())
@@ -171,7 +171,7 @@ def _load_library_objects(
 
 def _load_component_types(
     entry_points: bool, extra_modules: tuple[str, ...]
-) -> dict[PackageObjectKey, type[Component]]:
+) -> dict[PluginObjectKey, type[Component]]:
     return {
         key: obj
         for key, obj in _load_library_objects(entry_points, extra_modules).items()

python_modules/dagster/dagster/components/cli/scaffold.py

@@ -2,7 +2,7 @@
 from typing import Optional
 
 import click
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 from pydantic import TypeAdapter
 
 from dagster.components.component_scaffolding import scaffold_object
@@ -30,7 +30,7 @@ def scaffold_object_command(
     json_params: Optional[str],
     scaffold_format: str,
 ) -> None:
-    key = PackageObjectKey.from_typename(typename)
+    key = PluginObjectKey.from_typename(typename)
     obj = load_package_object(key)
 
     if json_params:

python_modules/dagster/dagster/components/core/defs_module.py

@@ -5,7 +5,7 @@
 from pathlib import Path
 from typing import Any, Optional, TypeVar
 
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 from dagster_shared.yaml_utils import parse_yaml_with_source_positions
 from pydantic import BaseModel, ConfigDict, TypeAdapter
 
@@ -245,7 +245,7 @@ def get_component(cls, context: ComponentLoadContext) -> Component:
 
         # find the component type
         type_str = context.normalize_component_type_str(component_file_model.type)
-        key = PackageObjectKey.from_typename(type_str)
+        key = PluginObjectKey.from_typename(type_str)
         obj = load_package_object(key)
         if not isinstance(obj, type) or not issubclass(obj, Component):
             raise DagsterInvalidDefinitionError(

python_modules/dagster/dagster/components/core/package_entry.py

@@ -5,7 +5,7 @@
 from collections.abc import Iterable, Sequence
 from types import ModuleType
 
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 
 from dagster._core.errors import DagsterError
 from dagster.components.utils import format_error_message
@@ -25,7 +25,7 @@ def get_entry_points_from_python_environment(group: str) -> Sequence[importlib.m
         return importlib.metadata.entry_points().get(group, [])
 
 
-def discover_entry_point_package_objects() -> dict[PackageObjectKey, object]:
+def discover_entry_point_package_objects() -> dict[PluginObjectKey, object]:
     """Discover package entries registered in the Python environment via the
     `dg_library` entry point group.
 
@@ -34,7 +34,7 @@ def discover_entry_point_package_objects() -> dict[PackageObjectKey, object]:
     component types and is loaded by default. Other entry points resolve to various sets of test
     component types. This method will only ever load one builtin package.
     """
-    objects: dict[PackageObjectKey, object] = {}
+    objects: dict[PluginObjectKey, object] = {}
     entry_points = get_entry_points_from_python_environment(DG_LIBRARY_ENTRY_POINT_GROUP)
 
     for entry_point in entry_points:
@@ -54,16 +54,16 @@ def discover_entry_point_package_objects() -> dict[PackageObjectKey, object]:
                 f"Value expected to be a module, got {root_module}."
             )
         for name, obj in get_package_objects_in_module(root_module):
-            key = PackageObjectKey(name=name, namespace=entry_point.value)
+            key = PluginObjectKey(name=name, namespace=entry_point.value)
             objects[key] = obj
     return objects
 
 
-def discover_package_objects(modules: Sequence[str]) -> dict[PackageObjectKey, object]:
-    objects: dict[PackageObjectKey, object] = {}
+def discover_package_objects(modules: Sequence[str]) -> dict[PluginObjectKey, object]:
+    objects: dict[PluginObjectKey, object] = {}
     for extra_module in modules:
         for name, obj in get_package_objects_in_module(importlib.import_module(extra_module)):
-            key = PackageObjectKey(name=name, namespace=extra_module)
+            key = PluginObjectKey(name=name, namespace=extra_module)
             objects[key] = obj
     return objects
 
@@ -77,7 +77,7 @@ def get_package_objects_in_module(
             yield attr, value
 
 
-def load_package_object(key: PackageObjectKey) -> object:
+def load_package_object(key: PluginObjectKey) -> object:
     module_name, attr = key.namespace, key.name
     try:
         module = importlib.import_module(module_name)

python_modules/dagster/dagster/components/core/snapshot.py

@@ -3,8 +3,8 @@
 
 from dagster_shared.serdes.objects.package_entry import (
     ComponentFeatureData,
-    PackageObjectKey,
-    PackageObjectSnap,
+    PluginObjectKey,
+    PluginObjectSnap,
     ScaffoldTargetTypeData,
 )
 
@@ -43,7 +43,7 @@ def _get_scaffold_target_type_data(scaffolder: Scaffolder) -> ScaffoldTargetType
     )
 
 
-def get_package_entry_snap(key: PackageObjectKey, obj: object) -> PackageObjectSnap:
+def get_package_entry_snap(key: PluginObjectKey, obj: object) -> PluginObjectSnap:
     type_data = []
     owners = []
     tags = []
@@ -56,7 +56,7 @@ def get_package_entry_snap(key: PackageObjectKey, obj: object) -> PackageObjectS
     if isinstance(scaffolder, Scaffolder):
         type_data.append(_get_scaffold_target_type_data(scaffolder))
     summary, description = _get_summary_and_description(obj)
-    return PackageObjectSnap(
+    return PluginObjectSnap(
         key=key,
         summary=summary,
         owners=owners,

python_modules/dagster/dagster_tests/components_tests/cli_tests/test_commands.py

@@ -8,8 +8,8 @@
 from dagster_shared import check
 from dagster_shared.serdes.objects import (
     ComponentFeatureData,
-    PackageObjectKey,
-    PackageObjectSnap,
+    PluginObjectKey,
+    PluginObjectSnap,
     ScaffoldTargetTypeData,
 )
 from dagster_shared.serdes.serdes import deserialize_value
@@ -40,7 +40,7 @@ def test_list_library_objects_from_module():
     # Now check what we get when we load directly from the test library. This has stable results.
     result = runner.invoke(cli, ["list", "library", "--no-entry-points", "dagster_test.components"])
     assert result.exit_code == 0
-    result = check.is_list(deserialize_value(result.output), PackageObjectSnap)
+    result = check.is_list(deserialize_value(result.output), PluginObjectSnap)
     assert len(result) > 1
 
     assert [obj.key.to_typename() for obj in result] == [
@@ -50,8 +50,8 @@ def test_list_library_objects_from_module():
         "dagster_test.components.SimplePipesScriptComponent",
     ]
 
-    assert result[2] == PackageObjectSnap(
-        key=PackageObjectKey(namespace="dagster_test.components", name="SimpleAssetComponent"),
+    assert result[2] == PluginObjectSnap(
+        key=PluginObjectKey(namespace="dagster_test.components", name="SimpleAssetComponent"),
         description="A simple asset that returns a constant string value.",
         summary="A simple asset that returns a constant string value.",
         owners=["[email protected]", "[email protected]"],
@@ -94,10 +94,8 @@ def test_list_library_objects_from_module():
         "type": "object",
     }
 
-    assert result[3] == PackageObjectSnap(
-        key=PackageObjectKey(
-            namespace="dagster_test.components", name="SimplePipesScriptComponent"
-        ),
+    assert result[3] == PluginObjectSnap(
+        key=PluginObjectKey(namespace="dagster_test.components", name="SimplePipesScriptComponent"),
         description="A simple asset that runs a Python script with the Pipes subprocess client.\n\nBecause it is a pipes asset, no value is returned.",
         summary="A simple asset that runs a Python script with the Pipes subprocess client.",
         owners=[],
@@ -134,7 +132,7 @@ def test_list_library_objects_from_project() -> None:
 
             result = deserialize_value(result.output, list)
             assert len(result) == 1
-            assert result[0].key == PackageObjectKey(
+            assert result[0].key == PluginObjectKey(
                 namespace=f"{location_name}.defs.local_component_sample",
                 name="MyComponent",
             )

python_modules/dagster/dagster_tests/components_tests/registry_tests/test_registry.py

@@ -13,7 +13,7 @@
 from dagster.components.core.package_entry import discover_entry_point_package_objects
 from dagster.components.core.snapshot import get_package_entry_snap
 from dagster_dg.utils import get_venv_executable
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 from dagster_shared.serdes.serdes import deserialize_value
 
 ensure_dagster_tests_import()
@@ -130,7 +130,7 @@ def test_all_components_have_defined_summary():
     for component_name, component_type in registry.items():
         if isinstance(component_type, type) and issubclass(component_type, Component):
             assert get_package_entry_snap(
-                PackageObjectKey("a", "a"), component_type
+                PluginObjectKey("a", "a"), component_type
             ).summary, f"Component {component_name} has no summary defined"
 
 

python_modules/dagster/dagster_tests/components_tests/unit_tests/test_library_object_key.py

@@ -1,21 +1,21 @@
 from pathlib import Path
 
 import pytest
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 
 dirpath = Path(".")
 
 
 @pytest.mark.parametrize(
     ["typename", "key"],
     [
-        ("foo.bar", PackageObjectKey("foo", "bar")),
-        ("foo.Bar", PackageObjectKey("foo", "Bar")),
-        ("foo.bar.baz", PackageObjectKey("foo.bar", "baz")),
+        ("foo.bar", PluginObjectKey("foo", "bar")),
+        ("foo.Bar", PluginObjectKey("foo", "Bar")),
+        ("foo.bar.baz", PluginObjectKey("foo.bar", "baz")),
     ],
 )
-def test_valid_keys(typename: str, key: PackageObjectKey) -> None:
-    assert PackageObjectKey.from_typename(typename) == key
+def test_valid_keys(typename: str, key: PluginObjectKey) -> None:
+    assert PluginObjectKey.from_typename(typename) == key
 
 
 @pytest.mark.parametrize(
@@ -28,4 +28,4 @@ def test_valid_keys(typename: str, key: PackageObjectKey) -> None:
 )
 def test_invalid_keys(typename: str) -> None:
     with pytest.raises(ValueError):
-        PackageObjectKey.from_typename(typename)
+        PluginObjectKey.from_typename(typename)

python_modules/libraries/dagster-components/dagster_components/__init__.py

@@ -34,7 +34,7 @@
     ScaffoldRequest as ScaffoldRequest,
 )
 from dagster_shared.libraries import DagsterLibraryRegistry
-from dagster_shared.serdes.objects import PackageObjectKey as PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey as PluginObjectKey
 
 from dagster_components.version import __version__ as __version__
 

python_modules/libraries/dagster-dg/dagster_dg/check.py

@@ -3,7 +3,7 @@
 from typing import Any, NamedTuple, Optional
 
 import click
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 from dagster_shared.yaml_utils import parse_yaml_with_source_positions
 from dagster_shared.yaml_utils.source_position import (
     LineCol,
@@ -15,11 +15,7 @@
 from yaml.scanner import ScannerError
 
 from dagster_dg.cli.check_utils import error_dict_to_formatted_error
-from dagster_dg.component import (
-    RemotePackageRegistry,
-    get_specified_env_var_deps,
-    get_used_env_vars,
-)
+from dagster_dg.component import RemotePluginRegistry, get_specified_env_var_deps, get_used_env_vars
 from dagster_dg.context import DgContext
 
 COMPONENT_FILE_SCHEMA = {
@@ -52,7 +48,7 @@ def _scaffold_value_and_source_position_tree(
 
 
 class ErrorInput(NamedTuple):
-    object_key: Optional[PackageObjectKey]
+    object_key: Optional[PluginObjectKey]
     error: ValidationError
     source_position_tree: ValueAndSourcePositionTree
 
@@ -67,7 +63,7 @@ def check_yaml(
     validation_errors: list[ErrorInput] = []
     all_specified_env_var_deps = set()
 
-    component_contents_by_key: dict[PackageObjectKey, Any] = {}
+    component_contents_by_key: dict[PluginObjectKey, Any] = {}
     modules_to_fetch = set()
     for component_dir in dg_context.defs_path.rglob("*"):
         if resolved_paths and not any(
@@ -136,7 +132,7 @@ def check_yaml(
             qualified_key = (
                 f"{component_instance_module}{raw_key}" if raw_key.startswith(".") else raw_key
             )
-            key = PackageObjectKey.from_typename(qualified_key)
+            key = PluginObjectKey.from_typename(qualified_key)
             component_contents_by_key[key] = component_doc_tree
 
             # We need to fetch components from any modules local to the project because these are
@@ -145,7 +141,7 @@ def check_yaml(
                 modules_to_fetch.add(key.namespace)
 
     # Fetch the local component types, if we need any local components
-    component_registry = RemotePackageRegistry.from_dg_context(
+    component_registry = RemotePluginRegistry.from_dg_context(
         dg_context, extra_modules=list(modules_to_fetch)
     )
     for key, component_doc_tree in component_contents_by_key.items():

python_modules/libraries/dagster-dg/dagster_dg/cli/__init__.py

@@ -14,7 +14,7 @@
 from dagster_dg.cli.scaffold import scaffold_group
 from dagster_dg.cli.shared_options import dg_global_options
 from dagster_dg.cli.utils import utils_group
-from dagster_dg.component import RemotePackageRegistry
+from dagster_dg.component import RemotePluginRegistry
 from dagster_dg.config import normalize_cli_config
 from dagster_dg.context import DgContext
 from dagster_dg.utils import DgClickGroup, exit_with_error
@@ -119,7 +119,7 @@ def _rebuild_component_registry(dg_context: DgContext):
     key = dg_context.get_cache_key("component_registry_data")
     dg_context.cache.clear_key(key)
     # This will trigger a rebuild of the component registry
-    RemotePackageRegistry.from_dg_context(dg_context)
+    RemotePluginRegistry.from_dg_context(dg_context)
 
 
 ENV_PREFIX = "DAGSTER_DG"

python_modules/libraries/dagster-dg/dagster_dg/cli/check_utils.py

@@ -4,7 +4,7 @@
 
 import click
 import typer
-from dagster_shared.serdes.objects import PackageObjectKey
+from dagster_shared.serdes.objects import PluginObjectKey
 from dagster_shared.yaml_utils.source_position import SourcePositionTree
 from jsonschema import ValidationError
 
@@ -39,7 +39,7 @@ def augment_error_path(error_details: ValidationError) -> Sequence[Union[str, in
 
 
 def error_dict_to_formatted_error(
-    key: Optional[PackageObjectKey],
+    key: Optional[PluginObjectKey],
     error_details: ValidationError,
     source_position_tree: SourcePositionTree,
     prefix: Sequence[str] = (),