Clarify package upgrade reasons in log output (#252)

* Output additional detail on why the current and (if applicable) upgraded versions are/aren't compatible

Author: chayac

.gitignore

@@ -22,4 +22,4 @@ project2/*
 .DS_Store
 
 # raw package hub output (because big)
-dbt_autofix/packages/scripts/output/*
+src/dbt_autofix/packages/scripts/output/*

pyproject.toml

@@ -13,7 +13,6 @@ dependencies = [
     "httpx>=0.27.0",
     "dbt-common>=1.27,<2",
     "dbt-extractor>=0.5.0,<=0.6",
-    "dbt-fusion-package-tools",
 ]
 
 dynamic = ["version"]
@@ -71,7 +70,7 @@ ignore = []
 
 [tool.pyright]
 include = [
-    "dbt_autofix",
+    "src/dbt_autofix",
     "tests",
 ]
 exclude = []

src/dbt_autofix/main.py

@@ -1,6 +1,5 @@
 import json
 from importlib.metadata import version
-import os
 from pathlib import Path
 from typing import List, Optional
 
@@ -22,7 +21,6 @@
 from dbt_autofix.packages.dbt_package_file import DbtPackageFile
 from dbt_autofix.refactor import apply_changesets, changeset_all_sql_yml_files
 from dbt_autofix.retrieve_schemas import SchemaSpecs
-from dbt_fusion_package_tools.package_example import output_package_name
 
 console = Console()
 error_console = Console(stderr=True)
@@ -38,10 +36,6 @@
 current_dir = Path.cwd()
 
 
-def print_package():
-    output_package_name()
-
-
 @app.command(name="list-yaml-duplicates")
 def identify_duplicate_keys(
     path: Annotated[Path, typer.Option("--path", "-p", help="The path to the dbt project")] = current_dir,

src/dbt_autofix/package_upgrade.py

@@ -3,16 +3,18 @@
 from rich.console import Console
 from dbt_autofix.packages.dbt_package_version import (
     DbtPackageVersion,
+)
+from dbt_common.semver import VersionSpecifier, VersionRange, versions_compatible
+from dbt_autofix.packages.manual_overrides import EXPLICIT_DISALLOW_ALL_VERSIONS, EXPLICIT_ALLOW_ALL_VERSIONS
+from dbt_autofix.packages.upgrade_status import PackageVersionFusionCompatibilityState, PackageFusionCompatibilityState
+from dbt_autofix.packages.fusion_version_compatibility_output import FUSION_VERSION_COMPATIBILITY_OUTPUT
+from dbt_autofix.packages.version_utils import (
     construct_version_list_from_raw,
     convert_optional_version_string_to_spec,
     convert_version_specifiers_to_range,
     convert_version_string_list_to_spec,
     get_version_specifiers,
 )
-from dbt_common.semver import VersionSpecifier, VersionRange, versions_compatible
-from dbt_autofix.packages.manual_overrides import EXPLICIT_DISALLOW_ALL_VERSIONS, EXPLICIT_ALLOW_ALL_VERSIONS
-from dbt_autofix.packages.upgrade_status import PackageVersionFusionCompatibilityState, PackageFusionCompatibilityState
-from dbt_autofix.packages.fusion_version_compatibility_output import FUSION_VERSION_COMPATIBILITY_OUTPUT
 
 
 console = Console()
@@ -149,7 +151,7 @@ def find_fusion_compatible_versions_in_requested_range(self) -> list[VersionSpec
                 version, self.project_config_version_range.start, self.project_config_version_range.end
             ):
                 compatible_versions.append(version)
-        sorted_versions = sorted(compatible_versions, reverse=True)
+        sorted_versions = sorted(compatible_versions)
         return sorted_versions
 
     def find_fusion_compatible_versions_above_requested_range(self) -> list[VersionSpecifier]:
@@ -177,7 +179,7 @@ def find_fusion_compatible_versions_above_requested_range(self) -> list[VersionS
                 and version > self.project_config_version_range.start
             ):
                 compatible_versions.append(version)
-        sorted_versions = sorted(compatible_versions, reverse=True)
+        sorted_versions = sorted(compatible_versions)
         return sorted_versions
 
     def find_fusion_incompatible_versions_in_requested_range(self) -> list[VersionSpecifier]:
@@ -191,7 +193,7 @@ def find_fusion_incompatible_versions_in_requested_range(self) -> list[VersionSp
                 version, self.project_config_version_range.start, self.project_config_version_range.end
             ):
                 incompatible_versions.append(version)
-        sorted_versions = sorted(incompatible_versions, reverse=True)
+        sorted_versions = sorted(incompatible_versions)
         return sorted_versions
 
     def find_fusion_unknown_versions_in_requested_range(self) -> list[VersionSpecifier]:
@@ -205,7 +207,7 @@ def find_fusion_unknown_versions_in_requested_range(self) -> list[VersionSpecifi
                 version, self.project_config_version_range.start, self.project_config_version_range.end
             ):
                 unknown_compatibility_versions.append(version)
-        sorted_versions = sorted(unknown_compatibility_versions, reverse=True)
+        sorted_versions = sorted(unknown_compatibility_versions)
         return sorted_versions
 
     def get_installed_package_version(self) -> str:

src/dbt_autofix/packages/dbt_package.py

@@ -9,83 +9,15 @@
 )
 
 from dbt_autofix.packages.upgrade_status import PackageVersionFusionCompatibilityState
+from dbt_autofix.packages.version_utils import (
+    FUSION_COMPATIBLE_VERSION,
+    construct_version_list_from_raw,
+    convert_version_specifiers_to_range,
+    get_version_specifiers,
+)
 
 console = Console()
 
-FUSION_COMPATIBLE_VERSION: VersionSpecifier = VersionSpecifier.from_version_string("2.0.0")
-
-# `float` also allows `int`, according to PEP484 (and jsonschema!)
-RawVersion = Union[str, float]
-
-
-def get_versions(version: Union[RawVersion, list[RawVersion]]) -> list[str]:
-    if isinstance(version, list):
-        return [str(v) for v in version]
-    else:
-        return [str(version)]
-
-
-def construct_version_list(raw_versions: Union[str, list[str], None]) -> list[str]:
-    if raw_versions is None:
-        return []
-    elif type(raw_versions) == str:
-        return [raw_versions]
-    elif type(raw_versions) == list:
-        return raw_versions
-    else:
-        return []
-
-
-def construct_version_list_from_raw(raw_versions: Any) -> list[str]:
-    if raw_versions is None:
-        return []
-    # isinstance is needed here because when ruyaml parses the YAML,
-    # sometimes it stores it as an instance of a class that extend str or list
-    elif isinstance(raw_versions, str):
-        return [str(raw_versions)]
-    elif isinstance(raw_versions, list):
-        versions = []
-        for version in raw_versions:
-            if isinstance(version, str) or isinstance(version, float):
-                versions.append(str(version))
-        return versions
-    else:
-        return []
-
-
-def get_version_specifiers(raw_version: list[str]) -> list[VersionSpecifier]:
-    return [VersionSpecifier.from_version_string(v) for v in raw_version]
-
-
-def convert_version_specifiers_to_range(specs: list[VersionSpecifier]) -> VersionRange:
-    if len(specs) == 0 or len(specs) > 2:
-        # assume any version compatible
-        any_version = VersionSpecifier.from_version_string(">=0.0.0")
-        return VersionRange(any_version, any_version)
-    elif len(specs) == 1:
-        return VersionRange(specs[0], specs[0])
-    elif specs[0] < specs[1]:
-        return VersionRange(specs[0], specs[1])
-    else:
-        return VersionRange(specs[1], specs[0])
-
-
-def convert_optional_version_string_to_spec(version_string: Optional[str]) -> Optional[VersionSpecifier]:
-    try:
-        if type(version_string) == str:
-            return VersionSpecifier.from_version_string(version_string)
-        else:
-            return None
-    except:
-        return None
-
-
-def convert_version_string_list_to_spec(version_string: list[str]) -> list[VersionSpecifier]:
-    if len(version_string) == 0:
-        return []
-    else:
-        return [VersionSpecifier.from_version_string(x) for x in version_string]
-
 
 @dataclass
 class DbtPackageVersion:

src/dbt_autofix/packages/dbt_package_version.py

@@ -4,12 +4,12 @@
 class PackageVersionFusionCompatibilityState(str, Enum):
     """String enum for Fusion compatibility of a specific version of a package."""
 
-    NO_DBT_VERSION_RANGE = "Package does not define required dbt version range"
-    DBT_VERSION_RANGE_EXCLUDES_2_0 = "Package's dbt version range excludes version 2.0"
-    DBT_VERSION_RANGE_INCLUDES_2_0 = "Package's dbt versions range include version 2.0"
-    EXPLICIT_ALLOW = "Package version has been verified as Fusion-compatible"
-    EXPLICIT_DISALLOW = "Package version has been verified as incompatible with Fusion"
-    UNKNOWN = "Package version state unknown"
+    NO_DBT_VERSION_RANGE = "require-dbt-version is not defined"
+    DBT_VERSION_RANGE_EXCLUDES_2_0 = "require-dbt-version excludes version 2.0"
+    DBT_VERSION_RANGE_INCLUDES_2_0 = "require-dbt-version includes version 2.0"
+    EXPLICIT_ALLOW = "Version has been verified by dbt as Fusion-compatible even though its declared require-dbt-version may not include 2.0"
+    EXPLICIT_DISALLOW = "Version has been verified by dbt as incompatible with Fusion"
+    UNKNOWN = "Version state unknown"
 
 
 class PackageFusionCompatibilityState(str, Enum):
@@ -27,7 +27,9 @@ class PackageVersionUpgradeType(str, Enum):
 
     NO_UPGRADE_REQUIRED = "Package is already compatible with Fusion"
     UPGRADE_AVAILABLE = "Package has Fusion-compatible version available"
-    PUBLIC_PACKAGE_MISSING_FUSION_ELIGIBILITY = "Public package has not defined Fusion eligibility"
+    PUBLIC_PACKAGE_MISSING_FUSION_ELIGIBILITY = (
+        "Public package has not defined require-dbt-version so Fusion eligibility cannot be determined"
+    )
     PUBLIC_PACKAGE_NOT_COMPATIBLE_WITH_FUSION = "Public package is not compatible with Fusion"
     PUBLIC_PACKAGE_FUSION_COMPATIBLE_VERSION_EXCEEDS_PROJECT_CONFIG = (
         "Public package has Fusion-compatible version that is outside the project's requested version range"

src/dbt_autofix/packages/upgrade_status.py

@@ -0,0 +1,75 @@
+from typing import Any, Optional, Union
+from dbt_common.semver import VersionRange, VersionSpecifier
+
+FUSION_COMPATIBLE_VERSION: VersionSpecifier = VersionSpecifier.from_version_string("2.0.0")
+# `float` also allows `int`, according to PEP484 (and jsonschema!)
+RawVersion = Union[str, float]
+
+
+def get_versions(version: Union[RawVersion, list[RawVersion]]) -> list[str]:
+    if isinstance(version, list):
+        return [str(v) for v in version]
+    else:
+        return [str(version)]
+
+
+def construct_version_list(raw_versions: Union[str, list[str], None]) -> list[str]:
+    if raw_versions is None:
+        return []
+    elif type(raw_versions) == str:
+        return [raw_versions]
+    elif type(raw_versions) == list:
+        return raw_versions
+    else:
+        return []
+
+
+def construct_version_list_from_raw(raw_versions: Any) -> list[str]:
+    if raw_versions is None:
+        return []
+    # isinstance is needed here because when ruyaml parses the YAML,
+    # sometimes it stores it as an instance of a class that extend str or list
+    elif isinstance(raw_versions, str):
+        return [str(raw_versions)]
+    elif isinstance(raw_versions, list):
+        versions = []
+        for version in raw_versions:
+            if isinstance(version, str) or isinstance(version, float):
+                versions.append(str(version))
+        return versions
+    else:
+        return []
+
+
+def get_version_specifiers(raw_version: list[str]) -> list[VersionSpecifier]:
+    return [VersionSpecifier.from_version_string(v) for v in raw_version]
+
+
+def convert_version_specifiers_to_range(specs: list[VersionSpecifier]) -> VersionRange:
+    if len(specs) == 0 or len(specs) > 2:
+        # assume any version compatible
+        any_version = VersionSpecifier.from_version_string(">=0.0.0")
+        return VersionRange(any_version, any_version)
+    elif len(specs) == 1:
+        return VersionRange(specs[0], specs[0])
+    elif specs[0] < specs[1]:
+        return VersionRange(specs[0], specs[1])
+    else:
+        return VersionRange(specs[1], specs[0])
+
+
+def convert_optional_version_string_to_spec(version_string: Optional[str]) -> Optional[VersionSpecifier]:
+    try:
+        if type(version_string) == str:
+            return VersionSpecifier.from_version_string(version_string)
+        else:
+            return None
+    except:
+        return None
+
+
+def convert_version_string_list_to_spec(version_string: list[str]) -> list[VersionSpecifier]:
+    if len(version_string) == 0:
+        return []
+    else:
+        return [VersionSpecifier.from_version_string(x) for x in version_string]

src/dbt_autofix/packages/version_utils.py

@@ -1,6 +1,6 @@
 from typing import Any, Optional
 import pytest
-from dbt_autofix.packages.dbt_package_version import DbtPackageVersion, construct_version_list_from_raw
+from dbt_autofix.packages.dbt_package_version import DbtPackageVersion
 from dbt_common.semver import VersionSpecifier, VersionRange, Matchers, versions_compatible, UnboundedVersionSpecifier
 
 
@@ -320,28 +320,3 @@ def test_fusion_compatible_from_raw(input_yaml: dict[Any, Any], expected_match:
 
     fusion_compatible: bool = package_version.is_require_dbt_version_fusion_compatible()
     assert fusion_compatible == expected_match
-
-
-@pytest.mark.parametrize(
-    "input_yaml,expected_match",
-    [
-        ([">=1.1.0", "<2.0.0"], [">=1.1.0", "<2.0.0"]),
-        (None, []),
-        (
-            [">=1.1.0"],
-            [">=1.1.0"],
-        ),
-        (
-            ">=1.1.0",
-            [">=1.1.0"],
-        ),
-        (">0.2.1", [">0.2.1"]),
-    ],
-)
-def test_convert_version_string_from_raw(input_yaml: Any, expected_match: list[str]):
-    version_list = construct_version_list_from_raw(input_yaml)
-
-    assert version_list == expected_match
-    if input_yaml is not None:
-        for i, version in enumerate(version_list):
-            assert version == expected_match[i]

tests/unit_tests/package_upgrades/test_dbt_package_version.py

@@ -0,0 +1,32 @@
+from dbt_autofix.packages.version_utils import construct_version_list_from_raw
+
+
+import pytest
+
+
+from typing import Any
+
+
+@pytest.mark.parametrize(
+    "input_yaml,expected_match",
+    [
+        ([">=1.1.0", "<2.0.0"], [">=1.1.0", "<2.0.0"]),
+        (None, []),
+        (
+            [">=1.1.0"],
+            [">=1.1.0"],
+        ),
+        (
+            ">=1.1.0",
+            [">=1.1.0"],
+        ),
+        (">0.2.1", [">0.2.1"]),
+    ],
+)
+def test_convert_version_string_from_raw(input_yaml: Any, expected_match: list[str]):
+    version_list = construct_version_list_from_raw(input_yaml)
+
+    assert version_list == expected_match
+    if input_yaml is not None:
+        for i, version in enumerate(version_list):
+            assert version == expected_match[i]