Update Fusion compatibility output (#268)

* allow dbt_artifacts versions 2.10+

Author: chayac

packages/dbt_fusion_package_tools/pyproject.toml

@@ -8,6 +8,8 @@ dependencies = [
     "mashumaro>=3.17",
     "pyyaml>=6.0.2",
     "dbt-protos>=1.0.410",
+    "requests>=2.32.5",
+    "rich>=14.0.0",
 ]
 
 dynamic = ["version"]

packages/dbt_fusion_package_tools/src/dbt_fusion_package_tools/fusion_version_compatibility_output.py

@@ -1101,10 +1101,10 @@
         "latest_version_incl_prerelease": "=0.5.3",
     },
     "bqbooster/dbt_bigquery_monitoring": {
-        "latest_version": "=0.23.1",
-        "oldest_fusion_compatible_version": None,
-        "latest_fusion_compatible_version": None,
-        "fusion_compatible_versions": [],
+        "latest_version": "=0.24.0",
+        "oldest_fusion_compatible_version": "=0.24.0",
+        "latest_fusion_compatible_version": "=0.24.0",
+        "fusion_compatible_versions": ["=0.24.0"],
         "fusion_incompatible_versions": [
             "=0.4.0",
             "=0.5.0",
@@ -1158,16 +1158,17 @@
             "=0.9.1",
         ],
         "unknown_compatibility_versions": ["=0.1.0", "=0.2.0", "=0.3.0"],
-        "package_latest_version_index_json": "=0.23.1",
+        "package_latest_version_index_json": "=0.24.0",
         "package_redirect_name": None,
         "package_redirect_namespace": None,
-        "latest_version_incl_prerelease": "=0.23.1",
+        "latest_version_incl_prerelease": "=0.24.0",
     },
     "brooklyn-data/dbt_artifacts": {
         "latest_version": "=2.10.0",
-        "oldest_fusion_compatible_version": "=0.2.0",
+        "oldest_fusion_compatible_version": "=2.10.0",
         "latest_fusion_compatible_version": "=2.10.0",
-        "fusion_compatible_versions": [
+        "fusion_compatible_versions": ["=2.10.0"],
+        "fusion_incompatible_versions": [
             "=0.2.0",
             "=0.2.1",
             "=0.3.0",
@@ -1188,14 +1189,11 @@
             "=2.0.0",
             "=2.1.0",
             "=2.1.1",
-            "=2.10.0",
             "=2.2.0",
             "=2.2.1",
             "=2.2.2",
             "=2.2.3",
             "=2.3.0",
-        ],
-        "fusion_incompatible_versions": [
             "=2.4.0",
             "=2.4.1",
             "=2.4.2",
@@ -2224,7 +2222,7 @@
         "latest_version_incl_prerelease": "=0.1.0",
     },
     "edanalytics/edu_edfi_source": {
-        "latest_version": "=0.5.3",
+        "latest_version": "=0.6.0",
         "oldest_fusion_compatible_version": None,
         "latest_fusion_compatible_version": None,
         "fusion_compatible_versions": [],
@@ -2267,15 +2265,16 @@
             "=0.5.1",
             "=0.5.2",
             "=0.5.3",
+            "=0.6.0",
         ],
         "unknown_compatibility_versions": ["=0.1.0", "=0.1.1", "=0.1.2", "=0.1.3", "=0.2.0", "=0.1.0"],
-        "package_latest_version_index_json": "=0.5.3",
+        "package_latest_version_index_json": "=0.6.0",
         "package_redirect_name": None,
         "package_redirect_namespace": None,
-        "latest_version_incl_prerelease": "=0.5.3",
+        "latest_version_incl_prerelease": "=0.6.0",
     },
     "edanalytics/edu_wh": {
-        "latest_version": "=0.5.3",
+        "latest_version": "=0.6.0",
         "oldest_fusion_compatible_version": None,
         "latest_fusion_compatible_version": None,
         "fusion_compatible_versions": [],
@@ -2304,12 +2303,13 @@
             "=0.5.1",
             "=0.5.2",
             "=0.5.3",
+            "=0.6.0",
         ],
         "unknown_compatibility_versions": ["=0.1.0", "=0.1.1-rc2", "=0.1.1", "=0.1.2", "=0.1.3", "=0.1.4", "=0.2.0"],
-        "package_latest_version_index_json": "=0.5.3",
+        "package_latest_version_index_json": "=0.6.0",
         "package_redirect_name": None,
         "package_redirect_namespace": None,
-        "latest_version_incl_prerelease": "=0.5.3",
+        "latest_version_incl_prerelease": "=0.6.0",
     },
     "elementary-data/elementary": {
         "latest_version": "=0.21.0",
@@ -8672,9 +8672,10 @@
     },
     "tailsdotcom/dbt_artifacts": {
         "latest_version": "=2.10.0",
-        "oldest_fusion_compatible_version": "=0.2.0",
+        "oldest_fusion_compatible_version": "=2.10.0",
         "latest_fusion_compatible_version": "=2.10.0",
-        "fusion_compatible_versions": [
+        "fusion_compatible_versions": ["=2.10.0"],
+        "fusion_incompatible_versions": [
             "=0.2.0",
             "=0.2.1",
             "=0.3.0",
@@ -8695,14 +8696,11 @@
             "=2.0.0",
             "=2.1.0",
             "=2.1.1",
-            "=2.10.0",
             "=2.2.0",
             "=2.2.1",
             "=2.2.2",
             "=2.2.3",
             "=2.3.0",
-        ],
-        "fusion_incompatible_versions": [
             "=2.4.0",
             "=2.4.1",
             "=2.4.2",

packages/dbt_fusion_package_tools/src/dbt_fusion_package_tools/manual_overrides.py

@@ -1,6 +1,5 @@
 EXPLICIT_DISALLOW_ALL_VERSIONS: set[str] = set(
     [
-        "brooklyn-data/dbt_artifacts",
         "Snowflake-Labs/dbt_constraints",
         "get-select/dbt_snowflake_query_tags",
         "data-mie/dbt_profiler",
@@ -55,5 +54,51 @@
     # dbt_project_evaluator version 1.1.0 has compatible
     # require dbt version but actually has bug that makes
     # package incompatible until fixed in 1.1.2
-    "dbt-labs/dbt_project_evaluator": set(["1.1.0", "1.1.1"])
+    "dbt-labs/dbt_project_evaluator": set(["1.1.0", "1.1.1"]),
+    "brooklyn-data/dbt_artifacts": set(
+        [
+            "0.1.0",
+            "0.2.0",
+            "0.2.1",
+            "0.3.0",
+            "0.4.0",
+            "0.4.1",
+            "0.4.2",
+            "0.4.3",
+            "0.4.4",
+            "0.5.0",
+            "0.6.0",
+            "0.7.0",
+            "0.8.0",
+            "1.0.0",
+            "1.1.0",
+            "1.1.1",
+            "1.1.2",
+            "1.2.0",
+            "2.0.0",
+            "2.1.0",
+            "2.1.1",
+            "2.2.0",
+            "2.2.1",
+            "2.2.2",
+            "2.2.3",
+            "2.3.0",
+            "2.4.0",
+            "2.4.1",
+            "2.4.2",
+            "2.4.3",
+            "2.5.0",
+            "2.6.0",
+            "2.6.1",
+            "2.6.2",
+            "2.6.3",
+            "2.6.4",
+            "2.7.0",
+            "2.8.0",
+            "2.9.0",
+            "2.9.1",
+            "2.9.2",
+            "2.9.3",
+        ]
+    ),
 }

packages/dbt_fusion_package_tools/src/dbt_fusion_package_tools/scripts/get_package_hub_files.py

@@ -197,6 +197,78 @@ def download_package_jsons_from_hub_repo(
     return packages
 
 
+def read_json_from_local_hub_repo(path: str, file_count_limit: int = 0):
+    """Read JSON files from a local copy of the hub repo and return a
+    defaultdict mapping package_id -> list[parsed outputs].
+
+    The `path` argument may be either:
+      - the repository root (so files are found under `data/packages/...`),
+      - or the `data/packages` directory itself, or
+      - a single JSON file path.
+
+    Behavior mirrors `download_package_jsons_from_hub_repo` where possible:
+      - JSON files are found recursively
+      - each file is parsed and passed to `process_json(file_path, parsed_json)`
+      - parsing/IO errors are warned and skipped
+    """
+    base = Path(path)
+    packages: defaultdict[str, list[dict[str, Any]]] = defaultdict(list)
+
+    if not base.exists():
+        warnings.warn(f"Path does not exist: {path}")
+        return packages
+
+    # Collect JSON files
+    json_files: List[Path]
+    if base.is_file():
+        if base.suffix.lower() == ".json":
+            json_files = [base]
+        else:
+            return packages
+    else:
+        json_files = sorted(base.rglob("*.json"), key=lambda p: str(p))
+
+    if file_count_limit > 0:
+        json_files = json_files[:file_count_limit]
+
+    if not json_files:
+        return packages
+
+    for file in json_files:
+        try:
+            with file.open("r", encoding="utf-8") as fh:
+                parsed = json.load(fh)
+
+            # Try to produce a repo-style path like 'data/packages/...'
+            file_path: str
+            parts = list(file.parts)
+            if "data" in parts:
+                idx = parts.index("data")
+                file_path = Path(*parts[idx:]).as_posix()
+            else:
+                try:
+                    # prefer path relative to provided base
+                    rel = file.relative_to(base)
+                    file_path = rel.as_posix()
+                except Exception:
+                    file_path = file.as_posix()
+
+            # If the user passed the `data/packages` directory itself,
+            # ensure returned path still starts with 'data/packages'
+            if not file_path.startswith("data/packages") and base.name == "packages" and base.parent.name == "data":
+                rel = file.relative_to(base)
+                file_path = Path("data") / "packages" / rel
+                file_path = file_path.as_posix()
+
+            output = process_json(file_path, parsed)
+            if output != {}:
+                packages[output["package_id_from_path"]].append(output)
+        except Exception as exc:
+            warnings.warn(f"Failed to read/parse {file}: {exc}")
+
+    return packages
+
+
 def reload_packages_from_file(
     file_path: Path,
 ) -> defaultdict[str, list[dict[str, Any]]]:
@@ -206,7 +278,8 @@ def reload_packages_from_file(
 
 def main():
     file_count_limit = 0
-    results = download_package_jsons_from_hub_repo(file_count_limit=file_count_limit)
+    # results = download_package_jsons_from_hub_repo(file_count_limit=file_count_limit)
+    results = read_json_from_local_hub_repo(path="~/workplace/hub.getdbt.com", file_count_limit=file_count_limit)
     print(f"Downloaded {len(results)} packages from hub.getdbt.com")
     output_path: Path = Path.cwd() / "src" / "dbt_fusion_package_tools" / "scripts" / "output"
     write_dict_to_json(results, output_path)

packages/dbt_fusion_package_tools/src/dbt_fusion_package_tools/version_utils.py

@@ -81,7 +81,7 @@ def to_version_string(self, skip_matcher: bool = False) -> str:
             build = "+" + self.build
 
         if not skip_matcher:
-            matcher = self.matcher
+            matcher = self.matcher.value
         return "{}{}.{}.{}{}{}".format(matcher, self.major, self.minor, self.patch, prerelease, build)
 
     @classmethod

tests/integration_tests/package_upgrades/mixed_versions/dbt_packages/dbt_artifacts/dbt_project.yml

@@ -0,0 +1,23 @@
+name: "dbt_artifacts"
+version: "2.9.3"
+config-version: 2
+require-dbt-version: [">=1.3.0", "<1.11.0"]
+profile: "dbt_artifacts"
+
+clean-targets: # folders to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+  - "logs"
+
+models:
+  dbt_artifacts:
+    +materialized: view
+    +file_format: '{{ "delta" if target.name == "databricks" else "" }}'
+    sources:
+      +materialized: incremental
+      +full_refresh: false
+      +persist_docs:
+        # Databricks and SQL Server don't offer column-level support for persisting docs
+        columns: '{{ target.name != "databricks" and target.type != "sqlserver" }}'
+        relation: '{{ target.type != "sqlserver" }}'
+      +as_columnstore: False

tests/integration_tests/package_upgrades/mixed_versions/packages.yml

@@ -28,4 +28,7 @@ packages:
     version: "1.1.0"
 # version is incompatible, but compatible version has different name
   - package: calogica/dbt_date
-    version: 0.10.1
+    version: 0.10.1
+# version 2.9.3 has manual override as incompatible, 2.10.0 is compatible
+  - package: brooklyn-data/dbt_artifacts
+    version: 2.9.3

tests/unit_tests/package_upgrades/test_fusion_compatible_versions.py

@@ -35,6 +35,9 @@ def test_check_renames(old_package_id, new_package_id):
         ("dbt-labs/dbt_project_evaluator", "=1.1.0", False),
         ("dbt-labs/dbt_project_evaluator", "=1.1.1", False),
         ("dbt-labs/dbt_project_evaluator", "=1.1.2", True),
+        ("brooklyn-data/dbt_artifacts", "=2.10.0", True),
+        ("brooklyn-data/dbt_artifacts", "=2.9.3", False),
+        ("brooklyn-data/dbt_artifacts", "=0.6.0", False),
     ],
 )
 def test_check_explicit_override_version(package_id, version, expected):

tests/unit_tests/package_upgrades/test_package_upgrade.py

@@ -16,7 +16,7 @@
 
 PROJECT_WITH_PACKAGES_PATH = Path("tests/integration_tests/package_upgrades/mixed_versions")
 # update if count changes
-PROJECT_DEPENDENCY_COUNT = 9
+PROJECT_DEPENDENCY_COUNT = 10
 
 # cases to test:
 
@@ -90,6 +90,9 @@ def test_generate_package_dependencies():
         elif package == "calogica/dbt_date":
             assert fusion_compatibility_state == PackageVersionFusionCompatibilityState.EXPLICIT_ALLOW
             assert package_fusion_compatibility_state == PackageFusionCompatibilityState.ALL_VERSIONS_COMPATIBLE
+        elif package == "brooklyn-data/dbt_artifacts":
+            assert fusion_compatibility_state == PackageVersionFusionCompatibilityState.EXPLICIT_DISALLOW
+            assert package_fusion_compatibility_state == PackageFusionCompatibilityState.SOME_VERSIONS_COMPATIBLE
 
 
 def test_check_for_package_upgrades():
@@ -125,6 +128,11 @@ def test_check_for_package_upgrades():
             )
         elif package == "calogica/dbt_date":
             assert fusion_compatibility_state == PackageVersionUpgradeType.NO_UPGRADE_REQUIRED
+        elif package == "brooklyn-data/dbt_artifacts":
+            assert (
+                fusion_compatibility_state
+                == PackageVersionUpgradeType.PUBLIC_PACKAGE_FUSION_COMPATIBLE_VERSION_EXCEEDS_PROJECT_CONFIG
+            )
 
 
 def test_upgrade_package_versions_no_force_update():
@@ -138,7 +146,7 @@ def test_upgrade_package_versions_no_force_update():
     assert output
     assert output.upgraded
     assert len(output.upgrades) == 1
-    assert len(output.unchanged) == 8
+    assert len(output.unchanged) == 9
     assert len(output.upgrades) + len(output.unchanged) == PROJECT_DEPENDENCY_COUNT
     output.print_to_console(json_output=False)
     output.print_to_console(json_output=True)
@@ -154,7 +162,7 @@ def test_upgrade_package_versions_with_force_update():
     )
     assert output
     assert output.upgraded
-    assert len(output.upgrades) == 3
+    assert len(output.upgrades) == 4
     assert len(output.unchanged) == 6
     assert len(output.upgrades) + len(output.unchanged) == PROJECT_DEPENDENCY_COUNT
     output.print_to_console(json_output=False)