Merge pull request #708 from hkad98/aac-productization

Accept config from analytics as a code

Author: hkad98

Makefile

@@ -33,8 +33,8 @@ format-diff:
 
 .PHONY: format-fix
 format-fix:
-	.venv/bin/ruff check .
 	.venv/bin/ruff format .
+	.venv/bin/ruff check . --fix --fixable I
 
 
 define download_client

gooddata-pandas/tests/good_pandas/profiles/profiles.yaml

@@ -1,30 +1,30 @@
 # (C) 2023 GoodData Corporation
 default:
   host: http://abc:3000
-  token: 123
+  token: "123"
   custom_headers:
     Host: localhost
   extra_user_agent: xyz
 correct_1:
   host: http://abc:3000
-  token: 123
+  token: "123"
 correct_2:
   host: http://abc:3000
-  token: 123
+  token: "123"
   custom_headers:
     Host: localhost
 correct_3:
   host: http://abc:3000
-  token: 123
+  token: "123"
   extra_user_agent: xyz
 correct_4:
   host: http://abc:3000
-  token: 123
+  token: "123"
   analytics_as_code_things:
     cool: True
 custom:
   host: http://xyz:3000
-  token: 456
+  token: "456"
   custom_headers:
     Host: localhost123
   extra_user_agent: abc

gooddata-pandas/tests/good_pandas/test_good_pandas.py

@@ -2,17 +2,18 @@
 from __future__ import annotations
 
 from pathlib import Path
-from typing import Any
+from typing import Any, Union
 
+import pytest
 import yaml
 from gooddata_pandas import GoodPandas
 
 _current_dir = Path(__file__).parent.absolute()
 PROFILES_PATH = _current_dir / "profiles" / "profiles.yaml"
 
 
-def load_profiles_content() -> dict:
-    with open(PROFILES_PATH, "r", encoding="utf-8") as f:
+def load_profiles_content(path: Union[str, Path]) -> dict:
+    with open(path, "r", encoding="utf-8") as f:
         return yaml.safe_load(f)
 
 
@@ -23,59 +24,24 @@ def are_same_check(profile_data: dict[str, Any], good_pandas: GoodPandas):
         assert profile_data["custom_headers"] == good_pandas.sdk.client._custom_headers
 
 
-def test_default_profile():
-    profile = "default"
-    good_pandas = GoodPandas.create_from_profile(profiles_path=PROFILES_PATH)
-    data = load_profiles_content()
-
-    are_same_check(data[profile], good_pandas)
-
-
-def test_other_profile():
-    profile = "custom"
+@pytest.mark.parametrize(
+    "profile",
+    [
+        "custom",
+        "default",
+        "correct_1",
+        "correct_2",
+        "correct_3",
+        "correct_4",
+    ],
+)
+def test_legacy_config(profile):
     good_pandas = GoodPandas.create_from_profile(profile=profile, profiles_path=PROFILES_PATH)
-    data = load_profiles_content()
-
+    data = load_profiles_content(PROFILES_PATH)
     are_same_check(data[profile], good_pandas)
 
 
 def test_wrong_profile():
     profile = "wrong"
-    try:
+    with pytest.raises(ValueError):
         GoodPandas.create_from_profile(profile=profile, profiles_path=PROFILES_PATH)
-    except ValueError:
-        assert True
-    else:
-        assert False, "The ValueError was expected to be raised."
-
-
-def test_correct_1_profile():
-    profile = "correct_1"
-    sdk = GoodPandas.create_from_profile(profile=profile, profiles_path=PROFILES_PATH)
-    data = load_profiles_content()
-
-    are_same_check(data[profile], sdk)
-
-
-def test_correct_2_profile():
-    profile = "correct_2"
-    sdk = GoodPandas.create_from_profile(profile=profile, profiles_path=PROFILES_PATH)
-    data = load_profiles_content()
-
-    are_same_check(data[profile], sdk)
-
-
-def test_correct_3_profile():
-    profile = "correct_3"
-    sdk = GoodPandas.create_from_profile(profile=profile, profiles_path=PROFILES_PATH)
-    data = load_profiles_content()
-
-    are_same_check(data[profile], sdk)
-
-
-def test_correct_4_profile():
-    profile = "correct_4"
-    sdk = GoodPandas.create_from_profile(profile=profile, profiles_path=PROFILES_PATH)
-    data = load_profiles_content()
-
-    are_same_check(data[profile], sdk)

gooddata-sdk/gooddata_sdk/config.py

@@ -0,0 +1,53 @@
+# (C) 2024 GoodData Corporation
+import os
+from typing import Any, Dict, Optional, Type, TypeVar
+
+import attrs
+from attrs import define
+from cattrs import structure
+from cattrs.errors import ClassValidationError
+from dotenv import load_dotenv
+
+T = TypeVar("T", bound="ConfigBase")
+
+
+@define
+class ConfigBase:
+    def to_dict(self) -> Dict[str, str]:
+        return attrs.asdict(self)
+
+    @classmethod
+    def from_dict(cls: Type[T], data: Dict[str, Any]) -> T:
+        return structure(data, cls)
+
+    @classmethod
+    def can_structure(cls: Type[T], data: Dict[str, Any]) -> bool:
+        try:
+            cls.from_dict(data)
+            return True
+        except ClassValidationError:
+            return False
+
+
+@define
+class Profile(ConfigBase):
+    host: str
+    token: str
+    custom_headers: Optional[Dict[str, str]] = None
+    extra_user_agent: Optional[str] = None
+
+    def to_dict(self, use_env: bool = False) -> Dict[str, str]:
+        load_dotenv()
+        if not use_env:
+            return attrs.asdict(self)
+        env_var = self.token[1:]
+        if env_var not in os.environ:
+            raise ValueError(f"Environment variable {env_var} not found")
+        return {**attrs.asdict(self), "token": os.environ[env_var]}
+
+
+@define
+class AacConfig(ConfigBase):
+    profiles: Dict[str, Profile]
+    default_profile: str
+    access: Dict[str, str]

gooddata-sdk/gooddata_sdk/utils.py

@@ -9,13 +9,17 @@
 from pathlib import Path
 from shutil import rmtree
 from typing import Any, Callable, Dict, List, NamedTuple, Tuple, Union, cast, no_type_check
+from warnings import warn
 from xml.etree import ElementTree as ET
 
 import yaml
+from cattrs import structure
+from cattrs.errors import ClassValidationError
 from gooddata_api_client import ApiAttributeError
 from gooddata_api_client.model_utils import OpenApiModel
 
 from gooddata_sdk.compute.model.base import ObjId
+from gooddata_sdk.config import AacConfig, Profile
 
 # Use typing collection types to support python < py3.9
 IdObjType = Union[str, ObjId, Dict[str, Dict[str, str]], Dict[str, str]]
@@ -241,6 +245,43 @@ def mandatory_profile_content_check(profile: str, profile_content_keys: KeysView
         raise ValueError(f"Profile {profile} is missing mandatory parameter or parameters {missing_str}.")
 
 
+def _create_profile_legacy(content: Dict) -> Dict:
+    try:
+        return structure(content, Profile).to_dict()
+    except ClassValidationError as e:
+        errors = []
+        for error in e.exceptions:
+            if isinstance(error, KeyError):
+                errors.append(f"Profile file does not contain mandatory parameter: {e}")
+        msg = "\n".join(errors)
+        if not msg:
+            msg = "An error occurred while parsing the profile file."
+        raise ValueError(msg)
+
+
+def _create_profile_aac(profile: str, content: Dict) -> Dict:
+    aac_config = AacConfig.from_dict(content)
+    selected_profile = aac_config.default_profile if profile == "default" else profile
+    if selected_profile not in aac_config.profiles:
+        raise ValueError(f"Profile file does not contain the specified profile: {profile}")
+    return aac_config.profiles[selected_profile].to_dict(use_env=True)
+
+
+def _get_profile(profile: str, content: Dict) -> Dict[str, Any]:
+    is_aac_config = AacConfig.can_structure(content)
+    if not is_aac_config and profile not in content:
+        raise ValueError("Configuration is invalid. Please check the documentation for the valid configuration.")
+    if is_aac_config:
+        return _create_profile_aac(profile, content)
+    else:
+        warn(
+            "Used configuration is deprecated and will be removed in the future. Please use the new configuration.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+        return _create_profile_legacy(content[profile])
+
+
 def profile_content(profile: str = "default", profiles_path: Path = PROFILES_FILE_PATH) -> dict[str, Any]:
     """Get the profile content from a given file.
 
@@ -263,10 +304,9 @@ def profile_content(profile: str = "default", profiles_path: Path = PROFILES_FIL
     if not profiles_path.exists():
         raise ValueError(f"There is no profiles file located for path {profiles_path}.")
     content = read_layout_from_file(profiles_path)
-    if not content.get(profile):
-        raise ValueError(f"Profiles file does not contain profile {profile}.")
-    mandatory_profile_content_check(profile, content[profile].keys())
-    return {key: content[profile][key] for key in content[profile] if key in SDK_PROFILE_KEYS}
+    if content is None:
+        raise ValueError(f"The config file is empty {profiles_path}.")
+    return _get_profile(profile, content)
 
 
 def good_pandas_profile_content(
@@ -288,7 +328,7 @@ def good_pandas_profile_content(
             The content and custom Headers.
     """
     content = profile_content(profile, profiles_path)
-    custom_headers = content.pop("custom_headers", {})
+    custom_headers = content.pop("custom_headers", {}) or {}
     content.pop("extra_user_agent", None)
     return content, custom_headers
 

gooddata-sdk/mypy.ini

@@ -14,3 +14,6 @@ ignore_missing_imports = True
 
 [mypy-requests.*]
 ignore_missing_imports = True
+
+[mypy-dotenv.*]
+ignore_missing_imports = True

gooddata-sdk/tests/conftest.py

@@ -1,5 +1,7 @@
 # (C) 2022 GoodData Corporation
+import os
 from pathlib import Path
+from unittest import mock
 
 import pytest
 import yaml
@@ -22,3 +24,12 @@ def test_config(request):
         config = yaml.safe_load(f)
 
     return config
+
+
+@pytest.fixture()
+def setenvvar(monkeypatch):
+    with mock.patch.dict(os.environ, clear=True):
+        envvars = {"OK_TOKEN": "secret_password", "ENV_VAR": "secret"}
+        for k, v in envvars.items():
+            monkeypatch.setenv(k, v)
+        yield

gooddata-sdk/tests/sdk/profiles/corrupted.yaml

@@ -0,0 +1,4 @@
+# (C) 2024 GoodData Corporation
+abc:
+  def:
+    xyz: 1

gooddata-sdk/tests/sdk/profiles/gooddata.yaml

@@ -0,0 +1,22 @@
+# (C) 2024 GoodData Corporation
+profiles:
+  abc:
+    host: https://abc.com/
+    token: $OK_TOKEN
+    workspace_id: demo
+    data_source_id: demo_ds
+  def:
+    host: https://def.com/
+    token: $NOT_EXISTING_ENV_VAR
+    workspace_id: demo
+    data_source_id: demo_ds
+  xyz:
+    host: https://xyz.com/
+    token: $ENV_VAR
+    workspace_id: demo
+    data_source_id: demo_ds
+source_dir: analytics
+default_profile: abc
+
+access:
+  demo_ds: $DS_PASSWORD

gooddata-sdk/tests/sdk/profiles/profiles.yaml

@@ -1,30 +1,30 @@
 # (C) 2022 GoodData Corporation
 default:
   host: http://abc:3000
-  token: 123
+  token: "123"
   custom_headers:
     Host: localhost
   extra_user_agent: xyz
 correct_1:
   host: http://abc:3000
-  token: 123
+  token: "123"
 correct_2:
   host: http://abc:3000
-  token: 123
+  token: "123"
   custom_headers:
     Host: localhost
 correct_3:
   host: http://abc:3000
-  token: 123
+  token: "123"
   extra_user_agent: xyz
 correct_4:
   host: http://abc:3000
-  token: 123
+  token: "123"
   analytics_as_code_things:
     cool: True
 custom:
   host: http://xyz:3000
-  token: 456
+  token: "456"
   custom_headers:
     Host: localhost123
   extra_user_agent: abc