Merge pull request #721 from hkad98/aac-productization

Integration with @gooddata/code-cli

Author: hkad98

.envrc

@@ -10,4 +10,5 @@ export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-sdk/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-pandas/"
 export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-dbt/"
 
+export PATH="${PATH}:${SCRIPT_DIR}/gooddata-sdk/bin"
 export PATH="${PATH}:${SCRIPT_DIR}/gooddata-dbt/bin"

gooddata-sdk/bin/gdc

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# (C) 2024 GoodData Corporation
+# -*- coding: utf-8 -*-
+import re
+import sys
+
+from gooddata_sdk.cli.gdc_core import main
+
+if __name__ == "__main__":
+    sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
+    sys.exit(main(sys.argv[1:]))

gooddata-sdk/gooddata_sdk/catalog/data_source/service.py

@@ -193,8 +193,10 @@ def put_declarative_data_sources(
         """
         if test_data_sources:
             self.test_data_sources_connection(declarative_data_sources, credentials_path, config_file)
-        credentials = self._credentials_from_file(credentials_path) if credentials_path is not None else dict()
-        self._layout_api.put_data_sources_layout(declarative_data_sources.to_api(credentials, config_file))
+        credentials = self._credentials_from_file(credentials_path) if credentials_path is not None else None
+        self._layout_api.put_data_sources_layout(
+            declarative_data_sources.to_api(credentials=credentials, config_file=config_file)
+        )
 
     def store_declarative_data_sources(self, layout_root_path: Path = Path.cwd()) -> None:
         """Store data sources layouts in a directory hierarchy.

gooddata-sdk/gooddata_sdk/catalog/user/declarative_model/user.py

@@ -47,6 +47,9 @@ def store_to_disk(self, layout_organization_folder: Path) -> None:
 @attr.s(auto_attribs=True, kw_only=True)
 class CatalogDeclarativeUser(Base):
     id: str
+    email: Optional[str] = None
+    firstname: Optional[str] = None
+    lastname: Optional[str] = None
     auth_id: Optional[str] = None
     user_groups: List[CatalogDeclarativeUserGroupIdentifier] = attr.field(factory=list)
     settings: List[CatalogDeclarativeSetting] = attr.field(factory=list)

gooddata-sdk/gooddata_sdk/cli/__init__.py

@@ -0,0 +1 @@
+# (C) 2024 GoodData Corporation

gooddata-sdk/gooddata_sdk/cli/clone.py

@@ -0,0 +1,108 @@
+# (C) 2024 GoodData Corporation
+import argparse
+import json
+import shutil
+import subprocess
+from pathlib import Path
+
+from gooddata_sdk import CatalogDeclarativeWorkspaces, GoodDataSdk
+from gooddata_sdk.cli.constants import (
+    BASE_DIR,
+    CONFIG_FILE,
+    DATA_SOURCES,
+    GD_COMMAND,
+    USER_GROUPS,
+    USERS,
+    WORKSPACES,
+    WORKSPACES_DATA_FILTERS,
+)
+from gooddata_sdk.cli.utils import (
+    Bcolors,
+    measure_clone,
+)
+
+
+def _call_gd_stream_in(workspace_objects: CatalogDeclarativeWorkspaces, path: Path) -> None:
+    """
+    Call 'gd stream-in' command to create workspaces file structure using Node.js CLI.
+    """
+    workspaces = json.dumps({WORKSPACES: workspace_objects.to_dict()[WORKSPACES]})
+    p = subprocess.Popen(
+        [GD_COMMAND, "stream-in"],
+        cwd=path,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    _, err = p.communicate(input=workspaces.encode())
+    if err:
+        print(f"{Bcolors.FAIL}Clone workspaces failed with the following error {err=}.{Bcolors.ENDC}")
+
+
+@measure_clone(step="workspaces")
+def _clone_workspaces(sdk: GoodDataSdk, path: Path) -> None:
+    assert (path / CONFIG_FILE).exists() and (path / BASE_DIR).exists()
+    workspace_objects = sdk.catalog_workspace.get_declarative_workspaces()
+    _call_gd_stream_in(workspace_objects, path)
+
+
+@measure_clone(step="data sources")
+def _clone_data_sources(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    data_sources = sdk.catalog_data_source.get_declarative_data_sources()
+    data_sources.store_to_disk(analytics_root_dir)
+
+
+@measure_clone(step="user groups")
+def _clone_user_groups(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    user_groups = sdk.catalog_user.get_declarative_user_groups()
+    user_groups.store_to_disk(analytics_root_dir)
+
+
+@measure_clone(step="users")
+def _clone_users(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    users = sdk.catalog_user.get_declarative_users()
+    users.store_to_disk(analytics_root_dir)
+
+
+@measure_clone(step="workspace data filters")
+def _clone_workspace_data_filters(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    workspace_data_filters = sdk.catalog_workspace.get_declarative_workspace_data_filters()
+    workspace_data_filters.store_to_disk(analytics_root_dir)
+
+
+def clone_all(path: Path) -> None:
+    init_file = path / CONFIG_FILE
+    sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
+    analytics_root_dir = path / BASE_DIR
+
+    # clean the directory
+    if analytics_root_dir.exists():
+        shutil.rmtree(analytics_root_dir)
+    # create directory
+    analytics_root_dir.mkdir()
+
+    print("Cloning the whole organization... ⏲️⏲️️⏲️️")
+    _clone_data_sources(sdk, analytics_root_dir)
+    _clone_user_groups(sdk, analytics_root_dir)
+    _clone_users(sdk, analytics_root_dir)
+    _clone_workspace_data_filters(sdk, analytics_root_dir)
+    _clone_workspaces(sdk, path)
+    print("Cloning finished 🚀🚀🚀")
+
+
+def clone_granular(path: Path, args: argparse.Namespace) -> None:
+    init_file = path / CONFIG_FILE
+    analytics_root_dir = path / "analytics"
+    config_directory = analytics_root_dir.parent
+    sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
+    selected_entities = set(args.only)
+    if DATA_SOURCES in selected_entities:
+        _clone_data_sources(sdk, analytics_root_dir)
+    if USER_GROUPS in selected_entities:
+        _clone_user_groups(sdk, analytics_root_dir)
+    if USERS in selected_entities:
+        _clone_users(sdk, analytics_root_dir)
+    if WORKSPACES_DATA_FILTERS in selected_entities:
+        _clone_workspace_data_filters(sdk, analytics_root_dir)
+    if WORKSPACES in selected_entities:
+        _clone_workspaces(sdk, config_directory)

gooddata-sdk/gooddata_sdk/cli/constants.py

@@ -0,0 +1,17 @@
+# (C) 2024 GoodData Corporation
+from pathlib import Path
+
+DATA_SOURCES = "data_sources"
+USER_GROUPS = "user_groups"
+USERS = "users"
+WORKSPACES_DATA_FILTERS = "workspaces_data_filters"
+WORKSPACES = "workspaces"
+WORKSPACE = "workspace"
+DATA_SOURCE = "data_source"
+
+CONFIG_FILE = "gooddata.yaml"
+BASE_DIR = "analytics"
+
+GD_ROOT = Path.home() / ".gooddata"
+GD_COMMAND = GD_ROOT / "node_modules/.bin/gd"
+GD_PACKAGE_JSON = GD_ROOT / "node_modules/@gooddata/code-cli/package.json"

gooddata-sdk/gooddata_sdk/cli/deploy.py

@@ -0,0 +1,118 @@
+# (C) 2024 GoodData Corporation
+import argparse
+import json
+import subprocess
+from pathlib import Path
+from typing import Any
+
+from gooddata_sdk import (
+    CatalogDeclarativeDataSources,
+    CatalogDeclarativeUserGroups,
+    CatalogDeclarativeUsers,
+    CatalogDeclarativeWorkspace,
+    CatalogDeclarativeWorkspaceDataFilters,
+    CatalogDeclarativeWorkspaces,
+    GoodDataSdk,
+)
+from gooddata_sdk.cli.constants import (
+    BASE_DIR,
+    CONFIG_FILE,
+    DATA_SOURCES,
+    GD_COMMAND,
+    USER_GROUPS,
+    USERS,
+    WORKSPACES,
+    WORKSPACES_DATA_FILTERS,
+)
+from gooddata_sdk.cli.utils import measure_deploy
+
+
+def _call_gd_stram_out(path: Path) -> dict[str, Any]:
+    """
+    Call 'gd stream-out' command to read workspaces file structure using Node.js CLI.
+    """
+    assert (path / CONFIG_FILE).exists() and (path / BASE_DIR).exists()
+    p = subprocess.Popen(
+        [GD_COMMAND, "stream-out", "--no-validate"],
+        cwd=path,
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    output, err = p.communicate()
+    if err:
+        print(f"Deploy workspaces failed with the following error {err=}.")
+    data = json.loads(output.decode())
+    if WORKSPACES not in data:
+        raise ValueError("No workspaces found in the output.")
+    return data
+
+
+@measure_deploy(step=WORKSPACES)
+def _deploy_workspaces_with_filters(sdk: GoodDataSdk, path: Path) -> None:
+    analytics_root_dir = path / BASE_DIR
+    data = _call_gd_stram_out(path)
+    workspaces = [CatalogDeclarativeWorkspace.from_dict(workspace_dict) for workspace_dict in data[WORKSPACES]]
+    # fetch this information first, so we do not lose them
+    workspace_data_filters = CatalogDeclarativeWorkspaceDataFilters.load_from_disk(analytics_root_dir)
+    workspaces_o = CatalogDeclarativeWorkspaces(
+        workspaces=workspaces, workspace_data_filters=workspace_data_filters.workspace_data_filters
+    )
+    sdk.catalog_workspace.put_declarative_workspaces(workspaces_o)
+
+
+@measure_deploy(step="data sources")
+def _deploy_data_sources(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    data_sources = CatalogDeclarativeDataSources.load_from_disk(analytics_root_dir)
+    sdk.catalog_data_source.put_declarative_data_sources(
+        data_sources, config_file=analytics_root_dir.parent / "gooddata.yaml"
+    )
+
+
+@measure_deploy(step="user groups")
+def _deploy_user_groups(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    user_groups = CatalogDeclarativeUserGroups.load_from_disk(analytics_root_dir)
+    sdk.catalog_user.put_declarative_user_groups(user_groups)
+
+
+@measure_deploy(step=USERS)
+def _deploy_users(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    users = CatalogDeclarativeUsers.load_from_disk(analytics_root_dir)
+    sdk.catalog_user.put_declarative_users(users)
+
+
+@measure_deploy(step="workspace data filters")
+def _deploy_workspace_data_filters(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
+    workspace_data_filters = CatalogDeclarativeWorkspaceDataFilters.load_from_disk(analytics_root_dir)
+    sdk.catalog_workspace.put_declarative_workspace_data_filters(workspace_data_filters)
+
+
+def deploy_all(path: Path) -> None:
+    init_file = path / CONFIG_FILE
+    sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
+
+    analytics_root_dir = path / BASE_DIR
+
+    print("Deploying the whole organization... ⏲️⏲️⏲️")
+    _deploy_data_sources(sdk, analytics_root_dir)
+    _deploy_user_groups(sdk, analytics_root_dir)
+    _deploy_users(sdk, analytics_root_dir)
+    _deploy_workspaces_with_filters(sdk, path)
+    print("Deployed 🚀🚀🚀")
+
+
+def deploy_granular(path: Path, args: argparse.Namespace) -> None:
+    init_file = path / CONFIG_FILE
+    analytics_root_dir = path / "analytics"
+    selected_entities = set(args.only)
+    sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
+    if DATA_SOURCES in selected_entities:
+        _deploy_data_sources(sdk, analytics_root_dir)
+    if USER_GROUPS in selected_entities:
+        _deploy_user_groups(sdk, analytics_root_dir)
+    if USERS in selected_entities:
+        _deploy_users(sdk, analytics_root_dir)
+    if WORKSPACES_DATA_FILTERS in selected_entities:
+        _deploy_workspace_data_filters(sdk, analytics_root_dir)
+    if WORKSPACES in selected_entities:
+        _deploy_workspaces_with_filters(sdk, analytics_root_dir.parent)