Skip to content

Commit 7668ff5

Browse files
hkad98hkad98
committed
feat: add PythonSDK CLI
PythonSDK CLI is an experimental feature utilizing Node.js CLI (@gooddata/code-cli) for managing workspace. It is necessary for GoodData's VSCode plugin to be able to understand the workspaces definition as code. JIRA: PSDK-186
1 parent 5e62dcb commit 7668ff5

File tree

11 files changed

+439
-0
lines changed

11 files changed

+439
-0
lines changed

‎.envrc

+1
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-sdk/"
1010
export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-pandas/"
1111
export PYTHONPATH="${PYTHONPATH}:${SCRIPT_DIR}/gooddata-dbt/"
1212

13+
export PATH="${PATH}:${SCRIPT_DIR}/gooddata-sdk/bin"
1314
export PATH="${PATH}:${SCRIPT_DIR}/gooddata-dbt/bin"

‎gooddata-sdk/bin/gdc

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env python3
2+
# (C) 2024 GoodData Corporation
3+
# -*- coding: utf-8 -*-
4+
import re
5+
import sys
6+
7+
from gooddata_sdk.cli.gdc_core import main
8+
9+
if __name__ == "__main__":
10+
sys.argv[0] = re.sub(r"(-script\.pyw?|\.exe)?$", "", sys.argv[0])
11+
sys.exit(main(sys.argv[1:]))

‎gooddata-sdk/gooddata_sdk/cli/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
# (C) 2024 GoodData Corporation

‎gooddata-sdk/gooddata_sdk/cli/clone.py

+108
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
# (C) 2024 GoodData Corporation
2+
import argparse
3+
import json
4+
import shutil
5+
import subprocess
6+
from pathlib import Path
7+
8+
from gooddata_sdk import CatalogDeclarativeWorkspaces, GoodDataSdk
9+
from gooddata_sdk.cli.constants import (
10+
BASE_DIR,
11+
CONFIG_FILE,
12+
DATA_SOURCES,
13+
GD_COMMAND,
14+
USER_GROUPS,
15+
USERS,
16+
WORKSPACES,
17+
WORKSPACES_DATA_FILTERS,
18+
)
19+
from gooddata_sdk.cli.utils import (
20+
Bcolors,
21+
measure_clone,
22+
)
23+
24+
25+
def _call_gd_stream_in(workspace_objects: CatalogDeclarativeWorkspaces, path: Path) -> None:
26+
"""
27+
Call 'gd stream-in' command to create workspaces file structure using Node.js CLI.
28+
"""
29+
workspaces = json.dumps({WORKSPACES: workspace_objects.to_dict()[WORKSPACES]})
30+
p = subprocess.Popen(
31+
[GD_COMMAND, "stream-in"],
32+
cwd=path,
33+
stdin=subprocess.PIPE,
34+
stdout=subprocess.PIPE,
35+
stderr=subprocess.PIPE,
36+
)
37+
_, err = p.communicate(input=workspaces.encode())
38+
if err:
39+
print(f"{Bcolors.FAIL}Clone workspaces failed with the following error {err=}.{Bcolors.ENDC}")
40+
41+
42+
@measure_clone(step="workspaces")
43+
def _clone_workspaces(sdk: GoodDataSdk, path: Path) -> None:
44+
assert (path / CONFIG_FILE).exists() and (path / BASE_DIR).exists()
45+
workspace_objects = sdk.catalog_workspace.get_declarative_workspaces()
46+
_call_gd_stream_in(workspace_objects, path)
47+
48+
49+
@measure_clone(step="data sources")
50+
def _clone_data_sources(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
51+
data_sources = sdk.catalog_data_source.get_declarative_data_sources()
52+
data_sources.store_to_disk(analytics_root_dir)
53+
54+
55+
@measure_clone(step="user groups")
56+
def _clone_user_groups(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
57+
user_groups = sdk.catalog_user.get_declarative_user_groups()
58+
user_groups.store_to_disk(analytics_root_dir)
59+
60+
61+
@measure_clone(step="users")
62+
def _clone_users(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
63+
users = sdk.catalog_user.get_declarative_users()
64+
users.store_to_disk(analytics_root_dir)
65+
66+
67+
@measure_clone(step="workspace data filters")
68+
def _clone_workspace_data_filters(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
69+
workspace_data_filters = sdk.catalog_workspace.get_declarative_workspace_data_filters()
70+
workspace_data_filters.store_to_disk(analytics_root_dir)
71+
72+
73+
def clone_all(path: Path) -> None:
74+
init_file = path / CONFIG_FILE
75+
sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
76+
analytics_root_dir = path / BASE_DIR
77+
78+
# clean the directory
79+
if analytics_root_dir.exists():
80+
shutil.rmtree(analytics_root_dir)
81+
# create directory
82+
analytics_root_dir.mkdir()
83+
84+
print("Cloning the whole organization... ⏲️⏲️️⏲️️")
85+
_clone_data_sources(sdk, analytics_root_dir)
86+
_clone_user_groups(sdk, analytics_root_dir)
87+
_clone_users(sdk, analytics_root_dir)
88+
_clone_workspace_data_filters(sdk, analytics_root_dir)
89+
_clone_workspaces(sdk, path)
90+
print("Cloning finished 🚀🚀🚀")
91+
92+
93+
def clone_granular(path: Path, args: argparse.Namespace) -> None:
94+
init_file = path / CONFIG_FILE
95+
analytics_root_dir = path / "analytics"
96+
config_directory = analytics_root_dir.parent
97+
sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
98+
selected_entities = set(args.only)
99+
if DATA_SOURCES in selected_entities:
100+
_clone_data_sources(sdk, analytics_root_dir)
101+
if USER_GROUPS in selected_entities:
102+
_clone_user_groups(sdk, analytics_root_dir)
103+
if USERS in selected_entities:
104+
_clone_users(sdk, analytics_root_dir)
105+
if WORKSPACES_DATA_FILTERS in selected_entities:
106+
_clone_workspace_data_filters(sdk, analytics_root_dir)
107+
if WORKSPACES in selected_entities:
108+
_clone_workspaces(sdk, config_directory)

‎gooddata-sdk/gooddata_sdk/cli/constants.py

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# (C) 2024 GoodData Corporation
2+
from pathlib import Path
3+
4+
DATA_SOURCES = "data_sources"
5+
USER_GROUPS = "user_groups"
6+
USERS = "users"
7+
WORKSPACES_DATA_FILTERS = "workspaces_data_filters"
8+
WORKSPACES = "workspaces"
9+
WORKSPACE = "workspace"
10+
DATA_SOURCE = "data_source"
11+
12+
CONFIG_FILE = "gooddata.yaml"
13+
BASE_DIR = "analytics"
14+
15+
GD_ROOT = Path.home() / ".gooddata"
16+
GD_COMMAND = GD_ROOT / "node_modules/.bin/gd"
17+
GD_PACKAGE_JSON = GD_ROOT / "node_modules/@gooddata/code-cli/package.json"

‎gooddata-sdk/gooddata_sdk/cli/deploy.py

+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
# (C) 2024 GoodData Corporation
2+
import argparse
3+
import json
4+
import subprocess
5+
from pathlib import Path
6+
from typing import Any
7+
8+
from gooddata_sdk import (
9+
CatalogDeclarativeDataSources,
10+
CatalogDeclarativeUserGroups,
11+
CatalogDeclarativeUsers,
12+
CatalogDeclarativeWorkspace,
13+
CatalogDeclarativeWorkspaceDataFilters,
14+
CatalogDeclarativeWorkspaces,
15+
GoodDataSdk,
16+
)
17+
from gooddata_sdk.cli.constants import (
18+
BASE_DIR,
19+
CONFIG_FILE,
20+
DATA_SOURCES,
21+
GD_COMMAND,
22+
USER_GROUPS,
23+
USERS,
24+
WORKSPACES,
25+
WORKSPACES_DATA_FILTERS,
26+
)
27+
from gooddata_sdk.cli.utils import measure_deploy
28+
29+
30+
def _call_gd_stram_out(path: Path) -> dict[str, Any]:
31+
"""
32+
Call 'gd stream-out' command to read workspaces file structure using Node.js CLI.
33+
"""
34+
assert (path / CONFIG_FILE).exists() and (path / BASE_DIR).exists()
35+
p = subprocess.Popen(
36+
[GD_COMMAND, "stream-out", "--no-validate"],
37+
cwd=path,
38+
stdin=subprocess.PIPE,
39+
stdout=subprocess.PIPE,
40+
stderr=subprocess.PIPE,
41+
)
42+
output, err = p.communicate()
43+
if err:
44+
print(f"Deploy workspaces failed with the following error {err=}.")
45+
data = json.loads(output.decode())
46+
if WORKSPACES not in data:
47+
raise ValueError("No workspaces found in the output.")
48+
return data
49+
50+
51+
@measure_deploy(step=WORKSPACES)
52+
def _deploy_workspaces_with_filters(sdk: GoodDataSdk, path: Path) -> None:
53+
analytics_root_dir = path / BASE_DIR
54+
data = _call_gd_stram_out(path)
55+
workspaces = [CatalogDeclarativeWorkspace.from_dict(workspace_dict) for workspace_dict in data[WORKSPACES]]
56+
# fetch this information first, so we do not lose them
57+
workspace_data_filters = CatalogDeclarativeWorkspaceDataFilters.load_from_disk(analytics_root_dir)
58+
workspaces_o = CatalogDeclarativeWorkspaces(
59+
workspaces=workspaces, workspace_data_filters=workspace_data_filters.workspace_data_filters
60+
)
61+
sdk.catalog_workspace.put_declarative_workspaces(workspaces_o)
62+
63+
64+
@measure_deploy(step="data sources")
65+
def _deploy_data_sources(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
66+
data_sources = CatalogDeclarativeDataSources.load_from_disk(analytics_root_dir)
67+
sdk.catalog_data_source.put_declarative_data_sources(
68+
data_sources, config_file=analytics_root_dir.parent / "gooddata.yaml"
69+
)
70+
71+
72+
@measure_deploy(step="user groups")
73+
def _deploy_user_groups(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
74+
user_groups = CatalogDeclarativeUserGroups.load_from_disk(analytics_root_dir)
75+
sdk.catalog_user.put_declarative_user_groups(user_groups)
76+
77+
78+
@measure_deploy(step=USERS)
79+
def _deploy_users(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
80+
users = CatalogDeclarativeUsers.load_from_disk(analytics_root_dir)
81+
sdk.catalog_user.put_declarative_users(users)
82+
83+
84+
@measure_deploy(step="workspace data filters")
85+
def _deploy_workspace_data_filters(sdk: GoodDataSdk, analytics_root_dir: Path) -> None:
86+
workspace_data_filters = CatalogDeclarativeWorkspaceDataFilters.load_from_disk(analytics_root_dir)
87+
sdk.catalog_workspace.put_declarative_workspace_data_filters(workspace_data_filters)
88+
89+
90+
def deploy_all(path: Path) -> None:
91+
init_file = path / CONFIG_FILE
92+
sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
93+
94+
analytics_root_dir = path / BASE_DIR
95+
96+
print("Deploying the whole organization... ⏲️⏲️⏲️")
97+
_deploy_data_sources(sdk, analytics_root_dir)
98+
_deploy_user_groups(sdk, analytics_root_dir)
99+
_deploy_users(sdk, analytics_root_dir)
100+
_deploy_workspaces_with_filters(sdk, path)
101+
print("Deployed 🚀🚀🚀")
102+
103+
104+
def deploy_granular(path: Path, args: argparse.Namespace) -> None:
105+
init_file = path / CONFIG_FILE
106+
analytics_root_dir = path / "analytics"
107+
selected_entities = set(args.only)
108+
sdk = GoodDataSdk.create_from_profile(profiles_path=init_file)
109+
if DATA_SOURCES in selected_entities:
110+
_deploy_data_sources(sdk, analytics_root_dir)
111+
if USER_GROUPS in selected_entities:
112+
_deploy_user_groups(sdk, analytics_root_dir)
113+
if USERS in selected_entities:
114+
_deploy_users(sdk, analytics_root_dir)
115+
if WORKSPACES_DATA_FILTERS in selected_entities:
116+
_deploy_workspace_data_filters(sdk, analytics_root_dir)
117+
if WORKSPACES in selected_entities:
118+
_deploy_workspaces_with_filters(sdk, analytics_root_dir.parent)

‎gooddata-sdk/gooddata_sdk/cli/gdc_core.py

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# (C) 2024 GoodData Corporation
2+
import argparse
3+
import shutil
4+
import subprocess
5+
import sys
6+
from pathlib import Path
7+
from typing import List
8+
9+
from gooddata_sdk.cli.clone import clone_all, clone_granular
10+
from gooddata_sdk.cli.constants import GD_COMMAND, GD_PACKAGE_JSON, GD_ROOT
11+
from gooddata_sdk.cli.deploy import deploy_all, deploy_granular
12+
from gooddata_sdk.cli.utils import _SUPPORTED, Bcolors
13+
from gooddata_sdk.utils import read_json
14+
15+
_CURRENT_DIR = Path(__file__).parent
16+
17+
18+
def get_manifest_directory() -> Path:
19+
"""
20+
Get the directory where the manifest file (gooddata.yaml) is located
21+
using the gd stream-manifest-path command.
22+
"""
23+
p = subprocess.Popen(
24+
[GD_COMMAND, "stream-manifest-path"],
25+
cwd=Path().resolve(),
26+
stdin=subprocess.PIPE,
27+
stdout=subprocess.PIPE,
28+
stderr=subprocess.PIPE,
29+
)
30+
output, err = p.communicate()
31+
if err:
32+
print(f"{Bcolors.FAIL}Manifest gooddata.yaml was not found.{Bcolors.ENDC}")
33+
sys.exit(1)
34+
return Path(output.decode()).parent
35+
36+
37+
def _deploy(path: Path, args: argparse.Namespace) -> None:
38+
"""
39+
Handles deploy command use cases.
40+
"""
41+
if not path.is_dir():
42+
raise ValueError(f"Path {path} is not a directory.")
43+
44+
if args.only is None:
45+
deploy_all(path)
46+
else:
47+
deploy_granular(path, args)
48+
49+
50+
def _clone(path: Path, args: argparse.Namespace) -> None:
51+
"""
52+
Handles clone command use cases.
53+
"""
54+
if args.only is None:
55+
clone_all(path)
56+
else:
57+
clone_granular(path, args)
58+
59+
60+
def _manage_node_cli() -> None:
61+
"""
62+
First, it checks if Node CLI is installed and if it is the correct version.
63+
If not, it installs the correct version. If it is not installed at all then it's installed
64+
locally in ~/.gooddata directory.
65+
"""
66+
requested_version = read_json(_CURRENT_DIR / "package.json")["dependencies"]["@gooddata/code-cli"]
67+
if GD_PACKAGE_JSON.exists():
68+
current_version = read_json(GD_PACKAGE_JSON)["version"]
69+
if current_version == requested_version:
70+
return
71+
else:
72+
print(
73+
f"Node.js @gooddata/code-cli version '{requested_version}' is required,"
74+
f" but version '{current_version}' is installed."
75+
)
76+
if not GD_ROOT.exists():
77+
GD_ROOT.mkdir()
78+
shutil.copyfile(_CURRENT_DIR / "package.json", GD_ROOT / "package.json")
79+
print(f"Installing @gooddata/code-cli version '{requested_version}' in {GD_ROOT}...")
80+
p = subprocess.Popen(
81+
["npm", "i"],
82+
cwd=GD_ROOT,
83+
stdin=subprocess.PIPE,
84+
stdout=subprocess.PIPE,
85+
stderr=subprocess.PIPE,
86+
)
87+
_, err = p.communicate()
88+
if err:
89+
print(f"{Bcolors.FAIL}An error has occurred during installation: {err.decode()}.{Bcolors.ENDC}")
90+
sys.exit(1)
91+
92+
93+
def main(cli_args: List[str]) -> None:
94+
"""
95+
The entrypoint for gdc cli.
96+
"""
97+
parser = argparse.ArgumentParser(
98+
prog="gdc",
99+
description="Process GoodData as code file structure. Utilize @gooddata/code-cli for workspaces. "
100+
"Note that this is an EXPERIMENTAL feature.",
101+
)
102+
parser.add_argument("action", help="Specify if you want to deploy or clone project.", choices=("deploy", "clone"))
103+
parser.add_argument("--only", help="Specify available granularity for action.", nargs="+", choices=_SUPPORTED)
104+
105+
_manage_node_cli()
106+
args = parser.parse_args(cli_args)
107+
manifest_directory = get_manifest_directory()
108+
if args.action == "clone":
109+
_clone(manifest_directory, args)
110+
elif args.action == "deploy":
111+
_deploy(manifest_directory, args)
112+
113+
114+
if __name__ == "__main__":
115+
main(sys.argv[1:])

‎gooddata-sdk/gooddata_sdk/cli/package.json

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"name": "test",
3+
"version": "1.0.0",
4+
"dependencies": {
5+
"@gooddata/code-cli": "1.0.0-alpha.2"
6+
}
7+
}

0 commit comments

Comments
 (0)