Skip to content

Commit 0a7c1a9

Browse files
eric-thomas-dagstereric-thomas-dagster
committed
chore(workspaces): recategorize data-fetching workspaces as 'ingestion'
The Dagster Designer's Add Data dialog surfaces components in the 'ingestion' category. Workspaces that convert vendor data into DataFrames belong there, so users can pick which SObjects / CRM objects / tables / databases / cubes / reports to bring in from a single component. Recategorized (integration → ingestion): - salesforce_workspace (SObjects → DataFrames) - hubspot_workspace (CRM objects → DataFrames) - jira_workspace (issues → DataFrames) - servicenow_workspace (tables → DataFrames) - notion_workspace (databases → DataFrames) - cognos_workspace (reports → DataFrames) - tm1_workspace (cubes/processes → DataFrames/executions) Kept as 'integration' (they run tasks, don't ingest data): - qlik_replicate_workspace (task control) - qlik_compose_workspace (workflow control) - jde_orchestrator_workspace (orchestration control) Bumps version to 0.10.37.
1 parent 5b31378 commit 0a7c1a9

9 files changed

Lines changed: 1425 additions & 319 deletions

File tree

‎integrations/cognos_workspace/schema.json‎

Lines changed: 95 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,101 @@
11
{
22
"component_type": "dagster_community_components.CognosWorkspaceComponent",
3-
"description": "Auto-emit one Dagster asset per Cognos report by enumerating reports in the specified folders via the Cognos Analytics REST API. StateBackedComponent — discovery cached to disk. Materializing an asset runs the underlying report. The workspace-shape peer of the cognos_* low-level components.",
4-
"category": "integration",
3+
"description": "Auto-emit one Dagster asset per Cognos report by enumerating reports in the specified folders via the Cognos Analytics REST API. StateBackedComponent \u2014 discovery cached to disk. Materializing an asset runs the underlying report. The workspace-shape peer of the cognos_* low-level components.",
4+
"category": "ingestion",
55
"icon": "AccountTree",
66
"attributes": {
7-
"base_url_env_var": {"type": "string", "label": "Base URL Env Var", "required": true},
8-
"username_env_var": {"type": "string", "label": "Username Env Var", "required": true},
9-
"password_env_var": {"type": "string", "label": "Password Env Var", "required": true},
10-
"namespace_env_var": {"type": "string", "label": "Namespace Env Var", "required": true},
11-
"verify_ssl": {"type": "boolean", "label": "Verify SSL", "required": false, "default": true, "ui:widget": "checkbox"},
12-
"folder_ids": {"type": "array", "label": "Folder IDs (searchPaths)", "required": false, "description": "Cognos folder searchPaths to enumerate. Omit for root."},
13-
"report_selector": {"type": "object", "label": "Report Selector", "required": false, "description": "Fivetran-shape selector: by_name / by_pattern / exclude_by_name / exclude_by_pattern."},
14-
"group_name": {"type": "string", "label": "Group Name", "required": false},
15-
"asset_key_prefix": {"type": "array", "label": "Asset Key Prefix", "required": false, "default": ["cognos", "report"]},
16-
"compute_kind": {"type": "string", "label": "Compute Kind", "required": false, "default": "cognos"},
17-
"output_format": {"type": "string", "label": "Output Format", "required": false, "default": "CSV", "description": "Report output format when materialized.", "enum": ["PDF", "HTML", "CSV", "XLSX", "XML", "JSON"], "ui:widget": "select"},
18-
"wait_for_completion": {"type": "boolean", "label": "Wait for Completion", "required": false, "default": true, "ui:widget": "checkbox"},
19-
"timeout_seconds": {"type": "integer", "label": "Timeout (s)", "required": false, "default": 600},
20-
"defs_state": {"type": "object", "label": "Defs State Config", "required": false}
7+
"base_url_env_var": {
8+
"type": "string",
9+
"label": "Base URL Env Var",
10+
"required": true
11+
},
12+
"username_env_var": {
13+
"type": "string",
14+
"label": "Username Env Var",
15+
"required": true
16+
},
17+
"password_env_var": {
18+
"type": "string",
19+
"label": "Password Env Var",
20+
"required": true
21+
},
22+
"namespace_env_var": {
23+
"type": "string",
24+
"label": "Namespace Env Var",
25+
"required": true
26+
},
27+
"verify_ssl": {
28+
"type": "boolean",
29+
"label": "Verify SSL",
30+
"required": false,
31+
"default": true,
32+
"ui:widget": "checkbox"
33+
},
34+
"folder_ids": {
35+
"type": "array",
36+
"label": "Folder IDs (searchPaths)",
37+
"required": false,
38+
"description": "Cognos folder searchPaths to enumerate. Omit for root."
39+
},
40+
"report_selector": {
41+
"type": "object",
42+
"label": "Report Selector",
43+
"required": false,
44+
"description": "Fivetran-shape selector: by_name / by_pattern / exclude_by_name / exclude_by_pattern."
45+
},
46+
"group_name": {
47+
"type": "string",
48+
"label": "Group Name",
49+
"required": false
50+
},
51+
"asset_key_prefix": {
52+
"type": "array",
53+
"label": "Asset Key Prefix",
54+
"required": false,
55+
"default": [
56+
"cognos",
57+
"report"
58+
]
59+
},
60+
"compute_kind": {
61+
"type": "string",
62+
"label": "Compute Kind",
63+
"required": false,
64+
"default": "cognos"
65+
},
66+
"output_format": {
67+
"type": "string",
68+
"label": "Output Format",
69+
"required": false,
70+
"default": "CSV",
71+
"description": "Report output format when materialized.",
72+
"enum": [
73+
"PDF",
74+
"HTML",
75+
"CSV",
76+
"XLSX",
77+
"XML",
78+
"JSON"
79+
],
80+
"ui:widget": "select"
81+
},
82+
"wait_for_completion": {
83+
"type": "boolean",
84+
"label": "Wait for Completion",
85+
"required": false,
86+
"default": true,
87+
"ui:widget": "checkbox"
88+
},
89+
"timeout_seconds": {
90+
"type": "integer",
91+
"label": "Timeout (s)",
92+
"required": false,
93+
"default": 600
94+
},
95+
"defs_state": {
96+
"type": "object",
97+
"label": "Defs State Config",
98+
"required": false
99+
}
21100
}
22101
}

‎integrations/hubspot_workspace/schema.json‎

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,58 @@
11
{
22
"component_type": "dagster_community_components.HubSpotWorkspaceComponent",
3-
"description": "Auto-emit one Dagster asset per HubSpot CRM object (contacts / companies / deals / tickets / products / line_items + custom objects). StateBackedComponent — discovery cached to disk. Materializing an asset fetches recent records via the CRM Objects API. Workspace-shape peer of hubspot_ingestion.",
4-
"category": "integration",
3+
"description": "Auto-emit one Dagster asset per HubSpot CRM object (contacts / companies / deals / tickets / products / line_items + custom objects). StateBackedComponent \u2014 discovery cached to disk. Materializing an asset fetches recent records via the CRM Objects API. Workspace-shape peer of hubspot_ingestion.",
4+
"category": "ingestion",
55
"icon": "AccountTree",
66
"attributes": {
7-
"access_token_env_var": {"type": "string", "label": "Access Token Env Var", "required": true, "description": "Env var with HubSpot private-app access token."},
8-
"verify_ssl": {"type": "boolean", "label": "Verify SSL", "required": false, "default": true, "ui:widget": "checkbox"},
9-
"object_selector": {"type": "object", "label": "Object Selector", "required": false, "description": "Fivetran-shape selector across standard + custom objects."},
10-
"group_name": {"type": "string", "label": "Group Name", "required": false},
11-
"asset_key_prefix": {"type": "array", "label": "Asset Key Prefix", "required": false, "default": ["hubspot"]},
12-
"compute_kind": {"type": "string", "label": "Compute Kind", "required": false, "default": "hubspot"},
13-
"page_limit": {"type": "integer", "label": "Page Limit", "required": false, "default": 100, "description": "Records per page for the initial fetch (HubSpot caps at 100)."},
14-
"defs_state": {"type": "object", "label": "Defs State Config", "required": false}
7+
"access_token_env_var": {
8+
"type": "string",
9+
"label": "Access Token Env Var",
10+
"required": true,
11+
"description": "Env var with HubSpot private-app access token."
12+
},
13+
"verify_ssl": {
14+
"type": "boolean",
15+
"label": "Verify SSL",
16+
"required": false,
17+
"default": true,
18+
"ui:widget": "checkbox"
19+
},
20+
"object_selector": {
21+
"type": "object",
22+
"label": "Object Selector",
23+
"required": false,
24+
"description": "Fivetran-shape selector across standard + custom objects."
25+
},
26+
"group_name": {
27+
"type": "string",
28+
"label": "Group Name",
29+
"required": false
30+
},
31+
"asset_key_prefix": {
32+
"type": "array",
33+
"label": "Asset Key Prefix",
34+
"required": false,
35+
"default": [
36+
"hubspot"
37+
]
38+
},
39+
"compute_kind": {
40+
"type": "string",
41+
"label": "Compute Kind",
42+
"required": false,
43+
"default": "hubspot"
44+
},
45+
"page_limit": {
46+
"type": "integer",
47+
"label": "Page Limit",
48+
"required": false,
49+
"default": 100,
50+
"description": "Records per page for the initial fetch (HubSpot caps at 100)."
51+
},
52+
"defs_state": {
53+
"type": "object",
54+
"label": "Defs State Config",
55+
"required": false
56+
}
1557
}
1658
}

‎integrations/jira_workspace/schema.json‎

Lines changed: 68 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,74 @@
11
{
22
"component_type": "dagster_community_components.JiraWorkspaceComponent",
3-
"description": "Auto-emit one Dagster asset per Jira project via /rest/api/3/project/search. StateBackedComponent — discovery cached to disk. Materializing an asset runs a JQL query in that project and emits issues as a DataFrame. Workspace-shape peer of jira_ingestion.",
4-
"category": "integration",
3+
"description": "Auto-emit one Dagster asset per Jira project via /rest/api/3/project/search. StateBackedComponent \u2014 discovery cached to disk. Materializing an asset runs a JQL query in that project and emits issues as a DataFrame. Workspace-shape peer of jira_ingestion.",
4+
"category": "ingestion",
55
"icon": "AccountTree",
66
"attributes": {
7-
"base_url_env_var": {"type": "string", "label": "Base URL Env Var", "required": true},
8-
"email_env_var": {"type": "string", "label": "Email Env Var", "required": true, "description": "Jira Cloud requires email + API-token for basic auth."},
9-
"api_token_env_var": {"type": "string", "label": "API Token Env Var", "required": true},
10-
"verify_ssl": {"type": "boolean", "label": "Verify SSL", "required": false, "default": true, "ui:widget": "checkbox"},
11-
"project_selector": {"type": "object", "label": "Project Selector", "required": false},
12-
"jql_template": {"type": "string", "label": "JQL Template", "required": false, "default": "project = {project_key} ORDER BY created DESC", "description": "JQL query template. {project_key} is substituted per asset."},
13-
"max_results": {"type": "integer", "label": "Max Results", "required": false, "default": 100},
14-
"group_name": {"type": "string", "label": "Group Name", "required": false},
15-
"asset_key_prefix": {"type": "array", "label": "Asset Key Prefix", "required": false, "default": ["jira", "project"]},
16-
"compute_kind": {"type": "string", "label": "Compute Kind", "required": false, "default": "jira"},
17-
"defs_state": {"type": "object", "label": "Defs State Config", "required": false}
7+
"base_url_env_var": {
8+
"type": "string",
9+
"label": "Base URL Env Var",
10+
"required": true
11+
},
12+
"email_env_var": {
13+
"type": "string",
14+
"label": "Email Env Var",
15+
"required": true,
16+
"description": "Jira Cloud requires email + API-token for basic auth."
17+
},
18+
"api_token_env_var": {
19+
"type": "string",
20+
"label": "API Token Env Var",
21+
"required": true
22+
},
23+
"verify_ssl": {
24+
"type": "boolean",
25+
"label": "Verify SSL",
26+
"required": false,
27+
"default": true,
28+
"ui:widget": "checkbox"
29+
},
30+
"project_selector": {
31+
"type": "object",
32+
"label": "Project Selector",
33+
"required": false
34+
},
35+
"jql_template": {
36+
"type": "string",
37+
"label": "JQL Template",
38+
"required": false,
39+
"default": "project = {project_key} ORDER BY created DESC",
40+
"description": "JQL query template. {project_key} is substituted per asset."
41+
},
42+
"max_results": {
43+
"type": "integer",
44+
"label": "Max Results",
45+
"required": false,
46+
"default": 100
47+
},
48+
"group_name": {
49+
"type": "string",
50+
"label": "Group Name",
51+
"required": false
52+
},
53+
"asset_key_prefix": {
54+
"type": "array",
55+
"label": "Asset Key Prefix",
56+
"required": false,
57+
"default": [
58+
"jira",
59+
"project"
60+
]
61+
},
62+
"compute_kind": {
63+
"type": "string",
64+
"label": "Compute Kind",
65+
"required": false,
66+
"default": "jira"
67+
},
68+
"defs_state": {
69+
"type": "object",
70+
"label": "Defs State Config",
71+
"required": false
72+
}
1873
}
1974
}

‎integrations/notion_workspace/schema.json‎

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,58 @@
11
{
22
"component_type": "dagster_community_components.NotionWorkspaceComponent",
3-
"description": "Auto-emit one Dagster asset per Notion database. Uses the Search API to enumerate databases the integration has access to. StateBackedComponent — discovery cached to disk. Materializing an asset queries the database and returns rows (with flattened property values) as a DataFrame.",
4-
"category": "integration",
3+
"description": "Auto-emit one Dagster asset per Notion database. Uses the Search API to enumerate databases the integration has access to. StateBackedComponent \u2014 discovery cached to disk. Materializing an asset queries the database and returns rows (with flattened property values) as a DataFrame.",
4+
"category": "ingestion",
55
"icon": "AccountTree",
66
"attributes": {
7-
"access_token_env_var": {"type": "string", "label": "Access Token Env Var", "required": true, "description": "Env var with a Notion internal-integration token."},
8-
"verify_ssl": {"type": "boolean", "label": "Verify SSL", "required": false, "default": true, "ui:widget": "checkbox"},
9-
"database_selector": {"type": "object", "label": "Database Selector", "required": false, "description": "Fivetran-shape selector matching against database titles."},
10-
"page_size": {"type": "integer", "label": "Page Size", "required": false, "default": 100, "description": "Records per query page (Notion caps at 100)."},
11-
"group_name": {"type": "string", "label": "Group Name", "required": false},
12-
"asset_key_prefix": {"type": "array", "label": "Asset Key Prefix", "required": false, "default": ["notion"]},
13-
"compute_kind": {"type": "string", "label": "Compute Kind", "required": false, "default": "notion"},
14-
"defs_state": {"type": "object", "label": "Defs State Config", "required": false}
7+
"access_token_env_var": {
8+
"type": "string",
9+
"label": "Access Token Env Var",
10+
"required": true,
11+
"description": "Env var with a Notion internal-integration token."
12+
},
13+
"verify_ssl": {
14+
"type": "boolean",
15+
"label": "Verify SSL",
16+
"required": false,
17+
"default": true,
18+
"ui:widget": "checkbox"
19+
},
20+
"database_selector": {
21+
"type": "object",
22+
"label": "Database Selector",
23+
"required": false,
24+
"description": "Fivetran-shape selector matching against database titles."
25+
},
26+
"page_size": {
27+
"type": "integer",
28+
"label": "Page Size",
29+
"required": false,
30+
"default": 100,
31+
"description": "Records per query page (Notion caps at 100)."
32+
},
33+
"group_name": {
34+
"type": "string",
35+
"label": "Group Name",
36+
"required": false
37+
},
38+
"asset_key_prefix": {
39+
"type": "array",
40+
"label": "Asset Key Prefix",
41+
"required": false,
42+
"default": [
43+
"notion"
44+
]
45+
},
46+
"compute_kind": {
47+
"type": "string",
48+
"label": "Compute Kind",
49+
"required": false,
50+
"default": "notion"
51+
},
52+
"defs_state": {
53+
"type": "object",
54+
"label": "Defs State Config",
55+
"required": false
56+
}
1557
}
1658
}

0 commit comments

Comments
 (0)