diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteCloud.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteCloud.ts index ab23e350c96e4..0e2b2aa121e15 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteCloud.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteCloud.ts @@ -25,4 +25,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis guide provides instructions for using Dagster with Airbyte Cloud using the `dagster-airbyte` library. Your Airbyte Cloud connection tables can be represented as assets in the Dagster asset graph, allowing you to track lineage and dependencies between Airbyte Cloud assets and data assets you are already modeling in Dagster. You can also use Dagster to orchestrate Airbyte Cloud connections, allowing you to trigger syncs for these on a cadence or based on upstream data changes.\n\n## What you\'ll learn\n\n- How to represent Airbyte Cloud assets in the Dagster asset graph, including lineage to other Dagster assets.\n- How to customize asset definition metadata for these Airbyte Cloud assets.\n- How to materialize Airbyte Cloud connection tables from Dagster.\n- How to customize how Airbyte Cloud connection tables are materialized.\n\n
\n Prerequisites\n\n- The `dagster` and `dagster-airbyte` libraries installed in your environment\n- Familiarity with asset definitions and the Dagster asset graph\n- Familiarity with Dagster resources\n- Familiarity with Airbyte Cloud concepts, like connections and connection tables\n- An Airbyte Cloud workspace\n- An Airbyte Cloud client ID and client secret. For more information, see [Configuring API Access](https://docs.airbyte.com/using-airbyte/configuring-api-access) in the Airbyte Cloud REST API documentation.\n\n
\n\n## Set up your environment\n\nTo get started, you\'ll need to install the `dagster` and `dagster-airbyte` Python packages:\n\n```bash\npip install dagster dagster-airbyte\n```\n\n## Represent Airbyte Cloud assets in the asset graph\n\nTo load Airbyte Cloud assets into the Dagster asset graph, you must first construct a resource, which allows Dagster to communicate with your Airbyte Cloud workspace. You\'ll need to supply your workspace ID, client ID and client secret. See [Configuring API Access](https://docs.airbyte.com/using-airbyte/configuring-api-access) in the Airbyte Cloud REST API documentation for more information on how to create your client ID and client secret.\n\nDagster can automatically load all connection tables from your Airbyte Cloud workspace as asset specs. Call the function, which returns list of s representing your Airbyte Cloud assets. You can then include these asset specs in your object:\n\n\n\n### Sync and materialize Airbyte Cloud assets\n\nYou can use Dagster to sync Airbyte Cloud connections and materialize Airbyte Cloud connection tables. You can use the factory to create all assets definitions for your Airbyte Cloud workspace.\n\n\n\n### Customize the materialization of Airbyte Cloud assets\n\nIf you want to customize the sync of your connections, you can use the decorator to do so. This allows you to execute custom code before and after the call to the Airbyte Cloud sync.\n\n\n\n### Customize asset definition metadata for Airbyte Cloud assets\n\nBy default, Dagster will generate asset specs for each Airbyte Cloud asset and populate default metadata. You can further customize asset properties by passing an instance of the custom to the function.\n\n\n\nNote that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it.\n\nYou can pass an instance of the custom to the decorator or the factory.\n\n### Load Airbyte Cloud assets from multiple workspaces\n\nDefinitions from multiple Airbyte Cloud workspaces can be combined by instantiating multiple resources and merging their specs. This lets you view all your Airbyte Cloud assets in a single asset graph:\n\n'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis guide provides instructions for using Dagster with Airbyte Cloud using the `dagster-airbyte` library. Your Airbyte Cloud connection tables can be represented as assets in the Dagster asset graph, allowing you to track lineage and dependencies between Airbyte Cloud assets and data assets you are already modeling in Dagster. You can also use Dagster to orchestrate Airbyte Cloud connections, allowing you to trigger syncs for these on a cadence or based on upstream data changes.\n\n## What you\'ll learn\n\n- How to represent Airbyte Cloud assets in the Dagster asset graph, including lineage to other Dagster assets.\n- How to customize asset definition metadata for these Airbyte Cloud assets.\n- How to materialize Airbyte Cloud connection tables from Dagster.\n- How to customize how Airbyte Cloud connection tables are materialized.\n\n
\n Prerequisites\n\n- The `dagster` and `dagster-airbyte` libraries installed in your environment\n- Familiarity with asset definitions and the Dagster asset graph\n- Familiarity with Dagster resources\n- Familiarity with Airbyte Cloud concepts, like connections and connection tables\n- An Airbyte Cloud workspace\n- An Airbyte Cloud client ID and client secret. For more information, see [Configuring API Access](https://docs.airbyte.com/using-airbyte/configuring-api-access) in the Airbyte Cloud REST API documentation.\n\n
\n\n## Set up your environment\n\nTo get started, you\'ll need to install the `dagster` and `dagster-airbyte` Python packages:\n\n```bash\npip install dagster dagster-airbyte\n```\n\n## Represent Airbyte Cloud assets in the asset graph\n\nTo load Airbyte Cloud assets into the Dagster asset graph, you must first construct a resource, which allows Dagster to communicate with your Airbyte Cloud workspace. You\'ll need to supply your workspace ID, client ID and client secret. See [Configuring API Access](https://docs.airbyte.com/using-airbyte/configuring-api-access) in the Airbyte Cloud REST API documentation for more information on how to create your client ID and client secret.\n\nDagster can automatically load all connection tables from your Airbyte Cloud workspace as asset specs. Call the function, which returns list of s representing your Airbyte Cloud assets. You can then include these asset specs in your object:\n\n\n```python\nfrom dagster_airbyte import AirbyteCloudWorkspace, load_airbyte_cloud_asset_specs\n\nimport dagster as dg\n\nairbyte_workspace = AirbyteCloudWorkspace(\n workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),\n client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),\n client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),\n)\n\n\nairbyte_cloud_specs = load_airbyte_cloud_asset_specs(airbyte_workspace)\ndefs = dg.Definitions(assets=airbyte_cloud_specs)\n```\n \n\n### Sync and materialize Airbyte Cloud assets\n\nYou can use Dagster to sync Airbyte Cloud connections and materialize Airbyte Cloud connection tables. You can use the factory to create all assets definitions for your Airbyte Cloud workspace.\n\n\n```python\nfrom dagster_airbyte import AirbyteCloudWorkspace, build_airbyte_assets_definitions\n\nimport dagster as dg\n\nairbyte_workspace = AirbyteCloudWorkspace(\n workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),\n client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),\n client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),\n)\n\nall_airbyte_assets = build_airbyte_assets_definitions(workspace=airbyte_workspace)\n\ndefs = dg.Definitions(\n assets=all_airbyte_assets,\n resources={"airbyte": airbyte_workspace},\n)\n```\n \n\n### Customize the materialization of Airbyte Cloud assets\n\nIf you want to customize the sync of your connections, you can use the decorator to do so. This allows you to execute custom code before and after the call to the Airbyte Cloud sync.\n\n\n```python\nfrom dagster_airbyte import AirbyteCloudWorkspace, airbyte_assets\n\nimport dagster as dg\n\nairbyte_workspace = AirbyteCloudWorkspace(\n workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),\n client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),\n client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),\n)\n\n\n@airbyte_assets(\n connection_id="airbyte_connection_id", # Replace with your connection ID\n workspace=airbyte_workspace,\n name="airbyte_connection_name", # Replace with your connection name\n group_name="airbyte_connection_name",\n)\ndef airbyte_connection_assets(\n context: dg.AssetExecutionContext, airbyte: AirbyteCloudWorkspace\n):\n # Do something before the materialization...\n yield from airbyte.sync_and_poll(context=context)\n # Do something after the materialization...\n\n\ndefs = dg.Definitions(\n assets=[airbyte_connection_assets],\n resources={"airbyte": airbyte_workspace},\n)\n```\n \n\n### Customize asset definition metadata for Airbyte Cloud assets\n\nBy default, Dagster will generate asset specs for each Airbyte Cloud asset and populate default metadata. You can further customize asset properties by passing an instance of the custom to the function.\n\n\n```python\nfrom dagster_airbyte import (\n AirbyteCloudWorkspace,\n AirbyteConnectionTableProps,\n DagsterAirbyteTranslator,\n load_airbyte_cloud_asset_specs,\n)\n\nimport dagster as dg\n\nairbyte_workspace = AirbyteCloudWorkspace(\n workspace_id=dg.EnvVar("AIRBYTE_CLOUD_WORKSPACE_ID"),\n client_id=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_ID"),\n client_secret=dg.EnvVar("AIRBYTE_CLOUD_CLIENT_SECRET"),\n)\n\n\n# A translator class lets us customize properties of the built\n# Airbyte Cloud assets, such as the owners or asset key\nclass MyCustomAirbyteTranslator(DagsterAirbyteTranslator):\n def get_asset_spec(self, props: AirbyteConnectionTableProps) -> dg.AssetSpec:\n # We create the default asset spec using super()\n default_spec = super().get_asset_spec(props)\n # We customize the metadata and asset key prefix for all assets\n return default_spec.replace_attributes(\n key=default_spec.key.with_prefix("prefix"),\n ).merge_attributes(metadata={"custom": "metadata"})\n\n\nairbyte_cloud_specs = load_airbyte_cloud_asset_specs(\n airbyte_workspace, dagster_airbyte_translator=MyCustomAirbyteTranslator()\n)\n\ndefs = dg.Definitions(assets=airbyte_cloud_specs)\n```\n \n\nNote that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it.\n\nYou can pass an instance of the custom to the decorator or the factory.\n\n### Load Airbyte Cloud assets from multiple workspaces\n\nDefinitions from multiple Airbyte Cloud workspaces can be combined by instantiating multiple resources and merging their specs. This lets you view all your Airbyte Cloud assets in a single asset graph:\n\n\n```python\nfrom dagster_airbyte import AirbyteCloudWorkspace, load_airbyte_cloud_asset_specs\n\nimport dagster as dg\n\nsales_airbyte_workspace = AirbyteCloudWorkspace(\n workspace_id=dg.EnvVar("AIRBYTE_CLOUD_SALES_WORKSPACE_ID"),\n client_id=dg.EnvVar("AIRBYTE_CLOUD_SALES_CLIENT_ID"),\n client_secret=dg.EnvVar("AIRBYTE_CLOUD_SALES_CLIENT_SECRET"),\n)\n\nmarketing_airbyte_workspace = AirbyteCloudWorkspace(\n workspace_id=dg.EnvVar("AIRBYTE_CLOUD_MARKETING_WORKSPACE_ID"),\n client_id=dg.EnvVar("AIRBYTE_CLOUD_MARKETING_CLIENT_ID"),\n client_secret=dg.EnvVar("AIRBYTE_CLOUD_MARKETING_CLIENT_SECRET"),\n)\n\nsales_airbyte_cloud_specs = load_airbyte_cloud_asset_specs(\n workspace=sales_airbyte_workspace\n)\nmarketing_airbyte_cloud_specs = load_airbyte_cloud_asset_specs(\n workspace=marketing_airbyte_workspace\n)\n\n# Merge the specs into a single set of definitions\ndefs = dg.Definitions(\n assets=[*sales_airbyte_cloud_specs, *marketing_airbyte_cloud_specs],\n)\n```'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteOss.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteOss.ts index d3f9194081e95..aa6f7b47b05f8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteOss.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/airbyteAirbyteOss.ts @@ -25,4 +25,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'Using this integration, you can trigger Airbyte syncs and orchestrate your Airbyte connections from within Dagster, making it easy to chain an Airbyte sync with upstream or downstream steps in your workflow.\n\n### Installation\n\n```bash\npip install dagster-airbyte\n```\n\n### Example\n\n\n\n### About Airbyte\n\n**Airbyte** is an open source data integration engine that helps you consolidate your SaaS application and database data into your data warehouses, lakes and databases.'; + 'Using this integration, you can trigger Airbyte syncs and orchestrate your Airbyte connections from within Dagster, making it easy to chain an Airbyte sync with upstream or downstream steps in your workflow.\n\n### Installation\n\n```bash\npip install dagster-airbyte\n```\n\n### Example\n\n\n```python\nfrom dagster_airbyte import AirbyteResource, load_assets_from_airbyte_instance\n\nimport dagster as dg\n\n# Load all assets from your Airbyte instance\nairbyte_assets = load_assets_from_airbyte_instance(\n # Connect to your OSS Airbyte instance\n AirbyteResource(\n host="localhost",\n port="8000",\n # If using basic auth, include username and password:\n username="airbyte",\n password=dg.EnvVar("AIRBYTE_PASSWORD"),\n )\n)\n\ndefs = dg.Definitions(\n assets=[airbyte_assets],\n)\n```\n \n\n### About Airbyte\n\n**Airbyte** is an open source data integration engine that helps you consolidate your SaaS application and database data into your data warehouses, lakes and databases.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/anthropic.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/anthropic.ts index 99e91ea50bb59..662ca3c045ffe 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/anthropic.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/anthropic.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-anthropic` library allows you to easily interact with the Anthropic REST API using the Anthropic Python API to build AI steps into your Dagster pipelines. You can also log Anthropic API usage metadata in Dagster Insights, giving you detailed observability on API call credit consumption.\n\nWhen paired with Dagster assets, the resource automatically logs Anthropic usage metadata in asset metadata.\n\n### Installation\n\n```bash\npip install dagster dagster-anthropic\n```\n\n### Example\n\n\n\n### About Anthropic\n\nAnthropic is an AI research company focused on developing safe and ethical AI systems. Their flagship product, Claude, is a language model known for its strong capabilities in analysis, writing, and coding tasks while maintaining high standards of truthfulness and safety.'; + 'The `dagster-anthropic` library allows you to easily interact with the Anthropic REST API using the Anthropic Python API to build AI steps into your Dagster pipelines. You can also log Anthropic API usage metadata in Dagster Insights, giving you detailed observability on API call credit consumption.\n\nWhen paired with Dagster assets, the resource automatically logs Anthropic usage metadata in asset metadata.\n\n### Installation\n\n```bash\npip install dagster dagster-anthropic\n```\n\n### Example\n\n\n```python\nfrom dagster_anthropic import AnthropicResource\n\nimport dagster as dg\n\n\n@dg.asset(compute_kind="anthropic")\ndef anthropic_asset(context: dg.AssetExecutionContext, anthropic: AnthropicResource):\n with anthropic.get_client(context) as client:\n response = client.messages.create(\n model="claude-3-5-sonnet-20241022",\n max_tokens=1024,\n messages=[{"role": "user", "content": "Say this is a test"}],\n )\n\n\ndefs = dg.Definitions(\n assets=[anthropic_asset],\n resources={\n "anthropic": AnthropicResource(api_key=dg.EnvVar("ANTHROPIC_API_KEY")),\n },\n)\n```\n \n\n### About Anthropic\n\nAnthropic is an AI research company focused on developing safe and ethical AI systems. Their flagship product, Claude, is a language model known for its strong capabilities in analysis, writing, and coding tasks while maintaining high standards of truthfulness and safety.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsAthena.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsAthena.ts index b3ea38ac09bfd..15e54ff6662d1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsAthena.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsAthena.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This integration allows you to connect to AWS Athena, a serverless interactive query service that makes it easy to analyze data in Amazon S3 using standard SQL. Using this integration, you can issue queries to Athena, fetch results, and handle query execution states within your Dagster pipelines.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS Athena\n\nAWS Athena is a serverless, interactive query service that allows you to analyze data directly in Amazon S3 using standard SQL. Athena is easy to use; point to your data in Amazon S3, define the schema, and start querying using standard SQL. Most results are delivered within seconds. With Athena, there are no infrastructure setups, and you pay only for the queries you run. It scales automatically—executing queries in parallel—so results are fast, even with large datasets and complex queries.'; + 'This integration allows you to connect to AWS Athena, a serverless interactive query service that makes it easy to analyze data in Amazon S3 using standard SQL. Using this integration, you can issue queries to Athena, fetch results, and handle query execution states within your Dagster pipelines.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nfrom dagster_aws.athena import AthenaClientResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef example_athena_asset(athena: AthenaClientResource):\n return athena.get_client().execute_query("SELECT 1", fetch_results=True)\n\n\ndefs = dg.Definitions(\n assets=[example_athena_asset], resources={"athena": AthenaClientResource()}\n)\n```\n \n\n### About AWS Athena\n\nAWS Athena is a serverless, interactive query service that allows you to analyze data directly in Amazon S3 using standard SQL. Athena is easy to use; point to your data in Amazon S3, define the schema, and start querying using standard SQL. Most results are delivered within seconds. With Athena, there are no infrastructure setups, and you pay only for the queries you run. It scales automatically—executing queries in parallel—so results are fast, even with large datasets and complex queries.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsCloudwatch.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsCloudwatch.ts index 8b31bf4f2ac51..04f952549ba6a 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsCloudwatch.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsCloudwatch.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Deprecated from \'@site/docs/partials/\\_Deprecated.md\';\n\n\n\nThis integration allows you to send Dagster logs to AWS CloudWatch, enabling centralized logging and monitoring of your Dagster jobs. By using AWS CloudWatch, you can take advantage of its powerful log management features, such as real-time log monitoring, log retention policies, and alerting capabilities.\n\nUsing this integration, you can configure your Dagster jobs to log directly to AWS CloudWatch, making it easier to track and debug your workflows. This is particularly useful for production environments where centralized logging is essential for maintaining observability and operational efficiency.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS CloudWatch\n\nAWS CloudWatch is a monitoring and observability service provided by Amazon Web Services (AWS). It allows you to collect, access, and analyze performance and operational data from a variety of AWS resources, applications, and services. With AWS CloudWatch, you can set up alarms, visualize logs and metrics, and gain insights into your infrastructure and applications to ensure they\'re running smoothly.\n\nAWS CloudWatch provides features such as:\n\n- Real-time monitoring: Track the performance of your applications and infrastructure in real-time.\n- Log management: Collect, store, and analyze log data from various sources.\n- Alarms and notifications: Set up alarms to automatically notify you of potential issues.\n- Dashboards: Create custom dashboards to visualize metrics and logs.\n- Integration with other AWS services: Seamlessly integrate with other AWS services for a comprehensive monitoring solution.'; + 'import Deprecated from \'@site/docs/partials/\\_Deprecated.md\';\n\n\n\nThis integration allows you to send Dagster logs to AWS CloudWatch, enabling centralized logging and monitoring of your Dagster jobs. By using AWS CloudWatch, you can take advantage of its powerful log management features, such as real-time log monitoring, log retention policies, and alerting capabilities.\n\nUsing this integration, you can configure your Dagster jobs to log directly to AWS CloudWatch, making it easier to track and debug your workflows. This is particularly useful for production environments where centralized logging is essential for maintaining observability and operational efficiency.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nfrom dagster_aws.cloudwatch import cloudwatch_logger\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_asset(context: dg.AssetExecutionContext):\n context.log.info("Hello, CloudWatch!")\n context.log.error("This is an error")\n context.log.debug("This is a debug message")\n\n\ndefs = dg.Definitions(\n assets=[my_asset],\n loggers={\n "cloudwatch_logger": cloudwatch_logger,\n },\n)\n```\n \n\n### About AWS CloudWatch\n\nAWS CloudWatch is a monitoring and observability service provided by Amazon Web Services (AWS). It allows you to collect, access, and analyze performance and operational data from a variety of AWS resources, applications, and services. With AWS CloudWatch, you can set up alarms, visualize logs and metrics, and gain insights into your infrastructure and applications to ensure they\'re running smoothly.\n\nAWS CloudWatch provides features such as:\n\n- Real-time monitoring: Track the performance of your applications and infrastructure in real-time.\n- Log management: Collect, store, and analyze log data from various sources.\n- Alarms and notifications: Set up alarms to automatically notify you of potential issues.\n- Dashboards: Create custom dashboards to visualize metrics and logs.\n- Integration with other AWS services: Seamlessly integrate with other AWS services for a comprehensive monitoring solution.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEcr.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEcr.ts index aebf28f127d52..65a41a4796915 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEcr.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEcr.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration allows you to connect to AWS Elastic Container Registry (ECR). It provides resources to interact with AWS ECR, enabling you to manage your container images.\n\nUsing this integration, you can seamlessly integrate AWS ECR into your Dagster pipelines, making it easier to manage and deploy containerized applications.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS ECR\n\nAWS Elastic Container Registry (ECR) is a fully managed Docker container registry that makes it easy for developers to store, manage, and deploy Docker container images. AWS ECR is integrated with Amazon Elastic Kubernetes Service (EKS), simplifying your development to production workflow. With ECR, you can securely store and manage your container images and easily integrate with your existing CI/CD pipelines. AWS ECR provides high availability and scalability, ensuring that your container images are always available when you need them.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration allows you to connect to AWS Elastic Container Registry (ECR). It provides resources to interact with AWS ECR, enabling you to manage your container images.\n\nUsing this integration, you can seamlessly integrate AWS ECR into your Dagster pipelines, making it easier to manage and deploy containerized applications.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nfrom dagster_aws.ecr import ECRPublicResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef get_ecr_login_password(ecr_public: ECRPublicResource):\n return ecr_public.get_client().get_login_password()\n\n\ndefs = dg.Definitions(\n assets=[get_ecr_login_password],\n resources={"ecr_public": ECRPublicResource()},\n)\n```\n \n\n### About AWS ECR\n\nAWS Elastic Container Registry (ECR) is a fully managed Docker container registry that makes it easy for developers to store, manage, and deploy Docker container images. AWS ECR is integrated with Amazon Elastic Kubernetes Service (EKS), simplifying your development to production workflow. With ECR, you can securely store and manage your container images and easily integrate with your existing CI/CD pipelines. AWS ECR provides high availability and scalability, ensuring that your container images are always available when you need them.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEmr.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEmr.ts index 687c8bf17a940..925dcbacc9b0b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEmr.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsEmr.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-aws` integration provides ways orchestrating data pipelines that leverage AWS services, including AWS EMR (Elastic MapReduce). This integration allows you to run and scale big data workloads using open source tools such as Apache Spark, Hive, Presto, and more.\n\nUsing this integration, you can:\n\n- Seamlessly integrate AWS EMR into your Dagster pipelines.\n- Utilize EMR for petabyte-scale data processing.\n- Easily manage and monitor EMR clusters and jobs from within Dagster.\n- Leverage Dagster\'s orchestration capabilities to handle complex data workflows involving EMR.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS EMR\n\n**AWS EMR** (Elastic MapReduce) is a cloud big data platform for processing vast amounts of data using open source tools such as Apache Spark, Apache Hive, Apache HBase, Apache Flink, Apache Hudi, and Presto. It simplifies running big data frameworks, allowing you to process and analyze large datasets quickly and cost-effectively. AWS EMR provides the scalability, flexibility, and reliability needed to handle complex data processing tasks, making it an ideal choice for data engineers and scientists.'; + 'The `dagster-aws` integration provides ways orchestrating data pipelines that leverage AWS services, including AWS EMR (Elastic MapReduce). This integration allows you to run and scale big data workloads using open source tools such as Apache Spark, Hive, Presto, and more.\n\nUsing this integration, you can:\n\n- Seamlessly integrate AWS EMR into your Dagster pipelines.\n- Utilize EMR for petabyte-scale data processing.\n- Easily manage and monitor EMR clusters and jobs from within Dagster.\n- Leverage Dagster\'s orchestration capabilities to handle complex data workflows involving EMR.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nfrom pathlib import Path\nfrom typing import Any\n\nfrom dagster_aws.emr import emr_pyspark_step_launcher\nfrom dagster_aws.s3 import S3Resource\nfrom dagster_pyspark import PySparkResource\nfrom pyspark.sql import DataFrame, Row\nfrom pyspark.sql.types import IntegerType, StringType, StructField, StructType\n\nimport dagster as dg\n\nemr_pyspark = PySparkResource(spark_config={"spark.executor.memory": "2g"})\n\n\n@dg.asset\ndef people(\n pyspark: PySparkResource, pyspark_step_launcher: dg.ResourceParam[Any]\n) -> DataFrame:\n schema = StructType(\n [StructField("name", StringType()), StructField("age", IntegerType())]\n )\n rows = [\n Row(name="Thom", age=51),\n Row(name="Jonny", age=48),\n Row(name="Nigel", age=49),\n ]\n return pyspark.spark_session.createDataFrame(rows, schema)\n\n\n@dg.asset\ndef people_over_50(\n pyspark_step_launcher: dg.ResourceParam[Any], people: DataFrame\n) -> DataFrame:\n return people.filter(people["age"] > 50)\n\n\ndefs = dg.Definitions(\n assets=[people, people_over_50],\n resources={\n "pyspark_step_launcher": emr_pyspark_step_launcher.configured(\n {\n "cluster_id": {"env": "EMR_CLUSTER_ID"},\n "local_pipeline_package_path": str(Path(__file__).parent),\n "deploy_local_pipeline_package": True,\n "region_name": "us-west-1",\n "staging_bucket": "my_staging_bucket",\n "wait_for_logs": True,\n }\n ),\n "pyspark": emr_pyspark,\n "s3": S3Resource(),\n },\n)\n```\n \n\n### About AWS EMR\n\n**AWS EMR** (Elastic MapReduce) is a cloud big data platform for processing vast amounts of data using open source tools such as Apache Spark, Apache Hive, Apache HBase, Apache Flink, Apache Hudi, and Presto. It simplifies running big data frameworks, allowing you to process and analyze large datasets quickly and cost-effectively. AWS EMR provides the scalability, flexibility, and reliability needed to handle complex data processing tasks, making it an ideal choice for data engineers and scientists.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsGlue.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsGlue.ts index 76009f93ddbc5..0ea06ee91ca89 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsGlue.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsGlue.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-aws` integration library provides the `PipesGlueClient` resource, enabling you to launch AWS Glue jobs directly from Dagster assets and ops. This integration allows you to pass parameters to Glue code while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS Glue\n\n**AWS Glue** is a fully managed cloud service designed to simplify and automate the process of discovering, preparing, and integrating data for analytics, machine learning, and application development. It supports a wide range of data sources and formats, offering seamless integration with other AWS services. AWS Glue provides the tools to create, run, and manage ETL (Extract, Transform, Load) jobs, making it easier to handle complex data workflows. Its serverless architecture allows for scalability and flexibility, making it a preferred choice for data engineers and analysts who need to process and prepare data efficiently.'; + 'The `dagster-aws` integration library provides the `PipesGlueClient` resource, enabling you to launch AWS Glue jobs directly from Dagster assets and ops. This integration allows you to pass parameters to Glue code while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nimport boto3\nfrom dagster_aws.pipes import (\n PipesGlueClient,\n PipesS3ContextInjector,\n PipesS3MessageReader,\n)\n\nimport dagster as dg\n\n\n@dg.asset\ndef glue_pipes_asset(\n context: dg.AssetExecutionContext, pipes_glue_client: PipesGlueClient\n):\n return pipes_glue_client.run(\n context=context,\n job_name="Example Job",\n arguments={"some_parameter_value": "1"},\n ).get_materialize_result()\n\n\ndefs = dg.Definitions(\n assets=[glue_pipes_asset],\n resources={\n "pipes_glue_client": PipesGlueClient(\n client=boto3.client("glue", region_name="us-east-1"),\n context_injector=PipesS3ContextInjector(\n client=boto3.client("s3"),\n bucket="my-bucket",\n ),\n message_reader=PipesS3MessageReader(\n client=boto3.client("s3"), bucket="my-bucket"\n ),\n )\n },\n)\n```\n \n\n### About AWS Glue\n\n**AWS Glue** is a fully managed cloud service designed to simplify and automate the process of discovering, preparing, and integrating data for analytics, machine learning, and application development. It supports a wide range of data sources and formats, offering seamless integration with other AWS services. AWS Glue provides the tools to create, run, and manage ETL (Extract, Transform, Load) jobs, making it easier to handle complex data workflows. Its serverless architecture allows for scalability and flexibility, making it a preferred choice for data engineers and analysts who need to process and prepare data efficiently.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsLambda.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsLambda.ts index 87c7f5e8bf713..a67463590138d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsLambda.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsLambda.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'Using this integration, you can leverage AWS Lambda to execute external code as part of your Dagster pipelines. This is particularly useful for running serverless functions that can scale automatically and handle various workloads without the need for managing infrastructure. The `PipesLambdaClient` class allows you to invoke AWS Lambda functions and stream logs and structured metadata back to Dagster\'s UI and tools.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS Lambda\n\n**AWS Lambda** is a serverless compute service provided by Amazon Web Services (AWS). It allows you to run code without provisioning or managing servers. AWS Lambda automatically scales your application by running code in response to each trigger, such as changes to data in an Amazon S3 bucket or an update to a DynamoDB table. You can use AWS Lambda to extend other AWS services with custom logic, or create your own backend services that operate at AWS scale, performance, and security.'; + 'Using this integration, you can leverage AWS Lambda to execute external code as part of your Dagster pipelines. This is particularly useful for running serverless functions that can scale automatically and handle various workloads without the need for managing infrastructure. The `PipesLambdaClient` class allows you to invoke AWS Lambda functions and stream logs and structured metadata back to Dagster\'s UI and tools.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nimport boto3\nfrom dagster_aws.pipes import PipesLambdaClient\n\nimport dagster as dg\n\nlambda_client = boto3.client("lambda", region_name="us-west-1")\n\nlambda_pipes_client = PipesLambdaClient(client=lambda_client)\n\n\n@dg.asset\ndef lambda_pipes_asset(\n context: dg.AssetExecutionContext, lambda_pipes_client: PipesLambdaClient\n):\n return lambda_pipes_client.run(\n context=context,\n function_name="your_lambda_function_name",\n event={"key": "value"},\n ).get_materialize_result()\n\n\ndefs = dg.Definitions(\n assets=[lambda_pipes_asset],\n resources={"lambda_pipes_client": lambda_pipes_client},\n)\n```\n \n\n### About AWS Lambda\n\n**AWS Lambda** is a serverless compute service provided by Amazon Web Services (AWS). It allows you to run code without provisioning or managing servers. AWS Lambda automatically scales your application by running code in response to each trigger, such as changes to data in an Amazon S3 bucket or an update to a DynamoDB table. You can use AWS Lambda to extend other AWS services with custom logic, or create your own backend services that operate at AWS scale, performance, and security.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsRedshift.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsRedshift.ts index 35b65a855d015..a6c36cd8dce02 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsRedshift.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsRedshift.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'Using this integration, you can connect to an AWS Redshift cluster and issue queries against it directly from your Dagster assets. This allows you to seamlessly integrate Redshift into your data pipelines, leveraging the power of Redshift\'s data warehousing capabilities within your Dagster workflows.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS Redshift\n\n**AWS Redshift** is a fully managed, petabyte-scale data warehouse service in the cloud. You can start with just a few hundred gigabytes of data and scale to a petabyte or more. This enables you to use your data to acquire new insights for your business and customers. Redshift offers fast query performance using SQL-based tools and business intelligence applications, making it a powerful tool for data warehousing and analytics.'; + 'Using this integration, you can connect to an AWS Redshift cluster and issue queries against it directly from your Dagster assets. This allows you to seamlessly integrate Redshift into your data pipelines, leveraging the power of Redshift\'s data warehousing capabilities within your Dagster workflows.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nfrom dagster_aws.redshift import RedshiftClientResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef example_redshift_asset(context, redshift: RedshiftClientResource):\n result = redshift.get_client().execute_query("SELECT 1", fetch_results=True)\n context.log.info(f"Query result: {result}")\n\n\nredshift_configured = RedshiftClientResource(\n host="my-redshift-cluster.us-east-1.redshift.amazonaws.com",\n port=5439,\n user="dagster",\n password=dg.EnvVar("DAGSTER_REDSHIFT_PASSWORD"),\n database="dev",\n)\n\ndefs = dg.Definitions(\n assets=[example_redshift_asset],\n resources={"redshift": redshift_configured},\n)\n```\n \n\n### About AWS Redshift\n\n**AWS Redshift** is a fully managed, petabyte-scale data warehouse service in the cloud. You can start with just a few hundred gigabytes of data and scale to a petabyte or more. This enables you to use your data to acquire new insights for your business and customers. Redshift offers fast query performance using SQL-based tools and business intelligence applications, making it a powerful tool for data warehousing and analytics.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsS3.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsS3.ts index 773c8582d9611..9dd9ac68af2de 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsS3.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsS3.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The AWS S3 integration allows data engineers to easily read, and write objects to the durable AWS S3 storage -- enabling engineers to a resilient storage layer when constructing their pipelines.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\nHere is an example of how to use the `S3Resource` in a Dagster job to interact with AWS S3:\n\n\n\n### About AWS S3\n\n**AWS S3** is an object storage service that offers industry-leading scalability, data availability, security, and performance. This means customers of all sizes and industries can use it to store and protect any amount of data for a range of use cases, such as data lakes, websites, mobile applications, backup and restore, archive, enterprise applications, IoT devices, and big data analytics. Amazon S3 provides easy-to-use management features so you can organize your data and configure finely tuned access controls to meet your specific business, organizational, and compliance requirements.'; + 'The AWS S3 integration allows data engineers to easily read, and write objects to the durable AWS S3 storage -- enabling engineers to a resilient storage layer when constructing their pipelines.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\nHere is an example of how to use the `S3Resource` in a Dagster job to interact with AWS S3:\n\n\n```python\nimport pandas as pd\nfrom dagster_aws.s3 import S3Resource\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_s3_asset(s3: S3Resource):\n df = pd.DataFrame({"column1": [1, 2, 3], "column2": ["A", "B", "C"]})\n\n csv_data = df.to_csv(index=False)\n\n s3_client = s3.get_client()\n\n s3_client.put_object(\n Bucket="my-cool-bucket",\n Key="path/to/my_dataframe.csv",\n Body=csv_data,\n )\n\n\ndefs = dg.Definitions(\n assets=[my_s3_asset],\n resources={"s3": S3Resource(region_name="us-west-2")},\n)\n```\n \n\n### About AWS S3\n\n**AWS S3** is an object storage service that offers industry-leading scalability, data availability, security, and performance. This means customers of all sizes and industries can use it to store and protect any amount of data for a range of use cases, such as data lakes, websites, mobile applications, backup and restore, archive, enterprise applications, IoT devices, and big data analytics. Amazon S3 provides easy-to-use management features so you can organize your data and configure finely tuned access controls to meet your specific business, organizational, and compliance requirements.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSecretsmanager.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSecretsmanager.ts index 12961a4c52d33..3d196795e39b8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSecretsmanager.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSecretsmanager.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration allows you to manage, retrieve, and rotate credentials, API keys, and other secrets using [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/).\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS Secrets Manager\n\n**AWS Secrets Manager** helps you protect access to your applications, services, and IT resources without the upfront cost and complexity of managing your own hardware security module infrastructure. With Secrets Manager, you can rotate, manage, and retrieve database credentials, API keys, and other secrets throughout their lifecycle. Users and applications retrieve secrets with a call to Secrets Manager APIs, eliminating the need to hardcode sensitive information in plain text.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration allows you to manage, retrieve, and rotate credentials, API keys, and other secrets using [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/).\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nfrom dagster_aws.secretsmanager import (\n SecretsManagerResource,\n SecretsManagerSecretsResource,\n)\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_asset(secretsmanager: SecretsManagerResource):\n secret_value = secretsmanager.get_client().get_secret_value(\n SecretId="arn:aws:secretsmanager:region:aws_account_id:secret:appauthexample-AbCdEf"\n )\n return secret_value\n\n\n@dg.asset\ndef my_other_asset(secrets: SecretsManagerSecretsResource):\n secret_value = secrets.fetch_secrets().get("my-secret-name")\n return secret_value\n\n\ndefs = dg.Definitions(\n assets=[my_asset, my_other_asset],\n resources={\n "secretsmanager": SecretsManagerResource(region_name="us-west-1"),\n "secrets": SecretsManagerSecretsResource(\n region_name="us-west-1",\n secrets_tag="dagster",\n ),\n },\n)\n```\n \n\n### About AWS Secrets Manager\n\n**AWS Secrets Manager** helps you protect access to your applications, services, and IT resources without the upfront cost and complexity of managing your own hardware security module infrastructure. With Secrets Manager, you can rotate, manage, and retrieve database credentials, API keys, and other secrets throughout their lifecycle. Users and applications retrieve secrets with a call to Secrets Manager APIs, eliminating the need to hardcode sensitive information in plain text.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSsm.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSsm.ts index c28c5b1646a0c..4334ef09ec77d 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSsm.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/awsSsm.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThe Dagster AWS Systems Manager (SSM) Parameter Store integration allows you to manage and retrieve parameters stored in AWS SSM Parameter Store directly within your Dagster pipelines. This integration provides resources to fetch parameters by name, tags, or paths, and optionally set them as environment variables for your operations.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n\n### About AWS Systems Parameter Store\n\n**AWS Systems Manager Parameter Store** is a secure storage service for configuration data management and secrets management. It allows you to store data such as passwords, database strings, and license codes as parameter values. You can then reference these parameters in your applications or scripts, ensuring that sensitive information isn\'t hard-coded or exposed in your codebase.\n\nAWS Systems Manager Parameter Store integrates with AWS Identity and Access Management (IAM) to control access to parameters, and it supports encryption using AWS Key Management Service (KMS) to protect sensitive data. This service is essential for maintaining secure and manageable configurations across your AWS environment.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThe Dagster AWS Systems Manager (SSM) Parameter Store integration allows you to manage and retrieve parameters stored in AWS SSM Parameter Store directly within your Dagster pipelines. This integration provides resources to fetch parameters by name, tags, or paths, and optionally set them as environment variables for your operations.\n\n### Installation\n\n```bash\npip install dagster-aws\n```\n\n### Examples\n\n\n```python\nfrom dagster_aws.ssm import ParameterStoreResource, ParameterStoreTag\n\nimport dagster as dg\n\n\n@dg.asset\ndef example_parameter_store_asset(parameter_store: ParameterStoreResource):\n parameter_value = parameter_store.fetch_parameters(\n parameters=["my-parameter-name"]\n ).get("my-parameter-name")\n return parameter_value\n\n\n@dg.asset\ndef example_parameter_store_asset_with_env(parameter_store: ParameterStoreResource):\n import os\n\n with parameter_store.parameters_in_environment():\n return os.getenv("my-other-parameter-name")\n\n\ndefs = dg.Definitions(\n assets=[example_parameter_store_asset, example_parameter_store_asset_with_env],\n resources={\n "parameter_store": ParameterStoreResource(\n region_name="us-west-1",\n parameter_tags=[\n ParameterStoreTag(key="my-tag-key", values=["my-tag-value"])\n ],\n with_decryption=True,\n )\n },\n)\n```\n \n\n### About AWS Systems Parameter Store\n\n**AWS Systems Manager Parameter Store** is a secure storage service for configuration data management and secrets management. It allows you to store data such as passwords, database strings, and license codes as parameter values. You can then reference these parameters in your applications or scripts, ensuring that sensitive information isn\'t hard-coded or exposed in your codebase.\n\nAWS Systems Manager Parameter Store integrates with AWS Identity and Access Management (IAM) to control access to parameters, and it supports encryption using AWS Key Management Service (KMS) to protect sensitive data. This service is essential for maintaining secure and manageable configurations across your AWS environment.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/azureAdls2.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/azureAdls2.ts index 57420d55706de..ef433590f1f6c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/azureAdls2.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/azureAdls2.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'Dagster helps you use Azure Storage Accounts as part of your data pipeline. Azure Data Lake Storage Gen 2 (ADLS2) is our primary focus but we also provide utilities for Azure Blob Storage.\n\n### Installation\n\n```bash\npip install dagster-azure\n```\n\n### Example\n\n\n\nIn this updated code, we use `ADLS2Resource` directly instead of `adls2_resource`. The configuration is passed to `ADLS2Resource` during its instantiation.\n\n### About Azure Data Lake Storage Gen 2 (ADLS2)\n\n**Azure Data Lake Storage Gen 2 (ADLS2)** is a set of capabilities dedicated to big data analytics, built on Azure Blob Storage. ADLS2 combines the scalability, cost-effectiveness, security, and rich capabilities of Azure Blob Storage with a high-performance file system that\'s built for analytics and is compatible with the Hadoop Distributed File System (HDFS). This makes it an ideal choice for data lakes and big data analytics.'; + 'Dagster helps you use Azure Storage Accounts as part of your data pipeline. Azure Data Lake Storage Gen 2 (ADLS2) is our primary focus but we also provide utilities for Azure Blob Storage.\n\n### Installation\n\n```bash\npip install dagster-azure\n```\n\n### Example\n\n\n```python\nimport pandas as pd\nfrom dagster_azure.adls2 import ADLS2Resource, ADLS2SASToken\n\nimport dagster as dg\n\n\n@dg.asset\ndef example_adls2_asset(adls2: ADLS2Resource):\n df = pd.DataFrame({"column1": [1, 2, 3], "column2": ["A", "B", "C"]})\n\n csv_data = df.to_csv(index=False)\n\n file_client = adls2.adls2_client.get_file_client(\n "my-file-system", "path/to/my_dataframe.csv"\n )\n file_client.upload_data(csv_data, overwrite=True)\n\n\ndefs = dg.Definitions(\n assets=[example_adls2_asset],\n resources={\n "adls2": ADLS2Resource(\n storage_account="my_storage_account",\n credential=ADLS2SASToken(token="my_sas_token"),\n )\n },\n)\n```\n \n\nIn this updated code, we use `ADLS2Resource` directly instead of `adls2_resource`. The configuration is passed to `ADLS2Resource` during its instantiation.\n\n### About Azure Data Lake Storage Gen 2 (ADLS2)\n\n**Azure Data Lake Storage Gen 2 (ADLS2)** is a set of capabilities dedicated to big data analytics, built on Azure Blob Storage. ADLS2 combines the scalability, cost-effectiveness, security, and rich capabilities of Azure Blob Storage with a high-performance file system that\'s built for analytics and is compatible with the Hadoop Distributed File System (HDFS). This makes it an ideal choice for data lakes and big data analytics.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/census.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/census.ts index ef8c17d21093f..d8654d3420355 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/census.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/census.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'With the `dagster-census` integration you can execute a Census sync and poll until that sync completes, raising an error if it\'s unsuccessful.\n\n### Installation\n\n```bash\npip install dagster-census\n```\n\n### Example\n\n\n\n### About Census\n\n**Census** syncs data from your cloud warehouse to the SaaS tools your organization uses. It allows everyone in your organization to take action with good data, no custom scripts or API integrations required.'; + 'With the `dagster-census` integration you can execute a Census sync and poll until that sync completes, raising an error if it\'s unsuccessful.\n\n### Installation\n\n```bash\npip install dagster-census\n```\n\n### Example\n\n\n```python\nfrom dagster_census import CensusResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef census_source(census: CensusResource):\n census.get_source(source_id=1)\n\n\ndefs = dg.Definitions(\n assets=[census_source],\n resources={"census": CensusResource(api_key=dg.EnvVar("CENSUS_API_KEY"))},\n)\n```\n \n\n### About Census\n\n**Census** syncs data from your cloud warehouse to the SaaS tools your organization uses. It allows everyone in your organization to take action with good data, no custom scripts or API integrations required.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/chroma.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/chroma.ts index ef516099563d8..f269f89e3169f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/chroma.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/chroma.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-chroma` library allows you to easily interact with Chroma\'s vector database capabilities to build AI-powered data pipelines in Dagster. You can perform vector similarity searches, manage schemas, and handle data operations directly from your Dagster assets.\n\n### Installation\n\n```bash\npip install dagster dagster-chroma\n```\n\n### Example\n\n\n\n### About Chroma\n\n**Chroma** is the open-source AI application database. Chroma makes it easy to build LLM apps by making knowledge, facts, and skills pluggable for LLMs. It provides a simple API for storing and querying embeddings, documents, and metadata. Chroma can be used to build semantic search, question answering, and other AI-powered applications. The database can run embedded in your application or as a separate service.'; + 'The `dagster-chroma` library allows you to easily interact with Chroma\'s vector database capabilities to build AI-powered data pipelines in Dagster. You can perform vector similarity searches, manage schemas, and handle data operations directly from your Dagster assets.\n\n### Installation\n\n```bash\npip install dagster dagster-chroma\n```\n\n### Example\n\n\n```python\nimport os\n\nfrom dagster_chroma import ChromaResource, HttpConfig, LocalConfig\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_table(chroma: ChromaResource):\n with chroma.get_client() as chroma_client:\n collection = chroma_client.create_collection("fruits")\n\n collection.add(\n documents=[\n "This is a document about oranges",\n "This is a document about pineapples",\n "This is a document about strawberries",\n "This is a document about cucumbers",\n ],\n ids=["oranges", "pineapples", "strawberries", "cucumbers"],\n )\n\n results = collection.query(\n query_texts=["hawaii"],\n n_results=1,\n )\n\n\ndefs = dg.Definitions(\n assets=[my_table],\n resources={\n "chroma": ChromaResource(\n connection_config=LocalConfig(persistence_path="./chroma")\n if os.getenv("DEV")\n else HttpConfig(host="192.168.0.10", port=8000)\n ),\n },\n)\n```\n \n\n### About Chroma\n\n**Chroma** is the open-source AI application database. Chroma makes it easy to build LLM apps by making knowledge, facts, and skills pluggable for LLMs. It provides a simple API for storing and querying embeddings, documents, and metadata. Chroma can be used to build semantic search, question answering, and other AI-powered applications. The database can run embedded in your application or as a separate service.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/cube.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/cube.ts index fe7c5e9fff18a..417e5ae271f78 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/cube.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/cube.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'With the `dagster_cube` integration you can setup Cube and Dagster to work together so that Dagster can push changes from upstream data sources to Cube using its integration API.\n\n### Installation\n\n```bash\npip install dagster_cube\n```\n\n### Example\n\n\n\n### About Cube\n\n**Cube.js** is the semantic layer for building data applications. It helps data engineers and application developers access data from modern data stores, organize it into consistent definitions, and deliver it to every application.'; + 'With the `dagster_cube` integration you can setup Cube and Dagster to work together so that Dagster can push changes from upstream data sources to Cube using its integration API.\n\n### Installation\n\n```bash\npip install dagster_cube\n```\n\n### Example\n\n\n```python\nfrom dagster_cube import CubeResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef cube_query_workflow(cube: CubeResource):\n response = cube.make_request(\n method="POST",\n endpoint="load",\n data={"query": {"measures": ["Orders.count"], "dimensions": ["Orders.status"]}},\n )\n\n return response\n\n\ndefs = dg.Definitions(\n assets=[cube_query_workflow],\n resources={\n "cube": CubeResource(\n instance_url="https://<>.cubecloudapp.dev/cubejs-api/v1/",\n api_key=dg.EnvVar("CUBE_API_KEY"),\n )\n },\n)\n```\n \n\n### About Cube\n\n**Cube.js** is the semantic layer for building data applications. It helps data engineers and application developers access data from modern data stores, organize it into consistent definitions, and deliver it to every application.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/databricks.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/databricks.ts index c8700c0f2b21b..a19667f681f4c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/databricks.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/databricks.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-databricks` integration library provides the `PipesDatabricksClient` resource, enabling you to launch Databricks jobs directly from Dagster assets and ops. This integration allows you to pass parameters to Databricks code while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-databricks\n```\n\n### Example\n\n\n\n\n\n### About Databricks\n\n**Databricks** is a unified data analytics platform that simplifies and accelerates the process of building big data and AI solutions. It integrates seamlessly with Apache Spark and offers support for various data sources and formats. Databricks provides powerful tools to create, run, and manage data pipelines, making it easier to handle complex data engineering tasks. Its collaborative and scalable environment is ideal for data engineers, scientists, and analysts who need to process and analyze large datasets efficiently.'; + 'The `dagster-databricks` integration library provides the `PipesDatabricksClient` resource, enabling you to launch Databricks jobs directly from Dagster assets and ops. This integration allows you to pass parameters to Databricks code while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-databricks\n```\n\n### Example\n\n\n```python\nimport sys\n\nfrom dagster_databricks import PipesDatabricksClient\nfrom databricks.sdk import WorkspaceClient\nfrom databricks.sdk.service import jobs\n\nimport dagster as dg\n\n\n@dg.asset\ndef databricks_asset(\n context: dg.AssetExecutionContext, pipes_databricks: PipesDatabricksClient\n):\n task = jobs.SubmitTask.from_dict(\n {\n # The cluster settings below are somewhat arbitrary. Dagster Pipes is\n # not dependent on a specific spark version, node type, or number of\n # workers.\n "new_cluster": {\n "spark_version": "12.2.x-scala2.12",\n "node_type_id": "i3.xlarge",\n "num_workers": 0,\n "cluster_log_conf": {\n "dbfs": {"destination": "dbfs:/cluster-logs-dir-noexist"},\n },\n },\n "libraries": [\n # Include the latest published version of dagster-pipes on PyPI\n # in the task environment\n {"pypi": {"package": "dagster-pipes"}},\n ],\n "task_key": "some-key",\n "spark_python_task": {\n "python_file": "dbfs:/my_python_script.py", # location of target code file\n "source": jobs.Source.WORKSPACE,\n },\n }\n )\n\n print("This will be forwarded back to Dagster stdout") # noqa: T201\n print("This will be forwarded back to Dagster stderr", file=sys.stderr) # noqa: T201\n\n extras = {"some_parameter": 100}\n\n return pipes_databricks.run(\n task=task,\n context=context,\n extras=extras,\n ).get_materialize_result()\n\n\npipes_databricks_resource = PipesDatabricksClient(\n client=WorkspaceClient(\n host="https://.cloud.databricks.com",\n token="",\n )\n)\n\ndefs = dg.Definitions(\n assets=[databricks_asset], resources={"pipes_databricks": pipes_databricks_resource}\n)\n```\n \n\n\n```python\nfrom dagster_pipes import (\n PipesDbfsContextLoader,\n PipesDbfsMessageWriter,\n open_dagster_pipes,\n)\n\n# Sets up communication channels and downloads the context data sent from Dagster.\n# Note that while other `context_loader` and `message_writer` settings are\n# possible, it is recommended to use `PipesDbfsContextLoader` and\n# `PipesDbfsMessageWriter` for Databricks.\nwith open_dagster_pipes(\n context_loader=PipesDbfsContextLoader(),\n message_writer=PipesDbfsMessageWriter(),\n) as pipes:\n # Access the `extras` dict passed when launching the job from Dagster.\n some_parameter_value = pipes.get_extra("some_parameter")\n\n # Stream log message back to Dagster\n pipes.log.info(f"Using some_parameter value: {some_parameter_value}")\n\n # ... your code that computes and persists the asset\n\n # Stream asset materialization metadata and data version back to Dagster.\n # This should be called after you\'ve computed and stored the asset value. We\n # omit the asset key here because there is only one asset in scope, but for\n # multi-assets you can pass an `asset_key` parameter.\n pipes.report_asset_materialization(\n metadata={\n "some_metric": {"raw_value": some_parameter_value + 1, "type": "int"}\n },\n data_version="alpha",\n )\n```\n \n\n### About Databricks\n\n**Databricks** is a unified data analytics platform that simplifies and accelerates the process of building big data and AI solutions. It integrates seamlessly with Apache Spark and offers support for various data sources and formats. Databricks provides powerful tools to create, run, and manage data pipelines, making it easier to handle complex data engineering tasks. Its collaborative and scalable environment is ideal for data engineers, scientists, and analysts who need to process and analyze large datasets efficiently.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/datadog.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/datadog.ts index c23390d7199fb..ea6c8649d42de 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/datadog.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/datadog.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - "import Beta from '@site/docs/partials/\\_Beta.md';\n\n\n\nWhile Dagster provides comprehensive monitoring and observability of the pipelines it orchestrates, many teams look to centralize all their monitoring across apps, processes and infrastructure using Datadog's 'Cloud Monitoring as a Service'. The `dagster-datadog` integration allows you to publish metrics to Datadog from within Dagster ops.\n\n### Installation\n\n```bash\npip install dagster-datadog\n```\n\n### Example\n\n\n\n### About Datadog\n\n**Datadog** is an observability service for cloud-scale applications, providing monitoring of servers, databases, tools, and services, through a SaaS-based data analytics platform."; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nWhile Dagster provides comprehensive monitoring and observability of the pipelines it orchestrates, many teams look to centralize all their monitoring across apps, processes and infrastructure using Datadog\'s \'Cloud Monitoring as a Service\'. The `dagster-datadog` integration allows you to publish metrics to Datadog from within Dagster ops.\n\n### Installation\n\n```bash\npip install dagster-datadog\n```\n\n### Example\n\n\n```python\nfrom dagster_datadog import DatadogResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef report_to_datadog(datadog: DatadogResource):\n datadog_client = datadog.get_client()\n datadog_client.event("Man down!", "This server needs assistance.")\n datadog_client.gauge("users.online", 1001, tags=["protocol:http"])\n datadog_client.increment("page.views")\n\n\ndefs = dg.Definitions(\n assets=[report_to_datadog],\n resources={\n "datadog": DatadogResource(\n api_key=dg.EnvVar("DATADOG_API_KEY"),\n app_key=dg.EnvVar("DATADOG_APP_KEY"),\n )\n },\n)\n```\n \n\n### About Datadog\n\n**Datadog** is an observability service for cloud-scale applications, providing monitoring of servers, databases, tools, and services, through a SaaS-based data analytics platform.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCloud.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCloud.ts index 426afd9db5bc0..f5db6ebfba32c 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCloud.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCloud.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nDagster allows you to run dbt Cloud jobs alongside other technologies. You can schedule them to run as a step in a larger pipeline and manage them as a data asset.\n\n### Installation\n\n```bash\npip install dagster-dbt\n```\n\n### Example\n\n\n\n### About dbt Cloud\n\n**dbt Cloud** is a hosted service for running dbt jobs. It helps data analysts and engineers productionize dbt deployments. Beyond dbt open source, dbt Cloud provides scheduling , CI/CD, serving documentation, and monitoring & alerting.\n\nIf you\'re currently using dbt Cloud™, you can also use Dagster to run `dbt-core` in its place. You can read more about [how to do that here](https://dagster.io/blog/migrate-off-dbt-cloud).'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nDagster allows you to run dbt Cloud jobs alongside other technologies. You can schedule them to run as a step in a larger pipeline and manage them as a data asset.\n\n### Installation\n\n```bash\npip install dagster-dbt\n```\n\n### Example\n\n\n```python\nimport os\n\nfrom dagster_dbt import dbt_cloud_resource, load_assets_from_dbt_cloud_job\n\nimport dagster as dg\n\n# set the auth token as an env var for demo purposes\nos.environ["DBT_CLOUD_AUTH_TOKEN"] = "abc"\n\n# configure a resource to connect to your dbt Cloud instance\ndbt_cloud = dbt_cloud_resource.configured(\n {"auth_token": {"env": "DBT_CLOUD_AUTH_TOKEN"}, "account_id": 11111}\n)\n\n# import assets from dbt\ndbt_cloud_assets = load_assets_from_dbt_cloud_job(\n dbt_cloud=dbt_cloud,\n job_id=33333,\n)\n\ndefs = dg.Definitions(assets=[dbt_cloud_assets])\n```\n \n\n### About dbt Cloud\n\n**dbt Cloud** is a hosted service for running dbt jobs. It helps data analysts and engineers productionize dbt deployments. Beyond dbt open source, dbt Cloud provides scheduling , CI/CD, serving documentation, and monitoring & alerting.\n\nIf you\'re currently using dbt Cloud™, you can also use Dagster to run `dbt-core` in its place. You can read more about [how to do that here](https://dagster.io/blog/migrate-off-dbt-cloud).'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCore.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCore.ts index 028235dc663e6..cbfc3b78f1ff7 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCore.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dbtDbtCore.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'Dagster orchestrates dbt alongside other technologies, so you can schedule dbt with Spark, Python, etc. in a single data pipeline.\n\nDagster assets understand dbt at the level of individual dbt models. This means that you can:\n\n- Use Dagster\'s UI or APIs to run subsets of your dbt models, seeds, and snapshots.\n- Track failures, logs, and run history for individual dbt models, seeds, and snapshots.\n- Define dependencies between individual dbt models and other data assets. For example, put dbt models after the Fivetran-ingested table that they read from, or put a machine learning after the dbt models that it\'s trained from.\n\n### Installation\n\n```bash\npip install dagster-dbt\n```\n\n### Example\n\n\n\n### About dbt\n\n**dbt** is a SQL-first transformation workflow that lets teams quickly and collaboratively deploy analytics code following software engineering best practices like modularity, portability, CI/CD, and documentation.\n\n'; + 'Dagster orchestrates dbt alongside other technologies, so you can schedule dbt with Spark, Python, etc. in a single data pipeline.\n\nDagster assets understand dbt at the level of individual dbt models. This means that you can:\n\n- Use Dagster\'s UI or APIs to run subsets of your dbt models, seeds, and snapshots.\n- Track failures, logs, and run history for individual dbt models, seeds, and snapshots.\n- Define dependencies between individual dbt models and other data assets. For example, put dbt models after the Fivetran-ingested table that they read from, or put a machine learning after the dbt models that it\'s trained from.\n\n### Installation\n\n```bash\npip install dagster-dbt\n```\n\n### Example\n\n\n```python\nfrom pathlib import Path\n\nfrom dagster_dbt import (\n DbtCliResource,\n DbtProject,\n build_schedule_from_dbt_selection,\n dbt_assets,\n)\n\nimport dagster as dg\n\nRELATIVE_PATH_TO_MY_DBT_PROJECT = "./my_dbt_project"\n\nmy_project = DbtProject(\n project_dir=Path(__file__)\n .joinpath("..", RELATIVE_PATH_TO_MY_DBT_PROJECT)\n .resolve(),\n)\nmy_project.prepare_if_dev()\n\n\n@dbt_assets(manifest=my_project.manifest_path)\ndef my_dbt_assets(context: dg.AssetExecutionContext, dbt: DbtCliResource):\n yield from dbt.cli(["build"], context=context).stream()\n\n\nmy_schedule = build_schedule_from_dbt_selection(\n [my_dbt_assets],\n job_name="materialize_dbt_models",\n cron_schedule="0 0 * * *",\n dbt_select="fqn:*",\n)\n\ndefs = dg.Definitions(\n assets=[my_dbt_assets],\n schedules=[my_schedule],\n resources={\n "dbt": DbtCliResource(project_dir=my_project),\n },\n)\n```\n \n\n### About dbt\n\n**dbt** is a SQL-first transformation workflow that lets teams quickly and collaboratively deploy analytics code following software engineering best practices like modularity, portability, CI/CD, and documentation.\n\n'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dlt.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dlt.ts index 71c3ec83244c8..6a834c5c0940f 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dlt.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/dlt.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This integration allows you to use [dlt](https://dlthub.com/) to easily ingest and replicate data between systems through Dagster.\n\n### Installation\n\n```bash\npip install dagster-dlt\n```\n\n### Example\n\n\n\n:::note\n\nIf you are using the [sql_database](https://dlthub.com/docs/api_reference/dlt/sources/sql_database/__init__) source, consider setting `defer_table_reflect=True` to reduce database reads. By default, the Dagster daemon will refresh definitions roughly every minute, which will query the database for resource definitions.\n\n:::\n\n### About dlt\n\n[Data Load Tool (dlt)](https://dlthub.com/) is an open source library for creating efficient data pipelines. It offers features like secret management, data structure conversion, incremental updates, and pre-built sources and destinations, simplifying the process of loading messy data into well-structured datasets.'; + 'This integration allows you to use [dlt](https://dlthub.com/) to easily ingest and replicate data between systems through Dagster.\n\n### Installation\n\n```bash\npip install dagster-dlt\n```\n\n### Example\n\n\n```python\nimport dlt\nfrom dagster_dlt import DagsterDltResource, dlt_assets\nfrom dlt_sources.github import github_reactions\n\nimport dagster as dg\n\n\n@dlt_assets(\n dlt_source=github_reactions("dagster-io", "dagster"),\n dlt_pipeline=dlt.pipeline(\n pipeline_name="github_issues",\n dataset_name="github",\n destination="snowflake",\n ),\n name="github",\n group_name="github",\n)\ndef github_issues_to_snowflake_assets(\n context: dg.AssetExecutionContext, dlt: DagsterDltResource\n):\n yield from dlt.run(context=context)\n\n\ndefs = dg.Definitions(\n assets=[\n github_issues_to_snowflake_assets,\n ],\n resources={\n "dlt": DagsterDltResource(),\n },\n)\n```\n \n\n:::note\n\nIf you are using the [sql_database](https://dlthub.com/docs/api_reference/dlt/sources/sql_database/__init__) source, consider setting `defer_table_reflect=True` to reduce database reads. By default, the Dagster daemon will refresh definitions roughly every minute, which will query the database for resource definitions.\n\n:::\n\n### About dlt\n\n[Data Load Tool (dlt)](https://dlthub.com/) is an open source library for creating efficient data pipelines. It offers features like secret management, data structure conversion, incremental updates, and pre-built sources and destinations, simplifying the process of loading messy data into well-structured datasets.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/docker.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/docker.ts index 0e6cc51d51674..f1fa7d92584e9 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/docker.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/docker.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-docker` integration library provides the `PipesDockerClient` resource, enabling you to launch Docker containers and execute external code directly from Dagster assets and ops. This integration allows you to pass parameters to Docker containers while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-docker\n```\n\n### Example\n\n\n\n### Deploying to Docker?\n\n- Deploying to Dagster+: Use with a Dagster+ Hybrid deployment, the Docker agent executes Dagster jobs on a Docker cluster. Checkout the [Dagster+ Docker Agent](https://docs.dagster.io/dagster-plus/deployment/deployment-types/hybrid/docker/) guide for more information.\n- Deploying to Open Source: Visit the [Deploying Dagster to Docker](https://docs.dagster.io/guides/deploy/deployment-options/docker) guide for more information.\n\n### About Docker\n\n**Docker** is a set of platform-as-a-service products that use OS-level virtualization to deliver software in packages called containers. The service has both free and premium tiers. The software that hosts the containers is called Docker Engine.'; + 'The `dagster-docker` integration library provides the `PipesDockerClient` resource, enabling you to launch Docker containers and execute external code directly from Dagster assets and ops. This integration allows you to pass parameters to Docker containers while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-docker\n```\n\n### Example\n\n\n```python\nfrom dagster_docker import PipesDockerClient\n\nimport dagster as dg\n\n\n@dg.asset\ndef docker_pipes_asset(\n context: dg.AssetExecutionContext, docker_pipes_client: PipesDockerClient\n):\n docker_image = "python:3.9-slim"\n return docker_pipes_client.run(\n image=docker_image,\n command=[\n "python",\n "-m",\n "my_module",\n ],\n context=context,\n ).get_results()\n\n\ndefs = dg.Definitions(\n assets=[docker_pipes_asset],\n resources={\n "docker_pipes_client": PipesDockerClient(),\n },\n)\n```\n \n\n### Deploying to Docker?\n\n- Deploying to Dagster+: Use with a Dagster+ Hybrid deployment, the Docker agent executes Dagster jobs on a Docker cluster. Checkout the [Dagster+ Docker Agent](https://docs.dagster.io/dagster-plus/deployment/deployment-types/hybrid/docker/) guide for more information.\n- Deploying to Open Source: Visit the [Deploying Dagster to Docker](https://docs.dagster.io/guides/deploy/deployment-options/docker) guide for more information.\n\n### About Docker\n\n**Docker** is a set of platform-as-a-service products that use OS-level virtualization to deliver software in packages called containers. The service has both free and premium tiers. The software that hosts the containers is called Docker Engine.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/duckdb.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/duckdb.ts index 168a4224cf096..0ddb120cbe73b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/duckdb.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/duckdb.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This library provides an integration with the DuckDB database, and allows for an out-of-the-box [I/O Manager](/guides/build/io-managers/) so that you can make DuckDB your storage of choice.\n\n### Installation\n\n```bash\npip install dagster-duckdb\n```\n\n### Example\n\n\n\n### About DuckDB\n\n**DuckDB** is a column-oriented in-process OLAP database. A typical OLTP relational database like SQLite is row-oriented. In row-oriented database, data is organised physically as consecutive tuples.'; + 'This library provides an integration with the DuckDB database, and allows for an out-of-the-box [I/O Manager](/guides/build/io-managers/) so that you can make DuckDB your storage of choice.\n\n### Installation\n\n```bash\npip install dagster-duckdb\n```\n\n### Example\n\n\n```python\nimport pandas as pd\nfrom dagster_duckdb_pandas import DuckDBPandasIOManager\n\nimport dagster as dg\n\n\n@dg.asset(\n key_prefix=["my_schema"] # will be used as the schema in duckdb\n)\ndef my_table() -> pd.DataFrame: # the name of the asset will be the table name\n return pd.DataFrame()\n\n\ndefs = dg.Definitions(\n assets=[my_table],\n resources={"io_manager": DuckDBPandasIOManager(database="my_db.duckdb")},\n)\n```\n \n\n### About DuckDB\n\n**DuckDB** is a column-oriented in-process OLAP database. A typical OLTP relational database like SQLite is row-oriented. In row-oriented database, data is organised physically as consecutive tuples.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/fivetran.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/fivetran.ts index 46acea29782d6..4e3a80ef170b8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/fivetran.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/fivetran.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis guide provides instructions for using Dagster with Fivetran using the `dagster-fivetran` library. Your Fivetran connector tables can be represented as assets in the Dagster asset graph, allowing you to track lineage and dependencies between Fivetran assets and data assets you are already modeling in Dagster. You can also use Dagster to orchestrate Fivetran connectors, allowing you to trigger syncs for these on a cadence or based on upstream data changes.\n\n:::note\n\nYour Fivetran connectors must have been synced at least once to be represented in Dagster.\n\n:::\n\n## What you\'ll learn\n\n- How to represent Fivetran assets in the Dagster asset graph, including lineage to other Dagster assets.\n- How to customize asset definition metadata for these Fivetran assets.\n- How to materialize Fivetran connector tables from Dagster.\n- How to customize how Fivetran connector tables are materialized.\n\n
\n Prerequisites\n\n- The `dagster` and `dagster-fivetran` libraries installed in your environment\n- Familiarity with asset definitions and the Dagster asset graph\n- Familiarity with Dagster resources\n- Familiarity with Fivetran concepts, like connectors and connector tables\n- A Fivetran workspace\n- A Fivetran API key and API secret. For more information, see [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation.\n\n
\n\n## Set up your environment\n\nTo get started, you\'ll need to install the `dagster` and `dagster-fivetran` Python packages:\n\n```bash\npip install dagster dagster-fivetran\n```\n\n## Represent Fivetran assets in the asset graph\n\nTo load Fivetran assets into the Dagster asset graph, you must first construct a resource, which allows Dagster to communicate with your Fivetran workspace. You\'ll need to supply your account ID, API key and API secret. See [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation for more information on how to create your API key and API secret.\n\nDagster can automatically load all connector tables from your Fivetran workspace as asset specs. Call the function, which returns list of s representing your Fivetran assets. You can then include these asset specs in your object:\n\n\n\n### Sync and materialize Fivetran assets\n\nYou can use Dagster to sync Fivetran connectors and materialize Fivetran connector tables. You can use the factory to create all assets definitions for your Fivetran workspace.\n\n\n\n### Customize the materialization of Fivetran assets\n\nIf you want to customize the sync of your connectors, you can use the decorator to do so. This allows you to execute custom code before and after the call to the Fivetran sync.\n\n\n\n### Customize asset definition metadata for Fivetran assets\n\nBy default, Dagster will generate asset specs for each Fivetran asset and populate default metadata. You can further customize asset properties by passing an instance of the custom to the function.\n\n\n\nNote that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it.\n\nYou can pass an instance of the custom to the decorator or the factory.\n\n### Fetching column-level metadata for Fivetran assets\n\nDagster allows you to emit column-level metadata, like [column schema](/guides/build/assets/metadata-and-tags/index.md#standard-metadata-types) and [column lineage](/guides/build/assets/metadata-and-tags/index.md#column-lineage), as [materialization metadata](/guides/build/assets/metadata-and-tags/index.md#runtime-metadata).\n\nWith this metadata, you can view documentation in Dagster for all columns in your Fivetran connector tables.\n\nTo enable this feature, call on the returned by the `sync_and_poll()` call on the resource.\n\n\n\n### Load Fivetran asset for selected connectors\n\nTo select a subset of Fivetran connectors for which your Fivetran assets will be loaded, you can use the callback and define your selection conditions.\n\n\n\n### Load Fivetran assets from multiple workspaces\n\nDefinitions from multiple Fivetran workspaces can be combined by instantiating multiple resources and merging their specs. This lets you view all your Fivetran assets in a single asset graph:\n\n\n\n### About Fivetran\n\n**Fivetran** ingests data from SaaS applications, databases, and servers. The data is stored and typically used for analytics.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis guide provides instructions for using Dagster with Fivetran using the `dagster-fivetran` library. Your Fivetran connector tables can be represented as assets in the Dagster asset graph, allowing you to track lineage and dependencies between Fivetran assets and data assets you are already modeling in Dagster. You can also use Dagster to orchestrate Fivetran connectors, allowing you to trigger syncs for these on a cadence or based on upstream data changes.\n\n:::note\n\nYour Fivetran connectors must have been synced at least once to be represented in Dagster.\n\n:::\n\n## What you\'ll learn\n\n- How to represent Fivetran assets in the Dagster asset graph, including lineage to other Dagster assets.\n- How to customize asset definition metadata for these Fivetran assets.\n- How to materialize Fivetran connector tables from Dagster.\n- How to customize how Fivetran connector tables are materialized.\n\n
\n Prerequisites\n\n- The `dagster` and `dagster-fivetran` libraries installed in your environment\n- Familiarity with asset definitions and the Dagster asset graph\n- Familiarity with Dagster resources\n- Familiarity with Fivetran concepts, like connectors and connector tables\n- A Fivetran workspace\n- A Fivetran API key and API secret. For more information, see [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation.\n\n
\n\n## Set up your environment\n\nTo get started, you\'ll need to install the `dagster` and `dagster-fivetran` Python packages:\n\n```bash\npip install dagster dagster-fivetran\n```\n\n## Represent Fivetran assets in the asset graph\n\nTo load Fivetran assets into the Dagster asset graph, you must first construct a resource, which allows Dagster to communicate with your Fivetran workspace. You\'ll need to supply your account ID, API key and API secret. See [Getting Started](https://fivetran.com/docs/rest-api/getting-started) in the Fivetran REST API documentation for more information on how to create your API key and API secret.\n\nDagster can automatically load all connector tables from your Fivetran workspace as asset specs. Call the function, which returns list of s representing your Fivetran assets. You can then include these asset specs in your object:\n\n\n```python\nfrom dagster_fivetran import FivetranWorkspace, load_fivetran_asset_specs\n\nimport dagster as dg\n\nfivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_API_SECRET"),\n)\n\nfivetran_specs = load_fivetran_asset_specs(fivetran_workspace)\ndefs = dg.Definitions(assets=fivetran_specs, resources={"fivetran": fivetran_workspace})\n```\n \n\n### Sync and materialize Fivetran assets\n\nYou can use Dagster to sync Fivetran connectors and materialize Fivetran connector tables. You can use the factory to create all assets definitions for your Fivetran workspace.\n\n\n```python\nfrom dagster_fivetran import FivetranWorkspace, build_fivetran_assets_definitions\n\nimport dagster as dg\n\nfivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_API_SECRET"),\n)\n\nall_fivetran_assets = build_fivetran_assets_definitions(workspace=fivetran_workspace)\n\ndefs = dg.Definitions(\n assets=all_fivetran_assets,\n resources={"fivetran": fivetran_workspace},\n)\n```\n \n\n### Customize the materialization of Fivetran assets\n\nIf you want to customize the sync of your connectors, you can use the decorator to do so. This allows you to execute custom code before and after the call to the Fivetran sync.\n\n\n```python\nfrom dagster_fivetran import FivetranWorkspace, fivetran_assets\n\nimport dagster as dg\n\nfivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_API_SECRET"),\n)\n\n\n@fivetran_assets(\n # Replace with your connector ID\n connector_id="fivetran_connector_id",\n name="fivetran_connector_id",\n group_name="fivetran_connector_id",\n workspace=fivetran_workspace,\n)\ndef fivetran_connector_assets(\n context: dg.AssetExecutionContext, fivetran: FivetranWorkspace\n):\n # Do something before the materialization...\n yield from fivetran.sync_and_poll(context=context)\n # Do something after the materialization...\n\n\ndefs = dg.Definitions(\n assets=[fivetran_connector_assets],\n resources={"fivetran": fivetran_workspace},\n)\n```\n \n\n### Customize asset definition metadata for Fivetran assets\n\nBy default, Dagster will generate asset specs for each Fivetran asset and populate default metadata. You can further customize asset properties by passing an instance of the custom to the function.\n\n\n```python\nfrom dagster_fivetran import (\n DagsterFivetranTranslator,\n FivetranConnectorTableProps,\n FivetranWorkspace,\n load_fivetran_asset_specs,\n)\n\nimport dagster as dg\n\nfivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_API_SECRET"),\n)\n\n\n# A translator class lets us customize properties of the built\n# Fivetran assets, such as the owners or asset key\nclass MyCustomFivetranTranslator(DagsterFivetranTranslator):\n def get_asset_spec(self, props: FivetranConnectorTableProps) -> dg.AssetSpec:\n # We create the default asset spec using super()\n default_spec = super().get_asset_spec(props)\n # We customize the metadata and asset key prefix for all assets\n return default_spec.replace_attributes(\n key=default_spec.key.with_prefix("prefix"),\n ).merge_attributes(metadata={"custom": "metadata"})\n\n\nfivetran_specs = load_fivetran_asset_specs(\n fivetran_workspace, dagster_fivetran_translator=MyCustomFivetranTranslator()\n)\n\ndefs = dg.Definitions(assets=fivetran_specs, resources={"fivetran": fivetran_workspace})\n```\n \n\nNote that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it.\n\nYou can pass an instance of the custom to the decorator or the factory.\n\n### Fetching column-level metadata for Fivetran assets\n\nDagster allows you to emit column-level metadata, like [column schema](/guides/build/assets/metadata-and-tags/index.md#standard-metadata-types) and [column lineage](/guides/build/assets/metadata-and-tags/index.md#column-lineage), as [materialization metadata](/guides/build/assets/metadata-and-tags/index.md#runtime-metadata).\n\nWith this metadata, you can view documentation in Dagster for all columns in your Fivetran connector tables.\n\nTo enable this feature, call on the returned by the `sync_and_poll()` call on the resource.\n\n\n```python\nfrom dagster_fivetran import FivetranWorkspace, fivetran_assets\n\nimport dagster as dg\n\nfivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_API_SECRET"),\n)\n\n\n@fivetran_assets(\n # Replace with your connector ID\n connector_id="fivetran_connector_id",\n workspace=fivetran_workspace,\n)\ndef fivetran_connector_assets(\n context: dg.AssetExecutionContext, fivetran: FivetranWorkspace\n):\n yield from fivetran.sync_and_poll(context=context).fetch_column_metadata()\n\n\ndefs = dg.Definitions(\n assets=[fivetran_connector_assets],\n resources={"fivetran": fivetran_workspace},\n)\n```\n \n\n### Load Fivetran asset for selected connectors\n\nTo select a subset of Fivetran connectors for which your Fivetran assets will be loaded, you can use the callback and define your selection conditions.\n\n\n```python\nfrom dagster_fivetran import FivetranWorkspace, build_fivetran_assets_definitions\n\nimport dagster as dg\n\nfivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_API_SECRET"),\n)\n\nall_fivetran_assets = build_fivetran_assets_definitions(\n workspace=fivetran_workspace,\n connector_selector_fn=(\n lambda connector: connector.id in {"some_connector_id", "another_connector_id"}\n ),\n)\n\ndefs = dg.Definitions(\n assets=all_fivetran_assets,\n resources={"fivetran": fivetran_workspace},\n)\n```\n \n\n### Load Fivetran assets from multiple workspaces\n\nDefinitions from multiple Fivetran workspaces can be combined by instantiating multiple resources and merging their specs. This lets you view all your Fivetran assets in a single asset graph:\n\n\n```python\nfrom dagster_fivetran import FivetranWorkspace, load_fivetran_asset_specs\n\nimport dagster as dg\n\nsales_fivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_SALES_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_SALES_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_SALES_API_SECRET"),\n)\nmarketing_fivetran_workspace = FivetranWorkspace(\n account_id=dg.EnvVar("FIVETRAN_MARKETING_ACCOUNT_ID"),\n api_key=dg.EnvVar("FIVETRAN_MARKETING_API_KEY"),\n api_secret=dg.EnvVar("FIVETRAN_MARKETING_API_SECRET"),\n)\n\nsales_fivetran_specs = load_fivetran_asset_specs(sales_fivetran_workspace)\nmarketing_fivetran_specs = load_fivetran_asset_specs(marketing_fivetran_workspace)\n\n# Merge the specs into a single set of definitions\ndefs = dg.Definitions(\n assets=[*sales_fivetran_specs, *marketing_fivetran_specs],\n resources={\n "marketing_fivetran": marketing_fivetran_workspace,\n "sales_fivetran": sales_fivetran_workspace,\n },\n)\n```\n \n\n### About Fivetran\n\n**Fivetran** ingests data from SaaS applications, databases, and servers. The data is stored and typically used for analytics.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpBigquery.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpBigquery.ts index a9879fc1f0c62..60028b314ed48 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpBigquery.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpBigquery.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThe Google Cloud Platform BigQuery integration allows data engineers to easily query and store data in the BigQuery data warehouse through the use of the `BigQueryResource`.\n\n### Installation\n\n```bash\npip install dagster-gcp\n```\n\n### Examples\n\n\n\n### About Google Cloud Platform BigQuery\n\nThe Google Cloud Platform BigQuery service, offers a fully managed enterprise data warehouse that enables fast SQL queries using the processing power of Google\'s infrastructure.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThe Google Cloud Platform BigQuery integration allows data engineers to easily query and store data in the BigQuery data warehouse through the use of the `BigQueryResource`.\n\n### Installation\n\n```bash\npip install dagster-gcp\n```\n\n### Examples\n\n\n```python\nfrom dagster_gcp import BigQueryResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_table(bigquery: BigQueryResource):\n with bigquery.get_client() as client:\n client.query("SELECT * FROM my_dataset.my_table")\n\n\ndefs = dg.Definitions(\n assets=[my_table], resources={"bigquery": BigQueryResource(project="my-project")}\n)\n```\n \n\n### About Google Cloud Platform BigQuery\n\nThe Google Cloud Platform BigQuery service, offers a fully managed enterprise data warehouse that enables fast SQL queries using the processing power of Google\'s infrastructure.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpDataproc.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpDataproc.ts index e808c248ae582..338d457f3558e 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpDataproc.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpDataproc.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nUsing this integration, you can manage and interact with Google Cloud Platform\'s Dataproc service directly from Dagster. This integration allows you to create, manage, and delete Dataproc clusters, and submit and monitor jobs on these clusters.\n\n### Installation\n\n```bash\npip install dagster-gcp\n```\n\n### Examples\n\n\n\n### About Google Cloud Platform Dataproc\n\nGoogle Cloud Platform\'s **Dataproc** is a fully managed and highly scalable service for running Apache Spark, Apache Hadoop, and other open source data processing frameworks. Dataproc simplifies the process of setting up and managing clusters, allowing you to focus on your data processing tasks without worrying about the underlying infrastructure. With Dataproc, you can quickly create clusters, submit jobs, and monitor their progress, all while benefiting from the scalability and reliability of Google Cloud Platform.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nUsing this integration, you can manage and interact with Google Cloud Platform\'s Dataproc service directly from Dagster. This integration allows you to create, manage, and delete Dataproc clusters, and submit and monitor jobs on these clusters.\n\n### Installation\n\n```bash\npip install dagster-gcp\n```\n\n### Examples\n\n\n```python\nfrom dagster_gcp import DataprocResource\n\nimport dagster as dg\n\ndataproc_resource = DataprocResource(\n project_id="your-gcp-project-id",\n region="your-gcp-region",\n cluster_name="your-cluster-name",\n cluster_config_yaml_path="path/to/your/cluster/config.yaml",\n)\n\n\n@dg.asset\ndef my_dataproc_asset(dataproc: DataprocResource):\n client = dataproc.get_client()\n job_details = {\n "job": {\n "placement": {"clusterName": dataproc.cluster_name},\n }\n }\n client.submit_job(job_details)\n\n\ndefs = dg.Definitions(\n assets=[my_dataproc_asset], resources={"dataproc": dataproc_resource}\n)\n```\n \n\n### About Google Cloud Platform Dataproc\n\nGoogle Cloud Platform\'s **Dataproc** is a fully managed and highly scalable service for running Apache Spark, Apache Hadoop, and other open source data processing frameworks. Dataproc simplifies the process of setting up and managing clusters, allowing you to focus on your data processing tasks without worrying about the underlying infrastructure. With Dataproc, you can quickly create clusters, submit jobs, and monitor their progress, all while benefiting from the scalability and reliability of Google Cloud Platform.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpGcs.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpGcs.ts index a8d1a40acaa73..63f01365aa4a3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpGcs.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gcpGcs.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This integration allows you to interact with Google Cloud Storage (GCS) using Dagster. It provides resources, I/O Managers, and utilities to manage and store data in GCS, making it easier to integrate GCS into your data pipelines.\n\n### Installation\n\n```bash\npip install dagster-gcp\n```\n\n### Examples\n\n\n\n### About Google Cloud Platform GCS\n\n**Google Cloud Storage (GCS)**, is a scalable and secure object storage service. GCS is designed for storing and accessing any amount of data at any time, making it ideal for data science, AI infrastructure, and frameworks for ML like AutoML. With this integration, you can leverage GCS for efficient data storage and retrieval within your Dagster pipelines.'; + 'This integration allows you to interact with Google Cloud Storage (GCS) using Dagster. It provides resources, I/O Managers, and utilities to manage and store data in GCS, making it easier to integrate GCS into your data pipelines.\n\n### Installation\n\n```bash\npip install dagster-gcp\n```\n\n### Examples\n\n\n```python\nimport pandas as pd\nfrom dagster_gcp.gcs import GCSResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_gcs_asset(gcs: GCSResource):\n df = pd.DataFrame({"column1": [1, 2, 3], "column2": ["A", "B", "C"]})\n\n csv_data = df.to_csv(index=False)\n\n gcs_client = gcs.get_client()\n\n bucket = gcs_client.bucket("my-cool-bucket")\n blob = bucket.blob("path/to/my_dataframe.csv")\n blob.upload_from_string(csv_data)\n\n\ndefs = dg.Definitions(\n assets=[my_gcs_asset],\n resources={"gcs": GCSResource(project="my-gcp-project")},\n)\n```\n \n\n### About Google Cloud Platform GCS\n\n**Google Cloud Storage (GCS)**, is a scalable and secure object storage service. GCS is designed for storing and accessing any amount of data at any time, making it ideal for data science, AI infrastructure, and frameworks for ML like AutoML. With this integration, you can leverage GCS for efficient data storage and retrieval within your Dagster pipelines.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gemini.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gemini.ts index 857a96237f8d2..702d3893a1441 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gemini.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/gemini.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-gemini` library allows you to easily interact with the Gemini REST API using the Gemini Python API to build AI steps into your Dagster pipelines. You can also log Gemini API usage metadata in Dagster Insights, giving you detailed observability on API call credit consumption.\n\nWhen paired with Dagster assets, the resource automatically logs Gemini usage metadata in asset metadata.\n\n### Installation\n\n```bash\npip install dagster dagster-gemini\n```\n\n### Example\n\n\n\n### About Gemini\n\nGemini is Google\'s most capable AI model family, designed to be multimodal from the ground up. It can understand and combine different types of information like text, code, audio, images, and video. Gemini comes in different sizes optimized for different use cases, from the lightweight Gemini Nano for on-device tasks to the powerful Gemini Ultra for complex reasoning. The model demonstrates strong performance across language understanding, coding, reasoning, and creative tasks.'; + 'The `dagster-gemini` library allows you to easily interact with the Gemini REST API using the Gemini Python API to build AI steps into your Dagster pipelines. You can also log Gemini API usage metadata in Dagster Insights, giving you detailed observability on API call credit consumption.\n\nWhen paired with Dagster assets, the resource automatically logs Gemini usage metadata in asset metadata.\n\n### Installation\n\n```bash\npip install dagster dagster-gemini\n```\n\n### Example\n\n\n```python\nfrom dagster_gemini import GeminiResource\n\nimport dagster as dg\n\n\n@dg.asset(compute_kind="gemini")\ndef gemini_asset(context: dg.AssetExecutionContext, gemini: GeminiResource):\n with gemini.get_model(context) as model:\n response = model.generate_content("Generate a short sentence on tests")\n\n\ndefs = dg.Definitions(\n assets=[gemini_asset],\n resources={\n "gemini": GeminiResource(\n api_key=dg.EnvVar("GEMINI_API_KEY"),\n generative_model_name="gemini-1.5-flash",\n ),\n },\n)\n```\n \n\n### About Gemini\n\nGemini is Google\'s most capable AI model family, designed to be multimodal from the ground up. It can understand and combine different types of information like text, code, audio, images, and video. Gemini comes in different sizes optimized for different use cases, from the lightweight Gemini Nano for on-device tasks to the powerful Gemini Ultra for complex reasoning. The model demonstrates strong performance across language understanding, coding, reasoning, and creative tasks.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/github.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/github.ts index 5f3d2769696df..3330c78bdfc0e 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/github.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/github.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Deprecated from \'@site/docs/partials/\\_Deprecated.md\';\n\n\n\nThis library provides an integration with _[GitHub Apps](https://docs.github.com/en/developers/apps/getting-started-with-apps/about-apps)_ by providing a thin wrapper on the GitHub v4 GraphQL API. This allows for automating operations within your GitHub repositories and with the tighter permissions scopes that GitHub Apps allow for vs using a personal token.\n\n### Installation\n\n```bash\npip install dagster-github\n```\n\n### Example\n\n\n\n### About GitHub\n\n**GitHub** provides a highly available git repo, access control, bug tracking, software feature requests, task management, continuous integration, and wikis for open source and commercial projects.'; + 'import Deprecated from \'@site/docs/partials/\\_Deprecated.md\';\n\n\n\nThis library provides an integration with _[GitHub Apps](https://docs.github.com/en/developers/apps/getting-started-with-apps/about-apps)_ by providing a thin wrapper on the GitHub v4 GraphQL API. This allows for automating operations within your GitHub repositories and with the tighter permissions scopes that GitHub Apps allow for vs using a personal token.\n\n### Installation\n\n```bash\npip install dagster-github\n```\n\n### Example\n\n\n```python\nfrom dagster_github import GithubResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef github_asset(github: GithubResource):\n github.get_client().create_issue(\n repo_name="dagster",\n repo_owner="dagster-io",\n title="Dagster\'s first github issue",\n body="this open source thing seems like a pretty good idea",\n )\n\n\ndefs = dg.Definitions(\n assets=[github_asset],\n resources={\n "github": GithubResource(\n github_app_id=dg.EnvVar.int("GITHUB_APP_ID"),\n github_app_private_rsa_key=dg.EnvVar("GITHUB_PRIVATE_KEY"),\n github_installation_id=dg.EnvVar.int("GITHUB_INSTALLATION_ID"),\n )\n },\n)\n```\n \n\n### About GitHub\n\n**GitHub** provides a highly available git repo, access control, bug tracking, software feature requests, task management, continuous integration, and wikis for open source and commercial projects.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hashicorp.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hashicorp.ts index 96908a97bf77e..4e139c06b51d0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hashicorp.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hashicorp.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'Package for integrating HashiCorp Vault into Dagster so that you can securely manage tokens and passwords.\n\n### Installation\n\n```bash\npip install dagster-hashicorp\n```\n\n### Example\n\n\n\n### About HashiCorp Vault\n\n**HashiCorp** provides open source tools and commercial products that enable developers, operators and security professionals to provision, secure, run and connect cloud-computing infrastructure. **HashiCorp Vault** secures, stores, and tightly controls access to tokens, passwords, certificates, API keys, and other secrets in modern computing.'; + 'Package for integrating HashiCorp Vault into Dagster so that you can securely manage tokens and passwords.\n\n### Installation\n\n```bash\npip install dagster-hashicorp\n```\n\n### Example\n\n\n```python\n# See the Resources docs to learn more: https://docs.dagster.io/concepts/resources\n\nimport os\n\nfrom dagster_hashicorp.vault import vault_resource\n\nimport dagster as dg\n\n\n@dg.asset(required_resource_keys={"vault"})\ndef example_asset(context):\n secret_data = context.resources.vault.read_secret(secret_path="secret/data/foo/bar")\n context.log.debug(f"Secret: {secret_data}")\n\n\ndefs = dg.Definitions(\n assets=[example_asset],\n resources={\n "vault": vault_resource.configured(\n {\n "url": "vault-host:8200",\n "auth_type": {"token": {"token": dg.EnvVar("VAULT_AUTH_TOKEN")}},\n }\n )\n },\n)\n```\n \n\n### About HashiCorp Vault\n\n**HashiCorp** provides open source tools and commercial products that enable developers, operators and security professionals to provision, secure, run and connect cloud-computing infrastructure. **HashiCorp Vault** secures, stores, and tightly controls access to tokens, passwords, certificates, API keys, and other secrets in modern computing.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hightouch.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hightouch.ts index cf3188de3dccb..365eb04a47506 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hightouch.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/hightouch.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'With this integration you can trigger Hightouch syncs and monitor them from within Dagster. Fine-tune when Hightouch syncs kick-off, visualize their dependencies, and monitor the steps in your data activation workflow.\n\nThis native integration helps your team more effectively orchestrate the last mile of data analytics—bringing that data from the warehouse back into the SaaS tools your business teams live in. With the `dagster-hightouch` integration, Hightouch users have more granular and sophisticated control over when data gets activated.\n\n### Installation\n\n```bash\npip install dagster-hightouch\n```\n\n### Example\n\n\n\n### About Hightouch\n\n**Hightouch** syncs data from any data warehouse into popular SaaS tools that businesses run on. Hightouch uses the power of Reverse ETL to transform core business applications from isolated data islands into powerful integrated solutions.'; + 'With this integration you can trigger Hightouch syncs and monitor them from within Dagster. Fine-tune when Hightouch syncs kick-off, visualize their dependencies, and monitor the steps in your data activation workflow.\n\nThis native integration helps your team more effectively orchestrate the last mile of data analytics—bringing that data from the warehouse back into the SaaS tools your business teams live in. With the `dagster-hightouch` integration, Hightouch users have more granular and sophisticated control over when data gets activated.\n\n### Installation\n\n```bash\npip install dagster-hightouch\n```\n\n### Example\n\n\n```python\nimport os\n\nfrom dagster_hightouch.ops import hightouch_sync_op\nfrom dagster_hightouch.resources import ht_resource\n\nimport dagster as dg\n\nHT_ORG = "39619"\n\nrun_ht_sync_orgs = hightouch_sync_op.configured(\n {"sync_id": HT_ORG}, name="hightouch_sfdc_organizations"\n)\n\n\n@dg.job\ndef ht_sfdc_job():\n ht_orgs = run_ht_sync_orgs()\n\n\ndefs = dg.Definitions(\n jobs=[ht_sfdc_job],\n resources={\n "hightouch": ht_resource.configured(\n {"api_key": dg.EnvVar("HIGHTOUCH_API_KEY")},\n ),\n },\n)\n```\n \n\n### About Hightouch\n\n**Hightouch** syncs data from any data warehouse into popular SaaS tools that businesses run on. Hightouch uses the power of Reverse ETL to transform core business applications from isolated data islands into powerful integrated solutions.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/kubernetes.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/kubernetes.ts index ddb0435c12478..f50899a69e4aa 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/kubernetes.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/kubernetes.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-k8s` integration library provides the `PipesK8sClient` resource, enabling you to launch Kubernetes pods and execute external code directly from Dagster assets and ops. This integration allows you to pass parameters to Kubernetes pods while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-k8s\n```\n\n### Example\n\n\n\n### Deploying to Kubernetes?\n\n- Deploying to Dagster+: Use with a Dagster+ Hybrid deployment, the Kubernetes agent executes Dagster jobs on a Kubernetes cluster. Checkout the [Dagster+ Kubernetes Agent](https://docs.dagster.io/dagster-plus/deployment/deployment-types/hybrid/kubernetes/) guide for more information.\n- Deploying to Open Source: Visit the [Deploying Dagster to Kubernetes](https://docs.dagster.io/guides/deploy/deployment-options/kubernetes/) guide for more information.\n\n### About Kubernetes\n\n**Kubernetes** is an open source container orchestration system for automating software deployment, scaling, and management. Google originally designed Kubernetes, but the Cloud Native Computing Foundation now maintains the project.'; + 'The `dagster-k8s` integration library provides the `PipesK8sClient` resource, enabling you to launch Kubernetes pods and execute external code directly from Dagster assets and ops. This integration allows you to pass parameters to Kubernetes pods while Dagster receives real-time events, such as logs, asset checks, and asset materializations, from the initiated jobs. With minimal code changes required on the job side, this integration is both efficient and easy to implement.\n\n### Installation\n\n```bash\npip install dagster-k8s\n```\n\n### Example\n\n\n```python\nfrom dagster_k8s import PipesK8sClient\n\nimport dagster as dg\n\n\n@dg.asset\ndef k8s_pipes_asset(\n context: dg.AssetExecutionContext, k8s_pipes_client: PipesK8sClient\n):\n return k8s_pipes_client.run(\n context=context,\n image="pipes-example:v1",\n ).get_materialize_result()\n\n\ndefs = dg.Definitions(\n assets=[k8s_pipes_asset],\n resources={\n "k8s_pipes_client": PipesK8sClient(),\n },\n)\n```\n \n\n### Deploying to Kubernetes?\n\n- Deploying to Dagster+: Use with a Dagster+ Hybrid deployment, the Kubernetes agent executes Dagster jobs on a Kubernetes cluster. Checkout the [Dagster+ Kubernetes Agent](https://docs.dagster.io/dagster-plus/deployment/deployment-types/hybrid/kubernetes/) guide for more information.\n- Deploying to Open Source: Visit the [Deploying Dagster to Kubernetes](https://docs.dagster.io/guides/deploy/deployment-options/kubernetes/) guide for more information.\n\n### About Kubernetes\n\n**Kubernetes** is an open source container orchestration system for automating software deployment, scaling, and management. Google originally designed Kubernetes, but the Cloud Native Computing Foundation now maintains the project.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/lakefs.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/lakefs.ts index 9d8022f922955..59e36c3be5dd0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/lakefs.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/lakefs.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'By integrating with lakeFS, a big data scale version control system, you can leverage the versioning capabilities of lakeFS to track changes to your data. This integration allows you to have a complete lineage of your data, from the initial raw data to the transformed and processed data, making it easier to understand and reproduce data transformations.\n\nWith lakeFS and Dagster integration, you can ensure that data flowing through your Dagster jobs is easily reproducible. lakeFS provides a consistent view of your data across different versions, allowing you to troubleshoot pipeline runs and ensure consistent results.\n\nFurthermore, with lakeFS branching capabilities, Dagster jobs can run on separate branches without additional storage costs, creating isolation and allowing promotion of only high-quality data to production leveraging a CI/CD pipeline for your data.\n\n### Installation\n\n```bash\npip install lakefs-client\n```\n\n### Example\n\n\n\n### About lakeFS\n\n**lakeFS** is on a mission to simplify the lives of data engineers, data scientists and analysts providing a data version control platform at scale.'; + 'By integrating with lakeFS, a big data scale version control system, you can leverage the versioning capabilities of lakeFS to track changes to your data. This integration allows you to have a complete lineage of your data, from the initial raw data to the transformed and processed data, making it easier to understand and reproduce data transformations.\n\nWith lakeFS and Dagster integration, you can ensure that data flowing through your Dagster jobs is easily reproducible. lakeFS provides a consistent view of your data across different versions, allowing you to troubleshoot pipeline runs and ensure consistent results.\n\nFurthermore, with lakeFS branching capabilities, Dagster jobs can run on separate branches without additional storage costs, creating isolation and allowing promotion of only high-quality data to production leveraging a CI/CD pipeline for your data.\n\n### Installation\n\n```bash\npip install lakefs-client\n```\n\n### Example\n\n\n```python\nimport lakefs_client\nfrom lakefs_client import models\nfrom lakefs_client.client import LakeFSClient\n\nimport dagster as dg\n\nlogger = dg.get_dagster_logger()\n\nconfiguration = lakefs_client.Configuration()\nconfiguration.username = "AAAA"\nconfiguration.password = "BBBBB"\nconfiguration.host = "https://my-org.us-east-1.lakefscloud.io"\n\n\n@dg.asset\ndef create_branch(client: dg.ResourceParam[LakeFSClient]):\n branch_id = client.branches.create_branch(\n repository="test-repo",\n branch_creation=models.BranchCreation(name="experiment", source="main"),\n )\n logger.info(branch_id)\n\n\n@dg.asset(deps=[create_branch])\ndef list_branches(client: dg.ResourceParam[LakeFSClient]):\n list_branches = client.branches.list_branches(repository="test-repo")\n logger.info(list_branches)\n\n\ndefs = dg.Definitions(\n assets=[create_branch, list_branches],\n resources={"client": LakeFSClient(configuration)},\n)\n```\n \n\n### About lakeFS\n\n**lakeFS** is on a mission to simplify the lives of data engineers, data scientists and analysts providing a data version control platform at scale.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/looker.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/looker.ts index bb43062531372..a1b13aea69ef1 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/looker.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/looker.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nDagster allows you to represent your Looker project as assets, alongside other your other technologies like dbt and Sling. This allows you to see how your Looker assets are connected to your other data assets, and how changes to other data assets might impact your Looker project.\n\n### Installation\n\n```bash\npip install dagster-looker\n```\n\n### Represent Looker assets in the asset graph\n\nTo load Looker assets into the Dagster asset graph, you must first construct a , which allows Dagster to communicate with your Looker instance. You\'ll need to supply your Looker instance URL and API credentials, which can be passed directly or accessed from the environment using .\n\nDagster can automatically load all views, explores, and dashboards from your Looker instance as asset specs. Call the function, which returns a list of representing your Looker assets. You can then include these asset specs in your object:\n\n\n\n### Load Looker assets from filtered dashboards and explores\n\nIt is possible to load a subset of your Looker assets by providing a to the function. All dashboards contained in the folders provided to your will be fetched. Additionally, only the explores used in these dashboards will be fetched by passing only_fetch_explores_used_in_dashboards=True to your .\n\nNote that the content and size of Looker instance may affect the performance of your Dagster deployments. Filtering the dashboards and explores selection from which your Looker assets will be loaded is particularly useful for improving loading times.\n\n### Customize asset definition metadata for Looker assets\n\n\n\nBy default, Dagster will generate asset specs for each Looker asset based on its type, and populate default metadata. You can further customize asset properties by passing a custom subclass to the function. This subclass can implement methods to customize the asset specs for each Looker asset type.\n\n\n\nNote that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it.\n\n### Materialize Looker PDTs from Dagster\n\nYou can use Dagster to orchestrate the materialization of Looker PDTs. To model PDTs as assets, build their asset definitions by passing a list of to function.\n\n\n\n### About Looker\n\n**Looker** is a modern platform for data analytics and visualization. It provides a unified interface for data exploration, modeling, and visualization, making it easier to understand and analyze data. Looker integrates with various data sources and can be used to create interactive reports, dashboards, and visualizations.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nDagster allows you to represent your Looker project as assets, alongside other your other technologies like dbt and Sling. This allows you to see how your Looker assets are connected to your other data assets, and how changes to other data assets might impact your Looker project.\n\n### Installation\n\n```bash\npip install dagster-looker\n```\n\n### Represent Looker assets in the asset graph\n\nTo load Looker assets into the Dagster asset graph, you must first construct a , which allows Dagster to communicate with your Looker instance. You\'ll need to supply your Looker instance URL and API credentials, which can be passed directly or accessed from the environment using .\n\nDagster can automatically load all views, explores, and dashboards from your Looker instance as asset specs. Call the function, which returns a list of representing your Looker assets. You can then include these asset specs in your object:\n\n\n```python\nfrom dagster_looker import LookerResource, load_looker_asset_specs\n\nimport dagster as dg\n\nlooker_resource = LookerResource(\n client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"),\n client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"),\n base_url=dg.EnvVar("LOOKERSDK_HOST_URL"),\n)\n\nlooker_specs = load_looker_asset_specs(looker_resource=looker_resource)\ndefs = dg.Definitions(assets=[*looker_specs], resources={"looker": looker_resource})\n```\n \n\n### Load Looker assets from filtered dashboards and explores\n\nIt is possible to load a subset of your Looker assets by providing a to the function. All dashboards contained in the folders provided to your will be fetched. Additionally, only the explores used in these dashboards will be fetched by passing only_fetch_explores_used_in_dashboards=True to your .\n\nNote that the content and size of Looker instance may affect the performance of your Dagster deployments. Filtering the dashboards and explores selection from which your Looker assets will be loaded is particularly useful for improving loading times.\n\n### Customize asset definition metadata for Looker assets\n\n\n```python\nfrom dagster_looker import LookerFilter, LookerResource, load_looker_asset_specs\n\nimport dagster as dg\n\nlooker_resource = LookerResource(\n client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"),\n client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"),\n base_url=dg.EnvVar("LOOKERSDK_HOST_URL"),\n)\n\nlooker_specs = load_looker_asset_specs(\n looker_resource=looker_resource,\n looker_filter=LookerFilter(\n dashboard_folders=[\n ["my_folder", "my_subfolder"],\n ["my_folder", "my_other_subfolder"],\n ],\n only_fetch_explores_used_in_dashboards=True,\n ),\n)\ndefs = dg.Definitions(assets=[*looker_specs], resources={"looker": looker_resource})\n```\n \n\nBy default, Dagster will generate asset specs for each Looker asset based on its type, and populate default metadata. You can further customize asset properties by passing a custom subclass to the function. This subclass can implement methods to customize the asset specs for each Looker asset type.\n\n\n```python\nfrom dagster_looker import (\n DagsterLookerApiTranslator,\n LookerApiTranslatorStructureData,\n LookerResource,\n LookerStructureType,\n load_looker_asset_specs,\n)\n\nimport dagster as dg\n\nlooker_resource = LookerResource(\n client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"),\n client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"),\n base_url=dg.EnvVar("LOOKERSDK_HOST_URL"),\n)\n\n\nclass CustomDagsterLookerApiTranslator(DagsterLookerApiTranslator):\n def get_asset_spec(\n self, looker_structure: LookerApiTranslatorStructureData\n ) -> dg.AssetSpec:\n # We create the default asset spec using super()\n default_spec = super().get_asset_spec(looker_structure)\n # We customize the team owner tag for all assets,\n # and we customize the asset key prefix only for dashboards.\n return default_spec.replace_attributes(\n key=(\n default_spec.key.with_prefix("looker")\n if looker_structure.structure_type == LookerStructureType.DASHBOARD\n else default_spec.key\n ),\n owners=["team:my_team"],\n )\n\n\nlooker_specs = load_looker_asset_specs(\n looker_resource, dagster_looker_translator=CustomDagsterLookerApiTranslator()\n)\ndefs = dg.Definitions(assets=[*looker_specs], resources={"looker": looker_resource})\n```\n \n\nNote that `super()` is called in each of the overridden methods to generate the default asset spec. It is best practice to generate the default asset spec before customizing it.\n\n### Materialize Looker PDTs from Dagster\n\nYou can use Dagster to orchestrate the materialization of Looker PDTs. To model PDTs as assets, build their asset definitions by passing a list of to function.\n\n\n```python\nfrom dagster_looker import (\n LookerResource,\n RequestStartPdtBuild,\n build_looker_pdt_assets_definitions,\n load_looker_asset_specs,\n)\n\nimport dagster as dg\n\nlooker_resource = LookerResource(\n client_id=dg.EnvVar("LOOKERSDK_CLIENT_ID"),\n client_secret=dg.EnvVar("LOOKERSDK_CLIENT_SECRET"),\n base_url=dg.EnvVar("LOOKERSDK_HOST_URL"),\n)\n\nlooker_specs = load_looker_asset_specs(looker_resource=looker_resource)\n\npdts = build_looker_pdt_assets_definitions(\n resource_key="looker",\n request_start_pdt_builds=[\n RequestStartPdtBuild(model_name="my_model", view_name="my_view")\n ],\n)\n\n\ndefs = dg.Definitions(\n assets=[*pdts, *looker_specs],\n resources={"looker": looker_resource},\n)\n```\n \n\n### About Looker\n\n**Looker** is a modern platform for data analytics and visualization. It provides a unified interface for data exploration, modeling, and visualization, making it easier to understand and analyze data. Looker integrates with various data sources and can be used to create interactive reports, dashboards, and visualizations.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/meltano.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/meltano.ts index cbef209dad1c4..9888ff1e075fd 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/meltano.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/meltano.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-meltano` library allows you to run Meltano using Dagster. Design and configure ingestion jobs using the popular [Singer.io](https://singer.io) specification.\n\n**Note** that this integration can also be [managed from the Meltano platform](https://hub.meltano.com/utilities/dagster/) using `meltano add utility dagster` and configured using `meltano config dagster set --interactive`.\n\n### Installation\n\n```bash\npip install dagster-meltano\n```\n\n### Example\n\n\n\n### About Meltano\n\n[Meltano](https://meltano.com/) provides data engineers with a set of tools for easily creating and managing pipelines as code by providing a wide array of composable connectors. Meltano\'s \'CLI for ELT+\' lets you test your changes before they go live.'; + 'The `dagster-meltano` library allows you to run Meltano using Dagster. Design and configure ingestion jobs using the popular [Singer.io](https://singer.io) specification.\n\n**Note** that this integration can also be [managed from the Meltano platform](https://hub.meltano.com/utilities/dagster/) using `meltano add utility dagster` and configured using `meltano config dagster set --interactive`.\n\n### Installation\n\n```bash\npip install dagster-meltano\n```\n\n### Example\n\n\n```python\nfrom dagster_meltano import meltano_resource, meltano_run_op\n\nimport dagster as dg\n\n\n@dg.job(resource_defs={"meltano": meltano_resource})\ndef meltano_run_job():\n tap_done = meltano_run_op("tap-1 target-1")()\n meltano_run_op("tap-2 target-2")(tap_done)\n\n\ndefs = dg.Definitions(jobs=[meltano_run_job])\n```\n \n\n### About Meltano\n\n[Meltano](https://meltano.com/) provides data engineers with a set of tools for easily creating and managing pipelines as code by providing a wide array of composable connectors. Meltano\'s \'CLI for ELT+\' lets you test your changes before they go live.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/microsoftTeams.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/microsoftTeams.ts index 9cf84ac083146..07fa333b03224 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/microsoftTeams.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/microsoftTeams.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'By configuring this resource, you can post messages to MS Teams from any Dagster op or asset.\n\n### Installation\n\n```bash\npip install dagster-msteams\n```\n\n### Example\n\n\n\n### About Microsoft Teams\n\n**Microsoft Teams** is a business communication platform. Teams offers workspace chat and videoconferencing, file storage, and application integration.'; + 'By configuring this resource, you can post messages to MS Teams from any Dagster op or asset.\n\n### Installation\n\n```bash\npip install dagster-msteams\n```\n\n### Example\n\n\n```python\n# Read the docs on Resources to learn more: https://docs.dagster.io/deployment/resources\nfrom dagster_msteams import Card, MSTeamsResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef microsoft_teams_message(msteams: MSTeamsResource):\n card = Card()\n card.add_attachment(text_message="Hello there!")\n msteams.get_client().post_message(payload=card.payload)\n\n\ndefs = dg.Definitions(\n assets=[microsoft_teams_message],\n resources={"msteams": MSTeamsResource(hook_url=dg.EnvVar("TEAMS_WEBHOOK_URL"))},\n)\n```\n \n\n### About Microsoft Teams\n\n**Microsoft Teams** is a business communication platform. Teams offers workspace chat and videoconferencing, file storage, and application integration.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/openai.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/openai.ts index e2bc1193f7580..c516c78294d83 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/openai.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/openai.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-openai` library allows you to easily interact with the OpenAI REST API using the OpenAI Python API to build AI steps into your Dagster pipelines. You can also log OpenAI API usage metadata in Dagster Insights, giving you detailed observability on API call credit consumption.\n\nWhen paired with Dagster assets, the resource automatically logs OpenAI usage metadata in asset metadata.\n\n### Installation\n\n```bash\npip install dagster dagster-openai\n```\n\n### Example\n\n\n\n### About OpenAI\n\nOpenAI is a U.S. based artificial intelligence (AI) research organization with the goal of developing "safe and beneficial" artificial general intelligence, which it defines as "highly autonomous systems that outperform humans at most economically valuable work".'; + 'The `dagster-openai` library allows you to easily interact with the OpenAI REST API using the OpenAI Python API to build AI steps into your Dagster pipelines. You can also log OpenAI API usage metadata in Dagster Insights, giving you detailed observability on API call credit consumption.\n\nWhen paired with Dagster assets, the resource automatically logs OpenAI usage metadata in asset metadata.\n\n### Installation\n\n```bash\npip install dagster dagster-openai\n```\n\n### Example\n\n\n```python\nfrom dagster_openai import OpenAIResource\n\nimport dagster as dg\n\n\n@dg.asset(compute_kind="OpenAI")\ndef openai_asset(context: dg.AssetExecutionContext, openai: OpenAIResource):\n with openai.get_client(context) as client:\n client.chat.completions.create(\n model="gpt-3.5-turbo",\n messages=[{"role": "user", "content": "Say this is a test."}],\n )\n\n\nopenai_asset_job = dg.define_asset_job(\n name="openai_asset_job", selection="openai_asset"\n)\n\ndefs = dg.Definitions(\n assets=[openai_asset],\n jobs=[openai_asset_job],\n resources={\n "openai": OpenAIResource(api_key=dg.EnvVar("OPENAI_API_KEY")),\n },\n)\n```\n \n\n### About OpenAI\n\nOpenAI is a U.S. based artificial intelligence (AI) research organization with the goal of developing "safe and beneficial" artificial general intelligence, which it defines as "highly autonomous systems that outperform humans at most economically valuable work".'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pagerduty.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pagerduty.ts index 31de3c39dd65b..cccdcf9ee102b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pagerduty.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pagerduty.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This library provides an integration between Dagster and PagerDuty to support creating alerts from your Dagster code.\n\n### Installation\n\n```bash\npip install dagster-pagerduty\n```\n\n### Example\n\n\n\n### About PagerDuty\n\n**PagerDuty** is a popular SaaS incident response platform. It integrates machine data & human intelligence to improve visibility & agility for Real-Time Operations.'; + 'This library provides an integration between Dagster and PagerDuty to support creating alerts from your Dagster code.\n\n### Installation\n\n```bash\npip install dagster-pagerduty\n```\n\n### Example\n\n\n```python\nfrom dagster_pagerduty import PagerDutyService\n\nimport dagster as dg\n\n\n@dg.asset\ndef pagerduty_alert(pagerduty: PagerDutyService):\n pagerduty.EventV2_create(\n summary="alert from dagster",\n source="localhost",\n severity="error",\n event_action="trigger",\n )\n\n\ndefs = dg.Definitions(\n assets=[pagerduty_alert],\n resources={\n "pagerduty": PagerDutyService(routing_key="0123456789abcdef0123456789abcdef")\n },\n)\n```\n \n\n### About PagerDuty\n\n**PagerDuty** is a popular SaaS incident response platform. It integrates machine data & human intelligence to improve visibility & agility for Real-Time Operations.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pandera.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pandera.ts index e3c06ffcdd90b..538825c065c9b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pandera.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/pandera.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThe `dagster-pandera` integration library provides an API for generating Dagster Types from [Pandera DataFrame schemas](https://pandera.readthedocs.io/en/stable/dataframe_schemas.html).\n\nLike all Dagster types, Dagster-Pandera-generated types can be used to annotate op inputs and outputs. This provides runtime type-checking with rich error reporting and allows Dagster UI to display information about a DataFrame\'s structure.\n\n:::note\n\nCurrently, `dagster-pandera` only supports pandas and Polars dataframes, despite Pandera supporting validation on other dataframe backends.\n\n:::\n\n### Installation\n\n```bash\npip install dagster-pandera\n```\n\n### Example\n\n\n\n### About Pandera\n\n**Pandera** is a statistical data testing toolkit, and a data validation library for scientists, engineers, and analysts seeking correctness.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThe `dagster-pandera` integration library provides an API for generating Dagster Types from [Pandera DataFrame schemas](https://pandera.readthedocs.io/en/stable/dataframe_schemas.html).\n\nLike all Dagster types, Dagster-Pandera-generated types can be used to annotate op inputs and outputs. This provides runtime type-checking with rich error reporting and allows Dagster UI to display information about a DataFrame\'s structure.\n\n:::note\n\nCurrently, `dagster-pandera` only supports pandas and Polars dataframes, despite Pandera supporting validation on other dataframe backends.\n\n:::\n\n### Installation\n\n```bash\npip install dagster-pandera\n```\n\n### Example\n\n\n```python\nimport random\n\nimport pandas as pd\nimport pandera as pa\nfrom dagster_pandera import pandera_schema_to_dagster_type\nfrom pandera.typing import Series\n\nimport dagster as dg\n\nAPPLE_STOCK_PRICES = {\n "name": ["AAPL", "AAPL", "AAPL", "AAPL", "AAPL"],\n "date": ["2018-01-22", "2018-01-23", "2018-01-24", "2018-01-25", "2018-01-26"],\n "open": [177.3, 177.3, 177.25, 174.50, 172.0],\n "close": [177.0, 177.04, 174.22, 171.11, 171.51],\n}\n\n\nclass StockPrices(pa.DataFrameModel):\n """Open/close prices for one or more stocks by day."""\n\n name: Series[str] = pa.Field(description="Ticker symbol of stock")\n date: Series[str] = pa.Field(description="Date of prices")\n open: Series[float] = pa.Field(ge=0, description="Price at market open")\n close: Series[float] = pa.Field(ge=0, description="Price at market close")\n\n\n@dg.asset(dagster_type=pandera_schema_to_dagster_type(StockPrices))\ndef apple_stock_prices_dirty():\n prices = pd.DataFrame(APPLE_STOCK_PRICES)\n i = random.choice(prices.index)\n prices.loc[i, "open"] = pd.NA\n prices.loc[i, "close"] = pd.NA\n return prices\n\n\ndefs = dg.Definitions(assets=[apple_stock_prices_dirty])\n```\n \n\n### About Pandera\n\n**Pandera** is a statistical data testing toolkit, and a data validation library for scientists, engineers, and analysts seeking correctness.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/prometheus.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/prometheus.ts index fc66ac66f585b..a7276757b6928 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/prometheus.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/prometheus.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration allows you to push metrics to the Prometheus gateway from within a Dagster pipeline.\n\n### Installation\n\n```bash\npip install dagster-prometheus\n```\n\n### Example\n\n\n\n### About Prometheus\n\n**Prometheus** is an open source systems monitoring and alerting toolkit. Originally built at SoundCloud, Prometheus joined the Cloud Native Computing Foundation in 2016 as the second hosted project, after Kubernetes.\n\nPrometheus collects and stores metrics as time series data along with the timestamp at which it was recorded, alongside optional key-value pairs called labels.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration allows you to push metrics to the Prometheus gateway from within a Dagster pipeline.\n\n### Installation\n\n```bash\npip install dagster-prometheus\n```\n\n### Example\n\n\n```python\nfrom dagster_prometheus import PrometheusResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef prometheus_metric(prometheus: PrometheusResource):\n prometheus.push_to_gateway(job="my_job_label")\n\n\ndefs = dg.Definitions(\n assets=[prometheus_metric],\n resources={\n "prometheus": PrometheusResource(gateway="http://pushgateway.example.org:9091")\n },\n)\n```\n \n\n### About Prometheus\n\n**Prometheus** is an open source systems monitoring and alerting toolkit. Originally built at SoundCloud, Prometheus joined the Cloud Native Computing Foundation in 2016 as the second hosted project, after Kubernetes.\n\nPrometheus collects and stores metrics as time series data along with the timestamp at which it was recorded, alongside optional key-value pairs called labels.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/qdrant.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/qdrant.ts index db0c5d6d0b362..ffe1e372eaae0 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/qdrant.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/qdrant.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-qdrant` library lets you integrate Qdrant\'s vector database with Dagster, making it easy to build AI-driven data pipelines. You can run vector searches and manage data directly within Dagster.\n\n### Installation\n\n```bash\npip install dagster dagster-qdrant\n```\n\n### Example\n\n\n\n### About Qdrant\n\nQdrant (read: quadrant) is a vector similarity search engine. It provides a production-ready service with a convenient API to store, search, and manage vectors with additional payload and extended filtering support. It makes it useful for all sorts of neural network or semantic-based matching, faceted search, and other applications.\n\nLearn more from the [Qdrant documentation](https://qdrant.tech/).'; + 'The `dagster-qdrant` library lets you integrate Qdrant\'s vector database with Dagster, making it easy to build AI-driven data pipelines. You can run vector searches and manage data directly within Dagster.\n\n### Installation\n\n```bash\npip install dagster dagster-qdrant\n```\n\n### Example\n\n\n```python\nfrom dagster_qdrant import QdrantConfig, QdrantResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_table(qdrant_resource: QdrantResource):\n with qdrant_resource.get_client() as qdrant:\n qdrant.add(\n collection_name="test_collection",\n documents=[\n "This is a document about oranges",\n "This is a document about pineapples",\n "This is a document about strawberries",\n "This is a document about cucumbers",\n ],\n )\n results = qdrant.query(\n collection_name="test_collection", query_text="hawaii", limit=3\n )\n\n\ndefs = dg.Definitions(\n assets=[my_table],\n resources={\n "qdrant_resource": QdrantResource(\n config=QdrantConfig(\n host="xyz-example.eu-central.aws.cloud.qdrant.io",\n api_key="",\n )\n )\n },\n)\n```\n \n\n### About Qdrant\n\nQdrant (read: quadrant) is a vector similarity search engine. It provides a production-ready service with a convenient API to store, search, and manage vectors with additional payload and extended filtering support. It makes it useful for all sorts of neural network or semantic-based matching, faceted search, and other applications.\n\nLearn more from the [Qdrant documentation](https://qdrant.tech/).'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sdf.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sdf.ts index 5459f59ffca7b..175748c114c51 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sdf.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sdf.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'SDF can integrate seamlessly with your existing Dagster projects, providing the best-in-class transformation layer while enabling you to schedule, orchestrate, and monitor your dags in Dagster.\n\nWhen it comes time to materialize your Dagster assets, you can be confident that SDF has successfully compiled your workspace, making it safe to execute locally or against your cloud data warehouse.\n\n### Installation\n\n```bash\npip install dagster-sdf\n```\n\n### Example\n\n\n\n### About SDF\n\n[SDF](https://www.sdf.com/) is a multi-dialect SQL compiler, transformation framework, and analytical database engine. It natively compiles SQL dialects, like Snowflake, and connects to their corresponding data warehouses to materialize models.'; + 'SDF can integrate seamlessly with your existing Dagster projects, providing the best-in-class transformation layer while enabling you to schedule, orchestrate, and monitor your dags in Dagster.\n\nWhen it comes time to materialize your Dagster assets, you can be confident that SDF has successfully compiled your workspace, making it safe to execute locally or against your cloud data warehouse.\n\n### Installation\n\n```bash\npip install dagster-sdf\n```\n\n### Example\n\n\n```python\nfrom pathlib import Path\n\nfrom dagster_sdf import SdfCliResource, SdfWorkspace, sdf_assets\n\nimport dagster as dg\n\nworkspace_dir = Path(__file__).joinpath("..", "./my_sdf_workspace").resolve()\ntarget_dir = workspace_dir.joinpath(\n "sdf_dagster_out"\n) # The destination for outputs generated by SDF during execution\nenvironment = "dbg" # Replace with your environment, e.g. "prod"\n\nworkspace = SdfWorkspace(\n workspace_dir=workspace_dir,\n target_dir=target_dir,\n environment=environment,\n)\n\n\n@sdf_assets(workspace=workspace)\ndef my_sdf_assets(context: dg.AssetExecutionContext, sdf: SdfCliResource):\n yield from sdf.cli(\n ["run", "--save", "info-schema"],\n target_dir=target_dir,\n environment=environment,\n context=context,\n ).stream()\n\n\ndefs = dg.Definitions(\n assets=[my_sdf_assets],\n resources={\n "sdf": SdfCliResource(workspace_dir=workspace_dir),\n },\n)\n```\n \n\n### About SDF\n\n[SDF](https://www.sdf.com/) is a multi-dialect SQL compiler, transformation framework, and analytical database engine. It natively compiles SQL dialects, like Snowflake, and connects to their corresponding data warehouses to materialize models.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/slack.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/slack.ts index a4ba72d0fcd2b..faf632bcc01d3 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/slack.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/slack.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This library provides an integration with Slack to support posting messages in your company\'s Slack workspace.\n\n### Installation\n\n```bash\npip install dagster-slack\n```\n\n### Example\n\n\n\n### About Slack\n\nThe **Slack** messaging app provides chat, video and voice communication tools and is used extensively across companies and communities. The Dagster slack community can be found at [dagster.io/slack](https://dagster.io/slack).'; + 'This library provides an integration with Slack to support posting messages in your company\'s Slack workspace.\n\n### Installation\n\n```bash\npip install dagster-slack\n```\n\n### Example\n\n\n```python\n# Read the docs on Resources to learn more: https://docs.dagster.io/deployment/resources\n\nfrom dagster_slack import SlackResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef slack_message(slack: SlackResource):\n slack.get_client().chat_postMessage(channel="#noise", text=":wave: hey there!")\n\n\ndefs = dg.Definitions(\n assets=[slack_message],\n resources={"slack": SlackResource(token=dg.EnvVar("SLACK_TOKEN"))},\n)\n```\n \n\n### About Slack\n\nThe **Slack** messaging app provides chat, video and voice communication tools and is used extensively across companies and communities. The Dagster slack community can be found at [dagster.io/slack](https://dagster.io/slack).'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sling.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sling.ts index e1a30f01ff870..f4c46b7ef6c43 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sling.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sling.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This integration allows you to use [Sling](https://slingdata.io/) to extract and load data from popular data sources to destinations with high performance and ease.\n\n### Installation\n\n```bash\npip install dagster-sling\n```\n\n### Example\n\n\n\n### About Sling\n\nSling provides an easy-to-use YAML configuration layer for loading data from files, replicating data between databases, exporting custom SQL queries to cloud storage, and much more.\n\n#### Key Features\n\n- **Data Movement**: Transfer data between different storage systems and databases efficiently\n\n- **Flexible Connectivity**: Support for numerous databases, data warehouses, and file storage systems\n\n- **Transformation Capabilities**: Built-in data transformation features during transfer\n\n- **Multiple Operation Modes**: Support for various replication modes including full-refresh, incremental, and snapshot\n\n- **Production-Ready**: Deployable with monitoring, scheduling, and error handling'; + 'This integration allows you to use [Sling](https://slingdata.io/) to extract and load data from popular data sources to destinations with high performance and ease.\n\n### Installation\n\n```bash\npip install dagster-sling\n```\n\n### Example\n\n\n```python\nfrom dagster_sling import SlingConnectionResource, SlingResource, sling_assets\n\nimport dagster as dg\n\nsource = SlingConnectionResource(\n name="MY_PG",\n type="postgres",\n host="localhost", # type: ignore\n port=5432, # type: ignore\n database="my_database", # type: ignore\n user="my_user", # type: ignore\n password=dg.EnvVar("PG_PASS"), # type: ignore\n)\n\ntarget = SlingConnectionResource(\n name="MY_SF",\n type="snowflake",\n host="hostname.snowflake", # type: ignore\n user="username", # type: ignore\n database="database", # type: ignore\n password=dg.EnvVar("SF_PASSWORD"), # type: ignore\n role="role", # type: ignore\n)\n\n\n@sling_assets(\n replication_config={\n "source": "MY_PG",\n "target": "MY_SF",\n "defaults": {\n "mode": "full-refresh",\n "object": "{stream_schema}_{stream_table}",\n },\n "streams": {\n "public.accounts": None,\n "public.users": None,\n "public.finance_departments": {"object": "departments"},\n },\n }\n)\ndef my_sling_assets(context, sling: SlingResource):\n yield from sling.replicate(context=context)\n\n\ndefs = dg.Definitions(\n assets=[my_sling_assets],\n resources={\n "sling": SlingResource(\n connections=[\n source,\n target,\n ]\n )\n },\n)\n```\n \n\n### About Sling\n\nSling provides an easy-to-use YAML configuration layer for loading data from files, replicating data between databases, exporting custom SQL queries to cloud storage, and much more.\n\n#### Key Features\n\n- **Data Movement**: Transfer data between different storage systems and databases efficiently\n\n- **Flexible Connectivity**: Support for numerous databases, data warehouses, and file storage systems\n\n- **Transformation Capabilities**: Built-in data transformation features during transfer\n\n- **Multiple Operation Modes**: Support for various replication modes including full-refresh, incremental, and snapshot\n\n- **Production-Ready**: Deployable with monitoring, scheduling, and error handling'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/snowflake.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/snowflake.ts index 5611ee2edbd02..4eb882674025b 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/snowflake.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/snowflake.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'This library provides an integration with the Snowflake data warehouse. Connect to Snowflake as a resource, then use the integration-provided functions to construct an op to establish connections and execute Snowflake queries. Read and write natively to Snowflake from Dagster assets.\n\n### Installation\n\n```bash\npip install dagster-snowflake\n```\n\n### Example\n\n\n\n### About Snowflake\n\nA cloud-based data storage and analytics service, generally termed "data-as-a-service". **Snowflake**\'s data warehouse is one of the most widely adopted cloud warehouses for analytics.'; + 'This library provides an integration with the Snowflake data warehouse. Connect to Snowflake as a resource, then use the integration-provided functions to construct an op to establish connections and execute Snowflake queries. Read and write natively to Snowflake from Dagster assets.\n\n### Installation\n\n```bash\npip install dagster-snowflake\n```\n\n### Example\n\n\n```python\nfrom dagster_snowflake import SnowflakeResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_table(snowflake: SnowflakeResource):\n with snowflake.get_connection() as conn:\n return conn.cursor().execute_query("SELECT * FROM foo")\n\n\ndefs = dg.Definitions(\n assets=[my_table],\n resources={\n "snowflake": SnowflakeResource(\n account="snowflake account",\n user="snowflake user",\n password=dg.EnvVar("SNOWFLAKE_PASSWORD"),\n database="snowflake database",\n schema="snowflake schema",\n warehouse="snowflake warehouse",\n )\n },\n)\n```\n \n\n### About Snowflake\n\nA cloud-based data storage and analytics service, generally termed "data-as-a-service". **Snowflake**\'s data warehouse is one of the most widely adopted cloud warehouses for analytics.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sshSftp.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sshSftp.ts index a8fda264c6542..174a812bcb199 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sshSftp.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/sshSftp.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration provides a resource for SSH remote execution using [Paramiko](https://github.com/paramiko/paramiko). It allows you to establish secure connections to networked resources and execute commands remotely. The integration also provides an SFTP client for secure file transfers between the local and remote systems.\n\n### Installation\n\n```bash\npip install dagster-ssh\n```\n\n### Example\n\n\n\n### About SSH SFTP\n\nThe **SSH protocol** allows for secure remote login with strong authentication to networked resources. It protects network connections with strong encryption. The Dagster library provides direct SSH and SFTP calls from within the execution of your pipelines.'; + 'import Beta from \'@site/docs/partials/\\_Beta.md\';\n\n\n\nThis integration provides a resource for SSH remote execution using [Paramiko](https://github.com/paramiko/paramiko). It allows you to establish secure connections to networked resources and execute commands remotely. The integration also provides an SFTP client for secure file transfers between the local and remote systems.\n\n### Installation\n\n```bash\npip install dagster-ssh\n```\n\n### Example\n\n\n```python\nfrom dagster_ssh import SSHResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef ssh_asset(ssh: SSHResource):\n ssh.sftp_get("/path/to/remote.csv", "path/to/local.csv")\n\n\ndefs = dg.Definitions(\n assets=[ssh_asset],\n resources={"ssh": SSHResource(remote_host="foo.com", key_file="path/to/id_rsa")},\n)\n```\n \n\n### About SSH SFTP\n\nThe **SSH protocol** allows for secure remote login with strong authentication to networked resources. It protects network connections with strong encryption. The Dagster library provides direct SSH and SFTP calls from within the execution of your pipelines.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/twilio.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/twilio.ts index a373198da08d5..cf9cae047f5c4 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/twilio.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/twilio.ts @@ -23,4 +23,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'Use your Twilio `Account SID` and `Auth Token` to build Twilio tasks right into your Dagster pipeline.\n\n### Installation\n\n```bash\npip install dagster-twilio\n```\n\n### Example\n\n\n\n### About Twilio\n\n**Twilio** provides communication APIs for phone calls, text messages, and other communication functions.'; + 'Use your Twilio `Account SID` and `Auth Token` to build Twilio tasks right into your Dagster pipeline.\n\n### Installation\n\n```bash\npip install dagster-twilio\n```\n\n### Example\n\n\n```python\n# Read the docs on Resources to learn more: https://docs.dagster.io/deployment/resources\nfrom dagster_twilio import TwilioResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef twilio_message(twilio: TwilioResource):\n twilio.get_client().messages.create(\n to="+15551234567", from_="+15558901234", body="Hello world!"\n )\n\n\ndefs = dg.Definitions(\n assets=[twilio_message],\n resources={\n "twilio": TwilioResource(\n account_sid=dg.EnvVar("TWILIO_ACCOUNT_SID"),\n auth_token=dg.EnvVar("TWILIO_AUTH_TOKEN"),\n )\n },\n)\n```\n \n\n### About Twilio\n\n**Twilio** provides communication APIs for phone calls, text messages, and other communication functions.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/weaviate.ts b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/weaviate.ts index 8f8217bc3f4f0..51a167134fbe8 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/weaviate.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/integrations/__generated__/weaviate.ts @@ -24,4 +24,4 @@ export const frontmatter: IntegrationFrontmatter = { }; export const content = - 'The `dagster-weaviate` library allows you to easily interact with Weaviate\'s vector database capabilities to build AI-powered data pipelines in Dagster. You can perform vector similarity searches, manage schemas, and handle data operations directly from your Dagster assets.\n\n### Installation\n\n```bash\npip install dagster dagster-weaviate\n```\n\n### Examples\n\n\n\n### About Weaviate\n\n**Weaviate** is an open-source vector database that enables you to store and manage vector embeddings at scale. You can start with a small dataset and scale up as your needs grow. This enables you to build powerful AI applications with semantic search and similarity matching capabilities. Weaviate offers fast query performance using vector-based search and GraphQL APIs, making it a powerful tool for AI-powered applications and machine learning workflows.'; + 'The `dagster-weaviate` library allows you to easily interact with Weaviate\'s vector database capabilities to build AI-powered data pipelines in Dagster. You can perform vector similarity searches, manage schemas, and handle data operations directly from your Dagster assets.\n\n### Installation\n\n```bash\npip install dagster dagster-weaviate\n```\n\n### Examples\n\n\n```python\nfrom dagster_weaviate import CloudConfig, WeaviateResource\n\nimport dagster as dg\n\n\n@dg.asset\ndef my_table(weaviate: WeaviateResource):\n with weaviate.get_client() as weaviate_client:\n questions = weaviate_client.collections.get("Question")\n questions.query.near_text(query="biology", limit=2)\n\n\ndefs = dg.Definitions(\n assets=[my_table],\n resources={\n "weaviate": WeaviateResource(\n connection_config=CloudConfig(cluster_url=dg.EnvVar("WCD_URL")),\n auth_credentials={"api_key": dg.EnvVar("WCD_API_KEY")},\n headers={\n "X-Cohere-Api-Key": dg.EnvVar("COHERE_API_KEY"),\n },\n ),\n },\n)\n```\n \n\n### About Weaviate\n\n**Weaviate** is an open-source vector database that enables you to store and manage vector embeddings at scale. You can start with a small dataset and scale up as your needs grow. This enables you to build powerful AI applications with semantic search and similarity matching capabilities. Weaviate offers fast query performance using vector-based search and GraphQL APIs, making it a powerful tool for AI-powered applications and machine learning workflows.'; diff --git a/js_modules/dagster-ui/packages/ui-core/src/scripts/generateIntegrationDocs.ts b/js_modules/dagster-ui/packages/ui-core/src/scripts/generateIntegrationDocs.ts index 06c0ced10ee46..68c276b3e5e83 100644 --- a/js_modules/dagster-ui/packages/ui-core/src/scripts/generateIntegrationDocs.ts +++ b/js_modules/dagster-ui/packages/ui-core/src/scripts/generateIntegrationDocs.ts @@ -9,9 +9,13 @@ import {IntegrationFrontmatter} from '../integrations/types'; const PATH_TO_INTEGRATION_DOCS = path.resolve('../../../../docs/docs/integrations/libraries'); const PATH_TO_INTEGRATION_LOGOS = path.resolve('../../../../docs/static'); +const PATH_TO_EXAMPLES = path.resolve('../../../../examples'); const OUTPUT_TARGET_DIR = path.resolve('./src/integrations/__generated__'); const OUTPUT_TARGET_LOGOS_DIR = path.join(OUTPUT_TARGET_DIR, 'logos'); +const CODE_EXAMPLE_PATH_REGEX = + /<(?:(?:CodeExample)|(?:CliInvocationExample))\s+[^>]*path=["']([^"']+)["'][^>]*language=["']([^"']+)["'][^>]*>/g; + function camelize(str: string) { return str.replace(/-([a-z])/g, (_, letter) => letter.toUpperCase()); } @@ -104,7 +108,33 @@ async function main() { } const outputPath = path.join(OUTPUT_TARGET_DIR, `${camelizedFileName}.ts`); - const content = String(file); + let content = String(file).trim(); + + const codeExampleMatches = []; + let foundMatches; + while ((foundMatches = CODE_EXAMPLE_PATH_REGEX.exec(content)) !== null) { + const [fullMatch, filePath, language] = foundMatches; + codeExampleMatches.push({fullMatch, filePath, language}); + } + + for (const {fullMatch, filePath, language} of codeExampleMatches) { + if (filePath) { + const codeFromFile = await fs.promises.readFile( + path.join(PATH_TO_EXAMPLES, filePath), + 'utf8', + ); + + content = content.replace( + fullMatch, + ` +\`\`\`${language} +${codeFromFile.trim()} +\`\`\` + `, + ); + } + } + const output = ` ${GENERATION_DISCLAIMER}