Skip to content

Issue Importing Cluster Using Instance Pool #115

@coreystokes-8451

Description

@coreystokes-8451

Hello, I have noticed an issue when importing a cluster that uses an instance pool:

2022-06-07 15:03:08 [INFO] │ Error: Conflicting configuration arguments
2022-06-07 15:03:08 [INFO] │
2022-06-07 15:03:08 [INFO] │   with databricks_cluster.databricks_cluster_0525_145639_efwf0ccj,
2022-06-07 15:03:08 [INFO] │   on databricks_cluster_0525_145639_efwf0ccj.tf.json line 13, in resource.databricks_cluster.databricks_cluster_0525_145639_efwf0ccj:
2022-06-07 15:03:08 [INFO] │   13:                 "driver_node_type_id": "${var.Standard_E8as_v4}",
2022-06-07 15:03:08 [INFO] │
2022-06-07 15:03:08 [INFO] │ "driver_node_type_id": conflicts with instance_pool_id
2022-06-07 15:03:08 [INFO] ╵
2022-06-07 15:03:08 [INFO] ╷
2022-06-07 15:03:08 [INFO] │ Error: Conflicting configuration arguments
2022-06-07 15:03:08 [INFO] │
2022-06-07 15:03:08 [INFO] │   with databricks_cluster.databricks_cluster_0525_145639_efwf0ccj,
2022-06-07 15:03:08 [INFO] │   on databricks_cluster_0525_145639_efwf0ccj.tf.json line 28, in resource.databricks_cluster.databricks_cluster_0525_145639_efwf0ccj:
2022-06-07 15:03:08 [INFO] │   28:                 "instance_pool_id": "${databricks_instance_pool.databricks_instance_pool_0525_144559_sorts3_pool_j050ndks.id}",
2022-06-07 15:03:08 [INFO] │
2022-06-07 15:03:08 [INFO] │ "instance_pool_id": conflicts with driver_node_type_id
2022-06-07 15:03:08 [INFO] ╵
2022-06-07 15:03:08 [INFO] ╷
2022-06-07 15:03:08 [INFO] │ Error: Conflicting configuration arguments
2022-06-07 15:03:08 [INFO] │
2022-06-07 15:03:08 [INFO] │   with databricks_cluster.databricks_cluster_0525_145639_efwf0ccj,
2022-06-07 15:03:08 [INFO] │   on databricks_cluster_0525_145639_efwf0ccj.tf.json line 29, in resource.databricks_cluster.databricks_cluster_0525_145639_efwf0ccj:
2022-06-07 15:03:08 [INFO] │   29:                 "node_type_id": "${var.Standard_E8as_v4}",

Here is the export json:

{
    "resource": {
        "databricks_cluster": {
            "databricks_cluster_0525_145639_efwf0ccj": {
                "autotermination_minutes": 30,
                "cluster_name": "test-cluster",
                "custom_tags": {
                    "ResourceClass": "SingleNode"
                },
                "depends_on": [
                    "databricks_global_init_script.databricks_global_init_scripts"
                ],
                "driver_node_type_id": "${var.Standard_E8as_v4}",
                "dynamic": [
                    {
                        "azure_attributes": {
                            "content": {
                                "availability": "SPOT_WITH_FALLBACK_AZURE",
                                "first_on_demand": 2147483647,
                                "spot_bid_max_price": -1.0
                            },
                            "for_each": "${upper(var.CLOUD) == \"AZURE\" ? [1] : []}"
                        }
                    }
                ],
                "enable_elastic_disk": "${upper(var.CLOUD) == \"AZURE\" ? true : true}",
                "enable_local_disk_encryption": false,
                "instance_pool_id": "${databricks_instance_pool.databricks_instance_pool_0525_144559_sorts3_pool_j050ndks.id}",
                "node_type_id": "${var.Standard_E8as_v4}",
                "num_workers": 0,
                "single_user_name": "[email protected]",
                "spark_conf": {
                    "spark.databricks.cluster.profile": "singleNode",
                    "spark.databricks.delta.preview.enabled": "true",
                    "spark.databricks.passthrough.enabled": "true",
                    "spark.master": "local[*, 4]"
                },
                "spark_env_vars": {
                    "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
                },
                "spark_version": "10.4.x-scala2.12"
            }
        }
    }
}

Here is the json of the cluster in Databricks:

{
    "num_workers": 0,
    "cluster_name": "test-cluster",
    "spark_version": "10.4.x-scala2.12",
    "spark_conf": {
        "spark.master": "local[*, 4]",
        "spark.databricks.cluster.profile": "singleNode"
    },
    "azure_attributes": {},
    "ssh_public_keys": [],
    "custom_tags": {
        "ResourceClass": "SingleNode"
    },
    "spark_env_vars": {
        "PYSPARK_PYTHON": "/databricks/python3/bin/python3"
    },
    "autotermination_minutes": 120,
    "cluster_source": "UI",
    "init_scripts": [],
    "instance_pool_id": "0607-154433-shaky52-pool-l08lynq8",
    "policy_id": "C9628FAD230000BE",
    "driver_instance_pool_id": "0607-154433-shaky52-pool-l08lynq8",
    "runtime_engine": "STANDARD"
}

Thanks

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions