Remove default spark_version from DatabricksResource (#57)

This is a breaking change that requires users to explicitly specify the
Databricks runtime version when creating a DatabricksResource. Previously,
the spark_version defaulted to "11.3.x-scala2.12".

This change:
- Removes the default value for spark_version field in DatabricksResource
- Updates pyproject.toml version from 2.10.2 to 3.0.0 (major version bump)
- Updates all tests to explicitly provide spark_version parameter

BREAKING CHANGE: Users must now explicitly provide spark_version when
instantiating DatabricksResource.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude <[email protected]>

Author: jhamet93

block_cascade/executors/databricks/resource.py

@@ -107,7 +107,6 @@ class DatabricksResource(BaseModel):
     spark_version: str
         Databricks runtime version. Tested on 11.3.x-scala2.12.
         https://docs.databricks.com/release-notes/runtime/releases.html
-        Default is 11.3.x-scala2.12
     data_security_mode: Optional[str]
         See `data_security_mode` at
         https://docs.databricks.com/administration-guide/clusters/policies.html#cluster-policy-attribute-paths
@@ -159,7 +158,7 @@ class DatabricksResource(BaseModel):
     storage_location: str
     worker_count: Union[int, DatabricksAutoscaleConfig] = 1
     machine: str = "i3.xlarge"
-    spark_version: str = "11.3.x-scala2.12"
+    spark_version: str
     data_security_mode: Optional[str] = "SINGLE_USER"
     cluster_spec_overrides: Optional[dict] = None
     cluster_policy: Optional[str] = None

pyproject.toml

@@ -3,7 +3,7 @@ name = "block-cascade"
 packages = [
     {include = "block_cascade"}
 ]
-version = "2.10.2"
+version = "3.0.0"
 description = "Library for model training in multi-cloud environment."
 readme = "README.md"
 authors = ["Block"]

tests/executors/databricks/resource/test_python_library.py

@@ -32,6 +32,7 @@ def test_databricks_resource_string_library_conversion():
 
     resource = DatabricksResource(
         storage_location="s3://test-bucket/cascade",
+        spark_version="11.3.x-scala2.12",
         python_libraries=["test-package"]
     )
 
@@ -41,8 +42,9 @@ def test_databricks_resource_string_library_conversion():
 
     resource = DatabricksResource(
         storage_location="s3://test-bucket/cascade",
+        spark_version="11.3.x-scala2.12",
         python_libraries=[
-            "package1", 
+            "package1",
             DatabricksPythonLibrary(name="package2", version="1.0.0")
         ]
     )
@@ -60,6 +62,7 @@ def test_databricks_resource_string_with_version_conversion():
 
     resource = DatabricksResource(
         storage_location="s3://test-bucket/cascade",
+        spark_version="11.3.x-scala2.12",
         python_libraries=["cloudpickle==0.10.0"]
     )
 
@@ -70,6 +73,7 @@ def test_databricks_resource_string_with_version_conversion():
 
     resource = DatabricksResource(
         storage_location="s3://test-bucket/cascade",
+        spark_version="11.3.x-scala2.12",
         python_libraries=[
             "numpy==1.22.4",
             "pandas==2.0.0",

tests/test_config.py

@@ -67,6 +67,7 @@ def databricks_resource():
         storage_location="s3://test-bucket/cascade",
         worker_count=DatabricksAutoscaleConfig(min_workers=5, max_workers=10),
         cloud_pickle_by_value=["a", "b"],
+        spark_version="11.3.x-scala2.12",
     )
 
 
@@ -120,13 +121,14 @@ def test_databricks_resource(
     configuration = f"""
 {test_job_name}:
     type: DatabricksResource
-    storage_location: {databricks_resource.storage_location}    
+    storage_location: {databricks_resource.storage_location}
     worker_count:
         min_workers: {databricks_resource.worker_count.min_workers}
         max_workers: {databricks_resource.worker_count.max_workers}
     cloud_pickle_by_value:
         - a
         - b
+    spark_version: {databricks_resource.spark_version}
 """
     fs.create_file(configuration_filename, contents=configuration)
     assert databricks_resource == find_default_configuration()[test_job_name]

tests/test_databricks_executor.py

@@ -19,6 +19,7 @@
 databricks_resource = DatabricksResource(
     storage_location="s3://test-bucket/cascade",
     group_name=DATABRICKS_GROUP,
+    spark_version="11.3.x-scala2.12",
 )