Remove change feed streaming scenarios from Databricks notebooks

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: xinlian12

sdk/cosmos/azure-cosmos-spark_3/test-databricks/notebooks/basicScenario.scala

@@ -111,43 +111,5 @@ df.filter(col("isAlive") === true)
 
 // COMMAND ----------
 
-// Change Feed - micro-batch structured streaming
-// This exercises the ChangeFeedInitialOffsetWriter and HDFSMetadataLog code paths
-// that can break on certain Spark distributions (e.g. Databricks Runtime 17.3+)
-
-val changeFeedCfg = Map("spark.cosmos.accountEndpoint" -> cosmosEndpoint,
-  "spark.cosmos.accountKey" -> cosmosMasterKey,
-  "spark.cosmos.database" -> cosmosDatabaseName,
-  "spark.cosmos.container" -> cosmosContainerName,
-  "spark.cosmos.read.inferSchema.enabled" -> "false",
-  "spark.cosmos.changeFeed.startFrom" -> "Beginning",
-  "spark.cosmos.changeFeed.mode" -> "Incremental",
-  "spark.cosmos.enforceNativeTransport" -> "true"
-)
-
-val testId = java.util.UUID.randomUUID().toString.replace("-", "")
-
-val changeFeedDF = spark
-  .readStream
-  .format("cosmos.oltp.changeFeed")
-  .options(changeFeedCfg)
-  .load()
-
-val microBatchQuery = changeFeedDF
-  .writeStream
-  .format("memory")
-  .queryName(testId)
-  .outputMode("append")
-  .start()
-
-microBatchQuery.processAllAvailable()
-microBatchQuery.stop()
-
-val sinkCount = spark.sql(s"SELECT * FROM $testId").count()
-println(s"Change Feed micro-batch streaming: $sinkCount records read via change feed")
-assert(sinkCount >= 2, s"Expected at least 2 records from change feed but found $sinkCount")
-
-// COMMAND ----------
-
 // cleanup
 spark.sql(s"DROP TABLE cosmosCatalog.${cosmosDatabaseName}.${cosmosContainerName};")

sdk/cosmos/azure-cosmos-spark_3/test-databricks/notebooks/basicScenarioAadManagedIdentity.scala

@@ -96,47 +96,5 @@ df.filter(col("isAlive") === true)
 
 // COMMAND ----------
 
-// Change Feed - micro-batch structured streaming
-// This exercises the ChangeFeedInitialOffsetWriter and HDFSMetadataLog code paths
-// that can break on certain Spark distributions (e.g. Databricks Runtime 17.3+)
-
-val changeFeedCfg = Map("spark.cosmos.accountEndpoint" -> cosmosEndpoint,
-  "spark.cosmos.auth.type" -> authType,
-  "spark.cosmos.account.subscriptionId" -> subscriptionId,
-  "spark.cosmos.account.tenantId" -> tenantId,
-  "spark.cosmos.account.resourceGroupName" -> resourceGroupName,
-  "spark.cosmos.database" -> cosmosDatabaseName,
-  "spark.cosmos.container" -> cosmosContainerName,
-  "spark.cosmos.read.inferSchema.enabled" -> "false",
-  "spark.cosmos.changeFeed.startFrom" -> "Beginning",
-  "spark.cosmos.changeFeed.mode" -> "Incremental",
-  "spark.cosmos.enforceNativeTransport" -> "true",
-  "spark.cosmos.read.consistencyStrategy" -> "LatestCommitted",
-)
-
-val testId = java.util.UUID.randomUUID().toString.replace("-", "")
-
-val changeFeedDF = spark
-  .readStream
-  .format("cosmos.oltp.changeFeed")
-  .options(changeFeedCfg)
-  .load()
-
-val microBatchQuery = changeFeedDF
-  .writeStream
-  .format("memory")
-  .queryName(testId)
-  .outputMode("append")
-  .start()
-
-microBatchQuery.processAllAvailable()
-microBatchQuery.stop()
-
-val sinkCount = spark.sql(s"SELECT * FROM $testId").count()
-println(s"Change Feed micro-batch streaming: $sinkCount records read via change feed")
-assert(sinkCount >= 2, s"Expected at least 2 records from change feed but found $sinkCount")
-
-// COMMAND ----------
-
 // cleanup
 spark.sql(s"DROP TABLE cosmosCatalogMI.${cosmosDatabaseName}.${cosmosContainerName};")