combined geojson and geojson_multi readers

Author: mjohns-databricks

src/main/resources/META-INF/services/org.apache.spark.sql.sources.DataSourceRegister

@@ -3,6 +3,5 @@ com.databricks.labs.gbx.vectorx.ds.ogr.OGR_DataSource
 com.databricks.labs.gbx.vectorx.ds.shp.ShapeFile_DataSource
 com.databricks.labs.gbx.vectorx.ds.gdb.FileGDB_DataSource
 com.databricks.labs.gbx.vectorx.ds.geojson.GeoJSON_DataSource
-com.databricks.labs.gbx.vectorx.ds.geojson.GeoJSONMulti_DataSource
 com.databricks.labs.gbx.vectorx.ds.gpkg.GPKG_DataSource
 com.databricks.labs.gbx.ds.register.RegisterDataSource

src/main/scala/com/databricks/labs/gbx/ds/DataSourceExtras.scala

@@ -6,15 +6,17 @@ import scala.jdk.CollectionConverters._
 
 trait DataSourceExtras {
 
-    def dsExtraMap(): Map[String, String]
+    def dsExtraMap(checkMap: Map[String, String] = Map.empty): Map[String, String]
 
     def extraJavaUtilMap(properties: java.util.Map[String, String]): java.util.Map[String, String] = {
-        val newProperties = properties.asScala.toMap ++ dsExtraMap()
-        newProperties.asJava
+        val cMap = properties.asScala.toMap
+        val newMap = cMap ++ dsExtraMap(checkMap = cMap)
+        newMap.asJava
     }
 
     def extraCaseInsensitiveStringMap(options: CaseInsensitiveStringMap): CaseInsensitiveStringMap = {
-        val newMap = options.asCaseSensitiveMap().asScala.toMap ++ dsExtraMap()
+        val cMap = options.asCaseSensitiveMap().asScala.toMap
+        val newMap = cMap ++ dsExtraMap(checkMap = cMap)
         new CaseInsensitiveStringMap(newMap.asJava)
     }
 

src/main/scala/com/databricks/labs/gbx/vectorx/ds/gdb/FileGDB_DataSource.scala

@@ -10,7 +10,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
 //noinspection ScalaUnusedSymbol
 class FileGDB_DataSource extends OGR_DataSource with DataSourceExtras {
 
-    override def dsExtraMap(): Map[String, String] = Map(
+    override def dsExtraMap(checkMap: Map[String, String] = Map.empty): Map[String, String] = Map(
         "driverName" -> "OpenFileGDB"
     )
 

src/main/scala/com/databricks/labs/gbx/vectorx/ds/geojson/GeoJSONMulti_DataSource.scala

@@ -11,9 +11,14 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
 //noinspection ScalaUnusedSymbol
 class GeoJSON_DataSource extends OGR_DataSource with DataSourceExtras{
 
-    override def dsExtraMap(): Map[String, String] = Map(
-        "driverName" -> "GeoJSON"
-    )
+    // default to multi = true given common use
+    override def dsExtraMap(checkMap: Map[String, String] = Map.empty): Map[String, String] = {
+        if (checkMap.getOrElse("multi", "true").toBoolean) {
+            Map("driverName" -> "GeoJSONSeq")
+        } else {
+            Map("driverName" -> "GeoJSON")
+        }
+    }
 
     override def shortName(): String = "geojson"
 

src/main/scala/com/databricks/labs/gbx/vectorx/ds/geojson/GeoJSON_DataSource.scala

@@ -11,7 +11,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
 //noinspection ScalaUnusedSymbol
 class GPKG_DataSource extends OGR_DataSource with DataSourceExtras{
 
-    override def dsExtraMap(): Map[String, String] = Map(
+    override def dsExtraMap(checkMap: Map[String, String] = Map.empty): Map[String, String] = Map(
         "driverName" -> "GPKG"
     )
 

src/main/scala/com/databricks/labs/gbx/vectorx/ds/gpkg/GPKG_DataSource.scala

@@ -11,7 +11,7 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
 //noinspection ScalaUnusedSymbol
 class ShapeFile_DataSource extends OGR_DataSource with DataSourceExtras{
 
-    override def dsExtraMap(): Map[String, String] = Map(
+    override def dsExtraMap(checkMap: Map[String, String] = Map.empty): Map[String, String] = Map(
         "driverName" -> "ESRI Shapefile"
     )
 

src/main/scala/com/databricks/labs/gbx/vectorx/ds/shp/ShapeFile_DataSource.scala

@@ -63,15 +63,16 @@ class OGR_DataSourceTest extends PlanTest with SilentSparkSession {
 
         val res_gj = spark.read
             .format("geojson")
+            .option("multi", "false")
             .load(gjPath)
 
         res_gj.count() shouldEqual 1L // single geom
 
         val res_gj1 = spark.read
-            .format("geojson_multi")
+            .format("geojson")
             .load(gjPath)
 
-        res_gj1.count() should be > 1L // individual geoms
+        res_gj1.count() should be > 1L // newline geoms
 
         // gpkg (zip fails)