Skip to content

Commit 20c029c

Browse files
KontinuationKontinuation
committed
Enable geospatial tests for parquet reader/writer, add more tests
1 parent f4fa79b commit 20c029c

File tree

11 files changed

+289
-29
lines changed

11 files changed

+289
-29
lines changed

‎api/src/test/java/org/apache/iceberg/util/RandomUtil.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020

2121
import java.math.BigDecimal;
2222
import java.math.BigInteger;
23+
import java.nio.ByteBuffer;
24+
import java.nio.ByteOrder;
2325
import java.util.Arrays;
2426
import java.util.List;
2527
import java.util.Map;
@@ -157,6 +159,19 @@ public static Object generatePrimitive(Type.PrimitiveType primitive, Random rand
157159
BigDecimal bigDecimal = new BigDecimal(unscaled, type.scale());
158160
return negate(choice) ? bigDecimal.negate() : bigDecimal;
159161

162+
case GEOMETRY:
163+
case GEOGRAPHY:
164+
// Generate a random point in range [0, 10) for both x and y coordinates
165+
double coordX = random.nextDouble() * 10;
166+
double coordY = random.nextDouble() * 10;
167+
ByteBuffer buffer = ByteBuffer.allocate(21);
168+
buffer.order(ByteOrder.LITTLE_ENDIAN);
169+
buffer.put((byte) 1); // Byte order (1 for Little Endian)
170+
buffer.putInt(1); // Geometry type (1 for Point)
171+
buffer.putDouble(coordX);
172+
buffer.putDouble(coordY);
173+
return buffer.flip();
174+
160175
default:
161176
throw new IllegalArgumentException(
162177
"Cannot generate random value for unknown type: " + primitive);

‎core/src/test/java/org/apache/iceberg/InternalTestHelpers.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,8 @@ private static void assertEquals(Type type, Object expected, Object actual) {
9191
case FIXED:
9292
case BINARY:
9393
case DECIMAL:
94+
case GEOMETRY:
95+
case GEOGRAPHY:
9496
assertThat(actual).as("Primitive value should be equal to expected").isEqualTo(expected);
9597
break;
9698
case STRUCT:

‎data/src/test/java/org/apache/iceberg/data/DataTestHelpers.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,8 @@ private static void assertEquals(Type type, Object expected, Object actual) {
120120
case UUID:
121121
case BINARY:
122122
case DECIMAL:
123+
case GEOMETRY:
124+
case GEOGRAPHY:
123125
assertThat(actual)
124126
.as("Primitive value should be equal to expected for type " + type)
125127
.isEqualTo(expected);

‎data/src/test/java/org/apache/iceberg/data/parquet/TestGenericData.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,11 @@ protected boolean supportsRowLineage() {
6868
return true;
6969
}
7070

71+
@Override
72+
protected boolean supportsGeospatial() {
73+
return true;
74+
}
75+
7176
@Override
7277
protected void writeAndValidate(Schema schema) throws IOException {
7378
writeAndValidate(schema, schema);

‎data/src/test/java/org/apache/iceberg/data/parquet/TestGeographyReadersAndWriters.java‎

Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -34,14 +34,13 @@
3434
import org.apache.iceberg.Schema;
3535
import org.apache.iceberg.Table;
3636
import org.apache.iceberg.TableProperties;
37-
import org.apache.iceberg.Tables;
37+
import org.apache.iceberg.TestTables;
3838
import org.apache.iceberg.data.GenericAppenderFactory;
3939
import org.apache.iceberg.data.GenericRecord;
4040
import org.apache.iceberg.data.IcebergGenerics;
4141
import org.apache.iceberg.data.Record;
4242
import org.apache.iceberg.hadoop.HadoopInputFile;
4343
import org.apache.iceberg.hadoop.HadoopOutputFile;
44-
import org.apache.iceberg.hadoop.HadoopTables;
4544
import org.apache.iceberg.io.CloseableIterable;
4645
import org.apache.iceberg.io.OutputFile;
4746
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
@@ -61,7 +60,6 @@
6160
public class TestGeographyReadersAndWriters {
6261
private final Schema schema;
6362
private static final Configuration CONF = new Configuration();
64-
private static final Tables TABLES = new HadoopTables(CONF);
6563

6664
@TempDir Path tempDir;
6765

@@ -97,15 +95,15 @@ private List<Record> prepareTestData() {
9795
@Test
9896
public void testWriteAndReadGeometryValues() throws IOException, ParseException {
9997
// Create a table
100-
File location = tempDir.resolve("geog-table-1").toFile();
98+
File location = tempDir.toFile();
10199
Table table =
102-
TABLES.create(
100+
TestTables.create(
101+
location,
102+
"geog_table",
103103
schema,
104104
PartitionSpec.unpartitioned(),
105-
ImmutableMap.of(
106-
TableProperties.FORMAT_VERSION, "3",
107-
TableProperties.DEFAULT_FILE_FORMAT, "parquet"),
108-
location.toString());
105+
3,
106+
ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, "parquet"));
109107

110108
// Write some data
111109
GenericAppenderFactory appenderFactory = new GenericAppenderFactory(table.schema());

‎data/src/test/java/org/apache/iceberg/data/parquet/TestGeometryReadersAndWriters.java‎

Lines changed: 17 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@
3838
import org.apache.iceberg.Schema;
3939
import org.apache.iceberg.Table;
4040
import org.apache.iceberg.TableProperties;
41-
import org.apache.iceberg.Tables;
41+
import org.apache.iceberg.TestTables;
4242
import org.apache.iceberg.data.GenericAppenderFactory;
4343
import org.apache.iceberg.data.GenericRecord;
4444
import org.apache.iceberg.data.IcebergGenerics;
@@ -49,7 +49,6 @@
4949
import org.apache.iceberg.geospatial.GeospatialBound;
5050
import org.apache.iceberg.hadoop.HadoopInputFile;
5151
import org.apache.iceberg.hadoop.HadoopOutputFile;
52-
import org.apache.iceberg.hadoop.HadoopTables;
5352
import org.apache.iceberg.io.CloseableIterable;
5453
import org.apache.iceberg.io.CloseableIterator;
5554
import org.apache.iceberg.io.FileAppender;
@@ -67,9 +66,10 @@ public class TestGeometryReadersAndWriters {
6766

6867
private final Schema schema;
6968
private static final Configuration CONF = new Configuration();
70-
private static final Tables TABLES = new HadoopTables(CONF);
7169
private final List<List<Record>> testData;
7270

71+
@TempDir java.nio.file.Path tempDir;
72+
7373
public TestGeometryReadersAndWriters() {
7474
this.schema =
7575
new Schema(
@@ -119,17 +119,16 @@ private List<List<Record>> prepareTestData() {
119119
}
120120

121121
@Test
122-
public void testFilterTableWithSpatialPredicates(@TempDir java.nio.file.Path tempDir)
123-
throws IOException {
124-
File location = tempDir.resolve("geos-table-2").toFile();
122+
public void testFilterTableWithSpatialPredicates() throws IOException {
123+
File location = tempDir.toFile();
125124
Table table =
126-
TABLES.create(
125+
TestTables.create(
126+
location,
127+
"geom_table",
127128
schema,
128129
PartitionSpec.unpartitioned(),
129-
ImmutableMap.of(
130-
TableProperties.FORMAT_VERSION, "3",
131-
TableProperties.DEFAULT_FILE_FORMAT, "parquet"),
132-
location.toString());
130+
3,
131+
ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, "parquet"));
133132

134133
AppendFiles append = table.newAppend();
135134
for (int i = 0; i < testData.size(); i++) {
@@ -174,17 +173,17 @@ public void testFilterTableWithSpatialPredicates(@TempDir java.nio.file.Path tem
174173
}
175174

176175
@Test
177-
public void testPartitionedGeometryTable(@TempDir java.nio.file.Path tempDir) throws IOException {
178-
File location = tempDir.resolve("geos-table-2-partitioned").toFile();
176+
public void testPartitionedGeometryTable() throws IOException {
177+
File location = tempDir.toFile();
179178
PartitionSpec spec = PartitionSpec.builderFor(schema).identity("part").build();
180179
Table table =
181-
TABLES.create(
180+
TestTables.create(
181+
location,
182+
"test_partitioned",
182183
schema,
183184
spec,
184-
ImmutableMap.of(
185-
TableProperties.FORMAT_VERSION, "3",
186-
TableProperties.DEFAULT_FILE_FORMAT, "parquet"),
187-
location.toString());
185+
3,
186+
ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, "parquet"));
188187

189188
AppendFiles append = table.newAppend();
190189
for (int i = 0; i < testData.size(); i++) {

‎data/src/test/java/org/apache/iceberg/data/parquet/TestParquetEncryptionWithWriteSupport.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,11 @@ protected boolean supportsVariant() {
7171
return true;
7272
}
7373

74+
@Override
75+
protected boolean supportsGeospatial() {
76+
return true;
77+
}
78+
7479
@Override
7580
protected void writeAndValidate(Schema schema) throws IOException {
7681
List<Record> expected = RandomGenericData.generate(schema, 100, 0L);

‎parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetrics.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -386,7 +386,7 @@ private FieldMetrics<ByteBuffer> geospatialBounds(
386386
}
387387

388388
BoundingBox boundingBox = geoStats.getBoundingBox();
389-
if (boundingBox == null || !boundingBox.isValid()) {
389+
if (boundingBox == null || !boundingBox.isValid() || boundingBox.isXYEmpty()) {
390390
isBoundValid = false;
391391
continue;
392392
}

‎parquet/src/main/java/org/apache/iceberg/parquet/ParquetMetricsRowGroupFilter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -581,7 +581,7 @@ public <T> Boolean stIntersects(BoundReference<T> ref, Literal<BoundingBox> lit)
581581

582582
org.apache.parquet.column.statistics.geospatial.BoundingBox boundingBox =
583583
colGeoStats.getBoundingBox();
584-
if (boundingBox == null || !boundingBox.isValid()) {
584+
if (boundingBox == null || !boundingBox.isValid() || boundingBox.isXYEmpty()) {
585585
// No valid geospatial bounds, we cannot make any assumptions about the geospatial data.
586586
return ROWS_MIGHT_MATCH;
587587
}

0 commit comments

Comments
 (0)