Skip to content

Commit 24955ad

Browse files
oskar-szwajkowskioskar-szwajkowski
committed
Add more columns to Iceberg $files system table
Add file_sequence_number, data_sequence_number, referenced_data_file, pos, manifest_location, first_row_id, content_offset, and content_size_in_bytes columns to the Iceberg $files system table. These columns expose additional metadata from Iceberg manifest entries, including delete-file-specific field like referenced_data_file and content_offset which are populated for deletion vectors.
1 parent 5a1c279 commit 24955ad

File tree

4 files changed

+130
-5
lines changed

4 files changed

+130
-5
lines changed

‎plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/system/FilesTable.java‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,14 @@ public final class FilesTable
7373
public static final String EQUALITY_IDS_COLUMN_NAME = "equality_ids";
7474
public static final String SORT_ORDER_ID_COLUMN_NAME = "sort_order_id";
7575
public static final String READABLE_METRICS_COLUMN_NAME = "readable_metrics";
76+
public static final String FILE_SEQUENCE_NUMBER_COLUMN_NAME = "file_sequence_number";
77+
public static final String DATA_SEQUENCE_NUMBER_COLUMN_NAME = "data_sequence_number";
78+
public static final String REFERENCED_DATA_FILE_COLUMN_NAME = "referenced_data_file";
79+
public static final String POS_COLUMN_NAME = "pos";
80+
public static final String MANIFEST_LOCATION_COLUMN_NAME = "manifest_location";
81+
public static final String FIRST_ROW_ID_COLUMN_NAME = "first_row_id";
82+
public static final String CONTENT_OFFSET_COLUMN_NAME = "content_offset";
83+
public static final String CONTENT_SIZE_IN_BYTES_COLUMN_NAME = "content_size_in_bytes";
7684

7785
private static final List<String> COLUMN_NAMES = ImmutableList.of(
7886
CONTENT_COLUMN_NAME,
@@ -92,7 +100,15 @@ public final class FilesTable
92100
SPLIT_OFFSETS_COLUMN_NAME,
93101
EQUALITY_IDS_COLUMN_NAME,
94102
SORT_ORDER_ID_COLUMN_NAME,
95-
READABLE_METRICS_COLUMN_NAME);
103+
READABLE_METRICS_COLUMN_NAME,
104+
FILE_SEQUENCE_NUMBER_COLUMN_NAME,
105+
DATA_SEQUENCE_NUMBER_COLUMN_NAME,
106+
REFERENCED_DATA_FILE_COLUMN_NAME,
107+
POS_COLUMN_NAME,
108+
MANIFEST_LOCATION_COLUMN_NAME,
109+
FIRST_ROW_ID_COLUMN_NAME,
110+
CONTENT_OFFSET_COLUMN_NAME,
111+
CONTENT_SIZE_IN_BYTES_COLUMN_NAME);
96112

97113
private final ConnectorTableMetadata tableMetadata;
98114
private final Table icebergTable;
@@ -159,9 +175,17 @@ public static Type getColumnType(String columnName, TypeManager typeManager)
159175
SORT_ORDER_ID_COLUMN_NAME,
160176
SPEC_ID_COLUMN_NAME -> INTEGER;
161177
case FILE_PATH_COLUMN_NAME,
162-
FILE_FORMAT_COLUMN_NAME -> VARCHAR;
178+
FILE_FORMAT_COLUMN_NAME,
179+
REFERENCED_DATA_FILE_COLUMN_NAME,
180+
MANIFEST_LOCATION_COLUMN_NAME -> VARCHAR;
163181
case RECORD_COUNT_COLUMN_NAME,
164-
FILE_SIZE_IN_BYTES_COLUMN_NAME -> BIGINT;
182+
FILE_SIZE_IN_BYTES_COLUMN_NAME,
183+
FILE_SEQUENCE_NUMBER_COLUMN_NAME,
184+
DATA_SEQUENCE_NUMBER_COLUMN_NAME,
185+
POS_COLUMN_NAME,
186+
FIRST_ROW_ID_COLUMN_NAME,
187+
CONTENT_OFFSET_COLUMN_NAME,
188+
CONTENT_SIZE_IN_BYTES_COLUMN_NAME -> BIGINT;
165189
case COLUMN_SIZES_COLUMN_NAME,
166190
NULL_VALUE_COUNTS_COLUMN_NAME,
167191
VALUE_COUNTS_COLUMN_NAME,

‎plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/system/files/FilesTablePageSource.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import io.trino.spi.type.RowType;
3333
import io.trino.spi.type.TypeManager;
3434
import org.apache.iceberg.ContentFile;
35+
import org.apache.iceberg.DeleteFile;
3536
import org.apache.iceberg.ManifestReader;
3637
import org.apache.iceberg.MetricsUtil;
3738
import org.apache.iceberg.PartitionField;
@@ -261,6 +262,30 @@ public SourcePage getNextSourcePage()
261262
// readable_metrics
262263
writeValueOrNull(pageBuilder, READABLE_METRICS_COLUMN_NAME, () -> metadataSchema.findField(MetricsUtil.READABLE_METRICS),
263264
(blkBldr, value) -> VARCHAR.writeString(blkBldr, readableMetricsToJson(readableMetricsStruct(schema, contentFile, value.type().asStructType()), primitiveFields)));
265+
// file_sequence_number
266+
writeValueOrNull(pageBuilder, FilesTable.FILE_SEQUENCE_NUMBER_COLUMN_NAME, contentFile::fileSequenceNumber, BIGINT::writeLong);
267+
// data_sequence_number
268+
writeValueOrNull(pageBuilder, FilesTable.DATA_SEQUENCE_NUMBER_COLUMN_NAME, contentFile::dataSequenceNumber, BIGINT::writeLong);
269+
if (contentFile instanceof DeleteFile deleteFile) {
270+
// referenced_data_file (DeleteFile specific)
271+
writeValueOrNull(pageBuilder, FilesTable.REFERENCED_DATA_FILE_COLUMN_NAME, deleteFile::referencedDataFile, VARCHAR::writeString);
272+
// content_offset (DeleteFile specific, v3+)
273+
writeValueOrNull(pageBuilder, FilesTable.CONTENT_OFFSET_COLUMN_NAME, deleteFile::contentOffset, BIGINT::writeLong);
274+
// content_size_in_bytes (DeleteFile specific, v3+)
275+
writeValueOrNull(pageBuilder, FilesTable.CONTENT_SIZE_IN_BYTES_COLUMN_NAME, deleteFile::contentSizeInBytes, BIGINT::writeLong);
276+
}
277+
else {
278+
// For non-delete files, these columns should be null
279+
writeValueOrNull(pageBuilder, FilesTable.REFERENCED_DATA_FILE_COLUMN_NAME, () -> null, VARCHAR::writeString);
280+
writeValueOrNull(pageBuilder, FilesTable.CONTENT_OFFSET_COLUMN_NAME, () -> null, BIGINT::writeLong);
281+
writeValueOrNull(pageBuilder, FilesTable.CONTENT_SIZE_IN_BYTES_COLUMN_NAME, () -> null, BIGINT::writeLong);
282+
}
283+
// pos
284+
writeValueOrNull(pageBuilder, FilesTable.POS_COLUMN_NAME, contentFile::pos, BIGINT::writeLong);
285+
// manifest_location
286+
writeValueOrNull(pageBuilder, FilesTable.MANIFEST_LOCATION_COLUMN_NAME, contentFile::manifestLocation, VARCHAR::writeString);
287+
// first_row_id
288+
writeValueOrNull(pageBuilder, FilesTable.FIRST_ROW_ID_COLUMN_NAME, contentFile::firstRowId, BIGINT::writeLong);
264289
readTimeNanos += System.nanoTime() - start;
265290
}
266291

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV2.java‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1123,7 +1123,13 @@ public void testFilesTable()
11231123
"CAST(upper_bounds AS JSON), " +
11241124
"key_metadata, " +
11251125
"split_offsets, " +
1126-
"equality_ids " +
1126+
"equality_ids, " +
1127+
"file_sequence_number, " +
1128+
"data_sequence_number, " +
1129+
"referenced_data_file, " +
1130+
"first_row_id, " +
1131+
"content_offset, " +
1132+
"content_size_in_bytes " +
11271133
"FROM \"" + tableName + "$files\"",
11281134
"""
11291135
VALUES
@@ -1138,6 +1144,12 @@ public void testFilesTable()
11381144
JSON '{"1":"24","2":"VIETNAM","3":"4","4":"y final packaget"}',
11391145
null,
11401146
ARRAY[4L],
1147+
null,
1148+
1L,
1149+
1L,
1150+
null,
1151+
null,
1152+
null,
11411153
null),
11421154
(0,
11431155
'ORC',
@@ -1150,6 +1162,12 @@ public void testFilesTable()
11501162
JSON '{"1":"4"}',
11511163
X'54 72 69 6e 6f',
11521164
ARRAY[4L],
1165+
null,
1166+
2L,
1167+
2L,
1168+
null,
1169+
null,
1170+
null,
11531171
null),
11541172
(2,
11551173
'PARQUET',
@@ -1162,8 +1180,17 @@ public void testFilesTable()
11621180
JSON '{"3":"1"}',
11631181
null,
11641182
ARRAY[4],
1165-
ARRAY[3])
1183+
ARRAY[3],
1184+
3L,
1185+
3L,
1186+
null,
1187+
null,
1188+
null,
1189+
null)
11661190
""");
1191+
// Verify columns with unpredictable exact values
1192+
assertThat(query("SELECT bool_and(manifest_location IS NOT NULL), bool_and(pos IS NOT NULL) FROM \"" + tableName + "$files\""))
1193+
.matches("VALUES (true, true)");
11671194
}
11681195

11691196
@Test

‎plugin/trino-iceberg/src/test/java/io/trino/plugin/iceberg/TestIcebergV3.java‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1358,6 +1358,30 @@ void testIcebergWritesAndTrinoReadsDeletionVector()
13581358
assertThat(query("SELECT * FROM " + tableName))
13591359
.matches("VALUES (1), (2), (4)");
13601360

1361+
// Verify new columns for data files: delete-specific columns are NULL
1362+
assertThat(query(
1363+
"SELECT referenced_data_file, content_offset, content_size_in_bytes, " +
1364+
"file_sequence_number IS NOT NULL, data_sequence_number IS NOT NULL, " +
1365+
"manifest_location IS NOT NULL, pos IS NOT NULL " +
1366+
"FROM \"" + tableName + "$files\" WHERE content = 0"))
1367+
.matches("VALUES (CAST(NULL AS VARCHAR), CAST(NULL AS BIGINT), CAST(NULL AS BIGINT), true, true, true, true)");
1368+
1369+
// Verify new columns for deletion vector: referenced_data_file matches the data file, content_offset and content_size_in_bytes are set
1370+
assertThat(query(
1371+
"SELECT referenced_data_file IS NOT NULL, content_offset IS NOT NULL, " +
1372+
"content_size_in_bytes IS NOT NULL AND content_size_in_bytes > 0, " +
1373+
"file_sequence_number IS NOT NULL, data_sequence_number IS NOT NULL, " +
1374+
"manifest_location IS NOT NULL, pos IS NOT NULL " +
1375+
"FROM \"" + tableName + "$files\" WHERE content = 1"))
1376+
.matches("VALUES (true, true, true, true, true, true, true)");
1377+
1378+
// Verify referenced_data_file for the DV matches the data file path
1379+
assertThat(query(
1380+
"SELECT dv.referenced_data_file = df.file_path " +
1381+
"FROM \"" + tableName + "$files\" dv " +
1382+
"JOIN \"" + tableName + "$files\" df ON dv.content = 1 AND df.content = 0"))
1383+
.matches("VALUES (true)");
1384+
13611385
assertUpdate("DROP TABLE " + tableName);
13621386
}
13631387

@@ -1394,6 +1418,31 @@ void testTrinoWritesAndReadsDeletionVector()
13941418
assertThat(query("SELECT sum(record_count), count(*), count_if(file_format = 'PUFFIN'), count(distinct file_path) FROM \"" + table.getName() + "$files\" WHERE content = 1"))
13951419
.matches("VALUES (BIGINT '500', BIGINT '10', BIGINT '10', BIGINT '1')");
13961420

1421+
// Verify new columns for DVs: delete-specific columns are set, sequence numbers and manifest_location present
1422+
assertThat(query(
1423+
"SELECT bool_and(referenced_data_file IS NOT NULL), " +
1424+
"bool_and(content_offset IS NOT NULL), " +
1425+
"bool_and(content_size_in_bytes IS NOT NULL AND content_size_in_bytes > 0), " +
1426+
"bool_and(file_sequence_number IS NOT NULL), " +
1427+
"bool_and(manifest_location IS NOT NULL) " +
1428+
"FROM \"" + table.getName() + "$files\" WHERE content = 1"))
1429+
.matches("VALUES (true, true, true, true, true)");
1430+
1431+
// Verify data files have NULL for delete-specific columns
1432+
assertThat(query(
1433+
"SELECT bool_and(referenced_data_file IS NULL), " +
1434+
"bool_and(content_offset IS NULL), " +
1435+
"bool_and(content_size_in_bytes IS NULL) " +
1436+
"FROM \"" + table.getName() + "$files\" WHERE content = 0"))
1437+
.matches("VALUES (true, true, true)");
1438+
1439+
// Verify referenced_data_file for each DV matches an actual data file path
1440+
assertThat(query(
1441+
"SELECT count(*) FROM \"" + table.getName() + "$files\" dv " +
1442+
"WHERE dv.content = 1 AND dv.referenced_data_file IN " +
1443+
"(SELECT file_path FROM \"" + table.getName() + "$files\" WHERE content = 0)"))
1444+
.matches("SELECT count(*) FROM \"" + table.getName() + "$files\" WHERE content = 1");
1445+
13971446
// delete multiples of 5 => 100 rows removed
13981447
assertUpdate("DELETE FROM " + table.getName() + " WHERE id % 5 = 0", 100);
13991448

0 commit comments

Comments
 (0)