Skip to content

Commit abff078

Browse files
JingsongLiJingsongLi
committed
fix
1 parent 75c1735 commit abff078

File tree

5 files changed

+42
-42
lines changed

5 files changed

+42
-42
lines changed

‎paimon-python/pypaimon/tests/binary_row_test.py‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -112,10 +112,10 @@ def test_is_null_append(self):
112112

113113
def test_is_not_null_append(self):
114114
table = self.catalog.get_table('default.test_append')
115-
starting_scanner = FileScanner(table, lambda: [])
116-
latest_snapshot = starting_scanner.snapshot_manager.get_latest_snapshot()
117-
manifest_files = starting_scanner.manifest_list_manager.read_all(latest_snapshot)
118-
manifest_entries = starting_scanner.manifest_file_manager.read(manifest_files[0].file_name)
115+
file_scanner = FileScanner(table, lambda: [])
116+
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
117+
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
118+
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
119119
self._transform_manifest_entries(manifest_entries, [])
120120
l = ['abc', 'abbc', 'bc', 'd', None]
121121
for i, entry in enumerate(manifest_entries):
@@ -125,7 +125,7 @@ def test_is_not_null_append(self):
125125
GenericRow([l[i]], [table.fields[1]]),
126126
[1 if l[i] is None else 0],
127127
)
128-
starting_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)
128+
file_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)
129129

130130
read_builder = table.new_read_builder()
131131
predicate_builder = read_builder.new_predicate_builder()
@@ -249,10 +249,10 @@ def test_append_multi_cols(self):
249249
table_write.close()
250250
table_commit.close()
251251

252-
starting_scanner = FileScanner(table, lambda: [])
253-
latest_snapshot = starting_scanner.snapshot_manager.get_latest_snapshot()
254-
manifest_files = starting_scanner.manifest_list_manager.read_all(latest_snapshot)
255-
manifest_entries = starting_scanner.manifest_file_manager.read(manifest_files[0].file_name)
252+
file_scanner = FileScanner(table, lambda: [])
253+
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
254+
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
255+
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
256256
self._transform_manifest_entries(manifest_entries, [])
257257
for i, entry in enumerate(manifest_entries):
258258
entry.file.value_stats_cols = ['f2', 'f6', 'f8']
@@ -261,7 +261,7 @@ def test_append_multi_cols(self):
261261
GenericRow([10 * (i + 1), 100 * (i + 1), 5 - i], [table.fields[2], table.fields[6], table.fields[8]]),
262262
[0, 0, 0],
263263
)
264-
starting_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)
264+
file_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)
265265
# Build multiple predicates and combine them
266266
read_builder = table.new_read_builder()
267267
predicate_builder = read_builder.new_predicate_builder()
@@ -288,10 +288,10 @@ def test_append_multi_cols(self):
288288
}
289289
self.assertEqual(expected_data, actual.to_pydict())
290290

291-
starting_scanner = FileScanner(table, lambda: [])
292-
latest_snapshot = starting_scanner.snapshot_manager.get_latest_snapshot()
293-
manifest_files = starting_scanner.manifest_list_manager.read_all(latest_snapshot)
294-
manifest_entries = starting_scanner.manifest_file_manager.read(manifest_files[0].file_name)
291+
file_scanner = FileScanner(table, lambda: [])
292+
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
293+
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
294+
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
295295
self._transform_manifest_entries(manifest_entries, [])
296296
for i, entry in enumerate(manifest_entries):
297297
entry.file.value_stats_cols = ['f2', 'f6', 'f8']
@@ -300,7 +300,7 @@ def test_append_multi_cols(self):
300300
GenericRow([0, 100 * (i + 1), 5 - i], [table.fields[2], table.fields[6], table.fields[8]]),
301301
[0, 0, 0],
302302
)
303-
starting_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)
303+
file_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)
304304
splits, actual = self._read_result(read_builder.with_filter(combined))
305305
self.assertFalse(actual)
306306

@@ -319,10 +319,10 @@ def _transform_manifest_entries(self, manifest_entries: List[ManifestEntry], tri
319319
trimmed_pk_fields)
320320

321321
def _overwrite_manifest_entry(self, table):
322-
starting_scanner = FileScanner(table, lambda: [])
323-
latest_snapshot = starting_scanner.snapshot_manager.get_latest_snapshot()
324-
manifest_files = starting_scanner.manifest_list_manager.read_all(latest_snapshot)
325-
manifest_entries = starting_scanner.manifest_file_manager.read(manifest_files[0].file_name)
322+
file_scanner = FileScanner(table, lambda: [])
323+
latest_snapshot = file_scanner.snapshot_manager.get_latest_snapshot()
324+
manifest_files = file_scanner.manifest_list_manager.read_all(latest_snapshot)
325+
manifest_entries = file_scanner.manifest_file_manager.read(manifest_files[0].file_name)
326326
self._transform_manifest_entries(manifest_entries, [])
327327
for i, entry in enumerate(manifest_entries):
328328
entry.file.value_stats_cols = ['f2']
@@ -331,4 +331,4 @@ def _overwrite_manifest_entry(self, table):
331331
GenericRow([6 + i], [table.fields[2]]),
332332
[0],
333333
)
334-
starting_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)
334+
file_scanner.manifest_file_manager.write(manifest_files[0].file_name, manifest_entries)

‎paimon-python/pypaimon/tests/py36/rest_ao_read_write_test.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -181,10 +181,10 @@ def test_full_data_types(self):
181181

182182
# to test GenericRow ability
183183
latest_snapshot = SnapshotManager(table).get_latest_snapshot()
184-
manifest_files = table_scan.starting_scanner.manifest_list_manager.read_all(latest_snapshot)
185-
manifest_entries = table_scan.starting_scanner.manifest_file_manager.read(
184+
manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot)
185+
manifest_entries = table_scan.file_scanner.manifest_file_manager.read(
186186
manifest_files[0].file_name,
187-
lambda row: table_scan.starting_scanner._filter_manifest_entry(row),
187+
lambda row: table_scan.file_scanner._filter_manifest_entry(row),
188188
drop_stats=False)
189189
# Python write does not produce value stats
190190
if stats_enabled:

‎paimon-python/pypaimon/tests/reader_base_test.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -227,9 +227,9 @@ def test_full_data_types(self):
227227

228228
# to test GenericRow ability
229229
latest_snapshot = SnapshotManager(table).get_latest_snapshot()
230-
manifest_files = table_scan.starting_scanner.manifest_list_manager.read_all(latest_snapshot)
231-
manifest_entries = table_scan.starting_scanner.manifest_file_manager.read(
232-
manifest_files[0].file_name, lambda row: table_scan.starting_scanner._filter_manifest_entry(row), False)
230+
manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot)
231+
manifest_entries = table_scan.file_scanner.manifest_file_manager.read(
232+
manifest_files[0].file_name, lambda row: table_scan.file_scanner._filter_manifest_entry(row), False)
233233

234234
# Python write does not produce value stats
235235
if stats_enabled:
@@ -513,10 +513,10 @@ def test_primary_key_value_stats_excludes_system_fields(self):
513513
pk_read_builder = pk_table.new_read_builder()
514514
pk_table_scan = pk_read_builder.new_scan()
515515
latest_snapshot = SnapshotManager(pk_table).get_latest_snapshot()
516-
pk_manifest_files = pk_table_scan.starting_scanner.manifest_list_manager.read_all(latest_snapshot)
517-
pk_manifest_entries = pk_table_scan.starting_scanner.manifest_file_manager.read(
516+
pk_manifest_files = pk_table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot)
517+
pk_manifest_entries = pk_table_scan.file_scanner.manifest_file_manager.read(
518518
pk_manifest_files[0].file_name,
519-
lambda row: pk_table_scan.starting_scanner._filter_manifest_entry(row),
519+
lambda row: pk_table_scan.file_scanner._filter_manifest_entry(row),
520520
False
521521
)
522522

@@ -530,7 +530,7 @@ def test_primary_key_value_stats_excludes_system_fields(self):
530530
f"table.fields should NOT contain system fields, but got: {pk_table_field_names}")
531531

532532
if pk_file_meta.value_stats_cols is None:
533-
pk_value_stats_fields = pk_table_scan.starting_scanner.manifest_file_manager._get_value_stats_fields(
533+
pk_value_stats_fields = pk_table_scan.file_scanner.manifest_file_manager._get_value_stats_fields(
534534
{'_VALUE_STATS_COLS': None},
535535
pk_table.fields
536536
)
@@ -583,10 +583,10 @@ def test_value_stats_empty_when_stats_disabled(self):
583583
read_builder = table.new_read_builder()
584584
table_scan = read_builder.new_scan()
585585
latest_snapshot = SnapshotManager(table).get_latest_snapshot()
586-
manifest_files = table_scan.starting_scanner.manifest_list_manager.read_all(latest_snapshot)
587-
manifest_entries = table_scan.starting_scanner.manifest_file_manager.read(
586+
manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot)
587+
manifest_entries = table_scan.file_scanner.manifest_file_manager.read(
588588
manifest_files[0].file_name,
589-
lambda row: table_scan.starting_scanner._filter_manifest_entry(row),
589+
lambda row: table_scan.file_scanner._filter_manifest_entry(row),
590590
False
591591
)
592592

@@ -1090,10 +1090,10 @@ def test_primary_key_value_stats(self):
10901090
read_builder = table.new_read_builder()
10911091
table_scan = read_builder.new_scan()
10921092
latest_snapshot = SnapshotManager(table).get_latest_snapshot()
1093-
manifest_files = table_scan.starting_scanner.manifest_list_manager.read_all(latest_snapshot)
1094-
manifest_entries = table_scan.starting_scanner.manifest_file_manager.read(
1093+
manifest_files = table_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot)
1094+
manifest_entries = table_scan.file_scanner.manifest_file_manager.read(
10951095
manifest_files[0].file_name,
1096-
lambda row: table_scan.starting_scanner._filter_manifest_entry(row),
1096+
lambda row: table_scan.file_scanner._filter_manifest_entry(row),
10971097
False
10981098
)
10991099

@@ -1146,7 +1146,7 @@ def test_primary_key_value_stats(self):
11461146
self.assertFalse(is_system_field,
11471147
f"value_stats_cols should not contain system field: {field_name}")
11481148

1149-
value_stats_fields = table_scan.starting_scanner.manifest_file_manager._get_value_stats_fields(
1149+
value_stats_fields = table_scan.file_scanner.manifest_file_manager._get_value_stats_fields(
11501150
{'_VALUE_STATS_COLS': file_meta.value_stats_cols},
11511151
table.fields
11521152
)

‎paimon-python/pypaimon/tests/reader_primary_key_test.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -358,10 +358,10 @@ def test_manifest_creation_time_timestamp(self):
358358
latest_snapshot = snapshot_manager.get_latest_snapshot()
359359
read_builder = table.new_read_builder()
360360
table_scan = read_builder.new_scan()
361-
manifest_list_manager = table_scan.starting_scanner.manifest_list_manager
361+
manifest_list_manager = table_scan.file_scanner.manifest_list_manager
362362
manifest_files = manifest_list_manager.read_all(latest_snapshot)
363363

364-
manifest_file_manager = table_scan.starting_scanner.manifest_file_manager
364+
manifest_file_manager = table_scan.file_scanner.manifest_file_manager
365365
creation_times_found = []
366366
for manifest_file_meta in manifest_files:
367367
entries = manifest_file_manager.read(manifest_file_meta.file_name, drop_stats=False)

‎paimon-python/pypaimon/tests/schema_evolution_read_test.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -259,14 +259,14 @@ def test_schema_evolution_with_scan_filter(self):
259259
schema_manager.commit(TableSchema.from_schema(schema_id=0, schema=schema))
260260
schema_manager.commit(TableSchema.from_schema(schema_id=1, schema=schema2))
261261
# scan filter for schema evolution
262-
latest_snapshot = table1.new_read_builder().new_scan().starting_scanner.snapshot_manager.get_latest_snapshot()
262+
latest_snapshot = table1.new_read_builder().new_scan().file_scanner.snapshot_manager.get_latest_snapshot()
263263
table2.table_path = table1.table_path
264264
new_read_buidler = table2.new_read_builder()
265265
predicate_builder = new_read_buidler.new_predicate_builder()
266266
predicate = predicate_builder.less_than('user_id', 3)
267267
new_scan = new_read_buidler.with_filter(predicate).new_scan()
268-
manifest_files = new_scan.starting_scanner.manifest_list_manager.read_all(latest_snapshot)
269-
entries = new_scan.starting_scanner.read_manifest_entries(manifest_files)
268+
manifest_files = new_scan.file_scanner.manifest_list_manager.read_all(latest_snapshot)
269+
entries = new_scan.file_scanner.read_manifest_entries(manifest_files)
270270
self.assertEqual(1, len(entries)) # verify scan filter success for schema evolution
271271

272272
def test_schema_evolution_with_read_filter(self):

0 commit comments

Comments
 (0)