Unify dictionaries

orf · orf · commit f18cf28277a2 · 2025-03-13T15:36:08.000Z
diff --git a/src/pypi_data/combine_parquet.py b/src/pypi_data/combine_parquet.py
@@ -111,7 +111,7 @@ async def fill_buffer(
         table = await asyncio.to_thread(
             lambda: pq.read_table(
                 pa.py_buffer(memoryview(dataset_bytes))
-            )#.combine_chunks()
+            ).unify_dictionaries()
         )
         del dataset_bytes
         log_system_stats(directory, message="Casting")

Original file line number	Diff line number	Diff line change
`@@ -111,7 +111,7 @@ async def fill_buffer(`
`111`	`111`	`table = await asyncio.to_thread(`
`112`	`112`	`lambda: pq.read_table(`
`113`	`113`	`pa.py_buffer(memoryview(dataset_bytes))`
`114`		`- )#.combine_chunks()`
	`114`	`+ ).unify_dictionaries()`
`115`	`115`	`)`
`116`	`116`	`del dataset_bytes`
`117`	`117`	`log_system_stats(directory, message="Casting")`