Skip to content

Commit ec17543

Browse files
authored
Log examples arrow failure (#3645)
Follow up: #3482 - when arrow conversion fails - print a couple of records.
1 parent b100de4 commit ec17543

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

lib/zephyr/src/zephyr/execution.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -396,10 +396,14 @@ def _write_stage_chunks(
396396
use_parquet = True
397397
logger.info("Using Parquet for scatter serialization for shard %d", source_shard)
398398
except Exception:
399+
sample_rows = str(test_envelope[:5]) if len(test_envelope) > 5 else str(test_envelope)
400+
if len(sample_rows) > 1000:
401+
sample_rows = sample_rows[:1000] + "...(truncated)"
399402
logger.warning(
400403
"Arrow scatter serialization failed for shard %d; "
401-
"falling back to pickle. Performance will be degraded.",
404+
"falling back to pickle. Performance will be degraded. Sample rows: %s",
402405
source_shard,
406+
sample_rows,
403407
exc_info=True,
404408
)
405409

0 commit comments

Comments
 (0)