Skip to content

Commit 98a55bd

Browse files
committed
fix: refine_value_previewable handle error on _set_file
1 parent 4e3bcde commit 98a55bd

File tree

2 files changed

+128
-54
lines changed

2 files changed

+128
-54
lines changed

lavender_data/server/dataset/preview.py

Lines changed: 34 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -147,35 +147,42 @@ def _set_file(content: bytes):
147147
return filename
148148

149149

150+
def refine_value_previewable(value: Any):
151+
if type(value) == bytes:
152+
if len(value) > 0:
153+
try:
154+
local_path = _set_file(value)
155+
return f"file://{local_path}"
156+
except ValueError:
157+
return f"<bytes>"
158+
else:
159+
return ""
160+
elif type(value) == dict:
161+
if value.get("bytes"):
162+
try:
163+
local_path = _set_file(value["bytes"])
164+
return f"file://{local_path}"
165+
except ValueError:
166+
return str(value)
167+
else:
168+
return str(value)
169+
elif type(value) == str:
170+
if any(
171+
value.startswith(prefix)
172+
for prefix in ["s3://", "hf://", "http://", "https://"]
173+
):
174+
return get_url(value)
175+
elif torch and isinstance(value, torch.Tensor):
176+
return f"<torch.Tensor shape={value.shape} dtype={value.dtype}>"
177+
elif isinstance(value, np.ndarray):
178+
return f"<numpy.ndarray shape={value.shape} dtype={value.dtype}>"
179+
180+
return value
181+
182+
150183
def refine_sample_previewable(sample: dict[str, Any]):
151184
for key in sample.keys():
152-
if type(sample[key]) == bytes:
153-
if len(sample[key]) > 0:
154-
local_path = _set_file(sample[key])
155-
sample[key] = f"file://{local_path}"
156-
# sample[key] = "<bytes>"
157-
else:
158-
sample[key] = ""
159-
if type(sample[key]) == dict:
160-
if sample[key].get("bytes"):
161-
local_path = _set_file(sample[key]["bytes"])
162-
sample[key] = f"file://{local_path}"
163-
else:
164-
sample[key] = str(sample[key])
165-
if type(sample[key]) == str:
166-
if any(
167-
sample[key].startswith(prefix)
168-
for prefix in ["s3://", "hf://", "http://", "https://"]
169-
):
170-
sample[key] = get_url(sample[key])
171-
if torch and isinstance(sample[key], torch.Tensor):
172-
sample[key] = (
173-
f"<torch.Tensor shape={sample[key].shape} dtype={sample[key].dtype}>"
174-
)
175-
if isinstance(sample[key], np.ndarray):
176-
sample[key] = (
177-
f"<numpy.ndarray shape={sample[key].shape} dtype={sample[key].dtype}>"
178-
)
185+
sample[key] = refine_value_previewable(sample[key])
179186
return sample
180187

181188

poetry.lock

Lines changed: 94 additions & 27 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)