@@ -147,35 +147,42 @@ def _set_file(content: bytes):
147147 return filename
148148
149149
150+ def refine_value_previewable (value : Any ):
151+ if type (value ) == bytes :
152+ if len (value ) > 0 :
153+ try :
154+ local_path = _set_file (value )
155+ return f"file://{ local_path } "
156+ except ValueError :
157+ return f"<bytes>"
158+ else :
159+ return ""
160+ elif type (value ) == dict :
161+ if value .get ("bytes" ):
162+ try :
163+ local_path = _set_file (value ["bytes" ])
164+ return f"file://{ local_path } "
165+ except ValueError :
166+ return str (value )
167+ else :
168+ return str (value )
169+ elif type (value ) == str :
170+ if any (
171+ value .startswith (prefix )
172+ for prefix in ["s3://" , "hf://" , "http://" , "https://" ]
173+ ):
174+ return get_url (value )
175+ elif torch and isinstance (value , torch .Tensor ):
176+ return f"<torch.Tensor shape={ value .shape } dtype={ value .dtype } >"
177+ elif isinstance (value , np .ndarray ):
178+ return f"<numpy.ndarray shape={ value .shape } dtype={ value .dtype } >"
179+
180+ return value
181+
182+
150183def refine_sample_previewable (sample : dict [str , Any ]):
151184 for key in sample .keys ():
152- if type (sample [key ]) == bytes :
153- if len (sample [key ]) > 0 :
154- local_path = _set_file (sample [key ])
155- sample [key ] = f"file://{ local_path } "
156- # sample[key] = "<bytes>"
157- else :
158- sample [key ] = ""
159- if type (sample [key ]) == dict :
160- if sample [key ].get ("bytes" ):
161- local_path = _set_file (sample [key ]["bytes" ])
162- sample [key ] = f"file://{ local_path } "
163- else :
164- sample [key ] = str (sample [key ])
165- if type (sample [key ]) == str :
166- if any (
167- sample [key ].startswith (prefix )
168- for prefix in ["s3://" , "hf://" , "http://" , "https://" ]
169- ):
170- sample [key ] = get_url (sample [key ])
171- if torch and isinstance (sample [key ], torch .Tensor ):
172- sample [key ] = (
173- f"<torch.Tensor shape={ sample [key ].shape } dtype={ sample [key ].dtype } >"
174- )
175- if isinstance (sample [key ], np .ndarray ):
176- sample [key ] = (
177- f"<numpy.ndarray shape={ sample [key ].shape } dtype={ sample [key ].dtype } >"
178- )
185+ sample [key ] = refine_value_previewable (sample [key ])
179186 return sample
180187
181188
0 commit comments