@@ -33,20 +33,27 @@ def _task_sort_key(x):
3333 return str (created_at )
3434
3535
36- def _dir_paths_from_ls (entries ) -> list [str ]:
36+ async def _dir_paths_from_ls (entries ) -> list [str ]:
3737 """Filter `storage.ls(detail=True)` output to directory paths only.
3838
39- Skips files like `.DS_Store` so they don't reach `get_metadata()`,
40- which would otherwise log a spurious error per cruft entry.
39+ Trusts fsspec's `type` field, with a defensive `storage.isdir` fallback
40+ if it's missing — matches the pattern in
41+ `migrate_tasks_to_experiment_dirs.py`. Skips files like `.DS_Store` so
42+ they don't reach `get_metadata()`.
4143 """
4244 paths : list [str ] = []
4345 for entry in entries :
4446 if not isinstance (entry , dict ):
4547 continue
46- if entry .get ("type" ) != "directory" :
48+ full = entry .get ("name" )
49+ if not full :
4750 continue
48- full = entry .get ("name" ) or entry .get ("path" )
49- if full :
51+ entry_type = entry .get ("type" )
52+ try :
53+ is_dir = entry_type == "directory" or await storage .isdir (full )
54+ except Exception :
55+ is_dir = entry_type == "directory"
56+ if is_dir :
5057 paths .append (full )
5158 return paths
5259
@@ -60,7 +67,7 @@ async def _read(full):
6067 try :
6168 return await TaskTemplate (entry , experiment_id = experiment_id ).get_metadata ()
6269 except Exception :
63- logger .error (f"Exception getting metadata for task : { entry } " )
70+ logger .warning (f"Skipping task without readable metadata : { entry } " )
6471 return None
6572
6673 gathered = await asyncio .gather (* (_read (full ) for full in entries ))
@@ -183,7 +190,7 @@ async def list_all():
183190 logger .error (f"Exception listing task directory: { e } " )
184191 return []
185192
186- results = await _gather_task_metadata (_dir_paths_from_ls (entries ))
193+ results = await _gather_task_metadata (await _dir_paths_from_ls (entries ))
187194 results .sort (key = _task_sort_key , reverse = True )
188195 return results
189196
@@ -205,7 +212,7 @@ async def list_by_experiment(experiment_id: int):
205212 logger .error (f"Exception listing experiment task directory: { e } " )
206213 return []
207214
208- results = await _gather_task_metadata (_dir_paths_from_ls (entries ), experiment_id = str (experiment_id ))
215+ results = await _gather_task_metadata (await _dir_paths_from_ls (entries ), experiment_id = str (experiment_id ))
209216 results .sort (key = _task_sort_key , reverse = True )
210217 return results
211218
0 commit comments