Skip to content

Commit bc12bcf

Browse files
YK-03YK-03
committed
fix: Ensure 'end' phase file_status events are always emitted
1 parent a9c76f7 commit bc12bcf

2 files changed

Lines changed: 125 additions & 122 deletions

File tree

‎src/borg/archiver/__init__.py‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ def print_warning_instance(self, warning):
156156
self.print_warning(msg, *args, wc=wc, wt="curly", msgid=msgid)
157157

158158
def print_file_status(self, status, path, *, phase=None, error=None):
159-
# START event (JSON only)
159+
# START lifecycle event (JSON only)
160160
if self.output_list and self.log_json and phase == "start" and status is None:
161161
json_data = {"type": "file_status", "phase": "start"}
162162
json_data |= text_to_json("path", path)
@@ -165,12 +165,23 @@ def print_file_status(self, status, path, *, phase=None, error=None):
165165
print(json.dumps(json_data), file=sys.stderr)
166166
return
167167

168-
# END event
168+
# END lifecycle event (JSON only)
169+
if self.output_list and self.log_json and phase == "end" and status is None:
170+
json_data = {"type": "file_status", "phase": "end"}
171+
json_data |= text_to_json("path", path)
172+
if error is not None:
173+
json_data["error"] = error
174+
# Always emit "end" so every "start" has a guaranteed closing pair
175+
print(json.dumps(json_data), file=sys.stderr)
176+
return
177+
178+
# regular status event (A, M, U, -, d, s, etc.)
169179
if self.output_list and status is not None and (self.output_filter is None or status in self.output_filter):
170180
if self.log_json:
171181
json_data = {"type": "file_status", "status": status}
172182
json_data |= text_to_json("path", path)
173-
json_data["phase"] = phase if phase is not None else "end"
183+
if phase is not None:
184+
json_data["phase"] = phase
174185
if error is not None:
175186
json_data["error"] = error
176187
print(json.dumps(json_data), file=sys.stderr)

‎src/borg/archiver/create_cmd.py‎

Lines changed: 111 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ def create_inner(archive, cache, fso):
9090
raise Error(f"{path!r}: {e}")
9191
else:
9292
status = "+" # included
93-
self.print_file_status(status, path, phase="end")
93+
self.print_file_status(status, path)
9494
elif args.paths_from_command or args.paths_from_shell_command or args.paths_from_stdin:
9595
paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n"
9696
if args.paths_from_command or args.paths_from_shell_command:
@@ -139,7 +139,7 @@ def create_inner(archive, cache, fso):
139139
status = "E"
140140
if status == "C":
141141
self.print_warning_instance(FileChangedWarning(path))
142-
self.print_file_status(status, path, phase="end")
142+
self.print_file_status(status, path)
143143
if not dry_run and status is not None:
144144
fso.stats.files_stats[status] += 1
145145
if args.paths_from_command or args.paths_from_shell_command:
@@ -167,7 +167,7 @@ def create_inner(archive, cache, fso):
167167
status = "E"
168168
else:
169169
status = "+" # included
170-
self.print_file_status(status, path, phase="end")
170+
self.print_file_status(status, path)
171171
if not dry_run and status is not None:
172172
fso.stats.files_stats[status] += 1
173173
continue
@@ -293,16 +293,17 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
293293
"""
294294
Call the right method on the given FilesystemObjectProcessor.
295295
"""
296-
297296
if dry_run:
298297
return "+" # included
298+
299299
# Types not archived: no list start/end pair (matches prior behavior of no status line).
300300
if stat.S_ISSOCK(st.st_mode):
301301
return
302302
elif stat.S_ISDOOR(st.st_mode):
303303
return
304304
elif stat.S_ISPORT(st.st_mode):
305305
return
306+
306307
m = st.st_mode
307308
if not (
308309
stat.S_ISREG(m)
@@ -314,134 +315,125 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
314315
):
315316
self.print_warning("Unknown file type: %s", path)
316317
return
317-
MAX_RETRIES = 10 # count includes the initial try (initial try == "retry 0")
318-
for retry in range(MAX_RETRIES):
319-
last_try = retry == MAX_RETRIES - 1
320-
try:
321-
if stat.S_ISREG(st.st_mode):
322-
if retry == 0:
323-
self.print_file_status(None, path, phase="start")
324-
return fso.process_file(
325-
path=path,
326-
parent_fd=parent_fd,
327-
name=name,
328-
st=st,
329-
cache=cache,
330-
last_try=last_try,
331-
strip_prefix=strip_prefix,
332-
)
333-
elif stat.S_ISDIR(st.st_mode):
334-
if retry == 0:
335-
self.print_file_status(None, path, phase="start")
336-
return fso.process_dir(path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix)
337-
elif stat.S_ISLNK(st.st_mode):
338-
if not read_special:
339-
if retry == 0:
340-
self.print_file_status(None, path, phase="start")
341-
return fso.process_symlink(
342-
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
318+
319+
# Emit START once, before any processing, before the retry loop.
320+
self.print_file_status(None, path, phase="start")
321+
322+
try:
323+
MAX_RETRIES = 10 # count includes the initial try (initial try == "retry 0")
324+
for retry in range(MAX_RETRIES):
325+
last_try = retry == MAX_RETRIES - 1
326+
try:
327+
if stat.S_ISREG(st.st_mode):
328+
return fso.process_file(
329+
path=path,
330+
parent_fd=parent_fd,
331+
name=name,
332+
st=st,
333+
cache=cache,
334+
last_try=last_try,
335+
strip_prefix=strip_prefix,
343336
)
344-
else:
345-
try:
346-
st_target = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=True)
347-
except OSError:
348-
special = False
337+
elif stat.S_ISDIR(st.st_mode):
338+
return fso.process_dir(path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix)
339+
elif stat.S_ISLNK(st.st_mode):
340+
if not read_special:
341+
return fso.process_symlink(
342+
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
343+
)
344+
else:
345+
try:
346+
st_target = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=True)
347+
except OSError:
348+
special = False
349+
else:
350+
special = is_special(st_target.st_mode)
351+
if special:
352+
return fso.process_file(
353+
path=path,
354+
parent_fd=parent_fd,
355+
name=name,
356+
st=st_target,
357+
cache=cache,
358+
flags=flags_special_follow,
359+
last_try=last_try,
360+
strip_prefix=strip_prefix,
361+
)
362+
else:
363+
return fso.process_symlink(
364+
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
365+
)
366+
elif stat.S_ISFIFO(st.st_mode):
367+
if not read_special:
368+
return fso.process_fifo(
369+
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
370+
)
349371
else:
350-
special = is_special(st_target.st_mode)
351-
if special:
352372
return fso.process_file(
353373
path=path,
354374
parent_fd=parent_fd,
355375
name=name,
356-
st=st_target,
376+
st=st,
357377
cache=cache,
358-
flags=flags_special_follow,
378+
flags=flags_special,
359379
last_try=last_try,
360380
strip_prefix=strip_prefix,
361381
)
382+
elif stat.S_ISCHR(st.st_mode):
383+
if not read_special:
384+
return fso.process_dev(
385+
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="c", strip_prefix=strip_prefix
386+
)
362387
else:
363-
return fso.process_symlink(
364-
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
388+
return fso.process_file(
389+
path=path,
390+
parent_fd=parent_fd,
391+
name=name,
392+
st=st,
393+
cache=cache,
394+
flags=flags_special,
395+
last_try=last_try,
396+
strip_prefix=strip_prefix,
397+
)
398+
elif stat.S_ISBLK(st.st_mode):
399+
if not read_special:
400+
return fso.process_dev(
401+
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="b", strip_prefix=strip_prefix
402+
)
403+
else:
404+
return fso.process_file(
405+
path=path,
406+
parent_fd=parent_fd,
407+
name=name,
408+
st=st,
409+
cache=cache,
410+
flags=flags_special,
411+
last_try=last_try,
412+
strip_prefix=strip_prefix,
365413
)
366-
elif stat.S_ISFIFO(st.st_mode):
367-
if not read_special:
368-
if retry == 0:
369-
self.print_file_status(None, path, phase="start")
370-
return fso.process_fifo(
371-
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
372-
)
373-
else:
374-
return fso.process_file(
375-
path=path,
376-
parent_fd=parent_fd,
377-
name=name,
378-
st=st,
379-
cache=cache,
380-
flags=flags_special,
381-
last_try=last_try,
382-
strip_prefix=strip_prefix,
383-
)
384-
elif stat.S_ISCHR(st.st_mode):
385-
if not read_special:
386-
if retry == 0:
387-
self.print_file_status(None, path, phase="start")
388-
return fso.process_dev(
389-
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="c", strip_prefix=strip_prefix
390-
)
391414
else:
392-
return fso.process_file(
393-
path=path,
394-
parent_fd=parent_fd,
395-
name=name,
396-
st=st,
397-
cache=cache,
398-
flags=flags_special,
399-
last_try=last_try,
400-
strip_prefix=strip_prefix,
401-
)
402-
elif stat.S_ISBLK(st.st_mode):
403-
if not read_special:
404-
if retry == 0:
405-
self.print_file_status(None, path, phase="start")
406-
return fso.process_dev(
407-
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="b", strip_prefix=strip_prefix
415+
self.print_warning("Unknown file type: %s", path)
416+
return
417+
except BackupItemExcluded:
418+
return "-"
419+
except BackupError as err:
420+
if isinstance(err, BackupOSError):
421+
if err.errno in (errno.EPERM, errno.EACCES):
422+
raise
423+
sleep_s = 1000.0 / 1e6 * 10 ** (retry / 2)
424+
time.sleep(sleep_s)
425+
if retry < MAX_RETRIES - 1:
426+
logger.warning(
427+
f"{path}: {err}, slept {sleep_s:.3f}s, next: retry: {retry + 1} of {MAX_RETRIES - 1}..."
408428
)
409429
else:
410-
return fso.process_file(
411-
path=path,
412-
parent_fd=parent_fd,
413-
name=name,
414-
st=st,
415-
cache=cache,
416-
flags=flags_special,
417-
last_try=last_try,
418-
strip_prefix=strip_prefix,
419-
)
420-
else:
421-
self.print_warning("Unknown file type: %s", path)
422-
return
423-
except BackupItemExcluded:
424-
return "-"
425-
except BackupError as err:
426-
if isinstance(err, BackupOSError):
427-
if err.errno in (errno.EPERM, errno.EACCES):
428-
# Do not try again, such errors can not be fixed by retrying.
429430
raise
430-
# sleep a bit, so temporary problems might go away...
431-
sleep_s = 1000.0 / 1e6 * 10 ** (retry / 2) # retry 0: 1ms, retry 6: 1s, ...
432-
time.sleep(sleep_s)
433-
if retry < MAX_RETRIES - 1:
434-
logger.warning(
435-
f"{path}: {err}, slept {sleep_s:.3f}s, next: retry: {retry + 1} of {MAX_RETRIES - 1}..."
436-
)
437-
else:
438-
# giving up with retries, error will be dealt with (logged) by upper error handler
439-
raise
440-
# we better do a fresh stat on the file, just to make sure to get the current file
441-
# mode right (which could have changed due to a race condition and is important for
442-
# dispatching) and also to get current inode number of that file.
443-
with backup_io("stat"):
444-
st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
431+
with backup_io("stat"):
432+
st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
433+
finally:
434+
# END is always emitted here — after ALL processing including chunked I/O,
435+
# even on exception, even on retry exhaustion.
436+
self.print_file_status(None, path, phase="end")
445437

446438
def _rec_walk(
447439
self,
@@ -477,7 +469,7 @@ def _rec_walk(
477469
with backup_io("stat"):
478470
st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
479471
else:
480-
self.print_file_status("-", path, phase="end") # excluded
472+
self.print_file_status("-", path) # excluded
481473
# get out here as quickly as possible:
482474
# we only need to continue if we shall recurse into an excluded directory.
483475
# if we shall not recurse, then do not even touch (stat()) the item, it
@@ -548,7 +540,7 @@ def _rec_walk(
548540
dry_run=dry_run,
549541
strip_prefix=strip_prefix,
550542
)
551-
self.print_file_status("-", path, phase="end") # excluded
543+
self.print_file_status("-", path) # excluded
552544
return
553545
if not recurse_excluded_dir:
554546
if not dry_run:
@@ -589,7 +581,7 @@ def _rec_walk(
589581
if status == "C":
590582
self.print_warning_instance(FileChangedWarning(path))
591583
if not recurse_excluded_dir:
592-
self.print_file_status(status, path, phase="end")
584+
self.print_file_status(status, path)
593585
if not dry_run and status is not None:
594586
fso.stats.files_stats[status] += 1
595587

0 commit comments

Comments
 (0)