Skip to content

Commit d80724b

Browse files
YK-03YK-03
committed
Add phase field to file_status JSON output (start/end lifecycle events)
1 parent 6bc4146 commit d80724b

3 files changed

Lines changed: 276 additions & 116 deletions

File tree

‎src/borg/archiver/__init__.py‎

Lines changed: 23 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -155,14 +155,34 @@ def print_warning_instance(self, warning):
155155
msg, msgid, args, wc = cls.__doc__, cls.__qualname__, warning.args, warning.exit_code
156156
self.print_warning(msg, *args, wc=wc, wt="curly", msgid=msgid)
157157

158-
def print_file_status(self, status, path):
159-
# if we get called with status == None, the final file status was already printed
158+
def print_file_status(self, status, path, *, phase=None, error=None):
159+
# START lifecycle event (JSON only)
160+
if self.output_list and self.log_json and phase == "start" and status is None:
161+
json_data = {"type": "file_status", "phase": "start"}
162+
json_data |= text_to_json("path", path)
163+
if error is not None:
164+
json_data["error"] = error
165+
print(json.dumps(json_data), file=sys.stderr)
166+
return
167+
# END lifecycle event (JSON only)
168+
if self.output_list and self.log_json and phase == "end" and status is None:
169+
json_data = {"type": "file_status", "phase": "end"}
170+
json_data |= text_to_json("path", path)
171+
if error is not None:
172+
json_data["error"] = error
173+
print(json.dumps(json_data), file=sys.stderr)
174+
return
175+
# regular status event (A, M, U, -, d, s, etc.)
160176
if self.output_list and status is not None and (self.output_filter is None or status in self.output_filter):
161177
if self.log_json:
162178
json_data = {"type": "file_status", "status": status}
163179
json_data |= text_to_json("path", path)
180+
if phase is not None:
181+
json_data["phase"] = phase
182+
if error is not None:
183+
json_data["error"] = error
164184
print(json.dumps(json_data), file=sys.stderr)
165-
else:
185+
elif not self.log_json and status is not None:
166186
logging.getLogger("borg.output.list").info("%1s %s", status, remove_surrogates(path))
167187

168188
def preprocess_args(self, args):

‎src/borg/archiver/create_cmd.py‎

Lines changed: 126 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ def create_inner(archive, cache, fso):
9090
raise Error(f"{path!r}: {e}")
9191
else:
9292
status = "+" # included
93-
self.print_file_status(status, path)
93+
self.print_file_status(status, path, phase="end")
9494
elif args.paths_from_command or args.paths_from_shell_command or args.paths_from_stdin:
9595
paths_sep = eval_escapes(args.paths_delimiter) if args.paths_delimiter is not None else "\n"
9696
if args.paths_from_command or args.paths_from_shell_command:
@@ -139,7 +139,7 @@ def create_inner(archive, cache, fso):
139139
status = "E"
140140
if status == "C":
141141
self.print_warning_instance(FileChangedWarning(path))
142-
self.print_file_status(status, path)
142+
self.print_file_status(status, path, phase="end")
143143
if not dry_run and status is not None:
144144
fso.stats.files_stats[status] += 1
145145
if args.paths_from_command or args.paths_from_shell_command:
@@ -167,7 +167,7 @@ def create_inner(archive, cache, fso):
167167
status = "E"
168168
else:
169169
status = "+" # included
170-
self.print_file_status(status, path)
170+
self.print_file_status(status, path, phase="end")
171171
if not dry_run and status is not None:
172172
fso.stats.files_stats[status] += 1
173173
continue
@@ -293,134 +293,147 @@ def _process_any(self, *, path, parent_fd, name, st, fso, cache, read_special, d
293293
"""
294294
Call the right method on the given FilesystemObjectProcessor.
295295
"""
296-
297296
if dry_run:
298297
return "+" # included
299-
MAX_RETRIES = 10 # count includes the initial try (initial try == "retry 0")
300-
for retry in range(MAX_RETRIES):
301-
last_try = retry == MAX_RETRIES - 1
302-
try:
303-
if stat.S_ISREG(st.st_mode):
304-
return fso.process_file(
305-
path=path,
306-
parent_fd=parent_fd,
307-
name=name,
308-
st=st,
309-
cache=cache,
310-
last_try=last_try,
311-
strip_prefix=strip_prefix,
312-
)
313-
elif stat.S_ISDIR(st.st_mode):
314-
return fso.process_dir(path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix)
315-
elif stat.S_ISLNK(st.st_mode):
316-
if not read_special:
317-
return fso.process_symlink(
318-
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
298+
299+
# Types not archived: no list start/end pair (matches prior behavior of no status line).
300+
if stat.S_ISSOCK(st.st_mode):
301+
return
302+
elif stat.S_ISDOOR(st.st_mode):
303+
return
304+
elif stat.S_ISPORT(st.st_mode):
305+
return
306+
307+
m = st.st_mode
308+
if not (
309+
stat.S_ISREG(m)
310+
or stat.S_ISDIR(m)
311+
or stat.S_ISLNK(m)
312+
or stat.S_ISFIFO(m)
313+
or stat.S_ISCHR(m)
314+
or stat.S_ISBLK(m)
315+
):
316+
self.print_warning("Unknown file type: %s", path)
317+
return
318+
319+
# Emit START once, before any processing, before the retry loop.
320+
self.print_file_status(None, path, phase="start")
321+
322+
try:
323+
MAX_RETRIES = 10 # count includes the initial try (initial try == "retry 0")
324+
for retry in range(MAX_RETRIES):
325+
last_try = retry == MAX_RETRIES - 1
326+
try:
327+
if stat.S_ISREG(st.st_mode):
328+
return fso.process_file(
329+
path=path,
330+
parent_fd=parent_fd,
331+
name=name,
332+
st=st,
333+
cache=cache,
334+
last_try=last_try,
335+
strip_prefix=strip_prefix,
319336
)
320-
else:
321-
try:
322-
st_target = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=True)
323-
except OSError:
324-
special = False
337+
elif stat.S_ISDIR(st.st_mode):
338+
return fso.process_dir(path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix)
339+
elif stat.S_ISLNK(st.st_mode):
340+
if not read_special:
341+
return fso.process_symlink(
342+
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
343+
)
344+
else:
345+
try:
346+
st_target = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=True)
347+
except OSError:
348+
special = False
349+
else:
350+
special = is_special(st_target.st_mode)
351+
if special:
352+
return fso.process_file(
353+
path=path,
354+
parent_fd=parent_fd,
355+
name=name,
356+
st=st_target,
357+
cache=cache,
358+
flags=flags_special_follow,
359+
last_try=last_try,
360+
strip_prefix=strip_prefix,
361+
)
362+
else:
363+
return fso.process_symlink(
364+
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
365+
)
366+
elif stat.S_ISFIFO(st.st_mode):
367+
if not read_special:
368+
return fso.process_fifo(
369+
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
370+
)
325371
else:
326-
special = is_special(st_target.st_mode)
327-
if special:
328372
return fso.process_file(
329373
path=path,
330374
parent_fd=parent_fd,
331375
name=name,
332-
st=st_target,
376+
st=st,
333377
cache=cache,
334-
flags=flags_special_follow,
378+
flags=flags_special,
335379
last_try=last_try,
336380
strip_prefix=strip_prefix,
337381
)
382+
elif stat.S_ISCHR(st.st_mode):
383+
if not read_special:
384+
return fso.process_dev(
385+
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="c", strip_prefix=strip_prefix
386+
)
338387
else:
339-
return fso.process_symlink(
340-
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
388+
return fso.process_file(
389+
path=path,
390+
parent_fd=parent_fd,
391+
name=name,
392+
st=st,
393+
cache=cache,
394+
flags=flags_special,
395+
last_try=last_try,
396+
strip_prefix=strip_prefix,
397+
)
398+
elif stat.S_ISBLK(st.st_mode):
399+
if not read_special:
400+
return fso.process_dev(
401+
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="b", strip_prefix=strip_prefix
402+
)
403+
else:
404+
return fso.process_file(
405+
path=path,
406+
parent_fd=parent_fd,
407+
name=name,
408+
st=st,
409+
cache=cache,
410+
flags=flags_special,
411+
last_try=last_try,
412+
strip_prefix=strip_prefix,
341413
)
342-
elif stat.S_ISFIFO(st.st_mode):
343-
if not read_special:
344-
return fso.process_fifo(
345-
path=path, parent_fd=parent_fd, name=name, st=st, strip_prefix=strip_prefix
346-
)
347-
else:
348-
return fso.process_file(
349-
path=path,
350-
parent_fd=parent_fd,
351-
name=name,
352-
st=st,
353-
cache=cache,
354-
flags=flags_special,
355-
last_try=last_try,
356-
strip_prefix=strip_prefix,
357-
)
358-
elif stat.S_ISCHR(st.st_mode):
359-
if not read_special:
360-
return fso.process_dev(
361-
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="c", strip_prefix=strip_prefix
362-
)
363414
else:
364-
return fso.process_file(
365-
path=path,
366-
parent_fd=parent_fd,
367-
name=name,
368-
st=st,
369-
cache=cache,
370-
flags=flags_special,
371-
last_try=last_try,
372-
strip_prefix=strip_prefix,
373-
)
374-
elif stat.S_ISBLK(st.st_mode):
375-
if not read_special:
376-
return fso.process_dev(
377-
path=path, parent_fd=parent_fd, name=name, st=st, dev_type="b", strip_prefix=strip_prefix
415+
self.print_warning("Unknown file type: %s", path)
416+
return
417+
except BackupItemExcluded:
418+
return "-"
419+
except BackupError as err:
420+
if isinstance(err, BackupOSError):
421+
if err.errno in (errno.EPERM, errno.EACCES):
422+
raise
423+
sleep_s = 1000.0 / 1e6 * 10 ** (retry / 2)
424+
time.sleep(sleep_s)
425+
if retry < MAX_RETRIES - 1:
426+
logger.warning(
427+
f"{path}: {err}, slept {sleep_s:.3f}s, next: retry: {retry + 1} of {MAX_RETRIES - 1}..."
378428
)
379429
else:
380-
return fso.process_file(
381-
path=path,
382-
parent_fd=parent_fd,
383-
name=name,
384-
st=st,
385-
cache=cache,
386-
flags=flags_special,
387-
last_try=last_try,
388-
strip_prefix=strip_prefix,
389-
)
390-
elif stat.S_ISSOCK(st.st_mode):
391-
# Ignore unix sockets
392-
return
393-
elif stat.S_ISDOOR(st.st_mode):
394-
# Ignore Solaris doors
395-
return
396-
elif stat.S_ISPORT(st.st_mode):
397-
# Ignore Solaris event ports
398-
return
399-
else:
400-
self.print_warning("Unknown file type: %s", path)
401-
return
402-
except BackupItemExcluded:
403-
return "-"
404-
except BackupError as err:
405-
if isinstance(err, BackupOSError):
406-
if err.errno in (errno.EPERM, errno.EACCES):
407-
# Do not try again, such errors can not be fixed by retrying.
408430
raise
409-
# sleep a bit, so temporary problems might go away...
410-
sleep_s = 1000.0 / 1e6 * 10 ** (retry / 2) # retry 0: 1ms, retry 6: 1s, ...
411-
time.sleep(sleep_s)
412-
if retry < MAX_RETRIES - 1:
413-
logger.warning(
414-
f"{path}: {err}, slept {sleep_s:.3f}s, next: retry: {retry + 1} of {MAX_RETRIES - 1}..."
415-
)
416-
else:
417-
# giving up with retries, error will be dealt with (logged) by upper error handler
418-
raise
419-
# we better do a fresh stat on the file, just to make sure to get the current file
420-
# mode right (which could have changed due to a race condition and is important for
421-
# dispatching) and also to get current inode number of that file.
422-
with backup_io("stat"):
423-
st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
431+
with backup_io("stat"):
432+
st = os_stat(path=path, parent_fd=parent_fd, name=name, follow_symlinks=False)
433+
finally:
434+
# END is always emitted here — after ALL processing including chunked I/O,
435+
# even on exception, even on retry exhaustion.
436+
self.print_file_status(None, path, phase="end")
424437

425438
def _rec_walk(
426439
self,

0 commit comments

Comments
 (0)