Skip to content

add improved error messaging in Worker class #741

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 10, 2025
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions osbenchmark/worker_coordinator/worker_coordinator.py
Original file line number Diff line number Diff line change
Expand Up @@ -1125,11 +1125,31 @@ def receiveMsg_WakeupMessage(self, msg, sender):
elif self.executor_future is not None and self.executor_future.done():
e = self.executor_future.exception(timeout=0)
if e:
self.logger.exception("Worker[%s] has detected a benchmark failure. Notifying master...",
str(self.worker_id), exc_info=e)
# the exception might be user-defined and not be on the load path of the master worker_coordinator. Hence, it cannot be
# deserialized on the receiver so we convert it here to a plain string.
self.send(self.master, actor.BenchmarkFailure("Error in load generator [{}]".format(self.worker_id), str(e)))
currentTasks = self.client_allocations.tasks(self.current_task_index)
detailed_error = (
f"Benchmark operation failed:\n"
f"Worker ID: {self.worker_id}\n"
f"Task: {', '.join(t.task.task.name for t in currentTasks)}\n"
f"Workload: {self.workload.name if self.workload else 'Unknown'}\n"
f"Test Procedure: {self.workload.selected_test_procedure_or_default}\n"
f"Cause: {e.cause if hasattr(e, 'cause') and e.cause is not None else 'Unknown'}"
)
detailed_error += f"\nError: {str(e)}"

self.logger.exception(
"Worker[%s] has detected a benchmark failure:\n%s",
str(self.worker_id),
detailed_error,
exc_info=e
)

self.send(
self.master,
actor.BenchmarkFailure(
detailed_error,
str(e)
)
)
else:
self.logger.info("Worker[%s] is ready for the next task.", str(self.worker_id))
self.executor_future = None
Expand Down
Loading