Skip to content

Commit eb58f45

Browse files
author
michalis1
committed
mmilaitis-fix-517-ltm
1 parent 7bad760 commit eb58f45

File tree

6 files changed

+98
-4
lines changed

6 files changed

+98
-4
lines changed

‎changes/517.fixed‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Added “fail job on task failure” so the job fails when one or more tasks fail.

‎nautobot_device_onboarding/jobs.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,11 @@ class Meta:
343343
required=False,
344344
description="Tenant to be applied to all synced devices.",
345345
)
346+
fail_job_on_task_failure = BooleanVar(
347+
description="If any tasks for any device fails, fail the entire job result.",
348+
required=False,
349+
default=False,
350+
)
346351

347352
template_name = "nautobot_device_onboarding/ssot_sync_devices.html"
348353

@@ -514,11 +519,13 @@ def run( # pylint: disable=too-many-positional-arguments
514519
ip_address_status=None,
515520
secrets_group=None,
516521
platform=None,
522+
fail_job_on_task_failure=None,
517523
):
518524
"""Run sync."""
519525
self.dryrun = dryrun
520526
self.memory_profiling = memory_profiling
521527
self.debug = debug
528+
self.fail_job_on_task_failure = fail_job_on_task_failure
522529

523530
if csv_file:
524531
self.ip_address_inventory = self._process_csv_data(csv_file=csv_file)
@@ -655,6 +662,11 @@ class Meta:
655662
required=False,
656663
description="Only update devices with the selected platform.",
657664
)
665+
fail_job_on_task_failure = BooleanVar(
666+
description="If any tasks for any device fails, fail the entire job result.",
667+
required=False,
668+
default=False,
669+
)
658670

659671
def load_source_adapter(self):
660672
"""Load network data adapter."""
@@ -686,6 +698,7 @@ def run( # pylint: disable=too-many-positional-arguments
686698
location=None,
687699
device_role=None,
688700
platform=None,
701+
fail_job_on_task_failure=None,
689702
):
690703
"""Run sync."""
691704
self.dryrun = dryrun
@@ -704,6 +717,7 @@ def run( # pylint: disable=too-many-positional-arguments
704717
self.location = location
705718
self.device_role = device_role
706719
self.platform = platform
720+
self.fail_job_on_task_failure = fail_job_on_task_failure
707721

708722
# Check for last_network_data_sync CustomField
709723
if self.debug:

‎nautobot_device_onboarding/nornir_plays/command_getter.py‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,8 @@ def netmiko_send_commands(task: Task, command_getter_yaml_data: Dict, command_ge
170170
if "Invalid input detected at" in current_result.result:
171171
task.results[result_idx].result = []
172172
task.results[result_idx].failed = False
173+
if nautobot_job.fail_job_on_task_failure:
174+
task.results[result_idx].failed = True
173175
else:
174176
if command["parser"] == "textfsm":
175177
try:
@@ -198,6 +200,10 @@ def netmiko_send_commands(task: Task, command_getter_yaml_data: Dict, command_ge
198200
task.results[result_idx].result = parsed_output
199201
task.results[result_idx].failed = False
200202
except Exception: # https://github.com/networktocode/ntc-templates/issues/369
203+
if nautobot_job.fail_job_on_task_failure:
204+
task.results[result_idx].result = []
205+
task.results[result_idx].failed = True
206+
raise
201207
try:
202208
if nautobot_job.debug:
203209
traceback_str = traceback.format_exc().replace("\n", "<br>")
@@ -226,6 +232,10 @@ def netmiko_send_commands(task: Task, command_getter_yaml_data: Dict, command_ge
226232
task.results[result_idx].result = json.loads(parsed_result)
227233
task.results[result_idx].failed = False
228234
except Exception:
235+
if nautobot_job.fail_job_on_task_failure:
236+
task.results[result_idx].result = []
237+
task.results[result_idx].failed = True
238+
raise
229239
task.results[result_idx].result = []
230240
task.results[result_idx].failed = False
231241
else:
@@ -239,8 +249,16 @@ def netmiko_send_commands(task: Task, command_getter_yaml_data: Dict, command_ge
239249
task.results[result_idx].result = jsonified
240250
task.results[result_idx].failed = False
241251
except Exception:
252+
if nautobot_job.fail_job_on_task_failure:
253+
task.results[result_idx].result = []
254+
task.results[result_idx].failed = True
255+
raise
242256
task.result.failed = False
243257
except NornirSubTaskError:
258+
if nautobot_job.fail_job_on_task_failure:
259+
task.results[result_idx].result = []
260+
task.results[result_idx].failed = True
261+
raise
244262
# These exceptions indicate that the device is unreachable or the credentials are incorrect.
245263
# We should fail the task early to avoid trying all commands on a device that is unreachable.
246264
if type(task.results[result_idx].exception).__name__ == "NetmikoAuthenticationException":
@@ -321,14 +339,18 @@ def sync_devices_command_getter(job, log_level):
321339
)
322340
continue
323341
nr_with_processors.inventory.hosts.update(single_host_inventory_constructed)
324-
nr_with_processors.run(
342+
result = nr_with_processors.run(
325343
task=netmiko_send_commands,
326344
command_getter_yaml_data=nr_with_processors.inventory.defaults.data["platform_parsing_info"],
327345
command_getter_job="sync_devices",
328346
logger=logger,
329347
nautobot_job=job,
330348
)
349+
if job.fail_job_on_task_failure and result.failed:
350+
raise RuntimeError(f"netmiko_send_commads task failed with {result.failed_hosts.items()}.")
331351
except Exception as err: # pylint: disable=broad-exception-caught
352+
if job.fail_job_on_task_failure:
353+
raise RuntimeError("Error During Sync Devices Command Getter.") from err
332354
try:
333355
if job.debug:
334356
traceback_str = format_log_message(traceback.format_exc())
@@ -369,13 +391,17 @@ def sync_network_data_command_getter(job, log_level):
369391
},
370392
) as nornir_obj:
371393
nr_with_processors = nornir_obj.with_processors([CommandGetterProcessor(logger, compiled_results, job)])
372-
nr_with_processors.run(
394+
result = nr_with_processors.run(
373395
task=netmiko_send_commands,
374396
command_getter_yaml_data=nr_with_processors.inventory.defaults.data["platform_parsing_info"],
375397
command_getter_job="sync_network_data",
376398
logger=logger,
377399
nautobot_job=job,
378400
)
379-
except Exception: # pylint: disable=broad-exception-caught
401+
if job.fail_job_on_task_failure and result.failed:
402+
raise RuntimeError(f"netmiko_send_commads task failed with {result.failed_hosts.items()}.")
403+
except Exception as err: # pylint: disable=broad-exception-caught
404+
if job.fail_job_on_task_failure:
405+
raise RuntimeError("Error During Sync Network Data Command Getter.") from err
380406
logger.info(f"Error During Sync Network Data Command Getter: {traceback.format_exc()}")
381407
return compiled_results

‎nautobot_device_onboarding/tests/test_jobs.py‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ def test_sync_devices__success(self, device_data):
5757
"secrets_group": self.testing_objects["secrets_group"].pk,
5858
"platform": None,
5959
"memory_profiling": False,
60+
"fail_job_on_task_failure": False,
6061
}
6162
job_result = create_job_result_and_run_job(
6263
module="nautobot_device_onboarding.jobs", name="SSOTSyncDevices", **job_form_inputs
@@ -159,11 +160,13 @@ def test_csv_process_pass_connectivity_test_flag(self, mock_sync_devices_command
159160
"update_devices_without_primary_ip": None,
160161
"device_role": None,
161162
"device_status": None,
163+
"device_tenant": None,
162164
"interface_status": None,
163165
"ip_address_status": None,
164166
"secrets_group": None,
165167
"platform": None,
166168
"memory_profiling": False,
169+
"fail_job_on_task_failure": False,
167170
}
168171

169172
create_job_result_and_run_job(
@@ -216,6 +219,7 @@ def test_sync_network_data__success(self, device_data):
216219
"device_role": None,
217220
"platform": None,
218221
"memory_profiling": False,
222+
"fail_job_on_task_failure": False,
219223
}
220224
job_result = create_job_result_and_run_job(
221225
module="nautobot_device_onboarding.jobs", name="SSOTSyncNetworkData", **job_form_inputs
@@ -273,12 +277,14 @@ def test_sync_network_devices_with_full_ssh(self):
273277
"update_devices_without_primary_ip": True,
274278
"device_role": self.testing_objects["device_role"].pk,
275279
"device_status": self.testing_objects["status"].pk,
280+
"device_tenant": None,
276281
"interface_status": self.testing_objects["status"].pk,
277282
"ip_address_status": self.testing_objects["status"].pk,
278283
"default_prefix_status": self.testing_objects["status"].pk,
279284
"secrets_group": self.testing_objects["secrets_group"].pk,
280285
"platform": self.testing_objects["platform_1"].pk,
281286
"memory_profiling": False,
287+
"fail_job_on_task_failure": False,
282288
}
283289
current_file_path = os.path.dirname(os.path.abspath(__file__))
284290
fake_ios_inventory = {
@@ -334,11 +340,13 @@ def test_sync_network_data_with_full_ssh_nxos_trunked_vlans(self):
334340
"device_role": self.testing_objects["device_role"].pk,
335341
"device_status": self.testing_objects["status"].pk,
336342
"interface_status": self.testing_objects["status"].pk,
343+
"device_tenant": None,
337344
"ip_address_status": self.testing_objects["status"].pk,
338345
"default_prefix_status": self.testing_objects["status"].pk,
339346
"secrets_group": self.testing_objects["secrets_group"].pk,
340347
"platform": self.testing_objects["platform_3"].pk,
341348
"memory_profiling": False,
349+
"fail_job_on_task_failure": False,
342350
}
343351
sync_network_data_job_form_inputs = {
344352
"debug": False,
@@ -356,6 +364,7 @@ def test_sync_network_data_with_full_ssh_nxos_trunked_vlans(self):
356364
"device_role": None,
357365
"platform": None,
358366
"memory_profiling": False,
367+
"fail_job_on_task_failure": False,
359368
}
360369

361370
initial_vlans = set(VLAN.objects.values_list("vid", flat=True))
@@ -447,12 +456,14 @@ def test_sync_network_data_with_full_ssh_cisco_xe_vrfs(self):
447456
"update_devices_without_primary_ip": True,
448457
"device_role": self.testing_objects["device_role"].pk,
449458
"device_status": self.testing_objects["status"].pk,
459+
"device_tenant": None,
450460
"interface_status": self.testing_objects["status"].pk,
451461
"ip_address_status": self.testing_objects["status"].pk,
452462
"default_prefix_status": self.testing_objects["status"].pk,
453463
"secrets_group": self.testing_objects["secrets_group"].pk,
454464
"platform": self.testing_objects["platform_2"].pk,
455465
"memory_profiling": False,
466+
"fail_job_on_task_failure": False,
456467
}
457468
sync_network_data_job_form_inputs = {
458469
"debug": False,
@@ -470,6 +481,7 @@ def test_sync_network_data_with_full_ssh_cisco_xe_vrfs(self):
470481
"device_role": None,
471482
"platform": None,
472483
"memory_profiling": False,
484+
"fail_job_on_task_failure": False,
473485
}
474486

475487
initial_vrfs = set(VRF.objects.values_list("name", flat=True))

‎nautobot_device_onboarding/tests/test_sync_devices_adapters.py‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
"""Test Cisco Support adapter."""
22

3-
from unittest.mock import patch
3+
from types import SimpleNamespace
4+
from unittest.mock import MagicMock, patch
45

56
from diffsync.exceptions import ObjectNotFound
67
from nautobot.apps.testing import TransactionTestCase
@@ -12,6 +13,7 @@
1213
SyncDevicesNetworkAdapter,
1314
)
1415
from nautobot_device_onboarding.jobs import SSOTSyncDevices
16+
from nautobot_device_onboarding.nornir_plays.command_getter import sync_devices_command_getter
1517
from nautobot_device_onboarding.tests import utils
1618
from nautobot_device_onboarding.tests.fixtures import sync_devices_fixture
1719

@@ -81,6 +83,42 @@ def test_load(self, device_data):
8183
self.assertEqual(data["mask_length"], diffsync_device.mask_length)
8284
self.assertEqual(data["serial"], diffsync_device.serial)
8385

86+
@patch("nautobot_device_onboarding.nornir_plays.command_getter.InitNornir")
87+
def test_command_getter_raises_when_fail_job_on_task_failure_true(self, init_nornir):
88+
self.job.fail_job_on_task_failure = True
89+
90+
nornir_obj = MagicMock()
91+
nr_with_processors = MagicMock()
92+
nornir_obj.with_processors.return_value = nr_with_processors
93+
94+
nr_with_processors.run.return_value = SimpleNamespace(
95+
failed=True,
96+
failed_hosts={"1.1.1.1": "DEVICE01"},
97+
)
98+
99+
init_nornir.return_value.__enter__.return_value = nornir_obj
100+
101+
with self.assertRaises(RuntimeError):
102+
sync_devices_command_getter(job=self.job, log_level="INFO")
103+
104+
@patch("nautobot_device_onboarding.nornir_plays.command_getter.InitNornir")
105+
def test_command_getter_does_not_raise_when_fail_job_on_task_failure_false(self, init_nornir):
106+
self.job.fail_job_on_task_failure = False
107+
108+
nornir_obj = MagicMock()
109+
nr_with_processors = MagicMock()
110+
nornir_obj.with_processors.return_value = nr_with_processors
111+
112+
nr_with_processors.run.return_value = SimpleNamespace(
113+
failed=True,
114+
failed_hosts={"1.1.1.1": "DEVICE01"},
115+
)
116+
117+
init_nornir.return_value.__enter__.return_value = nornir_obj
118+
119+
result = sync_devices_command_getter(job=self.job, log_level="INFO")
120+
self.assertEqual(result, {})
121+
84122

85123
class SyncDevicesNautobotAdapterTestCase(TransactionTestCase):
86124
"""Test SyncDevicesNautobotAdapter class."""

‎nautobot_device_onboarding/tests/test_sync_devices_models.py‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ def test_device_update__missing_primary_ip__success(self, device_data):
4545
"secrets_group": self.testing_objects["secrets_group_alternate"].pk,
4646
"platform": None,
4747
"memory_profiling": False,
48+
"fail_job_on_task_failure": False,
4849
}
4950
self.testing_objects["device_1"].primary_ip4 = None # test existing device with missing primary ip
5051
self.testing_objects["device_1"].validated_save()
@@ -98,6 +99,7 @@ def test_device_update__primary_ip_and_interface__success(self, device_data):
9899
"secrets_group": self.testing_objects["secrets_group"].pk,
99100
"platform": None,
100101
"memory_profiling": False,
102+
"fail_job_on_task_failure": False,
101103
}
102104
job_result = create_job_result_and_run_job(
103105
module="nautobot_device_onboarding.jobs", name="SSOTSyncDevices", **job_form_inputs
@@ -146,6 +148,7 @@ def test_device_update__interface_only__success(self, device_data):
146148
"secrets_group": self.testing_objects["secrets_group_alternate"].pk,
147149
"platform": None,
148150
"memory_profiling": False,
151+
"fail_job_on_task_failure": False,
149152
}
150153
job_result = create_job_result_and_run_job(
151154
module="nautobot_device_onboarding.jobs", name="SSOTSyncDevices", **job_form_inputs

0 commit comments

Comments
 (0)