GCU: Add path tracing support

config/main.py

@@ -33,6 +33,7 @@
 from sonic_py_common.general import getstatusoutput_noshell
 from sonic_py_common.interface import get_interface_table_name, get_port_table_name, get_intf_longname
 from sonic_yang_cfg_generator import SonicYangCfgDbGenerator
+from typing import IO, Optional
 from utilities_common import util_base
 from swsscommon import swsscommon
 from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, ConfigDBPipeConnector, \
@@ -1387,12 +1388,21 @@ def multiasic_save_to_singlefile(db, filename):
         json.dump(all_current_config, file, indent=4)
 
 
-def apply_patch_wrapper(args):
-    return apply_patch_for_scope(*args)
+def apply_patch_wrapper(args, **kwargs):
+    return apply_patch_for_scope(*args, **kwargs)
 
 
 # Function to apply patch for a single ASIC.
-def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path):
+def apply_patch_for_scope(
+    scope_changes,
+    results,
+    config_format,
+    verbose,
+    dry_run,
+    ignore_non_yang_tables,
+    ignore_path,
+    trace_io: Optional[IO] = None,
+):
     scope, changes = scope_changes
     # Replace localhost to DEFAULT_NAMESPACE which is db definition of Host
     if scope.lower() == HOST_NAMESPACE or scope == "":
@@ -1409,7 +1419,8 @@ def apply_patch_for_scope(scope_changes, results, config_format, verbose, dry_ru
                                                 verbose,
                                                 dry_run,
                                                 ignore_non_yang_tables,
-                                                ignore_path)
+                                                ignore_path,
+                                                trace_io=trace_io)
         results[scope_for_log] = {"success": True, "message": "Success"}
         log.log_notice(f"'apply-patch' executed successfully for {scope_for_log} by {changes} in thread:{thread_id}")
     except Exception as e:
@@ -1902,8 +1913,26 @@ def print_dry_run_message(dry_run):
 @click.option('-n', '--ignore-non-yang-tables', is_flag=True, default=False, help='ignore validation for tables without YANG models', hidden=True)
 @click.option('-i', '--ignore-path', multiple=True, help='ignore validation for config specified by given path which is a JsonPointer', hidden=True)
 @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing')
+@click.option(
+    '-t',
+    '--path-trace',
+    type=click.Path(writable=True),
+    help='fileneme to output decision path trace for patch generation as json',
+    hidden=True,
+)
+
 @click.pass_context
-def apply_patch(ctx, patch_file_path, format, dry_run, parallel, ignore_non_yang_tables, ignore_path, verbose):
+def apply_patch(
+    ctx,
+    patch_file_path,
+    format,
+    dry_run,
+    parallel,
+    ignore_non_yang_tables,
+    ignore_path,
+    verbose,
+    path_trace,
+):
     """Apply given patch of updates to Config. A patch is a JsonPatch which follows rfc6902.
        This command can be used do partial updates to the config with minimum disruption to running processes.
        It allows addition as well as deletion of configs. The patch file represents a diff of ConfigDb(ABNF)
@@ -1918,6 +1947,10 @@ def apply_patch(ctx, patch_file_path, format, dry_run, parallel, ignore_non_yang
             patch_as_json = json.loads(text)
             patch_ops = patch_as_json
 
+        trace_io = None
+        if path_trace is not None:
+            trace_io = open(path_trace, 'w')
+
         all_running_config = get_all_running_config()
 
         # Pre-process patch to append empty tables if required.
@@ -1968,7 +2001,7 @@ def apply_patch(ctx, patch_file_path, format, dry_run, parallel, ignore_non_yang
                              for scope_changes in changes_by_scope.items()]
 
                 # Submit all tasks and wait for them to complete
-                futures = [executor.submit(apply_patch_wrapper, args) for args in arguments]
+                futures = [executor.submit(apply_patch_wrapper, args, trace_io=trace_io) for args in arguments]
 
                 # Wait for all tasks to complete
                 concurrent.futures.wait(futures)
@@ -1979,11 +2012,15 @@ def apply_patch(ctx, patch_file_path, format, dry_run, parallel, ignore_non_yang
                                       config_format,
                                       verbose, dry_run,
                                       ignore_non_yang_tables,
-                                      ignore_path)
+                                      ignore_path,
+                                      trace_io=trace_io)
 
         # Check if any updates failed
         failures = [scope for scope, result in results.items() if not result['success']]
 
+        if trace_io is not None:
+            trace_io.close()
+
         if failures:
             failure_messages = '\n'.join([f"- {failed_scope}: {results[failed_scope]['message']}" for failed_scope in failures])
             raise GenericConfigUpdaterError(f"Failed to apply patch on the following scopes:\n{failure_messages}")
@@ -2004,8 +2041,15 @@ def apply_patch(ctx, patch_file_path, format, dry_run, parallel, ignore_non_yang
 @click.option('-n', '--ignore-non-yang-tables', is_flag=True, default=False, help='ignore validation for tables without YANG models', hidden=True)
 @click.option('-i', '--ignore-path', multiple=True, help='ignore validation for config specified by given path which is a JsonPointer', hidden=True)
 @click.option('-v', '--verbose', is_flag=True, default=False, help='print additional details of what the operation is doing')
+@click.option(
+    '-t',
+    '--path-trace',
+    type=click.Path(writable=True),
+    help='fileneme to output decision path trace for patch generation as json',
+    hidden=True,
+)
 @click.pass_context
-def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, ignore_path, verbose):
+def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, ignore_path, verbose, path_trace):
     """Replace the whole config with the specified config. The config is replaced with minimum disruption e.g.
        if ACL config is different between current and target config only ACL config is updated, and other config/services
        such as DHCP will not be affected.
@@ -2022,7 +2066,22 @@ def replace(ctx, target_file_path, format, dry_run, ignore_non_yang_tables, igno
 
         config_format = ConfigFormat[format.upper()]
 
-        GenericUpdater().replace(target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_path)
+        trace_io = None
+        if path_trace is not None:
+            trace_io = open(path_trace, 'w')
+
+        GenericUpdater().replace(
+            target_config,
+            config_format,
+            verbose,
+            dry_run,
+            ignore_non_yang_tables,
+            ignore_path,
+            trace_io=trace_io,
+        )
+
+        if trace_io is not None:
+            trace_io.close()
 
         click.secho("Config replaced successfully.", fg="cyan", underline=True)
     except Exception as ex:

generic_config_updater/generic_updater.py

@@ -5,6 +5,7 @@
 
 from datetime import datetime, timezone
 from enum import Enum
+from typing import IO, Optional
 from .gu_common import HOST_NAMESPACE, GenericConfigUpdaterError, EmptyTableError, ConfigWrapper, \
                     DryRunConfigWrapper, PatchWrapper, genericUpdaterLogging
 from .patch_sorter import StrictPatchSorter, NonStrictPatchSorter, ConfigSplitter, \
@@ -101,7 +102,7 @@ def __init__(self,
         self.patchsorter = patchsorter if patchsorter is not None else StrictPatchSorter(self.config_wrapper, self.patch_wrapper)
         self.changeapplier = changeapplier if changeapplier is not None else ChangeApplier(scope=self.scope)
 
-    def apply(self, patch, sort=True):
+    def apply(self, patch, sort=True, trace_io: Optional[IO] = None):
         scope = self.scope if self.scope else HOST_NAMESPACE
         self.logger.log_notice(f"{scope}: Patch application starting.")
         self.logger.log_notice(f"{scope}: Patch: {patch}")
@@ -130,7 +131,7 @@ def apply(self, patch, sort=True):
         # Generate list of changes to apply
         if sort:
             self.logger.log_notice(f"{scope}: sorting patch updates.")
-            changes = self.patchsorter.sort(patch)
+            changes = self.patchsorter.sort(patch, trace_io=trace_io)
         else:
             self.logger.log_notice(f"{scope}: converting patch to JsonChange.")
             changes = [JsonChange(jsonpatch.JsonPatch([element])) for element in patch]
@@ -166,7 +167,7 @@ def __init__(self, patch_applier=None, config_wrapper=None, patch_wrapper=None,
         self.config_wrapper = config_wrapper if config_wrapper is not None else ConfigWrapper(scope=self.scope)
         self.patch_wrapper = patch_wrapper if patch_wrapper is not None else PatchWrapper(scope=self.scope)
 
-    def replace(self, target_config):
+    def replace(self, target_config, trace_io: Optional[IO] = None):
         self.logger.log_notice("Config replacement starting.")
         self.logger.log_notice(f"Target config length: {len(json.dumps(target_config))}.")
 
@@ -178,7 +179,7 @@ def replace(self, target_config):
         self.logger.log_debug(f"Generated patch: {patch}.") # debug since the patch will printed again in 'patch_applier.apply'
 
         self.logger.log_notice("Applying patch using 'Patch Applier'.")
-        self.patch_applier.apply(patch)
+        self.patch_applier.apply(patch, trace_io=trace_io)
 
         self.logger.log_notice("Verifying config replacement is reflected on ConfigDB.")
         new_config = self.config_wrapper.get_config_db_as_json()
@@ -303,7 +304,7 @@ def __init__(self,
         self.scopelist = [HOST_NAMESPACE, *multi_asic.get_namespace_list()]
         super().__init__(patch_applier, config_wrapper, patch_wrapper, scope)
 
-    def replace(self, target_config):
+    def replace(self, target_config, trace_io: Optional[IO] = None):
         config_keys = set(target_config.keys())
         missing_scopes = set(self.scopelist) - config_keys
         if missing_scopes:
@@ -313,7 +314,7 @@ def replace(self, target_config):
             scope_config = target_config.pop(scope)
             if scope.lower() == HOST_NAMESPACE:
                 scope = multi_asic.DEFAULT_NAMESPACE
-            ConfigReplacer(scope=scope).replace(scope_config)
+            ConfigReplacer(scope=scope).replace(scope_config, trace_io=trace_io)
 
 
 class MultiASICConfigRollbacker(FileSystemConfigRollbacker):
@@ -420,11 +421,11 @@ def __init__(self,
         self.decorated_config_replacer = decorated_config_replacer
         self.decorated_config_rollbacker = decorated_config_rollbacker
 
-    def apply(self, patch):
-        self.decorated_patch_applier.apply(patch)
+    def apply(self, patch, sort=True, trace_io: Optional[IO] = None):
+        self.decorated_patch_applier.apply(patch, sort, trace_io=trace_io)
 
-    def replace(self, target_config):
-        self.decorated_config_replacer.replace(target_config)
+    def replace(self, target_config, trace_io: Optional[IO] = None):
+        self.decorated_config_replacer.replace(target_config, trace_io=trace_io)
 
     def rollback(self, checkpoint_name):
         self.decorated_config_rollbacker.rollback(checkpoint_name)
@@ -451,13 +452,13 @@ def __init__(self,
         self.patch_wrapper = patch_wrapper
         self.config_wrapper = config_wrapper
 
-    def apply(self, patch):
+    def apply(self, patch, sort=True, trace_io: Optional[IO] = None):
         config_db_patch = self.patch_wrapper.convert_sonic_yang_patch_to_config_db_patch(patch)
-        Decorator.apply(self, config_db_patch)
+        Decorator.apply(self, config_db_patch, sort, trace_io=trace_io)
 
-    def replace(self, target_config):
+    def replace(self, target_config, trace_io: Optional[IO] = None):
         config_db_target_config = self.config_wrapper.convert_sonic_yang_to_config_db(target_config)
-        Decorator.replace(self, config_db_target_config)
+        Decorator.replace(self, config_db_target_config, trace_io=trace_io)
 
 
 class ConfigLockDecorator(Decorator):
@@ -474,29 +475,36 @@ def __init__(self,
                            scope=scope)
         self.config_lock = config_lock
 
-    def apply(self, patch, sort=True):
-        self.execute_write_action(Decorator.apply, self, patch)
+    def apply(self, patch, sort=True, trace_io: Optional[IO] = None):
+        self.execute_write_action(Decorator.apply, self, patch, sort, trace_io=trace_io)
 
-    def replace(self, target_config):
-        self.execute_write_action(Decorator.replace, self, target_config)
+    def replace(self, target_config, trace_io: Optional[IO] = None):
+        self.execute_write_action(Decorator.replace, self, target_config, trace_io=trace_io)
 
     def rollback(self, checkpoint_name):
         self.execute_write_action(Decorator.rollback, self, checkpoint_name)
 
     def checkpoint(self, checkpoint_name):
         self.execute_write_action(Decorator.checkpoint, self, checkpoint_name)
 
-    def execute_write_action(self, action, *args):
+    def execute_write_action(self, action, *args, **kwargs):
         self.config_lock.acquire_lock()
-        action(*args)
+        action(*args, **kwargs)
         self.config_lock.release_lock()
 
 
 class GenericUpdateFactory:
     def __init__(self, scope=multi_asic.DEFAULT_NAMESPACE):
         self.scope = scope
 
-    def create_patch_applier(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths):
+    def create_patch_applier(
+        self,
+        config_format,
+        verbose,
+        dry_run,
+        ignore_non_yang_tables,
+        ignore_paths,
+    ):
         self.init_verbose_logging(verbose)
         config_wrapper = self.get_config_wrapper(dry_run)
         change_applier = self.get_change_applier(dry_run, config_wrapper)
@@ -523,7 +531,14 @@ def create_patch_applier(self, config_format, verbose, dry_run, ignore_non_yang_
 
         return patch_applier
 
-    def create_config_replacer(self, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths):
+    def create_config_replacer(
+        self,
+        config_format,
+        verbose,
+        dry_run,
+        ignore_non_yang_tables,
+        ignore_paths,
+    ):
         self.init_verbose_logging(verbose)
         config_wrapper = self.get_config_wrapper(dry_run)
         change_applier = self.get_change_applier(dry_run, config_wrapper)
@@ -622,13 +637,44 @@ def __init__(self, generic_update_factory=None, scope=multi_asic.DEFAULT_NAMESPA
         self.generic_update_factory = \
             generic_update_factory if generic_update_factory is not None else GenericUpdateFactory(scope=scope)
 
-    def apply_patch(self, patch, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths, sort=True):
-        patch_applier = self.generic_update_factory.create_patch_applier(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths)
-        patch_applier.apply(patch, sort)
-
-    def replace(self, target_config, config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths):
-        config_replacer = self.generic_update_factory.create_config_replacer(config_format, verbose, dry_run, ignore_non_yang_tables, ignore_paths)
-        config_replacer.replace(target_config)
+    def apply_patch(
+        self,
+        patch,
+        config_format,
+        verbose,
+        dry_run,
+        ignore_non_yang_tables,
+        ignore_paths,
+        sort=True,
+        trace_io: Optional[IO] = None,
+    ):
+        patch_applier = self.generic_update_factory.create_patch_applier(
+            config_format,
+            verbose,
+            dry_run,
+            ignore_non_yang_tables,
+            ignore_paths,
+        )
+        patch_applier.apply(patch, sort, trace_io=trace_io)
+
+    def replace(
+        self,
+        target_config,
+        config_format,
+        verbose,
+        dry_run,
+        ignore_non_yang_tables,
+        ignore_paths,
+        trace_io: Optional[IO] = None,
+    ):
+        config_replacer = self.generic_update_factory.create_config_replacer(
+            config_format,
+            verbose,
+            dry_run,
+            ignore_non_yang_tables,
+            ignore_paths,
+        )
+        config_replacer.replace(target_config, trace_io=trace_io)
 
     def rollback(self, checkpoint_name, verbose, dry_run, ignore_non_yang_tables, ignore_paths):
         config_rollbacker = self.generic_update_factory.create_config_rollbacker(verbose, dry_run, ignore_non_yang_tables, ignore_paths)

generic_config_updater/gu_common.py

@@ -152,7 +152,7 @@ def validate_config_db_config(self, config_db_as_json):
                 if not success:
                     return success, error
         except sonic_yang.SonicYangException as ex:
-            return False, ex
+            return False, str(ex)
 
         return True, None