add support for custom merge strategy (#104)

Author: oldgromit

README.md

@@ -136,6 +136,7 @@ usage: himl [-h] [--output-file OUTPUT_FILE] [--format OUTPUT_FORMAT]
              [--skip-interpolation-validation]
              [--skip-interpolation-resolving] [--enclosing-key ENCLOSING_KEY]
              [--cwd CWD]
+             [--list-merge-strategy {append,override,prepend,append_unique}]
              path
 ```
 
@@ -352,3 +353,38 @@ VA7:     !include configs/env=int/region=va7/kafka-brokers.yaml regionBrokers.VA
 ```
 
 This will replace the value after interpolation with the value of the regionBrokers.VA7 found under the configs/env=int/region=va7/kafka-brokers.yaml path.
+
+## Custom merge strategy
+An optional parameter `type_strategies` can be passed into ConfigProcessor to define custom merging behavior. It could be custom functions that fit your needs.
+Your function should take the arguments of (config, path, base, nxt) and return the merged result.
+
+Example:
+```py
+from himl import ConfigProcessor
+
+def strategy_merge_override(config, path, base, nxt):
+    """merge list of dicts. if objects have same id, nxt replaces base."""
+    """if remove flag is present in nxt item, remove base and not add nxt"""
+    result = deepcopy(base)
+    for nxto in nxt:
+        for baseo in result:
+            # if list is not a list of dicts, bail out and let the next strategy to execute
+            if not isinstance(baseo,dict) or not isinstance(nxto,dict):
+                return STRATEGY_END
+            if 'id' in baseo and 'id' in nxto and baseo['id'] == nxto['id']:
+                result.remove(baseo) #same id, remove previous item
+        if 'remove' not in nxto:
+            result.append(nxto)
+    return result
+
+config_processor = ConfigProcessor()
+path = "examples/simple/production"
+filters = () # can choose to output only specific keys
+exclude_keys = () # can choose to remove specific keys
+output_format = "yaml" # yaml/json
+
+config_processor.process(path=path, filters=filters, exclude_keys=exclude_keys,
+                         output_format=output_format, print_data=True,
+                         type_strategies= [(list, [strategy_merge_override,'append']), (dict, ["merge"])] ))
+
+```

examples/complex/default.yaml

@@ -8,7 +8,23 @@ cluster_info:
   # Interpolation example
   description: "This is cluster: {{cluster}}. It is using {{cluster_info.node_type}} instance type."
   node_type: c3.2xlarge # default value, which can be overridden by each cluster
-
+cluster_metrics:
+  - id: 1
+    metric: cpu
+    value: 90
+  - id: 2
+    metric: memory
+    value: 90   
+  - id: 3
+    metric: disk
+    value: 90
+metrics:
+  - cpu
+  - memory
+  - disk
+myList:
+ - id1
+ - id4
 # Fetching the secret value at runtime, from a secrets store (in this case AWS SSM).
 # passphrase: "{{ssm.path(/key/coming/from/aws/secrets/store/manager).aws_profile(myprofile)}}"
 

examples/complex/env=dev/region=us-east-1/cluster=cluster2/cluster.yaml

@@ -1 +1,21 @@
 cluster: cluster2
+cluster_metrics:
+  - id: 1
+    metric: cpu
+    value: 95
+  - id: 2
+    metric: memory
+    value: 95   
+  - id: 3
+    metric: disk
+    remove: True
+  - metric: exec
+    value: 5
+metrics:
+  - cpu
+  - exec
+myList:
+ - id1
+ - id2
+ - id3
+ 

himl/config_generator.py

@@ -28,7 +28,8 @@
 class ConfigProcessor(object):
 
     def process(self, cwd=None, path=None, filters=(), exclude_keys=(), enclosing_key=None, remove_enclosing_key=None, output_format="yaml",
-                print_data=False, output_file=None, skip_interpolations=False, skip_interpolation_validation=False, skip_secrets=False, multi_line_string=False):
+                print_data=False, output_file=None, skip_interpolations=False, skip_interpolation_validation=False, skip_secrets=False, multi_line_string=False,
+                type_strategies = [(list, ["append"]), (dict, ["merge"])], fallback_strategies = ["override"], type_conflict_strategies = ["override"]):
 
         path = self.get_relative_path(path)
 
@@ -41,7 +42,7 @@ def process(self, cwd=None, path=None, filters=(), exclude_keys=(), enclosing_ke
         if cwd is None:
             cwd = os.getcwd()
 
-        generator = ConfigGenerator(cwd, path, multi_line_string)
+        generator = ConfigGenerator(cwd, path, multi_line_string, type_strategies, fallback_strategies, type_conflict_strategies)
         generator.generate_hierarchy()
         generator.process_hierarchy()
 
@@ -120,13 +121,15 @@ class ConfigGenerator(object):
     will contain merged data on each layer.
     """
 
-    def __init__(self, cwd, path, multi_line_string):
+    def __init__(self, cwd, path, multi_line_string, type_strategies, fallback_strategies, type_conflict_strategies):
         self.cwd = cwd
         self.path = path
         self.hierarchy = self.generate_hierarchy()
         self.generated_data = OrderedDict()
         self.interpolation_validator = InterpolationValidator()
-
+        self.type_strategies = type_strategies
+        self.fallback_strategies = fallback_strategies
+        self.type_conflict_strategies = type_conflict_strategies
         if multi_line_string is True:
             yaml.representer.BaseRepresenter.represent_scalar = ConfigGenerator.custom_represent_scalar
 
@@ -176,22 +179,22 @@ def yaml_get_content(yaml_file):
         return content if content else {}
 
     @staticmethod
-    def merge_value(reference, new_value):
-        merger = Merger([(list, ["append"]), (dict, ["merge"])], ["override"], ["override"])
+    def merge_value(reference, new_value, type_strategies, fallback_strategies, type_conflict_strategies):
+        merger = Merger(type_strategies, fallback_strategies, type_conflict_strategies)
         if isinstance(new_value, (list, set, dict)):
             new_reference = merger.merge(reference, new_value)
         else:
             raise TypeError("Cannot handle merge_value of type {}".format(type(new_value)))
         return new_reference
 
     @staticmethod
-    def merge_yamls(values, yaml_content):
+    def merge_yamls(values, yaml_content, type_strategies, fallback_strategies, type_conflict_strategies):
         for key, value in iteritems(yaml_content):
             if key in values and type(values[key]) != type(value):
                 raise Exception("Failed to merge key '{}', because of mismatch in type: {} vs {}"
                                 .format(key, type(values[key]), type(value)))
             if key in values and not isinstance(value, primitive_types):
-                values[key] = ConfigGenerator.merge_value(values[key], value)
+                values[key] = ConfigGenerator.merge_value(values[key], value, type_strategies, fallback_strategies, type_conflict_strategies)
             else:
                 values[key] = value
 
@@ -224,7 +227,7 @@ def process_hierarchy(self):
         for yaml_files in self.hierarchy:
             for yaml_file in yaml_files:
                 yaml_content = self.yaml_get_content(yaml_file)
-                self.merge_yamls(merged_values, yaml_content)
+                self.merge_yamls(merged_values, yaml_content, self.type_strategies, self.fallback_strategies, self.type_conflict_strategies)
                 self.resolve_simple_interpolations(merged_values, yaml_file)
         self.generated_data = merged_values
 

himl/main.py

@@ -10,9 +10,17 @@
 
 import argparse
 import os
-
 from .config_generator import ConfigProcessor
+from enum import Enum
+
+class ListMergeStrategy(Enum):
+    append = 'append'
+    override = 'override'
+    prepend = 'prepend'
+    append_unique = 'append_unique' #WARNING: currently this strategy does not support list of dicts, only list of str
 
+    def __str__(self):
+        return self.value
 
 class ConfigRunner(object):
 
@@ -29,9 +37,11 @@ def do_run(self, opts):
             opts.print_data = True
 
         config_processor = ConfigProcessor()
+                                 
         config_processor.process(cwd, opts.path, filters, excluded_keys, opts.enclosing_key, opts.remove_enclosing_key,
                                  opts.output_format, opts.print_data, opts.output_file, opts.skip_interpolation_resolving,
-                                 opts.skip_interpolation_validation, opts.skip_secrets, opts.multi_line_string)
+                                 opts.skip_interpolation_validation, opts.skip_secrets, opts.multi_line_string,
+                                 type_strategies= [(list, [opts.merge_list_strategy.value]), (dict, ["merge"])] )
 
     @staticmethod
     def get_parser(parser=None):
@@ -63,10 +73,12 @@ def get_parser(parser=None):
                             help='the working directory')
         parser.add_argument('--multi-line-string', action='store_true',
                             help='will overwrite the global yaml dumper to use block style')
+        parser.add_argument('--list-merge-strategy', dest='merge_list_strategy', type=ListMergeStrategy, choices=list(ListMergeStrategy),
+                            default='append',
+                            help='override default merge strategy for list')
         parser.add_argument('--version', action='version', version='%(prog)s v{version}'.format(version="0.10.0"),
                             help='print himl version')
         return parser
 
-
 def run(args=None):
     ConfigRunner().run(args)