feat: add --allow-unicode flag for Unicode character handling (#207)

* feat: allow unicode characters in config generator

* docs: add documentation for Unicode support

* docs: revert formatting in README.md

Author: bensi94

README.md

@@ -46,8 +46,12 @@ Idea came from puppet's hiera.
       - [Vault](#vault)
     - [Merge with Terraform remote state](#merge-with-terraform-remote-state)
     - [Merge with env variables](#merge-with-env-variables)
+    - [Unicode Support](#unicode-support)
   - [himl config merger](#himl-config-merger)
+    - [Output filtering](#output-filtering)
     - [Extra merger features](#extra-merger-features)
+  - [Custom merge strategy](#custom-merge-strategy)
+  - [Development](#development)
 
 ## Installation
 
@@ -151,8 +155,9 @@ usage: himl [-h] [--output-file OUTPUT_FILE] [--format OUTPUT_FORMAT]
              [--filter FILTER] [--exclude EXCLUDE]
              [--skip-interpolation-validation]
              [--skip-interpolation-resolving] [--enclosing-key ENCLOSING_KEY]
-             [--cwd CWD]
+             [--cwd CWD] [--multi-line-string]
              [--list-merge-strategy {append,override,prepend,append_unique}]
+             [--allow-unicode]
              path
 ```
 
@@ -296,6 +301,63 @@ endpoint: "{{outputs.cluster_composition.output.value.redis_endpoint}}"
 kubeconfig_location: "{{env(KUBECONFIG)}}"
 ```
 
+### Unicode Support
+
+himl supports Unicode characters in configuration files, allowing you to use international languages, special characters, and emoji in your YAML configs.
+
+By default, Unicode characters are escaped in the output to ensure compatibility. You can preserve Unicode characters in their original form using the `--allow-unicode` flag.
+
+**Using the CLI:**
+```sh
+# With Unicode escaping (default)
+himl examples/simple/production --output-file config.yaml
+
+# Preserving Unicode characters
+himl examples/simple/production --output-file config.yaml --allow-unicode
+```
+
+**Using the Python module:**
+```py
+from himl import ConfigProcessor
+
+config_processor = ConfigProcessor()
+path = "examples/simple/production"
+
+# Process with Unicode preservation
+config = config_processor.process(
+    path=path,
+    output_format="yaml",
+    allow_unicode=True,  # Preserve Unicode characters
+    print_data=True
+)
+```
+
+**Example with Unicode content:**
+
+`config/default.yaml`:
+```yaml
+service:
+  name: "My Service"
+  description: "Multi-language support: English, 中文, العربية, Русский"
+
+messages:
+  welcome:
+    en: "Welcome"
+    zh: "欢迎"
+    ar: "مرحبا"
+    ru: "Добро пожаловать"
+
+team:
+  - name: "José García"
+    role: "Developer"
+  - name: "田中太郎"
+    role: "Designer"
+```
+
+When processed with `--allow-unicode`, the output preserves all Unicode characters. Without the flag, non-ASCII characters are escaped (e.g., `\u4e2d\u6587` for Chinese characters).
+
+**Note:** Some emoji and 4-byte UTF-8 characters may be escaped by the YAML library even with `--allow-unicode` enabled.
+
 
 ## himl-config-merger
 
@@ -394,6 +456,12 @@ Build the output with filtering:
 himl-config-merger examples/filters --output-dir merged_output --levels env region cluster --leaf-directories cluster --filter-rules-key _filters
 ```
 
+The `himl-config-merger` command also supports the `--allow-unicode` flag for preserving Unicode characters in the merged output files:
+
+```sh
+himl-config-merger examples/complex --output-dir merged_output --levels env region cluster --leaf-directories cluster --allow-unicode
+```
+
 ```yaml
 # output after filtering
 env: dev

himl/config_generator.py

@@ -42,6 +42,7 @@ def process(self, cwd=None,
                 skip_interpolation_validation=False,
                 skip_secrets=False,
                 multi_line_string=False,
+                allow_unicode=False,
                 type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
                 fallback_strategies=["override"],
                 type_conflict_strategies=["override"]):
@@ -53,7 +54,7 @@ def process(self, cwd=None,
         cwd = cwd or os.getcwd()
 
         generator = self._create_and_initialize_generator(
-            cwd, path, multi_line_string, type_strategies, fallback_strategies, type_conflict_strategies)
+            cwd, path, multi_line_string, allow_unicode, type_strategies, fallback_strategies, type_conflict_strategies)
 
         # Process data exclusions and interpolations
         self._process_exclusions(generator, exclude_keys)
@@ -73,10 +74,10 @@ def _should_skip_interpolation_validation(self, skip_interpolations, skip_secret
         """Determine if interpolation validation should be skipped."""
         return skip_interpolation_validation or skip_interpolations or skip_secrets
 
-    def _create_and_initialize_generator(self, cwd, path, multi_line_string, type_strategies,
+    def _create_and_initialize_generator(self, cwd, path, multi_line_string, allow_unicode, type_strategies,
                                          fallback_strategies, type_conflict_strategies):
         """Create and initialize the ConfigGenerator."""
-        generator = ConfigGenerator(cwd, path, multi_line_string, type_strategies, fallback_strategies,
+        generator = ConfigGenerator(cwd, path, multi_line_string, allow_unicode, type_strategies, fallback_strategies,
                                     type_conflict_strategies)
         generator.generate_hierarchy()
         generator.process_hierarchy()
@@ -179,12 +180,13 @@ class ConfigGenerator(object):
     will contain merged data on each layer.
     """
 
-    def __init__(self, cwd, path, multi_line_string, type_strategies, fallback_strategies, type_conflict_strategies):
+    def __init__(self, cwd, path, multi_line_string, allow_unicode, type_strategies, fallback_strategies, type_conflict_strategies):
         self.cwd = cwd
         self.path = path
         self.hierarchy = self.generate_hierarchy()
         self.generated_data = OrderedDict()
         self.interpolation_validator = InterpolationValidator()
+        self.allow_unicode = allow_unicode
         self.type_strategies = type_strategies
         self.fallback_strategies = fallback_strategies
         self.type_conflict_strategies = type_conflict_strategies
@@ -338,7 +340,7 @@ def get_values_from_dir_path(self):
 
     def output_yaml_data(self, data):
         return yaml.dump(data, Dumper=ConfigGenerator.yaml_dumper(), default_flow_style=False, width=200,
-                         sort_keys=False)
+                         sort_keys=False, allow_unicode=self.allow_unicode)
 
     def yaml_to_json(self, yaml_data):
         return json.dumps(yaml.load(yaml_data, Loader=yaml.SafeLoader), indent=4)

himl/config_merger.py

@@ -89,18 +89,19 @@ def __traverse_path(self, path: str, yaml_dict: dict):
 Loader.add_constructor('!include', Loader.include)
 
 
-def merge_configs(directories, levels, output_dir, enable_parallel, filter_rules):
+def merge_configs(directories, levels, output_dir, enable_parallel, filter_rules, allow_unicode):
     """
     Method for running the merge configuration logic under different formats
     :param directories: list of paths for leaf directories
     :param levels: list of hierarchy levels to traverse
     :param output_dir: where to save the generated configs
     :param enable_parallel: to enable parallel config generation
+    :param allow_unicode: allow unicode characters in output
     """
     config_processor = ConfigProcessor()
     process_config = []
     for path in directories:
-        process_config.append((config_processor, path, levels, output_dir, filter_rules))
+        process_config.append((config_processor, path, levels, output_dir, filter_rules, allow_unicode))
 
     if enable_parallel:
         logger.info("Processing config in parallel")
@@ -121,6 +122,7 @@ def merge_logic(process_params):
     levels = process_params[2]
     output_dir = process_params[3]
     filter_rules = process_params[4]
+    allow_unicode = process_params[5]
 
     # load the !include tag
     Loader.add_constructor('!include', Loader.include)
@@ -153,7 +155,7 @@ def merge_logic(process_params):
     logger.info("Found input config directory: %s", path)
     logger.info("Storing generated config to: %s", filename)
     with open(filename, "w+") as f:
-        f.write(yaml.dump(output))
+        f.write(yaml.dump(output, allow_unicode=allow_unicode))
 
 
 def is_leaf_directory(dir, leaf_directories):
@@ -203,6 +205,8 @@ def get_parser():
                         action='store_true', help='Process config using multiprocessing')
     parser.add_argument('--filter-rules-key', dest='filter_rules', default=None, type=str,
                         help='keep these keys from the generated data, based on the configured filter key')
+    parser.add_argument('--allow-unicode', dest='allow_unicode', default=False,
+                        action='store_true', help='allow unicode characters in output (default: False, outputs escape sequences)')
     return parser
 
 
@@ -219,4 +223,4 @@ def run(args=None):
 
     # merge the configs using HIML
     merge_configs(dirs, opts.hierarchy_levels,
-                  opts.output_dir, opts.enable_parallel, opts.filter_rules)
+                  opts.output_dir, opts.enable_parallel, opts.filter_rules, opts.allow_unicode)

himl/main.py

@@ -43,7 +43,7 @@ def do_run(self, opts):
         config_processor.process(cwd, opts.path, filters, excluded_keys, opts.enclosing_key,
                                  opts.remove_enclosing_key, opts.output_format, opts.print_data, opts.output_file,
                                  opts.skip_interpolation_resolving, opts.skip_interpolation_validation,
-                                 opts.skip_secrets, opts.multi_line_string,
+                                 opts.skip_secrets, opts.multi_line_string, opts.allow_unicode,
                                  type_strategies=[(list, [opts.merge_list_strategy.value]), (dict, ["merge"])])
 
     @staticmethod
@@ -79,6 +79,8 @@ def get_parser(parser=None):
         parser.add_argument('--list-merge-strategy', dest='merge_list_strategy', type=ListMergeStrategy,
                             choices=list(ListMergeStrategy), default='append_unique',
                             help='override default merge strategy for list')
+        parser.add_argument('--allow-unicode', dest='allow_unicode', action='store_true', default=False,
+                            help='allow unicode characters in output (default: False, outputs escape sequences)')
         parser.add_argument('--version', action='version', version='%(prog)s v{version}'.format(version="0.18.0"),
                             help='print himl version')
         return parser