feat: allow unicode characters in config generator

Author: bensi94

himl/config_generator.py

@@ -42,6 +42,7 @@ def process(self, cwd=None,
                 skip_interpolation_validation=False,
                 skip_secrets=False,
                 multi_line_string=False,
+                allow_unicode=False,
                 type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
                 fallback_strategies=["override"],
                 type_conflict_strategies=["override"]):
@@ -53,7 +54,7 @@ def process(self, cwd=None,
         cwd = cwd or os.getcwd()
 
         generator = self._create_and_initialize_generator(
-            cwd, path, multi_line_string, type_strategies, fallback_strategies, type_conflict_strategies)
+            cwd, path, multi_line_string, allow_unicode, type_strategies, fallback_strategies, type_conflict_strategies)
 
         # Process data exclusions and interpolations
         self._process_exclusions(generator, exclude_keys)
@@ -73,10 +74,10 @@ def _should_skip_interpolation_validation(self, skip_interpolations, skip_secret
         """Determine if interpolation validation should be skipped."""
         return skip_interpolation_validation or skip_interpolations or skip_secrets
 
-    def _create_and_initialize_generator(self, cwd, path, multi_line_string, type_strategies,
+    def _create_and_initialize_generator(self, cwd, path, multi_line_string, allow_unicode, type_strategies,
                                          fallback_strategies, type_conflict_strategies):
         """Create and initialize the ConfigGenerator."""
-        generator = ConfigGenerator(cwd, path, multi_line_string, type_strategies, fallback_strategies,
+        generator = ConfigGenerator(cwd, path, multi_line_string, allow_unicode, type_strategies, fallback_strategies,
                                     type_conflict_strategies)
         generator.generate_hierarchy()
         generator.process_hierarchy()
@@ -179,12 +180,13 @@ class ConfigGenerator(object):
     will contain merged data on each layer.
     """
 
-    def __init__(self, cwd, path, multi_line_string, type_strategies, fallback_strategies, type_conflict_strategies):
+    def __init__(self, cwd, path, multi_line_string, allow_unicode, type_strategies, fallback_strategies, type_conflict_strategies):
         self.cwd = cwd
         self.path = path
         self.hierarchy = self.generate_hierarchy()
         self.generated_data = OrderedDict()
         self.interpolation_validator = InterpolationValidator()
+        self.allow_unicode = allow_unicode
         self.type_strategies = type_strategies
         self.fallback_strategies = fallback_strategies
         self.type_conflict_strategies = type_conflict_strategies
@@ -338,7 +340,7 @@ def get_values_from_dir_path(self):
 
     def output_yaml_data(self, data):
         return yaml.dump(data, Dumper=ConfigGenerator.yaml_dumper(), default_flow_style=False, width=200,
-                         sort_keys=False)
+                         sort_keys=False, allow_unicode=self.allow_unicode)
 
     def yaml_to_json(self, yaml_data):
         return json.dumps(yaml.load(yaml_data, Loader=yaml.SafeLoader), indent=4)

himl/config_merger.py

@@ -89,18 +89,19 @@ def __traverse_path(self, path: str, yaml_dict: dict):
 Loader.add_constructor('!include', Loader.include)
 
 
-def merge_configs(directories, levels, output_dir, enable_parallel, filter_rules):
+def merge_configs(directories, levels, output_dir, enable_parallel, filter_rules, allow_unicode):
     """
     Method for running the merge configuration logic under different formats
     :param directories: list of paths for leaf directories
     :param levels: list of hierarchy levels to traverse
     :param output_dir: where to save the generated configs
     :param enable_parallel: to enable parallel config generation
+    :param allow_unicode: allow unicode characters in output
     """
     config_processor = ConfigProcessor()
     process_config = []
     for path in directories:
-        process_config.append((config_processor, path, levels, output_dir, filter_rules))
+        process_config.append((config_processor, path, levels, output_dir, filter_rules, allow_unicode))
 
     if enable_parallel:
         logger.info("Processing config in parallel")
@@ -121,6 +122,7 @@ def merge_logic(process_params):
     levels = process_params[2]
     output_dir = process_params[3]
     filter_rules = process_params[4]
+    allow_unicode = process_params[5]
 
     # load the !include tag
     Loader.add_constructor('!include', Loader.include)
@@ -153,7 +155,7 @@ def merge_logic(process_params):
     logger.info("Found input config directory: %s", path)
     logger.info("Storing generated config to: %s", filename)
     with open(filename, "w+") as f:
-        f.write(yaml.dump(output))
+        f.write(yaml.dump(output, allow_unicode=allow_unicode))
 
 
 def is_leaf_directory(dir, leaf_directories):
@@ -203,6 +205,8 @@ def get_parser():
                         action='store_true', help='Process config using multiprocessing')
     parser.add_argument('--filter-rules-key', dest='filter_rules', default=None, type=str,
                         help='keep these keys from the generated data, based on the configured filter key')
+    parser.add_argument('--allow-unicode', dest='allow_unicode', default=False,
+                        action='store_true', help='allow unicode characters in output (default: False, outputs escape sequences)')
     return parser
 
 
@@ -219,4 +223,4 @@ def run(args=None):
 
     # merge the configs using HIML
     merge_configs(dirs, opts.hierarchy_levels,
-                  opts.output_dir, opts.enable_parallel, opts.filter_rules)
+                  opts.output_dir, opts.enable_parallel, opts.filter_rules, opts.allow_unicode)

himl/main.py

@@ -43,7 +43,7 @@ def do_run(self, opts):
         config_processor.process(cwd, opts.path, filters, excluded_keys, opts.enclosing_key,
                                  opts.remove_enclosing_key, opts.output_format, opts.print_data, opts.output_file,
                                  opts.skip_interpolation_resolving, opts.skip_interpolation_validation,
-                                 opts.skip_secrets, opts.multi_line_string,
+                                 opts.skip_secrets, opts.multi_line_string, opts.allow_unicode,
                                  type_strategies=[(list, [opts.merge_list_strategy.value]), (dict, ["merge"])])
 
     @staticmethod
@@ -79,6 +79,8 @@ def get_parser(parser=None):
         parser.add_argument('--list-merge-strategy', dest='merge_list_strategy', type=ListMergeStrategy,
                             choices=list(ListMergeStrategy), default='append_unique',
                             help='override default merge strategy for list')
+        parser.add_argument('--allow-unicode', dest='allow_unicode', action='store_true', default=False,
+                            help='allow unicode characters in output (default: False, outputs escape sequences)')
         parser.add_argument('--version', action='version', version='%(prog)s v{version}'.format(version="0.18.0"),
                             help='print himl version')
         return parser

tests/test_config_generator.py

@@ -191,6 +191,52 @@ def test_output_formats(self):
         )
         assert json_result == config_data
 
+    def test_unicode_processing_disabled(self):
+        """Test Unicode processing with allow_unicode=False"""
+        config_data = {
+            'message': 'Hello 世界',
+            'emoji': '✨ sparkles',
+            'accents': 'café'
+        }
+        self.create_test_yaml('unicode.yaml', config_data)
+
+        result = self.config_processor.process(
+            cwd=self.temp_dir,
+            path='unicode.yaml',
+            allow_unicode=False,
+            print_data=False
+        )
+
+        # Data should be processed correctly regardless of Unicode settings
+        assert result['message'] == 'Hello 世界'
+        assert result['emoji'] == '✨ sparkles'
+        assert result['accents'] == 'café'
+
+    def test_unicode_processing_enabled(self):
+        """Test Unicode processing with allow_unicode=True"""
+        config_data = {
+            'message': 'Hello 世界',
+            'emoji': '✨ sparkles',
+            'accents': 'café',
+            'arabic': 'مرحبا',
+            'cyrillic': 'Привет'
+        }
+        self.create_test_yaml('unicode.yaml', config_data)
+
+        result = self.config_processor.process(
+            cwd=self.temp_dir,
+            path='unicode.yaml',
+            allow_unicode=True,
+            print_data=False
+        )
+
+        # Data should be processed correctly
+        assert result['message'] == 'Hello 世界'
+        assert result['emoji'] == '✨ sparkles'
+        assert result['accents'] == 'café'
+        assert result['arabic'] == 'مرحبا'
+        assert result['cyrillic'] == 'Привет'
+
 
 class TestConfigGenerator:
     """Test cases for ConfigGenerator class"""
@@ -218,6 +264,7 @@ def test_config_generator_initialization(self):
             cwd=self.temp_dir,
             path='test',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -238,6 +285,7 @@ def test_hierarchy_generation(self):
             cwd=self.temp_dir,
             path='production',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -256,6 +304,7 @@ def test_yaml_content_loading(self):
             cwd=self.temp_dir,
             path='test',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -270,6 +319,7 @@ def test_yaml_merging(self):
             cwd=self.temp_dir,
             path='test',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -298,6 +348,7 @@ def test_output_data_yaml(self):
             cwd=self.temp_dir,
             path='test',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -316,6 +367,7 @@ def test_output_data_json(self):
             cwd=self.temp_dir,
             path='test',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -335,6 +387,7 @@ def test_invalid_output_format(self):
             cwd=self.temp_dir,
             path='test',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -353,6 +406,7 @@ def test_values_from_dir_path(self):
             cwd=self.temp_dir,
             path='env=production/region=us-east-1/cluster=web',
             multi_line_string=False,
+            allow_unicode=False,
             type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
             fallback_strategies=["override"],
             type_conflict_strategies=["override"]
@@ -361,3 +415,88 @@ def test_values_from_dir_path(self):
         values = generator.get_values_from_dir_path()
         expected = {'env': 'production', 'region': 'us-east-1', 'cluster': 'web'}
         assert values == expected
+
+    def test_allow_unicode_false(self):
+        """Test that Unicode characters are escaped when allow_unicode=False"""
+        generator = ConfigGenerator(
+            cwd=self.temp_dir,
+            path='test',
+            multi_line_string=False,
+            allow_unicode=False,
+            type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
+            fallback_strategies=["override"],
+            type_conflict_strategies=["override"]
+        )
+
+        test_data = {
+            'greeting': 'Hello 世界',
+            'emoji': '🚀 rocket',
+            'special': 'café résumé naïve'
+        }
+        yaml_output = generator.output_yaml_data(test_data)
+
+        # When allow_unicode=False, Unicode should be escaped
+        assert '\\u' in yaml_output or '\\x' in yaml_output or 'greeting: Hello' in yaml_output
+
+    def test_allow_unicode_true(self):
+        """Test that Unicode characters are preserved when allow_unicode=True"""
+        generator = ConfigGenerator(
+            cwd=self.temp_dir,
+            path='test',
+            multi_line_string=False,
+            allow_unicode=True,
+            type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
+            fallback_strategies=["override"],
+            type_conflict_strategies=["override"]
+        )
+
+        test_data = {
+            'greeting': 'Hello 世界',
+            'emoji': '🚀 rocket',
+            'special': 'café résumé naïve'
+        }
+        yaml_output = generator.output_yaml_data(test_data)
+
+        # When allow_unicode=True, most Unicode should be preserved
+        # Note: PyYAML may still escape some 4-byte UTF-8 characters (emojis)
+        assert '世界' in yaml_output  # Chinese characters preserved
+        assert 'café' in yaml_output  # Accented characters preserved
+        assert 'résumé' in yaml_output  # Accented characters preserved
+        assert 'naïve' in yaml_output  # Accented characters preserved
+        # Emoji might be escaped as \U0001F680 even with allow_unicode=True
+        assert ('🚀' in yaml_output or '\\U0001F680' in yaml_output)
+
+    def test_unicode_in_nested_structures(self):
+        """Test Unicode handling in nested data structures"""
+        generator = ConfigGenerator(
+            cwd=self.temp_dir,
+            path='test',
+            multi_line_string=False,
+            allow_unicode=True,
+            type_strategies=[(list, ["append_unique"]), (dict, ["merge"])],
+            fallback_strategies=["override"],
+            type_conflict_strategies=["override"]
+        )
+
+        test_data = {
+            'users': [
+                {'name': 'José García', 'country': 'España'},
+                {'name': '田中太郎', 'country': '日本'},
+                {'name': 'François Müller', 'country': 'France'}
+            ],
+            'config': {
+                'title': 'Configuration — Настройки',
+                'description': 'Multi-language support: English, 中文, العربية, हिन्दी'
+            }
+        }
+        yaml_output = generator.output_yaml_data(test_data)
+
+        # Verify Unicode characters are preserved (excluding 4-byte emoji which may be escaped)
+        assert 'José García' in yaml_output
+        assert '田中太郎' in yaml_output
+        assert 'España' in yaml_output
+        assert '日本' in yaml_output
+        assert 'Настройки' in yaml_output
+        assert '中文' in yaml_output
+        assert 'العربية' in yaml_output
+        assert 'हिन्दी' in yaml_output