feat: configure default encoding via flag

.editorconfig

@@ -8,6 +8,9 @@ indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 
+[*.fixed-width]
+trim_trailing_whitespace = false
+
 [*.go]
 indent_style = tab
 

go.mod

@@ -4,6 +4,7 @@ go 1.24
 
 require (
 	github.com/Masterminds/sprig/v3 v3.3.0
+	github.com/google/go-cmp v0.6.0
 	github.com/mattn/go-isatty v0.0.14
 	github.com/rs/zerolog v1.28.0
 	github.com/spf13/cobra v1.9.1
@@ -16,7 +17,6 @@ require (
 	dario.cat/mergo v1.0.1 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.3.0 // indirect
-	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

internal/appli/command/fold.go

@@ -21,18 +21,20 @@ import (
 	"errors"
 	"io"
 
-	"github.com/cgi-fr/posimap/internal/infra/config"
+	"github.com/cgi-fr/posimap/internal/appli/config"
 	"github.com/cgi-fr/posimap/internal/infra/jsonline"
 	"github.com/cgi-fr/posimap/pkg/posimap/core/buffer"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+	"golang.org/x/text/encoding/charmap"
 )
 
 type Fold struct {
 	cmd *cobra.Command
 
 	configfile string
 	trim       bool
+	charset    string
 }
 
 func NewFoldCommand(rootname string, groupid string) *cobra.Command {
@@ -47,16 +49,19 @@ func NewFoldCommand(rootname string, groupid string) *cobra.Command {
 		},
 		configfile: "schema.yaml",
 		trim:       true,
+		charset:    charmap.ISO8859_1.String(),
 	}
 
 	fold.cmd.Flags().StringVarP(&fold.configfile, "config", "c", fold.configfile, "set the config file")
 	fold.cmd.Flags().BoolVarP(&fold.trim, "trim", "t", fold.trim, "trim the input records")
+	fold.cmd.Flags().StringVarP(&fold.charset, "charset", "C", fold.charset, "set the charset for the input records")
 
 	fold.cmd.Run = fold.execute
 
 	return fold.cmd
 }
 
+//nolint:cyclop
 func (f *Fold) execute(cmd *cobra.Command, _ []string) {
 	reader := buffer.NewBufferReader(cmd.InOrStdin())
 	writer := jsonline.NewWriter(cmd.OutOrStdout())
@@ -66,7 +71,10 @@ func (f *Fold) execute(cmd *cobra.Command, _ []string) {
 		log.Fatal().Err(err).Msg("Failed to load schema")
 	}
 
-	schema := cfg.Compile(config.Trim(f.trim))
+	schema, err := cfg.Compile(config.Trim(f.trim), config.Charset(f.charset))
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to compile config")
+	}
 
 	record, err := schema.Build()
 	if err != nil {

internal/appli/command/testdata/data.fixed-width

@@ -1,9 +1,9 @@
 RECORD-TYPE-1   001JOHN DOE                           1234 ELM STREET               SPRINGFIELD, IL 62704         1234560100123456     XXXXX0200123456     XXXXX0300123456     XXXXX0400123456     XXXXX0500123456     XXXXX
 RECORD-TYPE-1   002JANE SMITH                         56 MAPLE AVENUE               RIVERSIDE, CA 92501           9876540100123456     XXXXX0200123456     XXXXX0300123456     XXXXX0400123456     XXXXX0500123456     XXXXX
-RECORD-TYPE-2   01202504302025043020250430AAAAABBBBBCCCCC
-RECORD-TYPE-2   02202504302025043020250430AAAAABBBBBCCCCC
-RECORD-TYPE-2   03202504302025043020250430AAAAABBBBBCCCCC
-RECORD-TYPE-2   04202504302025043020250430AAAAABBBBBCCCCC
-RECORD-TYPE-2   11202504302025043020250430XXXXXXXYYYYYYYY
-RECORD-TYPE-2   12202504302025043020250430XXXXXXXYYYYYYYY
-RECORD-TYPE-2   13202504302025043020250430XXXXXXXYYYYYYYY
+RECORD-TYPE-2   01202504302025043020250430AAAAABBBBBCCCCC                                                                                                                                                                   
+RECORD-TYPE-2   02202504302025043020250430AAAAABBBBBCCCCC                                                                                                                                                                   
+RECORD-TYPE-2   03202504302025043020250430AAAAABBBBBCCCCC                                                                                                                                                                   
+RECORD-TYPE-2   04202504302025043020250430AAAAABBBBBCCCCC                                                                                                                                                                   
+RECORD-TYPE-2   11202504302025043020250430XXXXXXXYYYYYYYY                                                                                                                                                                   
+RECORD-TYPE-2   12202504302025043020250430XXXXXXXYYYYYYYY                                                                                                                                                                   
+RECORD-TYPE-2   13202504302025043020250430XXXXXXXYYYYYYYY                                                                                                                                                                   

internal/appli/command/testdata/schema-record-type-1.yaml

@@ -25,3 +25,5 @@ schema:
         length: 11
       - name: FILLER
         length: 5
+  - name: NL
+    length: 1

internal/appli/command/testdata/schema-record-type-2.yaml

@@ -25,3 +25,7 @@ schema:
         length: 7
       - name: DATA-2-2
         length: 8
+  - name: FILLER
+    length: 163
+  - name: NL
+    length: 1

internal/appli/command/unfold.go

@@ -21,17 +21,19 @@ import (
 	"errors"
 	"io"
 
-	"github.com/cgi-fr/posimap/internal/infra/config"
+	"github.com/cgi-fr/posimap/internal/appli/config"
 	"github.com/cgi-fr/posimap/internal/infra/jsonline"
 	"github.com/cgi-fr/posimap/pkg/posimap/core/buffer"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
+	"golang.org/x/text/encoding/charmap"
 )
 
 type Unfold struct {
 	cmd *cobra.Command
 
 	configfile string
+	charset    string
 }
 
 func NewUnfoldCommand(rootname string, groupid string) *cobra.Command {
@@ -45,9 +47,11 @@ func NewUnfoldCommand(rootname string, groupid string) *cobra.Command {
 			GroupID: groupid,
 		},
 		configfile: "schema.yaml",
+		charset:    charmap.ISO8859_1.String(),
 	}
 
 	unfold.cmd.Flags().StringVarP(&unfold.configfile, "config", "c", unfold.configfile, "set the config file")
+	unfold.cmd.Flags().StringVarP(&unfold.charset, "charset", "C", unfold.charset, "set the charset for the output records") //nolint:lll
 
 	unfold.cmd.Run = unfold.execute
 
@@ -63,7 +67,10 @@ func (u *Unfold) execute(cmd *cobra.Command, _ []string) {
 		log.Fatal().Err(err).Msg("Failed to load schema")
 	}
 
-	schema := cfg.Compile()
+	schema, err := cfg.Compile(config.Charset(u.charset))
+	if err != nil {
+		log.Fatal().Err(err).Msg("Failed to compile config")
+	}
 
 	record, err := schema.Build()
 	if err != nil {

internal/infra/config/load_test.go → internal/appli/config/load_test.go

@@ -4,7 +4,7 @@ package config_test
 import (
 	"testing"
 
-	"github.com/cgi-fr/posimap/internal/infra/config"
+	"github.com/cgi-fr/posimap/internal/appli/config"
 	"gotest.tools/assert"
 )
 

internal/appli/config/model.go

@@ -0,0 +1,129 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/cgi-fr/posimap/pkg/posimap/core/codec"
+	"github.com/cgi-fr/posimap/pkg/posimap/core/predicate"
+	"github.com/cgi-fr/posimap/pkg/posimap/core/schema"
+	"golang.org/x/text/encoding/charmap"
+)
+
+var ErrUnsupportedCharset = errors.New("unsupported charset")
+
+type Config struct {
+	Schema Schema `yaml:"schema"`
+}
+
+type Schema []Field
+
+type Field struct {
+	Name     string `yaml:"name"`
+	Occurs   int    `yaml:"occurs,omitempty"`
+	Redefine string `yaml:"redefine,omitempty"`
+	When     string `yaml:"when,omitempty"`
+
+	Length  int     `yaml:"length"`
+	Trim    *bool   `yaml:"trim,omitempty"`    // Trim can be nil if not set in the configuration file
+	Charset *string `yaml:"charset,omitempty"` // Charset can be nil if not set in the configuration file
+
+	Schema Either[string, Schema] `yaml:"schema"` // Schema is either a filename (external schema) or an embedded schema
+}
+
+func (f Field) IsRecord() bool {
+	return f.Schema.T2 != nil
+}
+
+func (f Field) CompileOptions() []schema.Option {
+	options := make([]schema.Option, 0)
+
+	if f.Occurs > 0 {
+		options = append(options, schema.Occurs(f.Occurs))
+	}
+
+	if f.Redefine != "" {
+		options = append(options, schema.Redefines(f.Redefine))
+	}
+
+	if f.When != "" {
+		options = append(options, schema.Condition(predicate.When(f.When)))
+	}
+
+	return options
+}
+
+func (f Field) CompileCharset() (*charmap.Charmap, error) {
+	for _, encoding := range charmap.All {
+		if charmap, ok := encoding.(*charmap.Charmap); ok && charmap.String() == *f.Charset {
+			return charmap, nil
+		}
+	}
+
+	return nil, fmt.Errorf("%w: %s", ErrUnsupportedCharset, *f.Charset)
+}
+
+func (f Field) Compile(record schema.Record, defaults ...Default) (schema.Record, error) {
+	if f.IsRecord() {
+		schema, err := f.Schema.T2.Compile(defaults...)
+		if err != nil {
+			return nil, err
+		}
+
+		record = record.WithRecord(f.Name, schema, f.CompileOptions()...)
+	} else {
+		charset, err := f.CompileCharset()
+		if err != nil {
+			return nil, err
+		}
+
+		record = record.WithField(f.Name, codec.NewString(charset, f.Length, *f.Trim), f.CompileOptions()...)
+	}
+
+	return record, nil
+}
+
+func (s Schema) Compile(defaults ...Default) (schema.Record, error) {
+	var err error
+
+	record := make(schema.Record, 0, len(s))
+
+	for _, field := range s {
+		for _, defaultFunc := range defaults {
+			field = defaultFunc(field)
+		}
+
+		record, err = field.Compile(record, defaults...)
+		if err != nil {
+			return nil, fmt.Errorf("failed to compile field %s: %w", field.Name, err)
+		}
+	}
+
+	return record, nil
+}
+
+func (c Config) Compile(defaults ...Default) (schema.Record, error) {
+	return c.Schema.Compile(defaults...)
+}
+
+type Default func(field Field) Field
+
+func Trim(enable bool) Default {
+	return func(field Field) Field {
+		if field.Trim == nil {
+			field.Trim = &enable
+		}
+
+		return field
+	}
+}
+
+func Charset(name string) Default {
+	return func(field Field) Field {
+		if field.Charset == nil {
+			field.Charset = &name
+		}
+
+		return field
+	}
+}

internal/appli/config/model_test.go

@@ -0,0 +1,71 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/cgi-fr/posimap/internal/appli/config"
+	"github.com/cgi-fr/posimap/pkg/posimap/core/codec"
+	"github.com/cgi-fr/posimap/pkg/posimap/core/schema"
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"golang.org/x/text/encoding/charmap"
+	"gotest.tools/assert"
+)
+
+func TestCompile(t *testing.T) {
+	t.Parallel()
+
+	//nolint:exhaustruct
+	tests := []struct {
+		name     string
+		schema   config.Schema
+		expected schema.Record
+	}{
+		{
+			name: "Simple schema",
+			schema: config.Schema{
+				config.Field{
+					Name:   "field1",
+					Length: 10,
+				},
+			},
+			expected: schema.NewSchema().WithField("field1", codec.NewString(charmap.ISO8859_1, 10, true)),
+		},
+		{
+			name: "Nested schema",
+			schema: config.Schema{
+				config.Field{
+					Name: "lvel1",
+					Schema: config.Either[string, config.Schema]{
+						T2: &config.Schema{
+							{
+								Name:   "lvel2",
+								Length: 10,
+							},
+						},
+					},
+				},
+			},
+			expected: schema.NewSchema().WithRecord("lvel1",
+				schema.NewSchema().WithField("lvel2", codec.NewString(charmap.ISO8859_1, 10, true)),
+			),
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			t.Parallel()
+
+			result, err := test.schema.Compile(config.Trim(true), config.Charset(charmap.ISO8859_1.String()))
+			if err != nil {
+				t.Fatalf("expected no error, got %v", err)
+			}
+
+			// Compare the loaded schema with the expected schema
+			assert.DeepEqual(t, result, test.expected,
+				cmp.AllowUnexported(schema.Field{}, schema.Definition{}, codec.String{}, charmap.Charmap{}),
+				cmpopts.IgnoreFields(charmap.Charmap{}, "decode"),
+			)
+		})
+	}
+}