feat: comp-3 decimal separator localization (#40)

* feat: comp-3 decimal separator handling

* style: lint

Author: adrienaury

CHANGELOG.md

@@ -14,10 +14,15 @@ Types of changes
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## [0.3.0]
+
+- `Added` COMP-3 codec now accepts both `.` and `,` as decimal separators from JSON input.
+- `Added` `usecomma` flag: when enabled, the COMP-3 codec encodes the decimal separator as `,` in JSON output.
+
 ## [0.2.0]
 
-- `Added` comp-3 numeric encoding support
-- `Fixed` always validate schema and automatically create missing fillers as non exported fields
+- `Added` COMP-3 numeric encoding support.
+- `Fixed` always validate schema and automatically create missing fillers as non exported fields.
 
 ## [0.1.0]
 

README.md

@@ -170,7 +170,10 @@ $ posimap fold --notrim < person.fixed-width
 
 ### Packed decimal (COMP-3)
 
-POSIMAP supports packed decimal (COMP-3) encoding, commonly used in mainframe systems. When working with packed decimal fields, you need to:
+POSIMAP supports packed decimal (COMP-3) encoding, commonly used in mainframe systems.
+Packed decimal fields typically use half the bytes of their displayed value, plus half a byte for the sign.
+
+When working with packed decimal fields, you need to:
 
 1. Specify the `codec: COMP-3` attribute
 2. Define the `picture` clause that describes the field format
@@ -189,7 +192,8 @@ The `picture` format follows standard COBOL notation:
 - `9(n)` represents n numeric digits
 - `V` represents an implied decimal point (no actual character in the data)
 
-Packed decimal fields typically use half the bytes of their displayed value, plus half a byte for the sign.
+The COMP-3 codec accepts both `.` and `,` as decimal separators from JSON input and outputs JSON using `.` as the default decimal separator.
+When the `usecomma` flag is enabled, the COMP-3 codec will encode the decimal separator as `,` in JSON output.
 
 ## Contributing
 

internal/appli/command/fold.go

@@ -38,6 +38,7 @@ type Fold struct {
 	configfile string
 	notrim     bool
 	charset    string
+	usecomma   bool
 }
 
 func NewFoldCommand(rootname string, groupid string) *cobra.Command {
@@ -53,11 +54,14 @@ func NewFoldCommand(rootname string, groupid string) *cobra.Command {
 		configfile: "schema.yaml",
 		notrim:     false,
 		charset:    charsets.ISO88591,
+		usecomma:   false,
 	}
 
 	fold.cmd.Flags().StringVarP(&fold.configfile, "schema", "s", fold.configfile, "set the schema file")
 	fold.cmd.Flags().BoolVarP(&fold.notrim, "notrim", "t", fold.notrim, "don't trim input  by default")
 	fold.cmd.Flags().StringVarP(&fold.charset, "charset", "c", fold.charset, "set the charset for input records")
+	fold.cmd.Flags().BoolVarP(&fold.usecomma, "usecomma", "u", fold.usecomma,
+		"use comma as decimal separator instead of dot")
 
 	fold.cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		if err := fold.execute(cmd, args); err != nil {
@@ -84,7 +88,12 @@ func (f *Fold) execute(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("failed to load configuration file : %w", err)
 	}
 
-	schema, err := cfg.Compile(config.Trim(!f.notrim), config.Charset(f.charset))
+	decimal := config.Comp3Dot()
+	if f.usecomma {
+		decimal = config.Comp3Comma()
+	}
+
+	schema, err := cfg.Compile(config.Trim(!f.notrim), config.Charset(f.charset), decimal)
 	if err != nil {
 		return fmt.Errorf("failed to compile configuration file : %w", err)
 	}

internal/appli/command/testdata/fold/13-comp3.yaml

@@ -5,6 +5,7 @@ description: >-
 stdin: testdata/fold/13-comp3/stdin.fixed-width
 flags:
   -s: testdata/fold/13-comp3/schema.yaml
+  -u: true # use comma as decimal separator
 loglevel: error
 expected:
   stdout: testdata/fold/13-comp3/stdout.jsonl

internal/appli/command/testdata/fold/13-comp3/stdout.jsonl

@@ -1,3 +1,3 @@
-{"compressed":"12345.67"}
-{"compressed":"+12345.67"}
-{"compressed":"-12345.67"}
+{"compressed":"12345,67"}
+{"compressed":"+12345,67"}
+{"compressed":"-12345,67"}

internal/appli/command/testdata/unfold/13-comp3/stdin.jsonl

@@ -1,4 +1,4 @@
 {"compressed":"12345.67"}
-{"compressed":"+12345.67"}
+{"compressed":"+12345,67"}
 {"compressed":"-12345.67"}
-{"compressed":"123.45"}
+{"compressed":"123,45"}

internal/appli/command/unfold.go

@@ -37,6 +37,7 @@ type Unfold struct {
 
 	configfile string
 	charset    string
+	usecomma   bool
 }
 
 func NewUnfoldCommand(rootname string, groupid string) *cobra.Command {
@@ -51,10 +52,13 @@ func NewUnfoldCommand(rootname string, groupid string) *cobra.Command {
 		},
 		configfile: "schema.yaml",
 		charset:    charsets.ISO88591,
+		usecomma:   false,
 	}
 
 	unfold.cmd.Flags().StringVarP(&unfold.configfile, "schema", "s", unfold.configfile, "set the schema file")
 	unfold.cmd.Flags().StringVarP(&unfold.charset, "charset", "c", unfold.charset, "set the charset for output records") //nolint:lll
+	unfold.cmd.Flags().BoolVarP(&unfold.usecomma, "usecomma", "u", unfold.usecomma,
+		"use comma as decimal separator instead of dot")
 
 	unfold.cmd.RunE = func(cmd *cobra.Command, args []string) error {
 		if err := unfold.execute(cmd, args); err != nil {
@@ -81,7 +85,12 @@ func (u *Unfold) execute(cmd *cobra.Command, _ []string) error {
 		return fmt.Errorf("failed to load configuration file : %w", err)
 	}
 
-	schema, err := cfg.Compile(config.Trim(true), config.Charset(u.charset))
+	decimal := config.Comp3Dot()
+	if u.usecomma {
+		decimal = config.Comp3Comma()
+	}
+
+	schema, err := cfg.Compile(config.Trim(true), config.Charset(u.charset), decimal)
 	if err != nil {
 		return fmt.Errorf("failed to compile configuration file : %w", err)
 	}

internal/appli/config/model.go

@@ -55,6 +55,7 @@ type Field struct {
 	Charset *string `yaml:"charset,omitempty"` // Charset can be nil if not set in the configuration file
 	Picture Picture `yaml:"picture,omitempty"` // Picture is an optional string representation of the format
 	Codec   string  `yaml:"codec,omitempty"`   // Codec is the codec to use for this field, default to String
+	Decimal rune    `yaml:"decimal,omitempty"` // Decimal is the decimal separator to use for this field, default to '.'
 
 	Schema Either[string, Schema] `yaml:"schema"` // Schema is either a filename (external schema) or an embedded schema
 }
@@ -115,7 +116,7 @@ func (f Field) Compile(record *schema.Record, defaults ...Default) (*schema.Reco
 				return nil, fmt.Errorf("failed to compile picture for field %s: %w", f.Name, err)
 			}
 
-			record = record.WithField(f.Name, codec.NewComp3(format.Length, format.Decimal, format.Signed),
+			record = record.WithField(f.Name, codec.NewComp3(format.Length, format.Decimal, format.Signed, f.Decimal),
 				f.CompileOptions()...)
 		default:
 			record = record.WithField(f.Name, codec.NewString(charset, f.Length, *f.Trim), f.CompileOptions()...)
@@ -193,3 +194,19 @@ func Charset(name string) Default {
 		return field
 	}
 }
+
+func Comp3Dot() Default {
+	return func(field Field) Field {
+		field.Decimal = '.'
+
+		return field
+	}
+}
+
+func Comp3Comma() Default {
+	return func(field Field) Field {
+		field.Decimal = ','
+
+		return field
+	}
+}

pkg/posimap/core/codec/comp3.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"slices"
 	"strings"
 
 	"github.com/cgi-fr/posimap/pkg/posimap/api"
@@ -38,11 +39,12 @@ type Comp3 struct {
 	intDigits int
 	decDigits int
 	signed    bool
-	size      int // number of bytes needed to store the COMP-3 value
-	length    int // number of characters in the string representation (not counting the sign)
+	size      int  // number of bytes needed to store the COMP-3 value
+	length    int  // number of characters in the string representation (not counting the sign)
+	sep       rune // decimal separator to use in the string representation while decoding comp-3
 }
 
-func NewComp3(intDigits, decDigits int, signed bool) *Comp3 {
+func NewComp3(intDigits, decDigits int, signed bool, sep rune) *Comp3 {
 	length := intDigits + decDigits
 	if decDigits > 0 {
 		length++
@@ -54,6 +56,7 @@ func NewComp3(intDigits, decDigits int, signed bool) *Comp3 {
 		signed:    signed,
 		size:      (intDigits + decDigits + 2) / 2, //nolint:mnd
 		length:    length,
+		sep:       sep,
 	}
 }
 
@@ -77,7 +80,7 @@ func (c *Comp3) Decode(buffer api.Buffer, offset int) (any, error) {
 
 	for byteIndex, byteVal := range bytes {
 		if byteIndex*2 == c.intDigits {
-			result.WriteRune('.')
+			result.WriteRune(c.sep)
 		}
 
 		if byteIndex == c.size-1 {
@@ -98,7 +101,7 @@ func (c *Comp3) Decode(buffer api.Buffer, offset int) (any, error) {
 		result.WriteRune(convertNibbleToRune((byteVal & highNibbleMask) >> nibbleShift))
 
 		if byteIndex*2+1 == c.intDigits {
-			result.WriteRune('.')
+			result.WriteRune(c.sep)
 		}
 
 		result.WriteRune(convertNibbleToRune(byteVal & lowNibbleMask))
@@ -113,13 +116,14 @@ func (c *Comp3) Encode(buffer api.Buffer, offset int, value any) error {
 		return err
 	}
 
-	if c.decDigits > 0 && str[c.intDigits] != '.' {
+	// accept both '.' and ',' as valid decimal separator while encoding ensure robustness
+	if c.decDigits > 0 && str[c.intDigits] != '.' && str[c.intDigits] != ',' {
 		return fmt.Errorf("%w: expected decimal separator at position %d, got %q",
 			ErrMisplacedDecimalSep, c.intDigits, str[c.intDigits])
 	}
 
 	// ensure that there is only one decimal separator
-	if strings.Count(str, ".") > 1 {
+	if strings.Count(str, ".")+strings.Count(str, ",") > 1 {
 		return fmt.Errorf("%w: too many decimal separators in COMP-3 encoding", ErrMisplacedDecimalSep)
 	}
 
@@ -171,7 +175,7 @@ func (c *Comp3) encode(buffer api.Buffer, offset int, str string, nibbleSign byt
 			return fmt.Errorf("%w: too many characters in COMP-3 encoding", ErrBufferTooShort)
 		}
 
-		if char == '.' {
+		if slices.Contains([]rune{'.', ','}, char) {
 			if charIndex-ignoreCount != c.intDigits {
 				return fmt.Errorf("%w", ErrMisplacedDecimalSep)
 			}