From a2204e77bc0efbde1f890335a27a39315d4a9d7a Mon Sep 17 00:00:00 2001 From: Adrien CABARBAYE Date: Wed, 27 May 2026 13:22:38 +0100 Subject: [PATCH] :gear: `[jsonschema]` Extend JSON Schema validation with schema options and support for schemas defined in YAML. --- changes/20260527132139.feature | 1 + changes/20260527132146.feature | 1 + changes/20260527132214.feature | 1 + utils/serialization/json/helpers.go | 12 ++++- utils/serialization/json/helpers_test.go | 35 +++++++++++++++ utils/serialization/yaml/helpers.go | 11 +++++ utils/serialization/yaml/helpers_test.go | 33 ++++++++++++++ .../jsonschema/testdata/person.schema.yaml | 13 ++++++ utils/validation/jsonschema/validation.go | 10 ++++- .../validation/jsonschema/validation_test.go | 45 +++++++++++++++++++ utils/validation/jsonschema/validator.go | 4 +- 11 files changed, 162 insertions(+), 4 deletions(-) create mode 100644 changes/20260527132139.feature create mode 100644 changes/20260527132146.feature create mode 100644 changes/20260527132214.feature create mode 100644 utils/validation/jsonschema/testdata/person.schema.yaml diff --git a/changes/20260527132139.feature b/changes/20260527132139.feature new file mode 100644 index 0000000000..a37d842e91 --- /dev/null +++ b/changes/20260527132139.feature @@ -0,0 +1 @@ +:sparkles: `[yaml]` Added helpers for checking valid files extensions diff --git a/changes/20260527132146.feature b/changes/20260527132146.feature new file mode 100644 index 0000000000..f7d8927592 --- /dev/null +++ b/changes/20260527132146.feature @@ -0,0 +1 @@ +:sparkles: `[json]` Added helpers for checking valid files extensions diff --git a/changes/20260527132214.feature b/changes/20260527132214.feature new file mode 100644 index 0000000000..ec04495fa7 --- /dev/null +++ b/changes/20260527132214.feature @@ -0,0 +1 @@ +:gear: `[jsonschema]` Extend JSON Schema validation with schema options and support for schemas defined in YAML. diff --git a/utils/serialization/json/helpers.go b/utils/serialization/json/helpers.go index 64456ec798..9cb25874b7 100644 --- a/utils/serialization/json/helpers.go +++ b/utils/serialization/json/helpers.go @@ -20,11 +20,16 @@ import ( "github.com/pquerna/ffjson/ffjson" sigsyaml "sigs.k8s.io/yaml" + "github.com/ARM-software/golang-utils/utils/collection" "github.com/ARM-software/golang-utils/utils/commonerrors" "github.com/ARM-software/golang-utils/utils/reflection" ) -var nullBytes = []byte("null") +var ( + nullBytes = []byte("null") + // JSONExtensions is the list of file extensions that are considered JSON files. + JSONExtensions = []string{".json"} +) // Marshal encodes a value to a JSON byte slice. // It matches encoding/json.Marshal. @@ -98,3 +103,8 @@ func ToYAML(rawJSON []byte) (yaml []byte, err error) { } return } + +// IsJSON returns true if the extension is a JSON file +func IsJSON(extension string) bool { + return collection.In(JSONExtensions, extension, collection.StringCleanCaseInsensitiveMatch) +} diff --git a/utils/serialization/json/helpers_test.go b/utils/serialization/json/helpers_test.go index a1d876f741..d7cca59580 100644 --- a/utils/serialization/json/helpers_test.go +++ b/utils/serialization/json/helpers_test.go @@ -88,6 +88,41 @@ func TestToYAML(t *testing.T) { assert.Contains(t, string(output), "count: 2") } +func TestToYAMLIntegrationInspiredByKubernetesSigsYAML(t *testing.T) { + // Tests inspired by https://github.com/kubernetes-sigs/yaml/blob/master/yaml_test.go + tests := map[string]struct { + json string + expectedContains []string + }{ + "string value": { + json: `{"t":"a"}`, + expectedContains: []string{"t: a"}, + }, + "boolean value": { + json: `{"t":true}`, + expectedContains: []string{"t: true"}, + }, + "array": { + json: `[{"t":"a"}]`, + expectedContains: []string{"- t: a"}, + }, + "large integer": { + json: `{"t":9007199254740993}`, + expectedContains: []string{"t: 9007199254740993"}, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + output, err := ToYAML([]byte(test.json)) + require.NoError(t, err) + for _, expected := range test.expectedContains { + assert.Contains(t, string(output), expected) + } + }) + } +} + func TestToYAMLInvalidJSON(t *testing.T) { _, err := ToYAML([]byte(`{"name":`)) require.Error(t, err) diff --git a/utils/serialization/yaml/helpers.go b/utils/serialization/yaml/helpers.go index c6a8d05671..e1fe0e5a24 100644 --- a/utils/serialization/yaml/helpers.go +++ b/utils/serialization/yaml/helpers.go @@ -20,10 +20,16 @@ import ( sigsyaml "sigs.k8s.io/yaml" + "github.com/ARM-software/golang-utils/utils/collection" "github.com/ARM-software/golang-utils/utils/commonerrors" jsonserialization "github.com/ARM-software/golang-utils/utils/serialization/json" //nolint:misspell ) +var ( + // YAMLExtensions is the list of file extensions that are considered YAML files. + YAMLExtensions = []string{".yaml", ".yml"} +) + // ToJSON converts YAML data to JSON. // // YAML parsing support comes from sigs.k8s.io/yaml, which uses yaml/go-yaml @@ -91,3 +97,8 @@ func Unmarshal(data []byte, v any) error { func UnmarshallWithContext(ctx context.Context, data []byte, v any) error { return NewDecoder(ctx, bytes.NewReader(data)).Decode(v) } + +// IsYAMLFile returns true if the given extension is a YAML file extension. +func IsYAMLFile(extension string) bool { + return collection.In(YAMLExtensions, extension, collection.StringCleanCaseInsensitiveMatch) +} diff --git a/utils/serialization/yaml/helpers_test.go b/utils/serialization/yaml/helpers_test.go index 7315ad6842..d2e90241f7 100644 --- a/utils/serialization/yaml/helpers_test.go +++ b/utils/serialization/yaml/helpers_test.go @@ -88,6 +88,39 @@ func TestToJSON(t *testing.T) { assert.JSONEq(t, `{"count":2,"name":"value"}`, string(output)) } +func TestToJSONIntegrationInspiredByKubernetesSigsYAML(t *testing.T) { + // Tests inspired by https://github.com/kubernetes-sigs/yaml/blob/master/yaml_test.go + tests := map[string]struct { + yaml string + json string + }{ + "string value": { + yaml: "t: a\n", + json: `{"t":"a"}`, + }, + "boolean value": { + yaml: "t: True\n", + json: `{"t":true}`, + }, + "array": { + yaml: "- t: a\n", + json: `[{"t":"a"}]`, + }, + "large integer": { + yaml: "t: 9007199254740993\n", + json: `{"t":9007199254740993}`, + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + output, err := ToJSON([]byte(test.yaml)) + require.NoError(t, err) + assert.JSONEq(t, test.json, string(output)) + }) + } +} + func TestToJSONInvalidYAML(t *testing.T) { _, err := ToJSON([]byte("name: [value\n")) require.Error(t, err) diff --git a/utils/validation/jsonschema/testdata/person.schema.yaml b/utils/validation/jsonschema/testdata/person.schema.yaml new file mode 100644 index 0000000000..8b193ab715 --- /dev/null +++ b/utils/validation/jsonschema/testdata/person.schema.yaml @@ -0,0 +1,13 @@ +$id: testdata/person.schema.yaml +type: object +$defs: + stringField: &stringField + type: string + integerField: &integerField + type: integer +properties: + name: *stringField + count: *integerField +required: + - name +additionalProperties: false diff --git a/utils/validation/jsonschema/validation.go b/utils/validation/jsonschema/validation.go index a6d2b66804..a12efef745 100644 --- a/utils/validation/jsonschema/validation.go +++ b/utils/validation/jsonschema/validation.go @@ -34,6 +34,7 @@ import ( "github.com/ARM-software/golang-utils/utils/filesystem" "github.com/ARM-software/golang-utils/utils/reflection" "github.com/ARM-software/golang-utils/utils/serialization/json" //nolint:misspell + "github.com/ARM-software/golang-utils/utils/serialization/yaml" //nolint:misspell ) // Schema describes a schema file that can be loaded and registered for @@ -120,11 +121,18 @@ func LoadSchemaSpec(ctx context.Context, schema *Schema) (*SchemaSpec, error) { if err != nil { return nil, err } + schemaPath := filesystem.FilePathClean(schema.Filesystem, schema.LocalPath) - data, err := schema.Filesystem.ReadFileWithContextAndLimits(ctx, filesystem.FilePathClean(schema.Filesystem, schema.LocalPath), schema.Limits) + data, err := schema.Filesystem.ReadFileWithContextAndLimits(ctx, schemaPath, schema.Limits) if err != nil { return nil, commonerrors.DescribeCircumstancef(err, "failed to load JSON Schema [%v] from [%v]", schema.Title, schema.LocalPath) } + if yaml.IsYAMLFile(filesystem.FilePathExt(schema.Filesystem, schemaPath)) { + data, err = yaml.ToJSON(data) + if err != nil { + return nil, commonerrors.DescribeCircumstancef(err, "failed to convert schema from YAML to JSON [%v]", schema.Title) + } + } return &SchemaSpec{ ID: schemaID(schema.ID, schema.LocalPath), diff --git a/utils/validation/jsonschema/validation_test.go b/utils/validation/jsonschema/validation_test.go index 6e2c3922b7..083eacf2fd 100644 --- a/utils/validation/jsonschema/validation_test.go +++ b/utils/validation/jsonschema/validation_test.go @@ -48,6 +48,15 @@ func newMissingSchema(t *testing.T, fs filesystem.FS) *Schema { ) } +func newYAMLValidSchema(t *testing.T, fs filesystem.FS) *Schema { + t.Helper() + return NewJSONSchemaFile( + WithTitle("person yaml schema"), + WithLocalPath(path.Join("testdata", "person.schema.yaml")), + WithFilesystem(fs), + ) +} + func newEmbeddedFilesystem(t *testing.T) filesystem.FS { t.Helper() fs, err := filesystem.NewEmbedFileSystem(&embeddedFS) @@ -150,6 +159,13 @@ func TestLoadSchemaSpecEmbeddedFS(t *testing.T) { assert.NotEmpty(t, spec.Specification) } +func TestLoadSchemaSpecYAMLSchemaFile(t *testing.T) { + spec, err := LoadSchemaSpec(context.Background(), newYAMLValidSchema(t, filesystem.GetGlobalFileSystem())) + require.NoError(t, err) + assert.NotEmpty(t, spec.Specification) + assert.JSONEq(t, `{"$id":"testdata/person.schema.yaml","type":"object","$defs":{"stringField":{"type":"string"},"integerField":{"type":"integer"}},"properties":{"name":{"type":"string"},"count":{"type":"integer"}},"required":["name"],"additionalProperties":false}`, string(spec.Specification)) +} + func TestLoadSchemaSpec_MissingSchemaPath(t *testing.T) { _, err := LoadSchemaSpec(context.Background(), newMissingSchema(t, filesystem.GetGlobalFileSystem())) require.Error(t, err) @@ -414,6 +430,30 @@ func TestNewJSONFileValidatorWithOptions_LowSchemaFileLimit(t *testing.T) { errortest.AssertError(t, err, commonerrors.ErrTooLarge) } +func TestValidateFileWithLimits(t *testing.T) { + v, err := NewJSONFileValidator(nil, *newValidSchema(t, filesystem.GetGlobalFileSystem())) + require.NoError(t, err) + + validator, ok := v.(*fileValidator) + require.True(t, ok) + + err = validator.ValidateFileWithLimits(context.Background(), path.Join("testdata", "valid.json"), filesystem.NewLimits(0, 1024, 1, 1, false)) + require.Error(t, err) + errortest.AssertError(t, err, commonerrors.ErrTooLarge) +} + +func TestValidateFileInFSWithLimits(t *testing.T) { + v, err := NewJSONFileValidator(nil, *newValidSchema(t, newEmbeddedFilesystem(t))) + require.NoError(t, err) + + validator, ok := v.(*fileValidator) + require.True(t, ok) + + err = validator.ValidateFileInFSWithLimits(context.Background(), newEmbeddedFilesystem(t), path.Join("testdata", "valid.json"), filesystem.NewLimits(0, 1024, 1, 1, false)) + require.Error(t, err) + errortest.AssertError(t, err, commonerrors.ErrTooLarge) +} + func TestValidateJSONFileAgainstSchema(t *testing.T) { err := ValidateJSONFileAgainstSchema(context.Background(), path.Join("testdata", "valid.json"), nil, *newValidSchema(t, filesystem.GetGlobalFileSystem())) require.NoError(t, err) @@ -461,6 +501,11 @@ func TestValidateYAMLFileAgainstSchema(t *testing.T) { require.NoError(t, err) } +func TestValidateJSONAgainstYAMLSchema(t *testing.T) { + err := ValidateJSONFileAgainstSchema(context.Background(), path.Join("testdata", "valid.json"), nil, *newYAMLValidSchema(t, filesystem.GetGlobalFileSystem())) + require.NoError(t, err) +} + func TestValidateYAMLFileAgainstSchemaOptions(t *testing.T) { err := ValidateYAMLFileAgainstSchemaOptions( context.Background(), diff --git a/utils/validation/jsonschema/validator.go b/utils/validation/jsonschema/validator.go index c4810e7c20..fac66dc29b 100644 --- a/utils/validation/jsonschema/validator.go +++ b/utils/validation/jsonschema/validator.go @@ -102,7 +102,7 @@ func newSchemaCreationFunc(schemaID *string, schema ...Schema) func(context.Cont func NewJSONFileValidator(schemaID *string, schema ...Schema) (v ISchemaValidator, err error) { v = &fileValidator{ schemaCreationFunc: newSchemaCreationFunc(schemaID, schema...), - expectedExtensions: []string{".json"}, + expectedExtensions: jsonserialization.JSONExtensions, convertFunc: nil, } return @@ -125,7 +125,7 @@ func NewJSONFileValidatorWithOptions(options ...SchemaOption) (ISchemaValidator, func NewYAMLFileValidator(schemaID *string, schema ...Schema) (v ISchemaValidator, err error) { v = &fileValidator{ schemaCreationFunc: newSchemaCreationFunc(schemaID, schema...), - expectedExtensions: []string{".yaml", ".yml"}, + expectedExtensions: yaml.YAMLExtensions, convertFunc: yaml.ToJSON, } return