Merge pull request #3341 from robnester-rh/EC-1836

feat(EC-1836): add ec.oci.parsed_blob builtin for cached JSON parsing

Author: robnester-rh

docs/modules/ROOT/pages/ec_oci_parsed_blob.adoc

@@ -0,0 +1,15 @@
+= ec.oci.parsed_blob
+
+Fetch a blob from an OCI registry and return the parsed JSON value. Results are cached per component.
+
+== Usage
+
+  value = ec.oci.parsed_blob(ref: string)
+
+== Parameters
+
+* `ref` (`string`): OCI blob reference
+
+== Return
+
+`value` (`any`): the parsed JSON value from the OCI blob

docs/modules/ROOT/pages/rego_builtins.adoc

@@ -26,6 +26,8 @@ information.
 |Discover artifacts attached to an image via OCI Referrers API.
 |xref:ec_oci_image_tag_refs.adoc[ec.oci.image_tag_refs]
 |Discover artifacts attached to an image via legacy tag-based discovery (cosign .sig, .att, .sbom suffixes).
+|xref:ec_oci_parsed_blob.adoc[ec.oci.parsed_blob]
+|Fetch a blob from an OCI registry and return the parsed JSON value. Results are cached per component.
 |xref:ec_purl_is_valid.adoc[ec.purl.is_valid]
 |Determine whether or not a given PURL is valid.
 |xref:ec_purl_parse.adoc[ec.purl.parse]

docs/modules/ROOT/partials/rego_nav.adoc

@@ -8,6 +8,7 @@
 ** xref:ec_oci_image_manifests.adoc[ec.oci.image_manifests]
 ** xref:ec_oci_image_referrers.adoc[ec.oci.image_referrers]
 ** xref:ec_oci_image_tag_refs.adoc[ec.oci.image_tag_refs]
+** xref:ec_oci_parsed_blob.adoc[ec.oci.parsed_blob]
 ** xref:ec_purl_is_valid.adoc[ec.purl.is_valid]
 ** xref:ec_purl_parse.adoc[ec.purl.parse]
 ** xref:ec_sigstore_verify_attestation.adoc[ec.sigstore.verify_attestation]

internal/rego/oci/oci.go

@@ -63,6 +63,7 @@ const (
 	ociImageIndexName          = "ec.oci.image_index"
 	ociImageTagRefsName        = "ec.oci.image_tag_refs"
 	ociImageReferrersName      = "ec.oci.image_referrers"
+	ociParsedBlobName          = "ec.oci.parsed_blob"
 	maxTarEntrySizeConst       = 500 * 1024 * 1024 // 500MB
 )
 
@@ -96,6 +97,28 @@ func registerOCIBlob() {
 	})
 }
 
+func registerOCIParsedBlob() {
+	decl := rego.Function{
+		Name: ociParsedBlobName,
+		Decl: types.NewFunction(
+			types.Args(
+				types.Named("ref", types.S).Description("OCI blob reference"),
+			),
+			types.Named("value", types.A).Description("the parsed JSON value from the OCI blob"),
+		),
+		Memoize:          true,
+		Nondeterministic: true,
+	}
+
+	rego.RegisterBuiltin1(&decl, ociParsedBlob)
+	ast.RegisterBuiltin(&ast.Builtin{
+		Name:             decl.Name,
+		Description:      "Fetch a blob from an OCI registry and return the parsed JSON value. Results are cached per component.",
+		Decl:             decl.Decl,
+		Nondeterministic: decl.Nondeterministic,
+	})
+}
+
 func registerOCIDescriptor() {
 	platform := types.NewObject(
 		[]*types.StaticProperty{
@@ -603,6 +626,71 @@ func ociBlobInternal(bctx rego.BuiltinContext, a *ast.Term, verifyDigest bool) (
 	return result.(*ast.Term), nil
 }
 
+func ociParsedBlob(bctx rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
+	logger := log.WithField("function", ociParsedBlobName)
+
+	uri, ok := a.Value.(ast.String)
+	if !ok {
+		logger.Error("input is not a string")
+		return nil, nil
+	}
+	refStr := string(uri)
+	logger = logger.WithField("ref", refStr)
+
+	cc := componentCacheFromContext(bctx.Context)
+
+	if cached, found := cc.parsedBlobCache.Load(refStr); found {
+		logger.Debug("Parsed blob served from cache")
+		return cached.(*ast.Term), nil
+	}
+
+	result, err, _ := cc.parsedBlobFlight.Do(refStr, func() (any, error) {
+		if cached, found := cc.parsedBlobCache.Load(refStr); found {
+			logger.Debug("Parsed blob served from cache (after singleflight)")
+			return cached, nil
+		}
+
+		rawTerm, err := ociBlobInternal(bctx, a, true)
+		if err != nil || rawTerm == nil {
+			return nil, nil //nolint:nilerr
+		}
+
+		rawStr, ok := rawTerm.Value.(ast.String)
+		if !ok {
+			logger.Error("blob value is not a string")
+			return nil, nil //nolint:nilerr
+		}
+
+		var parsed any
+		if err := json.Unmarshal([]byte(string(rawStr)), &parsed); err != nil {
+			logger.WithFields(log.Fields{
+				"action": "unmarshal",
+				"error":  err,
+			}).Error("failed to unmarshal blob as JSON")
+			return nil, nil //nolint:nilerr
+		}
+
+		value, err := ast.InterfaceToValue(parsed)
+		if err != nil {
+			logger.WithFields(log.Fields{
+				"action": "convert to ast",
+				"error":  err,
+			}).Error("failed to convert parsed JSON to AST value")
+			return nil, nil //nolint:nilerr
+		}
+
+		term := ast.NewTerm(value)
+		cc.parsedBlobCache.Store(refStr, term)
+		logger.Debug("Parsed blob cached")
+		return term, nil
+	})
+
+	if err != nil || result == nil {
+		return nil, nil
+	}
+	return result.(*ast.Term), nil
+}
+
 func ociDescriptor(bctx rego.BuiltinContext, a *ast.Term) (*ast.Term, error) {
 	logger := log.WithField("function", ociDescriptorName)
 
@@ -807,10 +895,12 @@ func ClearCaches() {
 // Lighter caches (manifests, descriptors, image indexes) remain global because they
 // are small and benefit from cross-component sharing (e.g., shared task bundle manifests).
 type ComponentCache struct {
-	blobCache   sync.Map
-	blobFlight  singleflight.Group
-	filesCache  sync.Map
-	filesFlight singleflight.Group
+	blobCache        sync.Map
+	blobFlight       singleflight.Group
+	filesCache       sync.Map
+	filesFlight      singleflight.Group
+	parsedBlobCache  sync.Map
+	parsedBlobFlight singleflight.Group
 }
 
 type componentCacheKey struct{}
@@ -1634,6 +1724,7 @@ func isNotFoundError(err error) bool {
 
 func init() {
 	registerOCIBlob()
+	registerOCIParsedBlob()
 	registerOCIBlobFiles()
 	registerOCIDescriptor()
 	registerOCIImageFiles()

internal/rego/oci/oci_test.go

@@ -129,6 +129,96 @@ func TestOCIBlob(t *testing.T) {
 	}
 }
 
+func TestOCIParsedBlob(t *testing.T) {
+	t.Cleanup(ClearCaches)
+	ClearCaches()
+
+	// Each ref must match the SHA256 of the test data content
+	// echo -n '{"spam": "maps"}' | sha256sum → 4bbf56a3...
+	validRef := ast.StringTerm("registry.local/spam@sha256:4bbf56a3a9231f752d3b9c174637975f0f83ed2b15e65799837c571e4ef3374b")
+	// echo -n 'not valid json' | sha256sum → 62eb7d4f...
+	invalidJSONRef := ast.StringTerm("registry.local/spam@sha256:62eb7d4ff39a69b09cf8fdaa37579468bf970290cb3ff1fe87554cba9d06cc50")
+
+	t.Run("valid JSON blob", func(t *testing.T) {
+		ClearCaches()
+		client := fake.FakeClient{}
+		layer := static.NewLayer([]byte(`{"spam": "maps"}`), types.OCIUncompressedLayer)
+		client.On("Layer", mock.Anything, mock.Anything).Return(layer, nil)
+
+		ctx := oci.WithClient(context.Background(), &client)
+		bctx := rego.BuiltinContext{Context: ctx}
+
+		result, err := ociParsedBlob(bctx, validRef)
+		require.NoError(t, err)
+		require.NotNil(t, result)
+
+		obj, ok := result.Value.(ast.Object)
+		require.True(t, ok)
+		val := obj.Get(ast.StringTerm("spam"))
+		require.NotNil(t, val)
+		require.Equal(t, ast.String("maps"), val.Value)
+	})
+
+	t.Run("invalid JSON blob", func(t *testing.T) {
+		ClearCaches()
+		client := fake.FakeClient{}
+		layer := static.NewLayer([]byte(`not valid json`), types.OCIUncompressedLayer)
+		client.On("Layer", mock.Anything, mock.Anything).Return(layer, nil)
+
+		ctx := oci.WithClient(context.Background(), &client)
+		bctx := rego.BuiltinContext{Context: ctx}
+
+		result, err := ociParsedBlob(bctx, invalidJSONRef)
+		require.NoError(t, err)
+		require.Nil(t, result)
+	})
+
+	t.Run("caching returns same result", func(t *testing.T) {
+		ClearCaches()
+		client := fake.FakeClient{}
+		layer := static.NewLayer([]byte(`{"spam": "maps"}`), types.OCIUncompressedLayer)
+		client.On("Layer", mock.Anything, mock.Anything).Return(layer, nil)
+
+		ctx := oci.WithClient(context.Background(), &client)
+		bctx := rego.BuiltinContext{Context: ctx}
+
+		result1, err := ociParsedBlob(bctx, validRef)
+		require.NoError(t, err)
+		require.NotNil(t, result1)
+
+		result2, err := ociParsedBlob(bctx, validRef)
+		require.NoError(t, err)
+		require.NotNil(t, result2)
+
+		require.Same(t, result1, result2)
+		client.AssertNumberOfCalls(t, "Layer", 1)
+	})
+
+	t.Run("unexpected uri type", func(t *testing.T) {
+		ClearCaches()
+		client := fake.FakeClient{}
+		ctx := oci.WithClient(context.Background(), &client)
+		bctx := rego.BuiltinContext{Context: ctx}
+
+		result, err := ociParsedBlob(bctx, ast.IntNumberTerm(42))
+		require.NoError(t, err)
+		require.Nil(t, result)
+	})
+
+	t.Run("remote error", func(t *testing.T) {
+		ClearCaches()
+		client := fake.FakeClient{}
+		client.On("Layer", mock.Anything, mock.Anything).Return(nil, errors.New("boom!"))
+
+		ctx := oci.WithClient(context.Background(), &client)
+		bctx := rego.BuiltinContext{Context: ctx}
+
+		result, err := ociParsedBlob(bctx, validRef)
+		require.NoError(t, err)
+		require.Nil(t, result)
+	})
+}
+
 func TestOCIBlobFiles(t *testing.T) {
 	t.Cleanup(ClearCaches)
 	ClearCaches() // Clear before test to avoid interference from previous tests
@@ -1258,6 +1348,7 @@ func TestFunctionsRegistered(t *testing.T) {
 		ociImageIndexName,
 		ociImageTagRefsName,
 		ociImageReferrersName,
+		ociParsedBlobName,
 	}
 	for _, name := range names {
 		t.Run(name, func(t *testing.T) {