Fix corpus test round-trip failures and achieve Rust escape_debug parity

.gitignore

@@ -3,6 +3,9 @@ tmp/
 .DS_Store
 coverage.out
 test/cedar-validation-tool/target/
+corpus-tests/
+corpus-tests-json-schemas/
+corpus-tests-validation/
 *.out
 .ai/
 ideas/

Makefile

@@ -13,11 +13,16 @@ linters:
 	go tool cover -func=coverage.out | sed 's/%$$//' | awk '$$2 == "isCedarType" { next } $$2 == "Entity" && $$1 ~ /entity\.go/ { next } $$2 == "typeOfExtensionCall" { next } { if ($$3 < 100.0) { printf "Insufficient code coverage for %s\n", $$0; failed=1 } } END { exit failed }'
 
 # Download the latest corpus tests tarball and overwrite corpus-tests.tar.gz if changed
-.PHONY: check-upstream-corpus
 check-upstream-corpus:
-	@tmp="$$(mktemp)" && \
-	curl -fL -o "$$tmp" https://raw.githubusercontent.com/cedar-policy/cedar-integration-tests/main/corpus-tests.tar.gz && \
-	if cmp -s "$$tmp" corpus-tests.tar.gz; then echo "corpus-tests.tar.gz is up to date."; rm -f "$$tmp"; else mv "$$tmp" corpus-tests.tar.gz; echo "corpus-tests.tar.gz updated."; fi
+	@tmpdir="$$(mktemp -d)" && \
+	trap 'rm -rf "$$tmpdir"' EXIT && \
+	curl -fL -o "$$tmpdir/corpus-tests.tar.gz" https://raw.githubusercontent.com/cedar-policy/cedar-integration-tests/main/corpus-tests.tar.gz && \
+	if cmp -s "$$tmpdir/corpus-tests.tar.gz" corpus-tests.tar.gz; then \
+		echo "corpus-tests.tar.gz is up to date."; \
+	else \
+		mv "$$tmpdir/corpus-tests.tar.gz" corpus-tests.tar.gz; \
+		echo "corpus-tests.tar.gz updated."; \
+	fi
 
 # Use an order-only prerequisite to check for changes. This allows other targets to
 # reference corpus-tests.tar.gz as a dependency so that they'll only be re-created
@@ -27,45 +32,46 @@ corpus-tests.tar.gz: | check-upstream-corpus
 # Convert Cedar schemas to JSON schemas
 corpus-tests-json-schemas.tar.gz: corpus-tests.tar.gz
 	@echo "Generating JSON schemas from Cedar schemas..."
-	@rm -rf /tmp/corpus-tests /tmp/corpus-tests-json-schemas
-	@mkdir -p /tmp/corpus-tests-json-schemas
-	@tar -xzf corpus-tests.tar.gz -C /tmp/
-	@for schema in /tmp/corpus-tests/*.cedarschema; do \
-		basename=$$(basename $$schema .cedarschema); \
-		echo "Converting $$basename.cedarschema..."; \
-		cedar translate-schema --direction cedar-to-json --schema "$$schema" > "/tmp/corpus-tests-json-schemas/$$basename.cedarschema.json" 2>&1; \
-	done
-	@tar -czf corpus-tests-json-schemas.tar.gz -C /tmp corpus-tests-json-schemas
-	@rm -rf /tmp/corpus-tests /tmp/corpus-tests-json-schemas
-	@echo "Done! Created corpus-tests-json-schemas.tar.gz"
+	@tmpdir="$$(mktemp -d)" && \
+	trap 'rm -rf "$$tmpdir"' EXIT && \
+	tar -xzf corpus-tests.tar.gz -C "$$tmpdir" && \
+	mkdir -p "$$tmpdir/corpus-tests-json-schemas" && \
+	for schema in "$$tmpdir"/corpus-tests/*.cedarschema; do \
+		basename=$$(basename "$$schema" .cedarschema); \
+		echo "  Converting $$basename.cedarschema..."; \
+		cedar translate-schema --direction cedar-to-json --schema "$$schema" \
+			> "$$tmpdir/corpus-tests-json-schemas/$$basename.cedarschema.json" 2>&1; \
+	done && \
+	tar -czf corpus-tests-json-schemas.tar.gz -C "$$tmpdir" corpus-tests-json-schemas && \
+	echo "Done! Created corpus-tests-json-schemas.tar.gz"
 
-# Build cedar-validation-tool and generate validation results
+# Build cedar-validation-tool
 test/cedar-validation-tool/target/release/cedar-validation-tool: test/cedar-validation-tool/src/main.rs test/cedar-validation-tool/Cargo.toml
 	@echo "Building cedar-validation-tool..."
 	@cd test/cedar-validation-tool && cargo build --release
 
+# Generate validation results from Rust Cedar
 corpus-tests-validation.tar.gz: corpus-tests.tar.gz test/cedar-validation-tool/target/release/cedar-validation-tool
 	@echo "Generating validation results from Rust Cedar..."
-	@rm -rf /tmp/corpus-tests /tmp/corpus-tests-validation
-	@mkdir -p /tmp/corpus-tests-validation
-	@tar -xzf corpus-tests.tar.gz -C /tmp/
-	@for testjson in /tmp/corpus-tests/*.json; do \
+	@tmpdir="$$(mktemp -d)" && \
+	trap 'rm -rf "$$tmpdir"' EXIT && \
+	tar -xzf corpus-tests.tar.gz -C "$$tmpdir" && \
+	mkdir -p "$$tmpdir/corpus-tests-validation" && \
+	for testjson in "$$tmpdir"/corpus-tests/*.json; do \
 		case "$$testjson" in *.entities.json) continue ;; esac; \
-		basename=$$(basename $$testjson .json); \
+		basename=$$(basename "$$testjson" .json); \
 		test/cedar-validation-tool/target/release/cedar-validation-tool \
-			"$$testjson" "/tmp/corpus-tests-validation/$${basename}.validation.json"; \
-	done
-	@cd /tmp && tar -czf corpus-tests-validation.tar.gz corpus-tests-validation/
-	@mv /tmp/corpus-tests-validation.tar.gz .
-	@rm -rf /tmp/corpus-tests /tmp/corpus-tests-validation
-	@echo "Done! Created corpus-tests-validation.tar.gz"
+			"$$testjson" "$$tmpdir/corpus-tests-validation/$${basename}.validation.json"; \
+	done && \
+	tar -czf corpus-tests-validation.tar.gz -C "$$tmpdir" corpus-tests-validation && \
+	echo "Done! Created corpus-tests-validation.tar.gz"
 
 # Regenerate validation data for x/exp/schema/validate/testdata
 testdata-validation: test/cedar-validation-tool/target/release/cedar-validation-tool
 	@echo "Regenerating testdata validation files..."
 	@for testjson in x/exp/schema/validate/testdata/*.json; do \
 		case "$$testjson" in *.entities.json|*.validation.json) continue ;; esac; \
-		basename=$$(basename $$testjson .json); \
+		basename=$$(basename "$$testjson" .json); \
 		echo "  Validating $$basename..."; \
 		test/cedar-validation-tool/target/release/cedar-validation-tool \
 			"$$testjson" "x/exp/schema/validate/testdata/$${basename}.validation.json"; \

corpus_test.go

@@ -276,88 +276,125 @@ func TestCorpus(t *testing.T) {
 			testvalidate.RunEntityChecks(t, rs, entities, cv)
 			testvalidate.RunRequestChecks(t, rs, cv, requests)
 
-			for _, request := range tt.Requests {
-				if len(request.Reasons) == 0 && request.Reasons != nil {
-					request.Reasons = nil
-				}
-				if len(request.Errors) == 0 && request.Errors != nil {
-					request.Errors = nil
-				}
-
-				t.Run(request.Desc, func(t *testing.T) {
-					t.Parallel()
-					ok, diag := policySet.IsAuthorized(
-						entities,
-						cedar.Request{
-							Principal: cedar.EntityUID(request.Principal),
-							Action:    cedar.EntityUID(request.Action),
-							Resource:  cedar.EntityUID(request.Resource),
-							Context:   request.Context,
-						})
+			// Round-trip the policy set through cedar marshal/unmarshal
+			cedarRTBytes := policySet.MarshalCedar()
+			cedarRTPolicySet, err := cedar.NewPolicySetFromBytes("policy.cedar", cedarRTBytes)
+			testutil.OK(t, wrapErr("parsing cedar round-tripped policy set", err))
+
+			// Round-trip the policy set through JSON marshal/unmarshal
+			jsonRTBytes, err := policySet.MarshalJSON()
+			testutil.OK(t, wrapErr("marshalling policy set to JSON", err))
+			var jsonRTPolicySet cedar.PolicySet
+			err = jsonRTPolicySet.UnmarshalJSON(jsonRTBytes)
+			testutil.OK(t, wrapErr("unmarshalling JSON round-tripped policy set", err))
+
+			// Round-trip the entities through JSON and assert equality
+			entitiesJSONBytes, err := entities.MarshalJSON()
+			testutil.OK(t, wrapErr("marshalling entities to JSON", err))
+			var rtEntities cedar.EntityMap
+			err = json.Unmarshal(entitiesJSONBytes, &rtEntities)
+			testutil.OK(t, wrapErr("unmarshalling JSON round-tripped entities", err))
+			testutil.Equals(t, rtEntities, entities)
+
+			policySets := map[string]*cedar.PolicySet{
+				"":                 policySet,
+				"/policy-cedar-rt": cedarRTPolicySet,
+				"/policy-json-rt":  &jsonRTPolicySet,
+			}
 
-					testutil.Equals(t, ok, request.Decision)
-					var errors []cedar.PolicyID
-					for _, n := range diag.Errors {
-						errors = append(errors, n.PolicyID)
+			for suffix, ps := range policySets {
+				ps := ps
+				suffix := suffix
+				for _, request := range tt.Requests {
+					if len(request.Reasons) == 0 && request.Reasons != nil {
+						request.Reasons = nil
 					}
-					testutil.Equals(t, errors, request.Errors)
-					var reasons []cedar.PolicyID
-					for _, n := range diag.Reasons {
-						reasons = append(reasons, n.PolicyID)
+					if len(request.Errors) == 0 && request.Errors != nil {
+						request.Errors = nil
 					}
-					testutil.Equals(t, reasons, request.Reasons)
-				})
 
-				t.Run(request.Desc+"/batch", func(t *testing.T) {
-					t.Parallel()
-					ctx := context.Background()
-					var res batch.Result
-					var total int
-					principal := cedar.EntityUID(request.Principal)
-					action := cedar.EntityUID(request.Action)
-					resource := cedar.EntityUID(request.Resource)
-					context := request.Context
-					err := batch.Authorize(ctx, policySet, entities, batch.Request{
-						Principal: batch.Variable("principal"),
-						Action:    batch.Variable("action"),
-						Resource:  batch.Variable("resource"),
-						Context:   batch.Variable("context"),
-						Variables: batch.Variables{
-							"principal": []cedar.Value{principal},
-							"action":    []cedar.Value{action},
-							"resource":  []cedar.Value{resource},
-							"context":   []cedar.Value{context},
-						},
-					}, func(r batch.Result) error {
-						res = r
-						total++
-						return nil
+					t.Run(request.Desc+suffix, func(t *testing.T) {
+						t.Parallel()
+						ok, diag := ps.IsAuthorized(
+							entities,
+							cedar.Request{
+								Principal: cedar.EntityUID(request.Principal),
+								Action:    cedar.EntityUID(request.Action),
+								Resource:  cedar.EntityUID(request.Resource),
+								Context:   request.Context,
+							})
+
+						testutil.Equals(t, ok, request.Decision)
+						var errors []cedar.PolicyID
+						for _, n := range diag.Errors {
+							errors = append(errors, n.PolicyID)
+						}
+						testutil.Equals(t, errors, request.Errors)
+						var reasons []cedar.PolicyID
+						for _, n := range diag.Reasons {
+							reasons = append(reasons, n.PolicyID)
+						}
+						testutil.Equals(t, reasons, request.Reasons)
 					})
-					testutil.OK(t, err)
-					testutil.Equals(t, total, 1)
-					testutil.Equals(t, res.Request.Principal, principal)
-					testutil.Equals(t, res.Request.Action, action)
-					testutil.Equals(t, res.Request.Resource, resource)
-					testutil.Equals(t, res.Request.Context, context)
-
-					ok, diag := res.Decision, res.Diagnostic
-					testutil.Equals(t, ok, request.Decision)
-					var errors []cedar.PolicyID
-					for _, n := range diag.Errors {
-						errors = append(errors, n.PolicyID)
-					}
-					testutil.Equals(t, errors, request.Errors)
-					var reasons []cedar.PolicyID
-					for _, n := range diag.Reasons {
-						reasons = append(reasons, n.PolicyID)
-					}
-					testutil.Equals(t, reasons, request.Reasons)
-				})
+
+					t.Run(request.Desc+"/batch"+suffix, func(t *testing.T) {
+						t.Parallel()
+						ctx := context.Background()
+						var res batch.Result
+						var total int
+						principal := cedar.EntityUID(request.Principal)
+						action := cedar.EntityUID(request.Action)
+						resource := cedar.EntityUID(request.Resource)
+						context := request.Context
+						err := batch.Authorize(ctx, ps, entities, batch.Request{
+							Principal: batch.Variable("principal"),
+							Action:    batch.Variable("action"),
+							Resource:  batch.Variable("resource"),
+							Context:   batch.Variable("context"),
+							Variables: batch.Variables{
+								"principal": []cedar.Value{principal},
+								"action":    []cedar.Value{action},
+								"resource":  []cedar.Value{resource},
+								"context":   []cedar.Value{context},
+							},
+						}, func(r batch.Result) error {
+							res = r
+							total++
+							return nil
+						})
+						testutil.OK(t, err)
+						testutil.Equals(t, total, 1)
+						testutil.Equals(t, res.Request.Principal, principal)
+						testutil.Equals(t, res.Request.Action, action)
+						testutil.Equals(t, res.Request.Resource, resource)
+						testutil.Equals(t, res.Request.Context, context)
+
+						ok, diag := res.Decision, res.Diagnostic
+						testutil.Equals(t, ok, request.Decision)
+						var errors []cedar.PolicyID
+						for _, n := range diag.Errors {
+							errors = append(errors, n.PolicyID)
+						}
+						testutil.Equals(t, errors, request.Errors)
+						var reasons []cedar.PolicyID
+						for _, n := range diag.Reasons {
+							reasons = append(reasons, n.PolicyID)
+						}
+						testutil.Equals(t, reasons, request.Reasons)
+					})
+				}
 			}
 		})
 	}
 }
 
+func wrapErr(msg string, err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("%s: %w", msg, err)
+}
+
 func normalizeJSON(t *testing.T, in []byte) []byte {
 	t.Helper()
 	var out any

internal/json/json.go

@@ -67,7 +67,7 @@ type ifThenElseJSON struct {
 
 type arrayJSON []nodeJSON
 
-type recordJSON map[string]nodeJSON
+type recordJSON map[string]*nodeJSON
 
 type extensionJSON map[string]arrayJSON
 
@@ -131,10 +131,10 @@ type nodeJSON struct {
 	IfThenElse *ifThenElseJSON `json:"if-then-else,omitempty"`
 
 	// Set
-	Set arrayJSON `json:"Set,omitempty"`
+	Set *arrayJSON `json:"Set,omitempty"`
 
 	// Record
-	Record recordJSON `json:"Record,omitempty"`
+	Record *recordJSON `json:"Record,omitempty"`
 
 	// Any other method: decimal, datetime, duration, ip, lessThan, lessThanOrEqual, greaterThan, greaterThanOrEqual, isIpv4, isIpv6, isLoopback, isMulticast, isInRange, toDate, toTime, toDays, toHours, toMinutes, toSeconds, toMilliseconds, offset, durationSince
 	ExtensionCall extensionJSON `json:"-"`

internal/json/json_marshal.go

@@ -103,7 +103,7 @@ func likeToJSON(dest **likeJSON, src ast.NodeTypeLike) {
 func recordToJSON(dest *recordJSON, src ast.NodeTypeRecord) {
 	res := recordJSON{}
 	for _, kv := range src.Elements {
-		var nn nodeJSON
+		nn := &nodeJSON{}
 		nn.FromNode(kv.Value)
 		res[string(kv.Key)] = nn
 	}
@@ -259,13 +259,15 @@ func (n *nodeJSON) FromNode(src ast.IsNode) {
 	// Set
 	// Set arrayJSON `json:"Set"`
 	case ast.NodeTypeSet:
-		arrayToJSON(&n.Set, t.Elements)
+		n.Set = &arrayJSON{}
+		arrayToJSON(n.Set, t.Elements)
 		return
 
 	// Record
 	// Record recordJSON `json:"Record"`
 	case ast.NodeTypeRecord:
-		recordToJSON(&n.Record, t)
+		n.Record = &recordJSON{}
+		recordToJSON(n.Record, t)
 		return
 
 	// Any other method: ip, decimal, lessThan, lessThanOrEqual, greaterThan, greaterThanOrEqual, isIpv4, isIpv6, isLoopback, isMulticast, isInRange

internal/json/json_unmarshal.go

@@ -259,11 +259,11 @@ func (n nodeJSON) ToNode() (ast.Node, error) {
 
 	// Set
 	case n.Set != nil:
-		return n.Set.ToNode()
+		return (*n.Set).ToNode()
 
 	// Record
 	case n.Record != nil:
-		return n.Record.ToNode()
+		return (*n.Record).ToNode()
 
 	// Any other method: lessThan, lessThanOrEqual, greaterThan, greaterThanOrEqual, isIpv4, isIpv6, isLoopback, isMulticast, isInRange
 	default: