feat: align manifest proto schema with handler parameter schemas (#1840)

* feat: add aligned fields to OrganizationDefinition and MarketDataSetDefinition

Add legal_name, display_name, external_reference, external_reference_type
to OrganizationDefinition proto to match party.register_organization handler
parameters directly. Add validation_expression and resolution_key_expression
to MarketDataSetDefinition to match market_information.register_data_set
handler parameters.

Existing name field retained for backward compatibility.

* feat: pass aligned fields through executor and simplify Starlark script

Update OrganizationInput and MarketDataSetInput structs with new fields.
Update extractPartyAndAccounts with fallback chain (legal_name -> name,
display_name -> legal_name, external_reference -> code).
Update extractMarketData to pass validation/resolution expressions.
Update buildSagaInput to include all new fields in saga input map.
Simplify v1.3.0.star to use direct field passthrough instead of ad-hoc
translation from name/attributes.

* test: add tests for aligned organization and market data set fields

Test new proto fields pass through buildExecutorInput and buildSagaInput.
Test backward compatibility fallback when new fields are empty.
Verify validation_expression and resolution_key_expression propagation.

* fix: regenerate JSON schema and frontend proto, fix gofmt

Regenerate manifest.v1.schema.json to include new MarketDataSetDefinition
and OrganizationDefinition fields. Regenerate frontend TypeScript proto
bindings. Fix gofmt alignment in executor.go and grpc_handler_test.go.

* fix: complete legal_name fallback chain to include code

Add missing code fallback when both legal_name and name are empty, matching
the documented fallback chain: legal_name -> name -> code.
Add test for code-only organization definition.

* fix: make new manifest proto fields optional to avoid required constraint

New organization and market data set fields (legal_name, display_name,
external_reference, external_reference_type, validation_expression,
resolution_key_expression) must be optional in proto3 so they don't
appear in the generated JSON schema's required arrays. This preserves
backward compatibility for existing manifests that don't specify them.

* fix: make legacy name field optional in OrganizationDefinition

The deprecated name field should also be optional so manifests can use
only legal_name and display_name without being forced to provide name.
Addresses remaining CodeRabbit review feedback.

* revert: keep name field non-optional to avoid proto breaking change

Reverting the name field optional change since it triggers the Proto
Breaking Change Detection check (implicit to explicit presence is a
cardinality change). The name field was already required before this PR.
Making it optional would be a separate, intentional breaking change.

---------

Co-authored-by: Ben Coombs <bjcoombs@users.noreply.github.com>

Author: bjcoombs

api/jsonschema/manifest.v1.schema.json

@@ -674,6 +674,14 @@
                 "description": {
                     "type": "string",
                     "description": "description provides additional context about this data set."
+                },
+                "validationExpression": {
+                    "type": "string",
+                    "description": "validation_expression is a CEL expression evaluated against each observation before acceptance. Must return a boolean. Observations are only stored when the expression returns true. Default: \"true\" (accept all observations). Example: \"value \u003e 0 \u0026\u0026 value \u003c 1000000\""
+                },
+                "resolutionKeyExpression": {
+                    "type": "string",
+                    "description": "resolution_key_expression is a CEL expression that determines the resolution key for deduplication and conflict resolution of observations. Default: \"observed_at\" (one observation per timestamp). Example: \"observed_at + ':' + source_code\""
                 }
             },
             "additionalProperties": false,
@@ -795,7 +803,7 @@
                 },
                 "name": {
                     "type": "string",
-                    "description": "name is the legal or display name of the organization."
+                    "description": "name is the legacy display name of the organization. Deprecated: Use legal_name and display_name instead. Retained for backward compatibility."
                 },
                 "partyType": {
                     "type": "string",
@@ -807,6 +815,22 @@
                     },
                     "type": "object",
                     "description": "attributes contains additional metadata for this organization as key-value pairs. Keys and values are validated against the party type's attribute_schema if defined."
+                },
+                "legalName": {
+                    "type": "string",
+                    "description": "legal_name is the official legal name of the organization. Maps directly to the party.register_organization handler's legal_name parameter. If empty, falls back to the name field for backward compatibility."
+                },
+                "displayName": {
+                    "type": "string",
+                    "description": "display_name is the human-readable display name of the organization. Maps directly to the party.register_organization handler's display_name parameter. If empty, falls back to legal_name, then name."
+                },
+                "externalReference": {
+                    "type": "string",
+                    "description": "external_reference is a stable identifier for this organization in an external system. Maps directly to the party.register_organization handler's external_reference parameter. If empty, falls back to the organization code."
+                },
+                "externalReferenceType": {
+                    "type": "string",
+                    "description": "external_reference_type identifies the type of external reference (e.g., \"LEI\", \"NATIONAL_ID\"). Accepts stripped enum names (e.g., \"LEI\") - the handler accepts both \"LEI\" and \"EXTERNAL_REFERENCE_TYPE_LEI\" formats."
                 }
             },
             "additionalProperties": false,

api/proto/meridian/control_plane/v1/manifest.pb.go

@@ -858,6 +858,18 @@ message MarketDataSetDefinition {
 
   // description provides additional context about this data set.
   string description = 6 [(buf.validate.field).string.max_len = 1024];
+
+  // validation_expression is a CEL expression evaluated against each observation before acceptance.
+  // Must return a boolean. Observations are only stored when the expression returns true.
+  // Default: "true" (accept all observations).
+  // Example: "value > 0 && value < 1000000"
+  optional string validation_expression = 7 [(buf.validate.field).string.max_len = 4096];
+
+  // resolution_key_expression is a CEL expression that determines the resolution key
+  // for deduplication and conflict resolution of observations.
+  // Default: "observed_at" (one observation per timestamp).
+  // Example: "observed_at + ':' + source_code"
+  optional string resolution_key_expression = 8 [(buf.validate.field).string.max_len = 4096];
 }
 
 // ========================================
@@ -876,11 +888,9 @@ message OrganizationDefinition {
     pattern: "^[A-Z][A-Z0-9_]*$"
   }];
 
-  // name is the legal or display name of the organization.
-  string name = 2 [(buf.validate.field).string = {
-    min_len: 1
-    max_len: 255
-  }];
+  // name is the legacy display name of the organization.
+  // Deprecated: Use legal_name and display_name instead. Retained for backward compatibility.
+  string name = 2 [(buf.validate.field).string.max_len = 255];
 
   // party_type is the party classification (e.g., "ORGANIZATION", "COUNTERPARTY").
   // Must match a party type code from the manifest's party_types or a built-in type.
@@ -893,6 +903,26 @@ message OrganizationDefinition {
   // attributes contains additional metadata for this organization as key-value pairs.
   // Keys and values are validated against the party type's attribute_schema if defined.
   map<string, string> attributes = 4;
+
+  // legal_name is the official legal name of the organization.
+  // Maps directly to the party.register_organization handler's legal_name parameter.
+  // If empty, falls back to the name field for backward compatibility.
+  optional string legal_name = 5 [(buf.validate.field).string.max_len = 255];
+
+  // display_name is the human-readable display name of the organization.
+  // Maps directly to the party.register_organization handler's display_name parameter.
+  // If empty, falls back to legal_name, then name.
+  optional string display_name = 6 [(buf.validate.field).string.max_len = 255];
+
+  // external_reference is a stable identifier for this organization in an external system.
+  // Maps directly to the party.register_organization handler's external_reference parameter.
+  // If empty, falls back to the organization code.
+  optional string external_reference = 7 [(buf.validate.field).string.max_len = 255];
+
+  // external_reference_type identifies the type of external reference (e.g., "LEI", "NATIONAL_ID").
+  // Accepts stripped enum names (e.g., "LEI") - the handler accepts both
+  // "LEI" and "EXTERNAL_REFERENCE_TYPE_LEI" formats.
+  optional string external_reference_type = 8 [(buf.validate.field).string.max_len = 64];
 }
 
 // ========================================

api/proto/meridian/control_plane/v1/manifest.proto

@@ -171,19 +171,19 @@ def execute_apply_manifest():
         })
 
     # Phase 55: Register organizations with Party service
+    # Fields are passed through directly from the manifest proto via buildSagaInput.
+    # Fallback resolution (legal_name from name, external_reference from code) is
+    # handled upstream in extractPartyAndAccounts, so the Starlark script receives
+    # pre-resolved values.
     for org in organizations:
         step(name="register_organization_" + org["code"])
-        org_name = org.get("name", org["code"])
-        org_attrs = org.get("attributes", {})
-        # Use external_reference from attributes if present, otherwise fall back to org code.
-        ext_ref = org_attrs.get("external_reference", org["code"])
         party.register_organization(
-            legal_name=org_name,
-            display_name=org_name,
+            legal_name=org.get("legal_name", org.get("name", org["code"])),
+            display_name=org.get("display_name", org.get("legal_name", org.get("name", org["code"]))),
             party_type=org.get("party_type", "ORGANIZATION"),
-            external_reference=ext_ref,
-            external_reference_type=org.get("external_reference_type", "NATIONAL_ID"),
-            attributes=org_attrs,
+            external_reference=org.get("external_reference", org["code"]),
+            external_reference_type=org.get("external_reference_type", ""),
+            attributes=org.get("attributes", {}),
         )
         registered_organizations.append({
             "code": org["code"],

frontend/src/api/gen/meridian/control_plane/v1/manifest_pb.ts

@@ -221,20 +221,26 @@ type MarketDataSourceInput struct {
 
 // MarketDataSetInput represents a market data set to register and activate.
 type MarketDataSetInput struct {
-	Code        string
-	Category    string
-	Unit        string
-	SourceCode  string
-	DisplayName string
-	Description string
+	Code                    string
+	Category                string
+	Unit                    string
+	SourceCode              string
+	DisplayName             string
+	Description             string
+	ValidationExpression    string
+	ResolutionKeyExpression string
 }
 
 // OrganizationInput represents an organization to register.
 type OrganizationInput struct {
-	Code       string
-	Name       string
-	PartyType  string
-	Attributes map[string]string
+	Code                  string
+	Name                  string
+	LegalName             string
+	DisplayName           string
+	ExternalReference     string
+	ExternalReferenceType string
+	PartyType             string
+	Attributes            map[string]string
 }
 
 // InternalAccountInput represents an internal account to initiate.
@@ -484,12 +490,14 @@ func (e *ManifestExecutor) buildSagaInput(input *ApplyManifestInput) map[string]
 	marketDataSets := make([]interface{}, len(input.MarketDataSets))
 	for i, ds := range input.MarketDataSets {
 		marketDataSets[i] = map[string]interface{}{
-			"code":         ds.Code,
-			"category":     ds.Category,
-			"unit":         ds.Unit,
-			"source_code":  ds.SourceCode,
-			"display_name": ds.DisplayName,
-			"description":  ds.Description,
+			"code":                      ds.Code,
+			"category":                  ds.Category,
+			"unit":                      ds.Unit,
+			"source_code":               ds.SourceCode,
+			"display_name":              ds.DisplayName,
+			"description":               ds.Description,
+			"validation_expression":     ds.ValidationExpression,
+			"resolution_key_expression": ds.ResolutionKeyExpression,
 		}
 	}
 	sagaInput["market_data_sets"] = marketDataSets
@@ -515,10 +523,14 @@ func (e *ManifestExecutor) buildSagaInput(input *ApplyManifestInput) map[string]
 			attrs[k] = v
 		}
 		organizations[i] = map[string]interface{}{
-			"code":       org.Code,
-			"name":       org.Name,
-			"party_type": org.PartyType,
-			"attributes": attrs,
+			"code":                    org.Code,
+			"name":                    org.Name,
+			"legal_name":              org.LegalName,
+			"display_name":            org.DisplayName,
+			"external_reference":      org.ExternalReference,
+			"external_reference_type": org.ExternalReferenceType,
+			"party_type":              org.PartyType,
+			"attributes":              attrs,
 		}
 	}
 	sagaInput["organizations"] = organizations

services/control-plane/internal/applier/defaults/apply_manifest/v1.3.0.star

@@ -140,20 +140,26 @@ func TestBuildSagaInput_NewResourceTypes(t *testing.T) {
 		},
 		MarketDataSets: []MarketDataSetInput{
 			{
-				Code:        "USD_EUR_FX",
-				Category:    "DATA_CATEGORY_FX_RATE",
-				Unit:        "USD/EUR",
-				SourceCode:  "BLOOMBERG",
-				DisplayName: "USD/EUR Spot Rate",
-				Description: "Spot FX rate",
+				Code:                    "USD_EUR_FX",
+				Category:                "DATA_CATEGORY_FX_RATE",
+				Unit:                    "USD/EUR",
+				SourceCode:              "BLOOMBERG",
+				DisplayName:             "USD/EUR Spot Rate",
+				Description:             "Spot FX rate",
+				ValidationExpression:    "value > 0",
+				ResolutionKeyExpression: "observed_at",
 			},
 		},
 		Organizations: []OrganizationInput{
 			{
-				Code:       "ACME_ENERGY",
-				Name:       "Acme Energy Corp",
-				PartyType:  "ORGANIZATION",
-				Attributes: map[string]string{"industry": "energy"},
+				Code:                  "ACME_ENERGY",
+				Name:                  "Acme Energy Corp",
+				LegalName:             "Acme Energy Corporation",
+				DisplayName:           "Acme Energy",
+				ExternalReference:     "LEI-ACME-001",
+				ExternalReferenceType: "LEI",
+				PartyType:             "ORGANIZATION",
+				Attributes:            map[string]string{"industry": "energy"},
 			},
 		},
 		InternalAccounts: []InternalAccountInput{
@@ -184,13 +190,19 @@ func TestBuildSagaInput_NewResourceTypes(t *testing.T) {
 	require.True(t, ok)
 	assert.Equal(t, "USD_EUR_FX", firstDS["code"])
 	assert.Equal(t, "BLOOMBERG", firstDS["source_code"])
+	assert.Equal(t, "value > 0", firstDS["validation_expression"])
+	assert.Equal(t, "observed_at", firstDS["resolution_key_expression"])
 
 	orgs, ok := sagaInput["organizations"].([]interface{})
 	require.True(t, ok)
 	require.Len(t, orgs, 1)
 	firstOrg, ok := orgs[0].(map[string]interface{})
 	require.True(t, ok)
 	assert.Equal(t, "ACME_ENERGY", firstOrg["code"])
+	assert.Equal(t, "Acme Energy Corporation", firstOrg["legal_name"])
+	assert.Equal(t, "Acme Energy", firstOrg["display_name"])
+	assert.Equal(t, "LEI-ACME-001", firstOrg["external_reference"])
+	assert.Equal(t, "LEI", firstOrg["external_reference_type"])
 	attrs, ok := firstOrg["attributes"].(map[string]interface{})
 	require.True(t, ok)
 	assert.Equal(t, "energy", attrs["industry"])

services/control-plane/internal/applier/executor.go

@@ -801,24 +801,51 @@ func extractMarketData(mf *controlplanev1.Manifest, input *ApplyManifestInput) {
 	}
 	for _, ds := range md.GetDatasets() {
 		input.MarketDataSets = append(input.MarketDataSets, MarketDataSetInput{
-			Code:        ds.GetCode(),
-			Category:    stripEnumPrefix(ds.GetCategory().String(), "DATA_CATEGORY_"),
-			Unit:        ds.GetUnit(),
-			SourceCode:  ds.GetSourceCode(),
-			DisplayName: ds.GetDisplayName(),
-			Description: ds.GetDescription(),
+			Code:                    ds.GetCode(),
+			Category:                stripEnumPrefix(ds.GetCategory().String(), "DATA_CATEGORY_"),
+			Unit:                    ds.GetUnit(),
+			SourceCode:              ds.GetSourceCode(),
+			DisplayName:             ds.GetDisplayName(),
+			Description:             ds.GetDescription(),
+			ValidationExpression:    ds.GetValidationExpression(),
+			ResolutionKeyExpression: ds.GetResolutionKeyExpression(),
 		})
 	}
 }
 
 // extractPartyAndAccounts converts organizations and internal accounts from the manifest proto.
 func extractPartyAndAccounts(mf *controlplanev1.Manifest, input *ApplyManifestInput) {
 	for _, org := range mf.GetOrganizations() {
+		// Resolve legal_name with fallback chain: legal_name -> name -> code
+		legalName := org.GetLegalName()
+		if legalName == "" {
+			legalName = org.GetName()
+		}
+		if legalName == "" {
+			legalName = org.GetCode()
+		}
+
+		// Resolve display_name with fallback chain: display_name -> legal_name
+		displayName := org.GetDisplayName()
+		if displayName == "" {
+			displayName = legalName
+		}
+
+		// Resolve external_reference with fallback: external_reference -> code
+		extRef := org.GetExternalReference()
+		if extRef == "" {
+			extRef = org.GetCode()
+		}
+
 		input.Organizations = append(input.Organizations, OrganizationInput{
-			Code:       org.GetCode(),
-			Name:       org.GetName(),
-			PartyType:  org.GetPartyType(),
-			Attributes: org.GetAttributes(),
+			Code:                  org.GetCode(),
+			Name:                  org.GetName(),
+			LegalName:             legalName,
+			DisplayName:           displayName,
+			ExternalReference:     extRef,
+			ExternalReferenceType: org.GetExternalReferenceType(),
+			PartyType:             org.GetPartyType(),
+			Attributes:            org.GetAttributes(),
 		})
 	}
 	for _, ia := range mf.GetInternalAccounts() {