fix(patch): include writeOnly fields and remove deprecated schema fields

Makefile

@@ -13,7 +13,9 @@ PKL_BIN_URL := https://github.com/apple/pkl/releases/download/${PKL_BUNDLE_VERSI
 # External plugin Git repositories to bundle
 EXTERNAL_PLUGIN_REPOS ?= \
     https://github.com/platform-engineering-labs/formae-plugin-aws.git \
+    https://github.com/platform-engineering-labs/formae-plugin-azure.git \
     https://github.com/platform-engineering-labs/formae-plugin-gcp.git \
+    https://github.com/platform-engineering-labs/formae-plugin-oci.git \
     https://github.com/platform-engineering-labs/formae-plugin-ovh.git
 
 # Directory for cloned plugins

docs/docs.go

@@ -509,9 +509,6 @@ const docTemplate = `{
                 "IndexField": {
                     "type": "string"
                 },
-                "Persist": {
-                    "type": "boolean"
-                },
                 "Required": {
                     "type": "boolean"
                 },
@@ -758,9 +755,6 @@ const docTemplate = `{
                 },
                 "Identifier": {
                     "type": "string"
-                },
-                "Nonprovisionable": {
-                    "type": "boolean"
                 }
             }
         },

internal/cli/dev/dev.go

@@ -80,7 +80,6 @@ func awsSupportedTypes() *cobra.Command {
 
 				fmt.Printf("Schema for AWS resource type '%s':\n\n", resourceType)
 				fmt.Printf("Identifier: %s\n", schema.Identifier)
-				fmt.Printf("Nonprovisionable: %t\n", schema.Nonprovisionable)
 				fmt.Printf("CreateOnly: %v\n", schema.CreateOnly())
 				fmt.Printf("Fields: %v\n", schema.Fields)
 

internal/metastructure/patch/patch_document.go

@@ -61,7 +61,7 @@ func generatePatch(document []byte, patch []byte, properties resolver.Resolvable
 		return nil, false, fmt.Errorf("unable to generate patch document for apply mode: %s", mode)
 	}
 
-	patchOps, err := createPatchDocument(flattenedDocument, flattenedPatch, schema.Fields, collectionSemanticsFromFieldHints(schema.Hints), defaultIgnoredFields, strategy)
+	patchOps, err := createPatchDocument(flattenedDocument, flattenedPatch, schema.Fields, schema.WriteOnly(), collectionSemanticsFromFieldHints(schema.Hints), defaultIgnoredFields, strategy)
 	if err != nil {
 		return nil, false, fmt.Errorf("failed to create patch document: %w", err)
 	}
@@ -79,21 +79,73 @@ func generatePatch(document []byte, patch []byte, properties resolver.Resolvable
 	return json.RawMessage(patchJson), needsReplacement, nil
 }
 
-func createPatchDocument(document []byte, patch []byte, schemaFields []string, collections jsonpatch.Collections, ignoredFields []jsonpatch.Path, strategy jsonpatch.PatchStrategy) ([]jsonpatch.JsonPatchOperation, error) {
+func createPatchDocument(document []byte, patch []byte, schemaFields []string, writeOnlyFields []string, collections jsonpatch.Collections, ignoredFields []jsonpatch.Path, strategy jsonpatch.PatchStrategy) ([]jsonpatch.JsonPatchOperation, error) {
 	patchWithSchemaFieldsOnly, err := removeNonSchemaFields(patch, schemaFields)
 	if err != nil {
 		return nil, err
 	}
 
+	// Remove writeOnly fields from the document (existing state).
+	// WriteOnly fields (like passwords) are never returned by the cloud provider's Read operation,
+	// but Formae stores them. By removing them from the document before comparison,
+	// jsonpatch will generate an "add" operation for these fields, ensuring they're always
+	// included in the patch sent to the cloud provider.
+	documentWithoutWriteOnly, err := removeWriteOnlyFields(document, writeOnlyFields)
+	if err != nil {
+		return nil, err
+	}
+
 	// Create the actual patch document
-	patchDoc, err := jsonpatch.CreatePatch(document, patchWithSchemaFieldsOnly, collections, ignoredFields, strategy)
+	patchDoc, err := jsonpatch.CreatePatch(documentWithoutWriteOnly, patchWithSchemaFieldsOnly, collections, ignoredFields, strategy)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create JSON patch: %w", err)
 	}
 
 	return patchDoc, nil
 }
 
+// removeWriteOnlyFields removes writeOnly fields from the document.
+// WriteOnly field paths can be nested (e.g., "LoginProfile.Password").
+func removeWriteOnlyFields(document []byte, writeOnlyFields []string) ([]byte, error) {
+	if len(writeOnlyFields) == 0 {
+		return document, nil
+	}
+
+	var deserialized map[string]any
+	if err := json.Unmarshal(document, &deserialized); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal document: %w", err)
+	}
+
+	for _, fieldPath := range writeOnlyFields {
+		removeNestedField(deserialized, strings.Split(fieldPath, "."))
+	}
+
+	serialized, err := json.Marshal(deserialized)
+	if err != nil {
+		return nil, err
+	}
+
+	return serialized, nil
+}
+
+// removeNestedField removes a field at the given path from a nested map structure.
+// For example, path ["LoginProfile", "Password"] removes the Password key from LoginProfile.
+func removeNestedField(obj map[string]any, path []string) {
+	if len(path) == 0 {
+		return
+	}
+
+	if len(path) == 1 {
+		delete(obj, path[0])
+		return
+	}
+
+	// Navigate to the nested object
+	if nested, ok := obj[path[0]].(map[string]any); ok {
+		removeNestedField(nested, path[1:])
+	}
+}
+
 func removeNonSchemaFields(patch []byte, schemaFields []string) ([]byte, error) {
 	var deserialized map[string]any
 	if err := json.Unmarshal(patch, &deserialized); err != nil {

internal/metastructure/patch/patch_document_test.go

@@ -157,7 +157,7 @@ func TestCreatePatchDocument_PrimitiveArray(t *testing.T) {
 
 	for _, tc := range testCases[1:2] {
 		t.Run(tc.name, func(t *testing.T) {
-			patches, err := createPatchDocument(tc.jsonA, tc.jsonB, []string{"label", "tags"}, jsonpatch.Collections{}, nil, jsonpatch.PatchStrategyEnsureExists)
+			patches, err := createPatchDocument(tc.jsonA, tc.jsonB, []string{"label", "tags"}, nil, jsonpatch.Collections{}, nil, jsonpatch.PatchStrategyEnsureExists)
 			if err != nil {
 				t.Fatalf("Error comparing JSONs: %v", err)
 			}
@@ -229,7 +229,7 @@ func TestCreatePatchDocument_ObjectArrayWithKeyValues(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			patches, err := createPatchDocument(tc.jsonA, tc.jsonB, []string{"label", "tags"}, jsonpatch.Collections{EntitySets: jsonpatch.EntitySets{jsonpatch.Path("$.tags"): jsonpatch.Key("key")}}, nil, jsonpatch.PatchStrategyEnsureExists)
+			patches, err := createPatchDocument(tc.jsonA, tc.jsonB, []string{"label", "tags"}, nil, jsonpatch.Collections{EntitySets: jsonpatch.EntitySets{jsonpatch.Path("$.tags"): jsonpatch.Key("key")}}, nil, jsonpatch.PatchStrategyEnsureExists)
 			if err != nil {
 				t.Fatalf("Error comparing JSONs: %v", err)
 			}
@@ -303,7 +303,7 @@ func TestCreatePatchDocument_ObjectArrayWithValues(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			patches, err := createPatchDocument(tc.jsonA, tc.jsonB, []string{"label", "tags"}, jsonpatch.Collections{}, nil, jsonpatch.PatchStrategyEnsureExists)
+			patches, err := createPatchDocument(tc.jsonA, tc.jsonB, []string{"label", "tags"}, nil, jsonpatch.Collections{}, nil, jsonpatch.PatchStrategyEnsureExists)
 			if err != nil {
 				t.Fatalf("Error comparing JSONs: %v", err)
 			}
@@ -561,6 +561,75 @@ func TestCollectionSemanticsFromFieldHints(t *testing.T) {
 	assert.Equal(t, expectedCollections, collections)
 }
 
+func TestGeneratePatch_WriteOnlyFieldsGenerateAddOperation(t *testing.T) {
+	// This simulates an AWS CloudControl scenario where:
+	// - Password is a writeOnly field (AWS never returns it)
+	// - Formae stores the password in its own state
+	// - When generating a patch, we need to always include writeOnly fields
+	//   even if they haven't changed, because AWS doesn't have them
+
+	// Existing state (what Formae has stored - includes Password)
+	document := []byte(`{
+		"LoginProfile": {
+			"Password": "secret123",
+			"PasswordResetRequired": false
+		},
+		"UserName": "testuser"
+	}`)
+
+	// Desired state (from PKL file - same password, but adding tags)
+	patch := []byte(`{
+		"LoginProfile": {
+			"Password": "secret123",
+			"PasswordResetRequired": false
+		},
+		"UserName": "testuser",
+		"Tags": [{"Key": "Env", "Value": "test"}]
+	}`)
+
+	schema := pkgmodel.Schema{
+		Fields: []string{"LoginProfile", "UserName", "Tags"},
+		Hints: map[string]pkgmodel.FieldHint{
+			"LoginProfile.Password": {
+				WriteOnly: true,
+			},
+		},
+	}
+	props := resolver.NewResolvableProperties()
+
+	patchDoc, needsReplacement, err := generatePatch(document, patch, props, schema, pkgmodel.FormaApplyModePatch)
+	require.NoError(t, err)
+	assert.False(t, needsReplacement)
+
+	var patches []jsonpatch.JsonPatchOperation
+	err = json.Unmarshal(patchDoc, &patches)
+	require.NoError(t, err)
+
+	// We expect:
+	// 1. An "add" operation for Tags
+	// 2. An "add" operation for LoginProfile/Password (because it's writeOnly and
+	//    must be re-added since AWS CloudControl won't have it in current state)
+	require.Len(t, patches, 2, "Expected 2 operations: one for Tags, one for writeOnly Password")
+
+	// Find the operations by path
+	var tagsOp, passwordOp *jsonpatch.JsonPatchOperation
+	for i := range patches {
+		switch patches[i].Path {
+		case "/Tags", "/Tags/0":
+			tagsOp = &patches[i]
+		case "/LoginProfile/Password":
+			passwordOp = &patches[i]
+		}
+	}
+
+	assert.NotNil(t, tagsOp, "Should have an operation for Tags")
+	assert.Equal(t, "add", tagsOp.Operation)
+
+	assert.NotNil(t, passwordOp, "Should have an add operation for writeOnly Password")
+	assert.Equal(t, "add", passwordOp.Operation, "WriteOnly fields should use 'add' operation")
+	assert.Equal(t, "secret123", passwordOp.Value, "Password value should be preserved")
+}
+
 func TestGeneratePatch_AddTagsWhileRetainingExisting(t *testing.T) {
 	// This is the existing resource state (from the database) - VPC with 1 tag
 	document := []byte(`{

pkg/model/schema.go

@@ -5,17 +5,15 @@
 package model
 
 type Schema struct {
-	Identifier       string               `json:"Identifier" pkl:"Identifier"`
-	Fields           []string             `json:"Fields" pkl:"Fields"`
-	Nonprovisionable bool                 `json:"Nonprovisionable" pkl:"Nonprovisionable"`
-	Hints            map[string]FieldHint `json:"Hints" pkl:"Hints"`
-	Discoverable     bool                 `json:"Discoverable" pkl:"Discoverable"`
-	Extractable      bool                 `json:"Extractable" pkl:"Extractable"`
+	Identifier   string               `json:"Identifier" pkl:"Identifier"`
+	Fields       []string             `json:"Fields" pkl:"Fields"`
+	Hints        map[string]FieldHint `json:"Hints" pkl:"Hints"`
+	Discoverable bool                 `json:"Discoverable" pkl:"Discoverable"`
+	Extractable  bool                 `json:"Extractable" pkl:"Extractable"`
 }
 
 type FieldHint struct {
 	CreateOnly       bool `json:"CreateOnly" pkl:"CreateOnly"`
-	Persist          bool `json:"Persist" pkl:"Persist"`
 	WriteOnly        bool `json:"WriteOnly" pkl:"WriteOnly"`
 	Required         bool `json:"Required" pkl:"Required"`
 	RequiredOnCreate bool `json:"RequiredOnCreate" pkl:"RequiredOnCreate"`

pkg/plugin/descriptors/Extractor.pkl

@@ -26,7 +26,6 @@ function extractDescriptors(): Listing<resourceDescriptor.ResourceDescriptor> =
                         new resourceDescriptor.ResourceDescriptor {
                             Type = clazz.annotations[0].getProperty("type")
                             Schema = new formae.Schema {
-                                Nonprovisionable = clazz.annotations[0].getProperty("nonprovisionable")
                                 Identifier = clazz.annotations[0].getProperty("identifier")
                                 Discoverable = clazz.annotations[0].getProperty("discoverable")
                                 Hints = new Mapping<String, formae.FieldHint> {

pkg/plugin/testutil/docs.go

@@ -25,14 +25,13 @@ type DocsResult struct {
 
 // ResourceDoc contains documentation for a single resource type.
 type ResourceDoc struct {
-	Type             string  `json:"type"`
-	Discoverable     bool    `json:"discoverable"`
-	Extractable      bool    `json:"extractable"`
-	Nonprovisionable bool    `json:"nonprovisionable"`
-	Identifier       string  `json:"identifier"`
-	DocComment       *string `json:"docComment"`
-	ModuleName       string  `json:"moduleName"`
-	ClassName        string  `json:"className"`
+	Type         string  `json:"type"`
+	Discoverable bool    `json:"discoverable"`
+	Extractable  bool    `json:"extractable"`
+	Identifier   string  `json:"identifier"`
+	DocComment   *string `json:"docComment"`
+	ModuleName   string  `json:"moduleName"`
+	ClassName    string  `json:"className"`
 }
 
 // GenerateDocs generates documentation for a plugin schema.

pkg/plugin/testutil/pkl/Docs.pkl

@@ -22,7 +22,6 @@ class ResourceDoc {
     type: String
     discoverable: Boolean
     extractable: Boolean
-    nonprovisionable: Boolean
     identifier: String
     docComment: String?
     moduleName: String
@@ -39,7 +38,6 @@ function getAllResourceDocs(): Listing<ResourceDoc> = new Listing {
                         type = v.annotations[0].getProperty("type")
                         discoverable = v.annotations[0].getProperty("discoverable")
                         extractable = v.annotations[0].getProperty("extractable")
-                        nonprovisionable = v.annotations[0].getProperty("nonprovisionable")
                         identifier = v.annotations[0].getProperty("identifier")
                         docComment = v.annotations[0].getPropertyOrNull("docComment")
                         moduleName = name
@@ -60,7 +58,6 @@ function generateDocs(): Map<String, Any> =
             ["type"] = r.type
             ["discoverable"] = r.discoverable
             ["extractable"] = r.extractable
-            ["nonprovisionable"] = r.nonprovisionable
             ["identifier"] = r.identifier
             ["docComment"] = r.docComment
             ["moduleName"] = r.moduleName

plugins/fake-aws/fake_aws.go

@@ -97,7 +97,6 @@ func (s FakeAWS) SchemaForResourceType(resourceType string) (model.Schema, error
 				"Ipv6NetmaskLength",
 				"Ipv6Pool",
 				"VpcId"},
-			Nonprovisionable: false,
 		}, nil
 	default:
 		return model.Schema{
@@ -123,7 +122,6 @@ func (s FakeAWS) SchemaForResourceType(resourceType string) (model.Schema, error
 				"Tags",
 				"VersioningConfiguration",
 				"WebsiteConfiguration"},
-			Nonprovisionable: false,
 		}, nil
 	}
 }

plugins/fake-aws/schema/pkl/route53/recordset.pkl

@@ -32,7 +32,6 @@ open class AliasTarget extends formae.SubResource {
 }
 open class RecordSet extends formae.Resource {
     @fakeaws.FieldHint {
-        persist = true
     }
     aliasTarget: AliasTarget?
 
@@ -41,7 +40,6 @@ open class RecordSet extends formae.Resource {
 
     @fakeaws.FieldHint {
         createOnly = true
-        persist = true
     }
     hostedZoneId: (String|formae.Resolvable)?
 
@@ -52,25 +50,21 @@ open class RecordSet extends formae.Resource {
 
     @fakeaws.FieldHint {
         createOnly = true
-        persist = true
     }
     name: String
 
     region: fakeaws.Region?
 
     @fakeaws.FieldHint {
-        persist = true
     }
     resourceRecords: Listing<String>?
 
     @fakeaws.FieldHint {
-        persist = true
         outputField = "TTL"
     }
     ttl: Number?
 
     @fakeaws.FieldHint {
-        persist = true
     }
     type: String