docs: data validation (NangoHQ#2468)

## Describe your changes

Fixes https://linear.app/nango/issue/NAN-1351/official-documentation
Fixes
https://linear.app/nango/issue/NAN-1201/option-to-generate-zod-models-in-the-nango-integrations-folder


- Documentation about data validation
I tried to provide some examples. I have tested the custom validations
generation but not really the JSON schema usage in other languages.
Feel free to propose rewrite suggestion, documentation is not my
strongest skill :D

Author: bodinsamuel

docs-v2/customize/guides/advanced/validate-input-and-output.mdx

@@ -0,0 +1,169 @@
+---
+title: 'Validate input and output'
+sidebarTitle: 'Validate input and output'
+description: 'How to automatically validate your input and ouput with JSON schema'
+---
+
+Nango automatically validates your integration inputs & outputs. It also offers ways to further customize data validation in code. The guide will walk you through each approach.
+
+## Automatic validation
+
+The validation is available during development and production, and do not require any configuration from you:
+
+- **CLI**: Dry Run validation errors are logged and will halt execution when using `--validation` command option
+- **Production**: Validation errors are logged but do not halt execution
+
+
+## Available schema files
+
+When you use Nango CLI, it automatically generates two schema files in the `.nango` folder:
+
+- `schema.ts` a TypeScript file that contains all your models.
+- `schema.json` a JSON Schema file that is used for automatic data validation.
+
+These files can be versioned and integrated into your own codebase, ensuring consistency and reliability across different environments.
+
+
+
+## Custom Validation
+
+For more advanced use cases, you can generate your own validation schemas using the available files with the tool of your choice.
+
+
+<Tabs>
+
+<Tab title="Typescript `zod`">
+
+To create `zod` schemas you can use [`ts-to-zod` npm package](https://www.npmjs.com/package/ts-to-zod). This tool converts TypeScript definitions into Zod schemas, which can be used for runtime validation in your own Typescript codebase.
+
+
+```bash
+npx ts-to-zod .nango/schema.ts schema.zod.ts
+```
+<Tip>
+You can use zod models in your scripts too
+</Tip>
+```typescript
+import { myZodModel } from '../../schema.zod.ts';
+
+export default async fetchData(nango: Nango) {
+  const response = await nango.get({ endpoint: '/tickets' });
+  const isValid = myZodModel.parse(response.json);
+  if (isValid) {
+    [...]
+  }
+}
+```
+</Tab>
+
+
+<Tab title="Golang">
+You can use [`go-jsonschema` golang package](https://github.com/omissis/go-jsonschema). This tool converts JSON Schema definitions into Golang struct. Note that some syntax are not supported by this package.
+
+```bash
+go-jsonschema -p main .nango/schema.json > test.go
+```
+
+</Tab>
+
+
+<Tab title="Rust">
+
+You can use [`typify` rust package](https://github.com/oxidecomputer/typify). This tool converts JSON Schema definitions into Rust types.  Note that some syntax are not supported by this package.
+
+```bash
+cargo typify .nango/schema.json
+```
+</Tab>
+</Tabs>
+
+
+## Using `schema.json` in your codebase
+
+JSON Schema is supported in most of the main software languages, here is a non-exhaustive list of how you can directly use this file to validate the records you receive from Nango.
+
+<Tabs>
+
+<Tab title="Typescript">
+
+```ts
+import { Ajv } from 'ajv';
+import addFormats from 'ajv-formats';
+import jsonSchema from '.nango/schema.json';
+
+// Initiate AJV
+const ajv = new Ajv({ allErrors: true, discriminator: true });
+addFormats(ajv);
+
+const modelToValidate = 'MyModelName';
+const myData = {"id": "hello-word"};
+
+// Compile the JSON schema
+const validate = ajv.compile({
+  ...jsonSchema,
+  ...jsonSchema['definitions'][modelToValidate]
+});
+
+// Validate your data
+validate(mydata);
+```
+</Tab>
+
+
+<Tab title="Golang">
+```go
+package main
+
+import (
+	"fmt"
+	"log"
+	"github.com/santhosh-tekuri/jsonschema/v5"
+)
+
+func main() {
+  sch, err := jsonschema.Compile(".nango/schema.json")
+  if err != nil {
+    log.Fatalf("%#v", err)
+  }
+
+  myData := map[string]interface{}{
+		"id": "hello-word",
+	}
+  if err = sch.Validate(v); err != nil {
+    log.Fatalf("%#v", err)
+  }
+}
+```
+</Tab>
+
+<Tab title="Rust">
+```rust
+use jsonschema::{Draft, JSONSchema};
+use serde_json::json;
+
+let data = fs::read_to_string(".nango/schema.json").expect("Unable to read file");
+let schema: serde_json::Value = serde_json::from_str(&data).expect("Unable to parse");
+
+let instance = json!("{'id': 'hello-word'}");
+let compiled = JSONSchema::compile(&schema).expect("A valid schema");
+let result = compiled.validate(&instance);
+```
+</Tab>
+
+
+<Tab title="Python">
+```py
+import json
+from jsonschema import validate
+
+with open('.nango/schema.json') as file:
+    schema = json.load(file)
+    print(schema)
+
+    validate(instance={"id": "hello-word"}, schema=schema)
+```
+</Tab>
+</Tabs>
+
+
+

docs-v2/mint.json

@@ -169,7 +169,8 @@
                             "pages": [
                                 "customize/guides/advanced/handle-rate-limits",
                                 "customize/guides/advanced/handle-large-datasets",
-                                "customize/guides/advanced/migrate-integration-configuration"
+                                "customize/guides/advanced/migrate-integration-configuration",
+                                "customize/guides/advanced/validate-input-and-output"
                             ]
                         }
                     ]

packages/cli/lib/services/local-integration.service.ts

@@ -69,7 +69,7 @@ class IntegrationService implements IntegrationServiceInterface {
                         jsonSchema: nangoProps.syncConfig.models_json_schema
                     });
                     if (Array.isArray(valInput)) {
-                        await nango.log('Invalid action input', { data: input, validation: valInput, model: nangoProps.syncConfig.input }, { level: 'error' });
+                        await nango.log('Invalid action input. Use `--validation` option to see the details', { level: 'warn' });
                         if (nangoProps.runnerFlags.validateActionInput) {
                             return {
                                 success: false,
@@ -89,7 +89,7 @@ class IntegrationService implements IntegrationServiceInterface {
                         jsonSchema: nangoProps.syncConfig.models_json_schema
                     });
                     if (Array.isArray(valOutput)) {
-                        await nango.log('Invalid action output', { data: output, validation: valOutput, model: modelNameOutput }, { level: 'error' });
+                        await nango.log('Invalid action output. Use `--validation` option to see the details', { level: 'warn' });
                         if (nangoProps.runnerFlags.validateActionOutput) {
                             return {
                                 success: false,

packages/shared/lib/sdk/sync.ts

@@ -836,14 +836,18 @@ export class NangoSync extends NangoAction {
 
         // Validate records
         for (const record of results) {
-            const validation = validateData({ input: record, jsonSchema: this.syncConfig!.models_json_schema, modelName: model });
+            const validation = validateData({ input: JSON.parse(JSON.stringify(record)), jsonSchema: this.syncConfig!.models_json_schema, modelName: model });
             if (validation === true) {
                 continue;
             }
 
             metrics.increment(metrics.Types.RUNNER_INVALID_SYNCS_RECORDS);
 
-            await this.log('Invalid record payload', { data: record, validation, model }, { level: 'warn' });
+            if (this.dryRun) {
+                await this.log('Invalid action input. Use `--validation` option to see the details', { level: 'warn' });
+            } else {
+                await this.log('Invalid record payload', { data: record, validation, model }, { level: 'warn' });
+            }
             if (this.runnerFlags?.validateSyncRecords) {
                 throw new NangoError(`invalid_sync_record`, { data: record, validation, model });
             }