Lightning to use sync v2 formats

CHANGELOG.md

@@ -17,15 +17,32 @@ and this project adheres to
 
 ### Added
 
+- V2 workflow and project YAML format that conforms to the OpenFn portability
+  spec, so files exported from Lightning are interchangeable with the CLI's
+  workflow format. Canonical V1 and V2 fixtures live under
+  `test/fixtures/portability/`, with CLI-deploy integration coverage.
+  [#4718](https://github.com/OpenFn/lightning/issues/4718)
+
 ### Changed
 
+- Project and workflow YAML serialization moved out of `Lightning.ExportUtils`
+  into a versioned `Lightning.Workflows.YamlFormat.V2` module, mirrored on the
+  frontend by `assets/js/yaml/v2.ts` (driving the inspector code view, template
+  publish panel, and YAML import editor).
+  [#4718](https://github.com/OpenFn/lightning/issues/4718)
+
+### Fixed
+
+- GitHub sync now prevents two projects in the same project tree (root,
+  sandboxes, siblings, and cousins) from claiming the same `(repo, branch)`
+  pair. Enforcement moved to a Postgres unique index on
+  `(root_project_id, repo, branch)`, closing a check-then-insert race that could
+  let two concurrent inserts both pass an in-memory ancestor check at READ
+  COMMITTED. [#4727](https://github.com/OpenFn/lightning/issues/4727)
 - `./bin/bootstrap` on aarch64 Linux now requires Rust upfront and builds the
   Rambo native binary via `mix compile.rambo` post-compile, matching the darwin
   path. x86_64 Linux is unchanged.
   [#4735](https://github.com/OpenFn/lightning/pull/4735)
-
-### Fixed
-
 - `mix lightning.install_runtime` no longer reports success when Rambo's binary
   fails to start; both `Rambo.run/2` calls now raise with the underlying reason.
   [#4735](https://github.com/OpenFn/lightning/pull/4735)

assets/js/collaborative-editor/components/inspector/CodeViewPanel.tsx

@@ -1,5 +1,4 @@
 import { useMemo } from 'react';
-import YAML from 'yaml';
 
 import { useCopyToClipboard } from '#/collaborative-editor/hooks/useCopyToClipboard';
 import {
@@ -8,8 +7,8 @@ import {
 } from '#/collaborative-editor/hooks/useWorkflow';
 import { useURLState } from '#/react/lib/use-url-state';
 import { cn } from '#/utils/cn';
+import { serializeWorkflow } from '#/yaml/format';
 import type { WorkflowState as YAMLWorkflowState } from '#/yaml/types';
-import { convertWorkflowStateToSpec } from '#/yaml/util';
 
 export function CodeViewPanel() {
   // Read workflow data from store - LoadingBoundary guarantees non-null
@@ -19,12 +18,13 @@ export function CodeViewPanel() {
   const edges = useWorkflowState(state => state.edges);
   const positions = useWorkflowState(state => state.positions);
 
-  // Generate YAML from current workflow state
+  // Generate v2 YAML from current workflow state. v2 is the CLI-aligned
+  // portability format; the panel content drives both the textarea and the
+  // Download button payload.
   const yamlCode = useMemo(() => {
     if (!workflow) return '';
 
     try {
-      // Build WorkflowState compatible with YAML utilities
       const workflowState: YAMLWorkflowState = {
         id: workflow.id,
         name: workflow.name,
@@ -34,9 +34,7 @@ export function CodeViewPanel() {
         positions,
       };
 
-      // Convert to spec without IDs (cleaner for export)
-      const spec = convertWorkflowStateToSpec(workflowState, false);
-      return YAML.stringify(spec);
+      return serializeWorkflow(workflowState);
     } catch (error) {
       console.error('Failed to generate YAML:', error);
       return '# Error generating YAML\n# Please check console for details';

assets/js/collaborative-editor/components/inspector/TemplatePublishPanel.tsx

@@ -1,5 +1,4 @@
 import { useMemo, useState } from 'react';
-import YAML from 'yaml';
 import { z } from 'zod';
 
 import { useAppForm } from '#/collaborative-editor/components/form';
@@ -12,8 +11,8 @@ import { notifications } from '#/collaborative-editor/lib/notifications';
 import { useURLState } from '#/react/lib/use-url-state';
 import { cn } from '#/utils/cn';
 import logger from '#/utils/logger';
+import { serializeWorkflow } from '#/yaml/format';
 import type { WorkflowState as YAMLWorkflowState } from '#/yaml/types';
-import { convertWorkflowStateToSpec } from '#/yaml/util';
 
 logger.ns('TemplatePublishPanel').seal();
 
@@ -97,7 +96,9 @@ export function TemplatePublishPanel() {
     setIsPublishing(true);
 
     try {
-      // Generate YAML code from current workflow state
+      // Generate v2 YAML code from current workflow state. New templates are
+      // written in the CLI-aligned portability format (v2); existing v1 rows
+      // continue to load via format-detection on read.
       const workflowState: YAMLWorkflowState = {
         id: workflow.id,
         name: workflow.name,
@@ -107,8 +108,7 @@ export function TemplatePublishPanel() {
         positions,
       };
 
-      const spec = convertWorkflowStateToSpec(workflowState, false);
-      const workflowCode = YAML.stringify(spec);
+      const workflowCode = serializeWorkflow(workflowState);
 
       // Parse comma-separated tags into array
       const formValues = form.state.values as TemplateFormValues;

assets/js/collaborative-editor/components/yaml-import/YAMLCodeEditor.tsx

@@ -13,6 +13,31 @@ interface YAMLCodeEditorProps {
   isValidating?: boolean;
 }
 
+// v2 (CLI-aligned portability format) example shown when the editor is
+// empty. Both v1 (legacy `jobs:`/`triggers:`/`edges:` maps) and v2 (unified
+// `steps:` array) are accepted by the importer; the v2 shape matches what
+// canvas Code panel exports and what `@openfn/cli` writes.
+const PLACEHOLDER_EXAMPLE = `# Paste your workflow YAML here, for example:
+#
+# name: My Workflow
+# steps:
+#   - id: webhook
+#     type: webhook
+#     webhook_reply: before_start
+#     enabled: true
+#     next:
+#       say-hello:
+#         condition: always
+#   - id: say-hello
+#     name: Say Hello
+#     adaptor: '@openfn/language-common@latest'
+#     expression: |
+#       fn(state => {
+#         console.log("Hello, world!");
+#         return state;
+#       })
+`;
+
 export function YAMLCodeEditor({ value, onChange }: YAMLCodeEditorProps) {
   const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
     onChange(e.target.value);
@@ -24,7 +49,7 @@ export function YAMLCodeEditor({ value, onChange }: YAMLCodeEditorProps) {
         id="yaml-editor"
         value={value}
         onChange={handleChange}
-        placeholder="Paste your YAML content here"
+        placeholder={PLACEHOLDER_EXAMPLE}
         className="focus:outline focus:outline-2 focus:outline-offset-1 rounded-md shadow-xs text-sm block w-full h-full focus:ring-0 sm:text-sm sm:leading-6 overflow-y-auto border-slate-300 focus:border-slate-400 focus:outline-primary-600 font-mono proportional-nums text-slate-200 bg-slate-700 resize-none text-nowrap overflow-x-auto"
       />
     </div>

assets/js/collaborative-editor/utils/workflowSerialization.ts

@@ -1,7 +1,5 @@
-import YAML from 'yaml';
-
+import { serializeWorkflow } from '../../yaml/format';
 import type { WorkflowState as YAMLWorkflowState } from '../../yaml/types';
-import { convertWorkflowStateToSpec } from '../../yaml/util';
 
 interface WorkflowMetadata {
   id: string;
@@ -108,62 +106,55 @@ export function prepareWorkflowForSerialization(
 /**
  * Serializes a workflow to YAML format for AI Assistant context.
  *
- * This utility converts the workflow state from the Zustand store into YAML format
- * that can be sent to the AI Assistant as context. It's used in multiple places:
+ * This utility converts the workflow state from the store into the v2
+ * (CLI-aligned portability format) YAML that can be sent to the AI Assistant
+ * as context. It's used in multiple places:
  * - Initial session connection with workflow context
  * - Sending messages with updated workflow state
  * - Creating new conversations
  * - Switching between sessions
  *
+ * The v2 format is a stateless interoperability format; UUIDs are not preserved. Steps
+ * are referenced by hyphenated name; the AI Assistant correlates back to
+ * persisted records by name.
+ *
  * @param workflow - The workflow data including jobs, triggers, edges, and positions
  * @returns YAML string representation of the workflow, or undefined if serialization fails
- *
- * @example
- * ```ts
- * const yaml = serializeWorkflowToYAML({
- *   id: workflow.id,
- *   name: workflow.name,
- *   jobs: jobs.map(job => ({ id: job.id, name: job.name, adaptor: job.adaptor, body: job.body })),
- *   triggers: triggers,
- *   edges: edges,
- *   positions: positions
- * });
- * ```
  */
 export function serializeWorkflowToYAML(
   workflow: SerializableWorkflow
 ): string | undefined {
   try {
-    const workflowSpec = convertWorkflowStateToSpec(
-      {
-        id: workflow.id,
-        name: workflow.name,
-        jobs: workflow.jobs,
-        triggers: workflow.triggers,
-        edges: workflow.edges.map(edge => ({
-          id: edge.id,
-          condition_type: edge.condition_type || 'always',
-          enabled: edge.enabled !== false,
-          target_job_id: edge.target_job_id,
-          ...(edge.source_job_id && {
-            source_job_id: edge.source_job_id,
-          }),
-          ...(edge.source_trigger_id && {
-            source_trigger_id: edge.source_trigger_id,
-          }),
-          ...(edge.condition_label && {
-            condition_label: edge.condition_label,
-          }),
-          ...(edge.condition_expression && {
-            condition_expression: edge.condition_expression,
-          }),
-        })),
-        positions: workflow.positions,
-      },
-      true // Include IDs so AI responses preserve them (matches legacy behavior)
-    );
+    const state: YAMLWorkflowState = {
+      id: workflow.id,
+      name: workflow.name,
+      jobs: workflow.jobs.map(job => ({
+        id: job.id,
+        name: job.name,
+        adaptor: job.adaptor,
+        body: job.body,
+        keychain_credential_id: null,
+        project_credential_id: null,
+      })),
+      triggers: workflow.triggers,
+      edges: workflow.edges.map(edge => ({
+        id: edge.id,
+        condition_type: edge.condition_type || 'always',
+        enabled: edge.enabled !== false,
+        target_job_id: edge.target_job_id,
+        ...(edge.source_job_id && { source_job_id: edge.source_job_id }),
+        ...(edge.source_trigger_id && {
+          source_trigger_id: edge.source_trigger_id,
+        }),
+        ...(edge.condition_label && { condition_label: edge.condition_label }),
+        ...(edge.condition_expression && {
+          condition_expression: edge.condition_expression,
+        }),
+      })),
+      positions: workflow.positions,
+    };
 
-    return YAML.stringify(workflowSpec);
+    return serializeWorkflow(state);
   } catch (error) {
     console.error('Failed to serialize workflow to YAML:', error);
     return undefined;

assets/js/yaml/WorkflowToYAML.ts

@@ -1,9 +1,7 @@
-import YAML from 'yaml';
-
 import type { PhoenixHook } from '../hooks/PhoenixHook';
 
+import { serializeWorkflow } from './format';
 import type { WorkflowState } from './types';
-import { convertWorkflowStateToSpec } from './util';
 
 interface WorkflowResponse {
   workflow_params: WorkflowState;
@@ -32,21 +30,14 @@ const WorkflowToYAML = {
     this.pushEvent('get-current-state', {}, (response: WorkflowResponse) => {
       const workflowState = response.workflow_params;
 
-      const workflowSpecWithoutIds = convertWorkflowStateToSpec(
-        workflowState,
-        false
-      );
-      const workflowSpecWithIds = convertWorkflowStateToSpec(
-        workflowState,
-        true
-      );
-
-      const yamlWithoutIds = YAML.stringify(workflowSpecWithoutIds);
-      const yamlWithIds = YAML.stringify(workflowSpecWithIds);
+      // v2 (CLI-aligned portability format) is stateless — no UUIDs in the
+      // canonical body. Both `code` and `code_with_ids` payloads carry the
+      // same v2 string after #4718's export cutover.
+      const yamlCode = serializeWorkflow(workflowState);
 
       this.pushEvent('workflow_code_generated', {
-        code: yamlWithoutIds,
-        code_with_ids: yamlWithIds,
+        code: yamlCode,
+        code_with_ids: yamlCode,
       });
     });
   },

assets/js/yaml/format.ts

@@ -0,0 +1,35 @@
+// Format façade — single boundary between Lightning's runtime workflow state
+// and YAML files. Knows about format versions; delegates to `./v1` or `./v2`.
+//
+// Phase 4 wiring: outbound serialization emits v2 (CLI-aligned portability
+// format) only — there is no v1 export path remaining in the codebase.
+// Inbound parsing dispatches by detected format and continues to accept both
+// v1 and v2 documents (Phase 5). See plan #4718.
+
+import YAML from 'yaml';
+
+import type { WorkflowSpec, WorkflowState } from './types';
+import * as v1 from './v1';
+import * as v2 from './v2';
+
+export type FormatVersion = 'v1' | 'v2';
+export type ParsedDoc = { format: FormatVersion; spec: WorkflowSpec };
+
+// Outbound: v2 only. v1 export was removed in Phase 4 of #4718.
+export const serializeWorkflow = (state: WorkflowState): string => {
+  return v2.serializeWorkflow(state);
+};
+
+// Inbound: detects format and dispatches.
+export const parseWorkflow = (yamlString: string): ParsedDoc => {
+  const parsed = YAML.parse(yamlString);
+  const format = detectFormat(parsed);
+  if (format === 'v2') {
+    return { format, spec: v2.parseWorkflow(parsed) };
+  }
+  return { format, spec: v1.parseWorkflow(parsed) };
+};
+
+export const detectFormat = (parsed: unknown): FormatVersion => {
+  return v2.detectFormat(parsed);
+};