feat: ai document parser now produces an output that can be consumed by the cma to create entries (#10277)

Author: ryunsong-contentful

apps/google-docs/functions/agents/contentTypeParserAgent/contentTypeParser.agent.ts

@@ -43,9 +43,6 @@ export async function analyzeContentTypes({
   return finalAnalysis;
 }
 
-/**
- * Builds the system prompt for the AI
- */
 function buildSystemPrompt(): string {
   return `You are an expert Contentful content modeling analyst. Your role is to analyze Contentful content type definitions and provide clear, actionable summaries.
 
@@ -61,9 +58,6 @@ Focus on clarity and actionability. Your summaries will be used by:
 - Content strategists planning content architecture`;
 }
 
-/**
- * Builds the analysis prompt from content type data
- */
 function buildAnalysisPrompt(contentTypes: ContentTypeProps[]): string {
   const contentTypeList = contentTypes.map((ct) => ct.name).join(', ');
   const totalFields = contentTypes.reduce((sum, ct) => sum + (ct.fields?.length || 0), 0);

apps/google-docs/functions/agents/contentTypeParserAgent/schema.ts

@@ -20,5 +20,4 @@ export const FinalContentTypesAnalysisSchema = z.object({
   complexity: z.string(),
 });
 
-export type ContentTypeSummary = z.infer<typeof ContentTypeAnalysisSchema>;
 export type FinalContentTypesResultSummary = z.infer<typeof FinalContentTypesAnalysisSchema>;

apps/google-docs/functions/agents/documentParserAgent/documentParser.agent.ts

@@ -1,58 +1,129 @@
 /**
- * INTEG-3263: Content Type Parser Agent
+ * INTEG-3263: Document Parser Agent
  *
- * Simple agent that takes JSON input and sends it to OpenAI/ChatGPT
- * to generate Contentful content type definitions.
+ * Agent that takes a Google Doc URL and content type definitions,
+ * then uses OpenAI to extract structured entries from the document
+ * that can be directly created in Contentful.
  * See https://contentful.atlassian.net/wiki/spaces/ECO/pages/5850955777/RFC+Google+Docs+V1+AI-Gen
  * for more details.
  */
 
-import { openai } from '@ai-sdk/openai';
-import { generateText } from 'ai';
+import { createOpenAI } from '@ai-sdk/openai';
+import { generateObject } from 'ai';
 import { ContentTypeProps } from 'contentful-management';
 import { fetchGoogleDoc } from '../../service/googleDriveService';
+import { FinalEntriesResultSchema, FinalEntriesResult } from './schema';
 
 /**
- * Configuration for the content type parser
+ * Configuration for the document parser
  */
-interface DocumentParserConfig {
-  // contentTypes: ContentTypeProps[];
+export interface DocumentParserConfig {
+  openAiApiKey: string;
   googleDocUrl: string;
+  contentTypes: ContentTypeProps[];
+  locale?: string;
 }
 
 /**
- * @param jsonData - JSON data to analyze
- * @param config - Parser configuration
- * @returns Promise resolving to LLM response
+ * AI Agent that parses a Google Doc and extracts structured entries
+ * based on provided Contentful content type definitions.
+ *
+ * @param config - Parser configuration including API key, document URL, and content types
+ * @returns Promise resolving to entries ready for CMA client
  */
-export async function createDocument(config: DocumentParserConfig) {
-  const { googleDocUrl } = config;
+export async function createDocument(config: DocumentParserConfig): Promise<FinalEntriesResult> {
+  // TODO: Double check these values and make sure they are compatible because not every user will have a key
+  // to access all models
+  const modelVersion = 'gpt-4o';
+  const temperature = 0.3;
+
+  const { googleDocUrl, openAiApiKey, contentTypes, locale = 'en-US' } = config;
   const googleDocContent = await fetchGoogleDoc(googleDocUrl);
-  // const prompt = buildPrompt(jsonData);
-  // return await callOpenAI(prompt, modelVersion, openaiApiKey);
-  return googleDocContent as string;
+
+  const openaiClient = createOpenAI({
+    apiKey: openAiApiKey,
+  });
+
+  const prompt = buildExtractionPrompt({ contentTypes, googleDocContent, locale });
+  const result = await generateObject({
+    model: openaiClient(modelVersion),
+    schema: FinalEntriesResultSchema,
+    temperature,
+    system: buildSystemPrompt(),
+    prompt,
+  });
+
+  return result.object as FinalEntriesResult;
 }
 
-function buildPrompt(jsonData: any): string {
-  // TODO: Create prompt template for the AI to consume
-  // 1. Add instructions for content type generation
-  // 2. Include JSON data
-  // 3. Specify output format
-  return `Parse the document and create a json object that represents the document for the Contetful API to consume: ${JSON.stringify(
-    jsonData,
-    null,
-    2
-  )}`;
+function buildSystemPrompt(): string {
+  return `You are an expert content extraction AI that analyzes documents and extracts structured content based on Contentful content type definitions.
+
+Your role is to:
+1. Carefully read and understand the document content
+2. Analyze the provided Contentful content type definitions (their fields, types, and validations)
+3. Extract relevant information from the document that matches the content type structure
+4. Create properly formatted entries that are ready to be created in Contentful via the CMA API
+
+Important guidelines:
+- Each entry must have a contentTypeId that matches one of the provided content types
+- Fields must be in the correct format: { "fieldId": { "locale": value } }
+- Respect field types (Text, Symbol, RichText, Number, Boolean, Date, Reference, etc.)
+- Only include fields that exist in the content type definition
+- Extract all relevant content from the document - don't skip entries
+- If a field is required in the content type, ensure it's populated
+- For rich text fields, extract formatted content when possible
+- Be thorough and extract as many valid entries as you can find in the document`;
 }
 
-async function callOpenAI(prompt: string, modelVersion: string, openaiApiKey: string) {
-  const model = openai(modelVersion);
+function buildExtractionPrompt({
+  contentTypes,
+  googleDocContent,
+  locale,
+}: {
+  contentTypes: ContentTypeProps[];
+  googleDocContent: string;
+  locale: string;
+}): string {
+  const contentTypeList = contentTypes.map((ct) => `${ct.name} (ID: ${ct.sys.id})`).join(', ');
+  const totalFields = contentTypes.reduce((sum, ct) => sum + (ct.fields?.length || 0), 0);
 
-  // ai-sdk documentation: https://ai-sdk.dev/docs/reference/ai-sdk-core/generate-text
-  // Select the appropriate core function from the list of options
-  const result = await generateText({
-    model,
-    prompt,
-  });
-  return result;
+  // Create a simplified view of content types for the prompt
+  const contentTypeDefinitions = contentTypes.map((ct) => ({
+    id: ct.sys.id,
+    name: ct.name,
+    description: ct.description,
+    fields:
+      ct.fields?.map((field) => ({
+        id: field.id,
+        name: field.name,
+        type: field.type,
+        required: field.required,
+        localized: field.localized,
+        validations: field.validations,
+      })) || [],
+  }));
+
+  return `Extract structured entries from the following document based on the provided Contentful content type definitions.
+
+AVAILABLE CONTENT TYPES: ${contentTypeList}
+TOTAL CONTENT TYPES: ${contentTypes.length}
+TOTAL FIELDS ACROSS ALL TYPES: ${totalFields}
+LOCALE TO USE: ${locale}
+
+CONTENT TYPE DEFINITIONS:
+${JSON.stringify(contentTypeDefinitions, null, 2)}
+
+DOCUMENT CONTENT:
+${googleDocContent}
+
+INSTRUCTIONS:
+1. Analyze the document and identify content that matches the provided content type structures
+2. Extract all relevant entries from the document
+3. For each entry, use the contentTypeId that best matches the content
+4. Format fields correctly: { "fieldId": { "${locale}": value } }
+5. Ensure all required fields are populated
+6. Be thorough - extract all valid content from the document
+
+Return the extracted entries in the specified JSON schema format.`;
 }

apps/google-docs/functions/agents/documentParserAgent/schema.ts

@@ -0,0 +1,29 @@
+import { z } from 'zod';
+
+// Schema Definitions for the Document Parser Agent
+
+// Schema for a single field value in Contentful format (locale-specific)
+// Contentful expects fields in format: { 'en-US': value }
+const LocalizedFieldSchema = z.record(z.string(), z.any());
+
+// Schema for a single entry that will be created in Contentful
+export const EntryToCreateSchema = z.object({
+  contentTypeId: z.string().describe('The ID of the content type for this entry'),
+  fields: z
+    .record(z.string(), LocalizedFieldSchema)
+    .describe(
+      'Fields with localized values, e.g., { "title": { "en-US": "My Title" }, "body": { "en-US": "Content..." } }'
+    ),
+});
+
+// The final output schema - array of entries ready for CMA client
+export const FinalEntriesResultSchema = z.object({
+  entries: z
+    .array(EntryToCreateSchema)
+    .describe('Array of entries extracted from the document, ready to be created in Contentful'),
+  summary: z.string().describe('Brief summary of what was extracted from the document'),
+  totalEntries: z.number().describe('Total number of entries extracted'),
+});
+
+export type EntryToCreate = z.infer<typeof EntryToCreateSchema>;
+export type FinalEntriesResult = z.infer<typeof FinalEntriesResultSchema>;

apps/google-docs/functions/createEntriesFromDocument.ts

@@ -33,7 +33,6 @@ export const handler: FunctionEventHandler<
   const { contentTypeIds, googleDocUrl } = event.body;
   const { openAiApiKey } = context.appInstallationParameters as AppInstallationParameters;
   // INTEG-3262 and INTEG-3263: Take in Content Type, Prompt, and Upload File from user
-
   const cma = initContentfulManagementClient(context);
   const contentTypes = await fetchContentTypes(cma, new Set<string>(contentTypeIds));
 
@@ -44,16 +43,28 @@ export const handler: FunctionEventHandler<
   // createContentTypeObservationsFromLLMResponse()
 
   // INTEG-3263: Implement the document parser agent
-  const aiDocumentResponse = await createDocument({ googleDocUrl });
+  // Pass the content types to the document parser so it can extract entries based on the structure
+  const aiDocumentResponse = await createDocument({
+    googleDocUrl,
+    openAiApiKey,
+    contentTypes,
+  });
 
   // INTEG-3261: Pass the ai document response to the observer for analysis
   // createDocumentObservationsFromLLMResponse()
 
   // INTEG-3264: Create the entries in Contentful using the entry service
-  // await createEntries();
+  // The aiDocumentResponse.entries is now ready to be passed to the CMA client
+  // await createEntries(aiDocumentResponse.entries, { spaceId, environmentId, accessToken });
 
   // INTEG-3265: Create the assets in Contentful using the asset service
   // await createAssets()
 
-  return { success: true, response: { contentTypeParserAgentResult, aiDocumentResponse } };
+  return {
+    success: true,
+    response: {
+      contentTypeParserAgentResult,
+      entriesReadyForCreation: aiDocumentResponse.entries,
+    },
+  };
 };

apps/google-docs/functions/service/entryService.ts

@@ -1,21 +1,50 @@
-import { EntryProps, ContentTypeProps } from 'contentful-management';
+import { PlainClientAPI, EntryProps } from 'contentful-management';
+import { EntryToCreate } from '../agents/documentParserAgent/schema';
 
 /**
- * INTEG-3264: Implement this file to create entries in Contentful using the Contentful Management API
+ * INTEG-3264: Service for creating entries in Contentful using the Contentful Management API
+ *
+ * This service takes the output from the Document Parser Agent (which extracts entries from documents)
+ * and creates them in Contentful using the CMA client.
  */
-interface EntryServiceParams {
-  spaceId: string;
-  environmentId: string;
-  accessToken: string;
-}
 
-export async function createEntry(aiDocumentResponse: any): Promise<EntryProps> {
+/**
+ * Creates a single entry in Contentful
+ *
+ * @param cma - Contentful Management API client
+ * @param entry - Entry data from Document Parser Agent (matches EntryToCreate schema)
+ * @returns Promise resolving to the created entry
+ */
+export async function createEntry(cma: PlainClientAPI, entry: EntryToCreate): Promise<EntryProps> {
+  // TODO: Implement entry creation using the CMA client
+  // Example implementation:
+  // const createdEntry = await cma.entry.create(
+  //   { spaceId, environmentId },
+  //   {
+  //     contentTypeId: entry.contentTypeId,
+  //     fields: entry.fields
+  //   }
+  // );
+  // return createdEntry;
   throw new Error('Not implemented');
 }
 
+/**
+ * Creates multiple entries in Contentful
+ *
+ * @param cma - Contentful Management API client
+ * @param entries - Array of entries from Document Parser Agent output
+ * @returns Promise resolving to array of created entries
+ */
 export async function createEntries(
-  entries: Array<{ contentTypeId: string; fields: Record<string, any> }>,
-  config: EntryServiceParams
+  cma: PlainClientAPI,
+  entries: EntryToCreate[]
 ): Promise<EntryProps[]> {
+  // TODO: Implement batch entry creation
+  // Consider implementing with proper error handling and rate limiting
+  // const createdEntries = await Promise.all(
+  //   entries.map(entry => createEntry(cma, entry))
+  // );
+  // return createdEntries;
   throw new Error('Not implemented');
 }

apps/google-docs/src/utils/appFunctionUtils.ts

@@ -15,9 +15,7 @@ export async function getAppActionId(
     environmentId: sdk.ids.environment,
     spaceId: sdk.ids.space,
   });
-
   const appAction = appActions.items.find((action) => action.name === actionName);
-
   if (!appAction) {
     throw new Error(`App action "${actionName}" not found`);
   }
@@ -38,7 +36,6 @@ export const createEntriesFromDocumentAction = async (
     }
 
     const appActionId = await getAppActionId(sdk, 'createEntriesFromDocumentAction');
-
     const result = await sdk.cma.appActionCall.createWithResult(
       {
         appDefinitionId,
@@ -55,6 +52,7 @@ export const createEntriesFromDocumentAction = async (
 
     return result;
   } catch (error) {
+    console.error('Error creating entries from document', error);
     throw new Error(error instanceof Error ? error.message : 'Failed to analyze content types');
   }
 };