Enhance document processing flow by adding checks for syllabus completion and improving logging; normalize document types for consistency in formatting and status updates.

Novapool · Novapool · commit 472fcd7e4c8a · 2025-03-27T05:58:57.000-04:00
diff --git a/frontend/src/components/DocumentProcessingStatus.tsx b/frontend/src/components/DocumentProcessingStatus.tsx
@@ -21,6 +21,7 @@ const DocumentProcessingStatus: React.FC<DocumentProcessingStatusProps> = ({ onP
   const { currentUser } = useAuth();
   const [documents, setDocuments] = useState<Document[]>([]);
   const [isFormatting, setIsFormatting] = useState<boolean>(false);
+  const [processingComplete, setProcessingComplete] = useState<boolean>(false);
   const [error, setError] = useState<string | null>(null);
   const [status, setStatus] = useState<string>('');
   
@@ -35,44 +36,88 @@ const DocumentProcessingStatus: React.FC<DocumentProcessingStatusProps> = ({ onP
     const documentsRef = collection(userDocRef, 'documents');
     const q = query(documentsRef);
     
+    console.log('Setting up document status listener');
+    
     const unsubscribe = onSnapshot(q, (snapshot) => {
       const docs: Document[] = [];
+      
+      console.log(`Received ${snapshot.docs.length} documents from Firestore`);
+      
       snapshot.forEach(doc => {
+        const data = doc.data();
+        console.log(`Document ${doc.id}, Type: ${data.documentType}, Status: ${data.status}`);
+        
         docs.push({
           id: doc.id,
-          ...doc.data() as Omit<Document, 'id'>
+          ...data as Omit<Document, 'id'>
         });
       });
       
+      // Update the documents state
       setDocuments(docs);
       
-      // Check if all documents are processed
-      const allProcessed = docs.length > 0 && docs.every(doc => doc.status === 'processed');
-      if (allProcessed && onProcessingComplete) {
-        onProcessingComplete();
+      if (docs.length > 0) {
+        // Force case-insensitive comparison for document types
+        const documentTypeCount = {
+          syllabus: docs.filter(doc => doc.documentType?.toLowerCase() === DOCUMENT_TYPES.SYLLABUS).length,
+          transcript: docs.filter(doc => doc.documentType?.toLowerCase() === DOCUMENT_TYPES.TRANSCRIPT).length,
+          grades: docs.filter(doc => doc.documentType?.toLowerCase() === DOCUMENT_TYPES.GRADES).length
+        };
+        
+        // Force case-insensitive comparison for document status
+        const statusCount = {
+          uploaded: docs.filter(d => d.status?.toLowerCase() === 'uploaded').length,
+          extracted: docs.filter(d => d.status?.toLowerCase() === 'extracted').length,
+          processed: docs.filter(d => d.status?.toLowerCase() === 'processed').length,
+          error: docs.filter(d => d.status?.toLowerCase() === 'error').length
+        };
+        
+        console.log('Document counts by type:', documentTypeCount);
+        console.log('Document counts by status:', statusCount);
+        
+        // Check for processing completion based on processed syllabus
+        const hasSyllabus = docs.some(doc => 
+          doc.documentType?.toLowerCase() === DOCUMENT_TYPES.SYLLABUS && 
+          doc.status?.toLowerCase() === 'processed'
+        );
+        
+        console.log('Has processed syllabus:', hasSyllabus);
+        
+        // Trigger completion if we have a processed syllabus
+        if (hasSyllabus && onProcessingComplete && !processingComplete) {
+          console.log('Processing complete condition met - has processed syllabus');
+          onProcessingComplete();
+          setProcessingComplete(true);
+        }
       }
+    }, (error) => {
+      console.error('Error in document snapshot listener:', error);
+      setError('Error monitoring document status');
     });
     
     return () => unsubscribe();
   }, [currentUser, db, onProcessingComplete]);
 
-  // Count documents by status
+  // Calculate document counts with case-insensitive comparison
   const documentCounts = {
-    uploaded: documents.filter(doc => doc.status === 'uploaded').length,
-    extracted: documents.filter(doc => doc.status === 'extracted').length,
-    processed: documents.filter(doc => doc.status === 'processed').length,
-    error: documents.filter(doc => doc.status === 'error').length
+    uploaded: documents.filter(doc => doc.status?.toLowerCase() === 'uploaded').length,
+    extracted: documents.filter(doc => doc.status?.toLowerCase() === 'extracted').length,
+    processed: documents.filter(doc => doc.status?.toLowerCase() === 'processed').length,
+    error: documents.filter(doc => doc.status?.toLowerCase() === 'error').length
   };
 
-  // Count documents by type (case-insensitive)
+  // Count documents by type with case-insensitive comparison
   const documentTypeCount = {
-    syllabus: documents.filter(doc => doc.documentType.toLowerCase() === DOCUMENT_TYPES.SYLLABUS).length,
-    transcript: documents.filter(doc => doc.documentType.toLowerCase() === DOCUMENT_TYPES.TRANSCRIPT).length,
-    grades: documents.filter(doc => doc.documentType.toLowerCase() === DOCUMENT_TYPES.GRADES).length
+    syllabus: documents.filter(doc => doc.documentType?.toLowerCase() === DOCUMENT_TYPES.SYLLABUS).length,
+    transcript: documents.filter(doc => doc.documentType?.toLowerCase() === DOCUMENT_TYPES.TRANSCRIPT).length,
+    grades: documents.filter(doc => doc.documentType?.toLowerCase() === DOCUMENT_TYPES.GRADES).length
   };
 
-  // Check if we have the minimum required documents (case-insensitive)
-  const hasSyllabus = documents.some(doc => doc.documentType.toLowerCase() === DOCUMENT_TYPES.SYLLABUS);
+  // Check if we have the minimum required documents with case-insensitive comparison
+  const hasSyllabus = documents.some(doc => 
+    doc.documentType?.toLowerCase() === DOCUMENT_TYPES.SYLLABUS &&
+    doc.status?.toLowerCase() === 'processed'
+  );
   const hasMinimumDocuments = hasSyllabus;
 
   // Handle manual formatting
@@ -123,10 +168,12 @@ const DocumentProcessingStatus: React.FC<DocumentProcessingStatusProps> = ({ onP
     }
   };
 
-  // Calculate overall progress
+  // Calculate overall progress with case-insensitive comparison and logging
   const calculateProgress = () => {
     if (documents.length === 0) return 0;
     
+    console.log('Calculating progress with', documents.length, 'documents');
+    
     const totalSteps = documents.length * 2; // Upload + Process for each document
     let completedSteps = 0;
     
@@ -135,12 +182,14 @@ const DocumentProcessingStatus: React.FC<DocumentProcessingStatusProps> = ({ onP
       completedSteps += 1;
       
       // Count processing step for extracted or processed documents
-      if (doc.status === 'extracted' || doc.status === 'processed') {
+      if (doc.status?.toLowerCase() === 'extracted' || doc.status?.toLowerCase() === 'processed') {
         completedSteps += 1;
       }
     });
     
-    return Math.round((completedSteps / totalSteps) * 100);
+    const progress = Math.round((completedSteps / totalSteps) * 100);
+    console.log(`Progress calculation: ${completedSteps}/${totalSteps} = ${progress}%`);
+    return progress;
   };
 
   const progress = calculateProgress();
@@ -245,23 +294,23 @@ const DocumentProcessingStatus: React.FC<DocumentProcessingStatusProps> = ({ onP
                   <td style={styles.tableCell}>
                     <span style={{
                       ...styles.statusBadge,
-                      backgroundColor: doc.status === 'processed' ? '#4caf50' : 
-                                      doc.status === 'extracted' ? '#ff9800' :
-                                      doc.status === 'error' ? '#f44336' : '#2196f3'
+                      backgroundColor: doc.status?.toLowerCase() === 'processed' ? '#4caf50' : 
+                                      doc.status?.toLowerCase() === 'extracted' ? '#ff9800' :
+                                      doc.status?.toLowerCase() === 'error' ? '#f44336' : '#2196f3'
                     }}>
                       {doc.status}
                     </span>
                   </td>
                   <td style={styles.tableCell}>
-                    {doc.status === 'uploaded' && (
+                    {doc.status?.toLowerCase() === 'uploaded' && (
                       <button
                         onClick={() => handleRetryProcessing(doc.id)}
                         style={styles.actionButton}
                       >
                         Process Document
                       </button>
                     )}
-                    {doc.status === 'error' && (
+                    {doc.status?.toLowerCase() === 'error' && (
                       <button
                         onClick={() => handleRetryProcessing(doc.id)}
                         style={styles.actionButton}
diff --git a/functions-node/formatDocumentsData.js b/functions-node/formatDocumentsData.js
@@ -3,11 +3,6 @@ const functions = require('firebase-functions');
 const admin = require('firebase-admin');
 const { DOCUMENT_TYPES, normalizeDocumentType } = require('./constants/documentTypes');
 
-/**
- * Formats all document data using a single OpenAI API call to ensure consistent structure
- * @param {string} userId - The user ID
- * @returns {Promise<Object>} Formatted data for calculations and predictions
- */
 /**
  * Formats all document data using a single OpenAI API call to ensure consistent structure
  * @param {string} userId - The user ID
@@ -17,7 +12,7 @@ exports.formatDocumentsData = async (userId) => {
   console.log(`Formatting all document data for user ${userId} using OpenAI`);
   
   try {
-    // Get all processed documents with extracted text
+    // Get all documents with extracted text
     const db = admin.firestore();
     const documentsRef = db.collection('users').doc(userId).collection('documents');
     const snapshot = await documentsRef.where('status', '==', 'extracted').get();
@@ -27,12 +22,16 @@ exports.formatDocumentsData = async (userId) => {
       return null;
     }
     
+    console.log(`Found ${snapshot.size} documents with status 'extracted'`);
+    
     // Organize documents by type
     const documentsByType = {};
     snapshot.forEach(doc => {
       const data = doc.data();
       if (data.documentType && data.text) {
-        documentsByType[data.documentType] = {
+        // Use normalized document type
+        const normalizedType = normalizeDocumentType(data.documentType);
+        documentsByType[normalizedType] = {
           id: doc.id,
           text: data.text,
           ...data
@@ -92,7 +91,8 @@ exports.formatDocumentsData = async (userId) => {
     await storeFormattedData(userId, formattedData);
     
     // Update the status of all processed documents
-    await updateDocumentStatus(userId, snapshot.docs);
+    const updateResult = await updateDocumentStatus(userId, snapshot.docs);
+    console.log(`Document status update result: ${updateResult}`);
     
     return formattedData;
   } catch (error) {
@@ -103,21 +103,16 @@ exports.formatDocumentsData = async (userId) => {
   }
 };
 
-/**
- * Creates the prompt for OpenAI formatting
- * @param {Object} documentsByType - Documents organized by type
- * @returns {string} Formatted prompt
- */
 /**
  * Creates the prompt for OpenAI formatting
  * @param {Object} documentsByType - Documents organized by type
  * @returns {string} Formatted prompt
  */
 function createFormattingPrompt(documentsByType) {
   // Extract the document texts
-  const syllabusText = documentsByType.syllabus ? documentsByType.syllabus.text : '';
-  const gradesText = documentsByType.grades ? documentsByType.grades.text : '';
-  const transcriptText = documentsByType.transcript ? documentsByType.transcript.text : '';
+  const syllabusText = documentsByType[DOCUMENT_TYPES.SYLLABUS]?.text || '';
+  const gradesText = documentsByType[DOCUMENT_TYPES.GRADES]?.text || '';
+  const transcriptText = documentsByType[DOCUMENT_TYPES.TRANSCRIPT]?.text || '';
   
   return `
 I need you to format educational document data into a consistent structure for grade calculations and predictions.
@@ -190,12 +185,6 @@ For the academicHistory.relevantCourses, analyze the transcript to find courses
 `;
 }
 
-/**
- * Stores the formatted data in Firestore
- * @param {string} userId - The user ID
- * @param {Object} formattedData - The formatted data
- * @returns {Promise<void>}
- */
 /**
  * Stores the formatted data in Firestore
  * @param {string} userId - The user ID
@@ -223,27 +212,27 @@ async function storeFormattedData(userId, formattedData) {
  * Updates the status of processed documents
  * @param {string} userId - The user ID
  * @param {Array} documents - The document snapshots
- * @returns {Promise<void>}
- */
-/**
- * Updates the status of processed documents
- * @param {string} userId - The user ID
- * @param {Array} documents - The document snapshots
- * @returns {Promise<void>}
+ * @returns {Promise<boolean>} True if any documents were updated
  */
 async function updateDocumentStatus(userId, documents) {
-  console.log(`Updating status for ${documents.length} documents`);
+  console.log(`Attempting to update status for ${documents.length} documents`);
   const db = admin.firestore();
   const batch = db.batch();
   
   let updateCount = 0;
   
   documents.forEach(doc => {
+    // Add extra logging to debug
+    console.log(`Processing document for status update: ${doc.id}`);
+    
     const docRef = db.collection('users').doc(userId).collection('documents').doc(doc.id);
-    const docData = doc.data();
     
-    // Only update documents that are in 'extracted' status
-    if (docData.status === 'extracted') {
+    // Handle different document object formats
+    const docData = doc.data ? doc.data() : doc;
+    console.log(`Document status before update: ${docData.status}`);
+    
+    // Only update documents that are in 'extracted' status (case-insensitive)
+    if (docData.status?.toLowerCase() === 'extracted') {
       batch.update(docRef, { 
         status: 'processed',
         processedAt: admin.firestore.FieldValue.serverTimestamp()
@@ -256,20 +245,20 @@ async function updateDocumentStatus(userId, documents) {
   });
   
   if (updateCount > 0) {
-    await batch.commit();
-    console.log(`Successfully updated ${updateCount} document statuses to processed`);
-    return true;
+    try {
+      await batch.commit();
+      console.log(`Successfully updated ${updateCount} document statuses to processed`);
+      return true;
+    } catch (error) {
+      console.error(`Error committing batch update: ${error}`);
+      throw error;
+    }
   } else {
     console.log('No documents to update');
     return false;
   }
 }
 
-/**
- * Creates a fallback formatted data structure if OpenAI fails
- * @param {Object} documentsByType - Documents organized by type
- * @returns {Object} Fallback formatted data
- */
 /**
  * Creates a fallback formatted data structure if OpenAI fails
  * @param {Object} documentsByType - Documents organized by type
@@ -279,7 +268,7 @@ function createFallbackFormattedData(documentsByType) {
   console.log('Creating fallback formatted data');
   
   // Extract basic information using regex patterns
-  const syllabusText = documentsByType.syllabus ? documentsByType.syllabus.text : '';
+  const syllabusText = documentsByType[DOCUMENT_TYPES.SYLLABUS]?.text || '';
   
   // Extract course name
   const courseNameMatch = syllabusText.match(/course(?:\s+title)?:?\s*([^\n]+)/i);
@@ -297,7 +286,7 @@ function createFallbackFormattedData(documentsByType) {
   const gradeWeights = extractGradeWeights(syllabusText);
   
   // Extract GPA from transcript
-  const transcriptText = documentsByType.transcript ? documentsByType.transcript.text : '';
+  const transcriptText = documentsByType[DOCUMENT_TYPES.TRANSCRIPT]?.text || '';
   const gpaMatch = transcriptText.match(/gpa:?\s*([\d\.]+)/i);
   const gpa = gpaMatch ? gpaMatch[1].trim() : "3.0";
   
@@ -321,11 +310,6 @@ function createFallbackFormattedData(documentsByType) {
   };
 }
 
-/**
- * Extract grade weights using regex patterns
- * @param {string} text - Text to extract grade weights from
- * @returns {Array} Array of {name, weight} objects
- */
 /**
  * Extract grade weights using regex patterns
  * @param {string} text - Text to extract grade weights from
@@ -372,9 +356,6 @@ function extractGradeWeights(text) {
   }
 }
 
-/**
- * Helper function to get OpenAI API key
- */
 /**
  * Helper function to get OpenAI API key
  */
diff --git a/functions-node/index.js b/functions-node/index.js
