add additional sanity checks for data returned by loaders

Author: jonsch1

src/content/loaders/funding.ts

@@ -62,6 +62,16 @@ export function fundingLoader(options: { orcid: string }): Loader {
 
         logger.info(`Successfully processed ${grants.length} grants`);
 
+        // Sanity checks
+        if (grants.length === 0) {
+          throw new Error('Funding loader returned 0 grants - this is likely an error');
+        }
+
+        const grantsWithoutName = grants.filter((g: any) => !g.name || g.name.trim() === '' || g.name === 'Untitled Grant');
+        if (grantsWithoutName.length > 0) {
+          throw new Error(`${grantsWithoutName.length} grants missing proper title - data quality issue`);
+        }
+
         store.set({
           id: 'funding_loaded',
           data: {

src/content/loaders/github.ts

@@ -132,6 +132,21 @@ export function githubLoader(options: {
 
       logger.info(`Total teams processed: ${allTeams.length}`);
 
+      // Sanity checks
+      if (allTeams.length === 0) {
+        throw new Error('GitHub loader returned 0 teams - this is likely an error');
+      }
+
+      const totalMembers = allTeams.reduce((sum, team) => sum + team.members.length, 0);
+      if (totalMembers === 0) {
+        throw new Error('GitHub loader returned 0 team members across all teams - this is likely an error');
+      }
+
+      const teamsWithoutMembers = allTeams.filter(team => team.members.length === 0);
+      if (teamsWithoutMembers.length > 0) {
+        logger.warn(`${teamsWithoutMembers.length} teams have no members: ${teamsWithoutMembers.map(t => t.name).join(', ')}`);
+      }
+
       store.set({
         id: 'teams_loaded',
         data: {

src/content/loaders/google-sheets.ts

@@ -26,7 +26,12 @@ export function googleSheetsLoader(config: GoogleSheetsConfig): Loader {
 
         const csvText = await response.text();
         const rows = csvText.split('\n').filter(row => row.trim());
-        
+
+        // Sanity check: ensure we got data
+        if (rows.length <= 1) {
+          throw new Error(`Google Sheets loader for "${sheetName}" returned no data rows - expected at least 1`);
+        }
+
         // Skip header row and process data rows
         for (let i = 1; i < rows.length; i++) {
           const row = parseCSVRow(rows[i]);

src/content/loaders/orcid.ts

@@ -241,6 +241,16 @@ export function orcidLoader(options: { orcid: string }): Loader {
 
         logger.info(`Successfully processed ${allPublications.length} publications from ORCID`);
 
+        // Sanity checks
+        if (allPublications.length < 10) {
+          throw new Error(`ORCID loader returned only ${allPublications.length} publications - expected at least 10`);
+        }
+
+        const publicationsWithoutTitle = allPublications.filter(p => !p.title || p.title.trim() === '');
+        if (publicationsWithoutTitle.length > 0) {
+          throw new Error(`${publicationsWithoutTitle.length} publications missing title - data quality issue`);
+        }
+
         allPublications.sort((a, b) => {
           // Publications without year go to the end
           if (!a.year && !b.year) {