sql view for latest versions

Author: davidpomerenke

sql/latest_versions_view.sql

@@ -0,0 +1,56 @@
+-- View for latest versions of each report series
+-- Run: psql $DATABASE_URL -f sql/latest_versions_view.sql
+
+--------------------------------------------------------------------------------
+-- LATEST VERSIONS VIEW
+--------------------------------------------------------------------------------
+
+DROP VIEW IF EXISTS sg_reports_survey.latest_versions;
+
+CREATE VIEW sg_reports_survey.latest_versions AS
+WITH 
+-- Count versions per series
+version_counts AS (
+  SELECT proper_title, COUNT(*)::int as version_count
+  FROM sg_reports_survey.reports
+  WHERE proper_title IS NOT NULL
+    AND symbol NOT LIKE '%/CORR.%'
+    AND symbol NOT LIKE '%/REV.%'
+  GROUP BY proper_title
+),
+-- Rank versions to find latest
+ranked AS (
+  SELECT 
+    r.id, r.symbol, r.proper_title, r.title, r.date_year, r.publication_date,
+    r.un_body, r.subject_terms, r.embedding,
+    re.entity,
+    COALESCE(r.date_year, 
+      CASE WHEN r.publication_date ~ '^\d{4}' 
+      THEN SUBSTRING(r.publication_date FROM 1 FOR 4)::int END
+    ) as effective_year,
+    ROW_NUMBER() OVER (
+      PARTITION BY r.proper_title 
+      ORDER BY 
+        COALESCE(r.date_year, 
+          CASE WHEN r.publication_date ~ '^\d{4}' 
+          THEN SUBSTRING(r.publication_date FROM 1 FOR 4)::int END
+        ) DESC NULLS LAST,
+        r.publication_date DESC NULLS LAST,
+        r.symbol DESC
+    ) as rn
+  FROM sg_reports_survey.reports r
+  LEFT JOIN sg_reports_survey.reporting_entities re ON r.symbol = re.symbol
+  WHERE r.proper_title IS NOT NULL
+    AND r.symbol NOT LIKE '%/CORR.%'
+    AND r.symbol NOT LIKE '%/REV.%'
+)
+SELECT 
+  r.id, r.symbol, r.proper_title, r.title, r.date_year, r.publication_date,
+  r.un_body, r.subject_terms, r.embedding, r.entity, r.effective_year,
+  vc.version_count
+FROM ranked r
+JOIN version_counts vc ON r.proper_title = vc.proper_title
+WHERE r.rn = 1;
+
+COMMENT ON VIEW sg_reports_survey.latest_versions IS 
+  'Latest version of each report series (by proper_title), with version_count';

src/app/api/sg-reports/route.ts

@@ -187,39 +187,33 @@ export async function GET(req: NextRequest) {
        WHERE ${whereClause}`,
       params
     ),
-    // Body counts by distinct proper_title
+    // Body counts (from latest_versions view - one per series)
     query<{ body: string; count: number }>(
-      `SELECT un_body as body, COUNT(DISTINCT proper_title)::int as count 
-       FROM ${DB_SCHEMA}.reports 
-       WHERE un_body IS NOT NULL AND proper_title IS NOT NULL
+      `SELECT un_body as body, COUNT(*)::int as count 
+       FROM ${DB_SCHEMA}.latest_versions 
+       WHERE un_body IS NOT NULL
        GROUP BY un_body ORDER BY count DESC`
     ),
-    // Year range (min/max)
+    // Year range (from latest_versions view)
     query<{ min_year: number; max_year: number }>(
-      `SELECT 
-        MIN(COALESCE(date_year, CASE WHEN publication_date ~ '^\\d{4}' THEN SUBSTRING(publication_date FROM 1 FOR 4)::int END))::int as min_year,
-        MAX(COALESCE(date_year, CASE WHEN publication_date ~ '^\\d{4}' THEN SUBSTRING(publication_date FROM 1 FOR 4)::int END))::int as max_year
-       FROM ${DB_SCHEMA}.reports 
-       WHERE proper_title IS NOT NULL`
+      `SELECT MIN(effective_year)::int as min_year, MAX(effective_year)::int as max_year
+       FROM ${DB_SCHEMA}.latest_versions`
     ),
-    // Get subject term counts - count by unique report title (not by version/symbol)
-    // Only include subjects that appear in more than one report (excluding credentials)
+    // Subject term counts (from latest_versions - one per series, excluding credentials)
     query<SubjectCount>(
-      `SELECT subject as subject, COUNT(DISTINCT proper_title)::int as count
-       FROM ${DB_SCHEMA}.reports, unnest(subject_terms) as subject
-       WHERE proper_title IS NOT NULL
-         AND subject != 'Representative''s credentials'
+      `SELECT subject, COUNT(*)::int as count
+       FROM ${DB_SCHEMA}.latest_versions, unnest(subject_terms) as subject
+       WHERE subject != 'Representative''s credentials'
        GROUP BY subject
-       HAVING COUNT(DISTINCT proper_title) > 1
+       HAVING COUNT(*) > 1
        ORDER BY count DESC, subject`
     ),
-    // Entity counts by distinct proper_title
+    // Entity counts (from latest_versions view)
     query<{ entity: string; count: number }>(
-      `SELECT re.entity, COUNT(DISTINCT r.proper_title)::int as count 
-       FROM ${DB_SCHEMA}.reporting_entities re
-       JOIN ${DB_SCHEMA}.reports r ON re.symbol = r.symbol
-       WHERE re.entity IS NOT NULL AND r.proper_title IS NOT NULL
-       GROUP BY re.entity ORDER BY count DESC`
+      `SELECT entity, COUNT(*)::int as count 
+       FROM ${DB_SCHEMA}.latest_versions 
+       WHERE entity IS NOT NULL
+       GROUP BY entity ORDER BY count DESC`
     ),
   ]);
 

src/app/api/similar-reports/route.ts

@@ -24,7 +24,7 @@ export async function GET(req: NextRequest) {
 
   try {
     // Find similar reports using vector similarity search
-    // Excludes other versions of the same report series (same proper_title)
+    // Source from reports table (any version), results from latest_versions only
     const similar = await query<SimilarReport>(
       `WITH source AS (
         SELECT embedding, proper_title
@@ -33,31 +33,23 @@ export async function GET(req: NextRequest) {
         AND embedding IS NOT NULL
       )
       SELECT 
-        r.symbol,
-        r.proper_title,
-        COALESCE(
-          r.date_year,
-          CASE WHEN r.publication_date ~ '^\\d{4}' 
-          THEN SUBSTRING(r.publication_date FROM 1 FOR 4)::int END
-        ) as year,
-        1 - (r.embedding <=> s.embedding) as similarity,
-        re.entity
-      FROM ${DB_SCHEMA}.reports r
+        lv.symbol,
+        lv.proper_title,
+        lv.effective_year as year,
+        1 - (lv.embedding <=> s.embedding) as similarity,
+        lv.entity
+      FROM ${DB_SCHEMA}.latest_versions lv
       CROSS JOIN source s
-      LEFT JOIN ${DB_SCHEMA}.reporting_entities re ON r.symbol = re.symbol
-      WHERE r.embedding IS NOT NULL
-        AND (s.proper_title IS NULL OR r.proper_title IS NULL OR TRIM(r.proper_title) != TRIM(s.proper_title))
-        AND r.symbol != $1
-        AND r.symbol NOT LIKE '%/CORR.%'
-        AND r.symbol NOT LIKE '%/REV.%'
-      ORDER BY r.embedding <=> s.embedding
+      WHERE lv.embedding IS NOT NULL
+        AND (s.proper_title IS NULL OR lv.proper_title IS NULL OR TRIM(lv.proper_title) != TRIM(s.proper_title))
+        AND lv.symbol != $1
+      ORDER BY lv.embedding <=> s.embedding
       LIMIT $2`,
       [symbol, limit]
     );
 
     // If no embedding exists for the source report, return empty
     if (similar.length === 0) {
-      // Check if the source report has an embedding
       const hasEmbedding = await query<{ has_embedding: boolean }>(
         `SELECT embedding IS NOT NULL as has_embedding 
          FROM ${DB_SCHEMA}.reports