v0.1.3

Author: alejandromav

CHANGELOG.md

@@ -13,9 +13,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed -->
 
+## [0.1.3] - 2025-12-26
+
+### Changed
+
+- Better error reporting, and general refactor
+
 ## [0.1.2] - 2025-12-24
 
-### Added
+### Changed
 
 - Enable Quiver scrapper by default
 
@@ -24,6 +30,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - Better error reporting
+
+### Changed
+
 - Reddit time window set to 10 min
 
 ## [0.1.0] - 2025-12-23
@@ -32,7 +41,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - First working version
 
-[unreleased]: https://github.com/alejandromav/quantarded/compare/v0.1.2...HEAD
+[unreleased]: https://github.com/alejandromav/quantarded/compare/v0.1.3...HEAD
+[0.1.3]: https://github.com/alejandromav/quantarded/compare/v0.2.1...v0.1.3
 [0.1.2]: https://github.com/alejandromav/quantarded/compare/v0.1.1...v0.1.2
 [0.1.1]: https://github.com/alejandromav/quantarded/compare/v0.1.0...v0.1.1
 [0.1.0]: https://github.com/alejandromav/quantarded/releases/tag/v0.1.0

README.md

@@ -119,16 +119,17 @@ src/
 │   ├── normalize.ts  # Reddit event normalization
 │   ├── normalize-quiver.ts  # Quiver trade normalization
 │   ├── reddit.ts     # Reddit API client with pagination
-│   ├── quiver.ts        # Quiver API client with pagination
+│   ├── quiver.ts     # Quiver API client with pagination
 │   ├── job-runner.ts # Common job utilities (summary, job IDs)
 │   └── tinybird.ts   # Tinybird API client
 ├── scripts/          # CLI entry points
 │   ├── reddit-scraper.ts  # Reddit scraper script
-│   ├── quiver-scraper.ts    # Quiver scraper script
-│   └── print-config.ts     # Configuration printer
+│   ├── quiver-scraper.ts  # Quiver scraper script
+│   └── print-config.ts    # Configuration printer
+├── utils/            # Utility functions
+│   └── utils.ts      # HTTP client, hashing, date utilities
 ├── config.ts         # Configuration constants
-├── types.ts          # TypeScript type definitions
-└── utils.ts          # Utility functions
+└── types.ts          # TypeScript type definitions
 tinybird/             # Tinybird datasource schemas
 ├── events_landing__v0.datasource    # Events datasource
 ├── job_runs__v0.datasource          # Job runs datasource

infra/scripts/deploy-to-server.sh

@@ -38,7 +38,6 @@ services:
       TIME_WINDOW_MINUTES: "10"
       SCRAPER_INTERVAL_MINUTES: "5"
       CLASSIFY_CONCURRENCY: "3"
-      QUIVER_SCRAPER_ENABLED: "false"
       QUIVER_SCRAPER_CRON: "0 */6 * * *"
       NODE_ENV: "production"
     logging:

src/config.ts

@@ -19,16 +19,13 @@ export const CONFIG = {
   tinybirdDatasource: process.env.TINYBIRD_DATASOURCE ?? "events_landing__v0",
   tinybirdJobRunsDatasource: process.env.TINYBIRD_JOB_RUNS_DATASOURCE ?? "job_runs__v0",
   scraperIntervalMinutes: parseInt(process.env.SCRAPER_INTERVAL_MINUTES ?? "5", 10),
-  // Quiver Quant Configuration
   quiverApiKey: process.env.QUIVER_API_KEY ?? "",
   quiverApiUrl: process.env.QUIVER_API_URL ?? "https://api.quiverquant.com/beta",
   quiverCongressionalEndpoint: process.env.QUIVER_CONGRESSIONAL_ENDPOINT ?? "bulk/congresstrading",
   quiverPageSize: parseInt(process.env.QUIVER_PAGE_SIZE ?? "100", 10),
   quiverBatchSize: parseInt(process.env.QUIVER_BATCH_SIZE ?? "100", 10),
-  // Job scheduling
   redditScraperEnabled: process.env.REDDIT_SCRAPER_ENABLED === "true",
   quiverScraperEnabled: process.env.QUIVER_SCRAPER_ENABLED === "true",
-  quiverScraperCron: process.env.QUIVER_SCRAPER_CRON ?? "0 */6 * * *", // Default: every 6 hours
-  // Reddit proxy (optional, only used for Reddit requests to minimize traffic)
+  quiverScraperCron: process.env.QUIVER_SCRAPER_CRON ?? "0 */6 * * *",
   redditProxy: process.env.REDDIT_PROXY ?? "",
 } as const;

src/lib/classify.ts

@@ -135,20 +135,16 @@ export async function classifyBatch(items: ClassifyItem[]): Promise<ClassifyResu
     );
   }
 
-  // Handle case where LLM returns a single object instead of { results: [...] }
-  // This can happen when there's only one item or the LLM doesn't follow the format exactly
   if (!parsed || typeof parsed !== "object") {
     console.error(
       `[CLASSIFY] Invalid response: not an object. Content: ${content.substring(0, 500)}`
     );
-    // Return empty results for all items instead of crashing
     return items.map((item) => ({
       item_id: item.item_id,
       tickers: [],
     }));
   }
 
-  // Type guard for single result object
   type SingleResult = {
     item_id: string;
     tickers: Array<{
@@ -179,7 +175,6 @@ export async function classifyBatch(items: ClassifyItem[]): Promise<ClassifyResu
     );
   }
 
-  // Check if it's a single result object (missing results wrapper)
   let classifyResponse: ClassifyResponse;
   if (isSingleResult(parsed)) {
     console.error(
@@ -192,17 +187,14 @@ export async function classifyBatch(items: ClassifyItem[]): Promise<ClassifyResu
     console.error(
       `[CLASSIFY] Invalid response structure: ${JSON.stringify(parsed).substring(0, 500)}`
     );
-    // Return empty results for all items instead of crashing
     return items.map((item) => ({
       item_id: item.item_id,
       tickers: [],
     }));
   }
 
-  // Create a map of results by item_id for quick lookup
   const resultsMap = new Map<string, ClassifyResult>();
   for (const resultItem of classifyResponse.results) {
-    // Type guard to ensure result has the expected structure
     if (
       !resultItem ||
       typeof resultItem !== "object" ||
@@ -260,7 +252,6 @@ export async function classifyBatch(items: ClassifyItem[]): Promise<ClassifyResu
     });
   }
 
-  // Ensure all items in the batch have a result, even if empty
   return items.map((item) => {
     const result = resultsMap.get(item.item_id);
     return (

src/lib/job-runner.ts

@@ -5,9 +5,6 @@ export type SummaryItem = {
   value: string;
 };
 
-/**
- * Formats and prints a job execution summary table.
- */
 export function printJobSummary(summary: SummaryItem[]): void {
   if (summary.length === 0) {
     return;
@@ -29,25 +26,15 @@ export function printJobSummary(summary: SummaryItem[]): void {
   console.error("");
 }
 
-/**
- * Creates a deterministic job ID from job type and timestamp.
- */
 export function createJobId(jobType: string, timestamp: Date): string {
   return sha256(`job|${jobType}|${timestamp.toISOString()}`);
 }
 
-/**
- * Formats elapsed time in seconds with 2 decimal places.
- */
 export function formatElapsedSeconds(startedAt: Date, endedAt: Date): string {
   const elapsedMs = endedAt.getTime() - startedAt.getTime();
   return (elapsedMs / 1000).toFixed(2);
 }
 
-/**
- * Formats an error into a detailed error message string for job tracking.
- * Includes stack trace if available and truncates to maxLength to avoid payload size issues.
- */
 export function formatErrorMessage(err: unknown, maxLength = 5000): string {
   let errorMessage: string;
   if (err instanceof Error) {

src/lib/normalize-quiver.ts

@@ -1,52 +1,33 @@
 import { sha256 } from "../utils/utils.js";
 import type { NormalizedEvent, QuiverTrade } from "../types.js";
 
-/**
- * Normalizes a Quiver congressional trade into a NormalizedEvent.
- * Creates a stable event_id from trade data and preserves the full Quiver payload.
- * Maintains the same schema structure for Tinybird compatibility.
- *
- * @param trade The Quiver trade data
- * @param ingestedAt When the trade was ingested
- * @param disclosedAt The date used to query the API (YYYYMMDD format)
- */
 export function normalizeQuiverTrade(
   trade: QuiverTrade,
   ingestedAt: Date,
   disclosedAt?: string
 ): NormalizedEvent {
-  // Create stable event ID from trade fields
-  // Use ticker, transaction_date, representative, and amount for uniqueness
   const ticker = trade.Ticker || trade.ticker || trade.Symbol || trade.symbol || "UNKNOWN";
   const transactionDate = trade.TransactionDate || trade.transaction_date || trade.Date || "";
   const representative = trade.Representative || trade.representative || trade.Name || "";
   const amount = trade.Amount || trade.amount || "";
   const transactionType = trade.Transaction || trade.transaction || trade.TransactionType || "";
 
-  // Create deterministic ID from stable fields
   const eventIdInput = `quiver-daily|${ticker}|${transactionDate}|${representative}|${amount}|${transactionType}`;
   const eventId = sha256(eventIdInput);
 
-  // Use transaction_date if available, otherwise use ingested_at
   let timestamp: string;
   const dateField = transactionDate || trade.Date || trade.date;
   if (dateField) {
     try {
-      // Quiver dates might be in various formats
       const date = new Date(dateField);
-      if (!isNaN(date.getTime())) {
-        timestamp = date.toISOString();
-      } else {
-        timestamp = ingestedAt.toISOString();
-      }
+      timestamp = !isNaN(date.getTime()) ? date.toISOString() : ingestedAt.toISOString();
     } catch {
       timestamp = ingestedAt.toISOString();
     }
   } else {
     timestamp = ingestedAt.toISOString();
   }
 
-  // Convert disclosedAt from YYYYMMDD to ISO format if provided
   let disclosedAtIso: string | undefined;
   if (disclosedAt && disclosedAt.length === 8) {
     try {
@@ -58,7 +39,7 @@ export function normalizeQuiverTrade(
         disclosedAtIso = date.toISOString();
       }
     } catch {
-      // If conversion fails, leave undefined
+      // Conversion failed, leave undefined
     }
   }
 
@@ -69,9 +50,7 @@ export function normalizeQuiverTrade(
     timestamp,
     version: "1",
     payload: {
-      // Preserve full Quiver trade data
       ...trade,
-      // Add metadata
       ingested_at: ingestedAt.toISOString(),
       ...(disclosedAtIso && { disclosed_at: disclosedAtIso }),
     },

src/lib/normalize.ts

@@ -13,7 +13,6 @@ export function normalize(child: RedditListingChild): NormalizedEvent {
   const body = child.kind === "t3" ? child.data.selftext || "" : child.data.body || "";
   let content = [title, body].filter(Boolean).join(" ").trim();
 
-  // Truncate content to max length to avoid excessive token usage
   if (content.length > CONFIG.maxContentLength) {
     content = content.substring(0, CONFIG.maxContentLength - 3).trim() + "...";
   }

src/lib/quiver.ts

@@ -20,19 +20,13 @@ export type OnPageCallback = (
   disclosedAt: string
 ) => Promise<void>;
 
-/**
- * Formats a date as YYYYMMDD string.
- */
 function formatDateYYYYMMDD(date: Date): string {
   const year = date.getFullYear();
   const month = String(date.getMonth() + 1).padStart(2, "0");
   const day = String(date.getDate()).padStart(2, "0");
   return `${year}${month}${day}`;
 }
 
-/**
- * Builds request headers for Quiver API.
- */
 function buildHeaders(): Record<string, string> {
   return {
     Accept: "application/json",
@@ -46,21 +40,16 @@ function buildHeaders(): Record<string, string> {
   };
 }
 
-/**
- * Handles HTTP error responses from Quiver API.
- * @returns true if the error was handled and we should retry, false otherwise
- */
 async function handleErrorResponse(
   response: Response,
   page: number,
   errorText: string
 ): Promise<boolean> {
   if (response.status === 429) {
-    let waitSeconds = 30; // Default fallback
+    let waitSeconds = 30;
     try {
       const errorJson = JSON.parse(errorText) as { detail?: string };
       const detail = errorJson.detail || "";
-      // Extract "Expected available in X seconds" from the detail message
       const match = detail.match(/Expected available in (\d+)\s*seconds?/i);
       if (match && match[1]) {
         const parsedSeconds = parseInt(match[1], 10);
@@ -69,14 +58,14 @@ async function handleErrorResponse(
         }
       }
     } catch {
-      // If parsing fails, use default wait time
+      // Use default wait time if parsing fails
     }
 
     console.error(
       `[QUIVER] Page ${page}: Rate limited (429). Waiting ${waitSeconds} seconds before retry...`
     );
     await sleep(waitSeconds * 1000);
-    return true; // Indicates we should retry
+    return true;
   }
 
   if (response.status === 500) {
@@ -94,14 +83,6 @@ async function handleErrorResponse(
   throw new Error(`HTTP ${response.status} ${response.statusText}: ${errorText}`);
 }
 
-/**
- * Fetches all congressional trades from Quiver Quant API.
- * Uses the bulk congressional trading endpoint.
- * See: https://api.quiverquant.com/docs/#/operations/beta_bulk_congresstrading_retrieve
- *
- * @param onPage Optional callback to process each page as it's fetched (for memory efficiency)
- * @returns Promise with total trades fetched and pages fetched
- */
 export async function fetchCongressionalTradesPaginated(
   onPage?: OnPageCallback
 ): Promise<FetchCongressionalTradesResult> {
@@ -116,7 +97,6 @@ export async function fetchCongressionalTradesPaginated(
   const pageSize = CONFIG.quiverPageSize || 100;
   const headers = buildHeaders();
 
-  // Use yesterday's date (ensures complete data for the day)
   const targetDate = new Date();
   targetDate.setDate(targetDate.getDate() - 1);
   const dateStr = formatDateYYYYMMDD(targetDate);
@@ -157,7 +137,6 @@ export async function fetchCongressionalTradesPaginated(
 
       totalTradesFetched += data.length;
 
-      // Process page immediately if callback provided (memory efficient)
       if (onPage) {
         await onPage(data, page, dateStr);
       } else {
@@ -176,16 +155,13 @@ export async function fetchCongressionalTradesPaginated(
         `[QUIVER] Page ${page}: Fetched ${data.length} trades (${totalTradesFetched} total)`
       );
 
-      // Add delay between requests to respect rate limits (1-3 seconds)
       await sleep(Math.floor(Math.random() * 2000) + 1000);
     } catch (error) {
       const errorMessage = error instanceof Error ? error.message : String(error);
-      // If it's a 404 or empty response, we've reached the end
       if (errorMessage.includes("404") || errorMessage.includes("No data")) {
         console.error(`[QUIVER] Page ${page}: No more data available, stopping pagination`);
         break;
       }
-      // For other errors, log and stop (could be rate limit or API issue)
       console.error(`[QUIVER] Page ${page}: Error fetching data: ${errorMessage}`);
       throw error;
     }

src/lib/reddit.ts

@@ -21,7 +21,6 @@ async function fetchPaginated(
 
     console.error(`[REDDIT] ${label}: Fetching page ${pageCount}...`);
 
-    // Add small random delay to mimic human behavior (100-500ms)
     if (pageCount > 1) {
       const delay = Math.floor(Math.random() * 400) + 100;
       await new Promise((resolve) => setTimeout(resolve, delay));
@@ -49,7 +48,7 @@ async function fetchPaginated(
 
     if (itemsInWindow === 0) {
       console.error(
-        `[REDDIT] ${label}: Page ${pageCount}: No items in window, stopping pagination (Reddit returns items sorted by time, so all subsequent pages are older)`
+        `[REDDIT] ${label}: Page ${pageCount}: No items in window, stopping pagination`
       );
       break;
     }