feat: optimise database and query

Author: tduyng

core/stats/achievements.go

@@ -51,7 +51,7 @@ var AchievementConfigs = []AchievementConfig{
 }
 
 // CalculateAchievements determines which achievements are unlocked based on stats and activities
-func CalculateAchievements(periodStats PeriodStats, activities []Activity, streakInfo StreakInfo) []Achievement {
+func CalculateAchievements(periodStats PeriodStats, streakInfo StreakInfo) []Achievement {
 	var achievements []Achievement
 
 	// Count unique languages (only code languages)
@@ -62,10 +62,6 @@ func CalculateAchievements(periodStats PeriodStats, activities []Activity, strea
 		}
 	}
 
-	// Group activities into sessions for session achievements
-	sm := NewSessionManager(15*60, 1*60) // 15 min timeout, 1 min minimum
-	sessions := sm.GroupActivitiesIntoSessions(activities)
-
 	for _, cfg := range AchievementConfigs {
 		unlocked := false
 
@@ -91,7 +87,7 @@ func CalculateAchievements(periodStats PeriodStats, activities []Activity, strea
 			}
 
 		case "session":
-			for _, s := range sessions {
+			for _, s := range periodStats.Sessions {
 				if s.Duration >= float64(cfg.MinSession) {
 					unlocked = true
 					break

core/stats/bridge.go

@@ -3,6 +3,7 @@ package stats
 
 import (
 	"database/sql"
+	"fmt"
 	"time"
 )
 
@@ -20,6 +21,15 @@ type APIStats struct {
 	DailyActivity map[string]DailyStat `json:"daily_activity"`
 	WeeklyHeatmap []HeatmapDay         `json:"weekly_heatmap"`
 	GeneratedAt   time.Time            `json:"generated_at"`
+	Meta          APIMeta              `json:"_meta"` // NEW: Performance metadata
+}
+
+// APIMeta contains performance and diagnostic information
+type APIMeta struct {
+	LoadedActivities int     `json:"loaded_activities"`
+	TotalActivities  int     `json:"total_activities"`
+	QueryTimeMs      float64 `json:"query_time_ms"`
+	DataWindow       string  `json:"data_window"` // e.g., "last_365_days"
 }
 
 // APIPeriodStats is simplified version of PeriodStats for API responses
@@ -90,22 +100,50 @@ type APISession struct {
 	EndTime    time.Time `json:"end_time"`
 	Duration   float64   `json:"duration"`
 	Projects   []string  `json:"projects"`
-	Languages  []string  `json:"languages,omitempty"` // Top languages used in session
+	Languages  []string  `json:"languages,omitempty"`
 	IsActive   bool      `json:"is_active"`
-	BreakAfter float64   `json:"break_after,omitempty"` // Duration until next session
+	BreakAfter float64   `json:"break_after,omitempty"`
 }
 
 type APIComparisonResult struct {
 	Trend string `json:"trend"`
 }
 
-// GetAPIStats returns simplified stats optimized for API consumers
+const LOOKBACK_DAYS = 180
+
+// GetAPIStats with smart data loading
 func GetAPIStats(db *sql.DB) (*APIStats, error) {
-	activities, err := loadActivitiesFromDB(db)
+	return GetAPIStatsWithOptions(db, APIStatsOptions{
+		LoadRecentDays: LOOKBACK_DAYS, // Default: only load last x days
+	})
+}
+
+// APIStatsOptions controls how much data to load
+type APIStatsOptions struct {
+	LoadRecentDays int  // Load only last N days (0 = all time)
+	IncludeAllTime bool // Include all-time stats
+}
+
+// GetAPIStatsWithOptions allows custom data loading
+func GetAPIStatsWithOptions(db *sql.DB, opts APIStatsOptions) (*APIStats, error) {
+	startTime := time.Now()
+
+	if opts.LoadRecentDays == 0 {
+		opts.LoadRecentDays = LOOKBACK_DAYS
+	}
+
+	now := time.Now()
+	cutoffDate := now.AddDate(0, 0, -opts.LoadRecentDays)
+
+	// Only load recent activities
+	activities, err := loadActivitiesSince(db, cutoffDate)
 	if err != nil {
 		return nil, err
 	}
 
+	// Get total count for metadata
+	totalCount, _ := getTotalActivityCount(db)
+
 	calc := NewCalculator(time.Local)
 	fullStats, err := calc.Calculate(activities)
 	if err != nil {
@@ -114,9 +152,71 @@ func GetAPIStats(db *sql.DB) (*APIStats, error) {
 
 	apiStats := convertToAPIStats(fullStats)
 
+	// Add performance metadata
+	queryTime := time.Since(startTime).Seconds() * 1000 // Convert to ms
+	apiStats.Meta = APIMeta{
+		LoadedActivities: len(activities),
+		TotalActivities:  totalCount,
+		QueryTimeMs:      queryTime,
+		DataWindow:       fmt.Sprintf("last_%d_days", opts.LoadRecentDays),
+	}
+
 	return apiStats, nil
 }
 
+// Load only activities since a specific date
+func loadActivitiesSince(db *sql.DB, since time.Time) ([]Activity, error) {
+	query := `
+		SELECT id, timestamp, lines, language, project,
+		       editor, file, COALESCE(branch, ''), is_write
+		FROM activities
+		WHERE timestamp >= ?
+		ORDER BY timestamp ASC
+	`
+
+	rows, err := db.Query(query, since.Unix())
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+
+	activities := []Activity{}
+	for rows.Next() {
+		var a Activity
+		var timestampUnix int64
+		var isWriteInt int
+
+		err := rows.Scan(
+			&a.ID,
+			&timestampUnix,
+			&a.Lines,
+			&a.Language,
+			&a.Project,
+			&a.Editor,
+			&a.File,
+			&a.Branch,
+			&isWriteInt,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		a.Timestamp = time.Unix(timestampUnix, 0)
+		a.IsWrite = isWriteInt == 1
+
+		activities = append(activities, a)
+	}
+
+	return activities, rows.Err()
+}
+
+// Get total activity count (for metadata)
+func getTotalActivityCount(db *sql.DB) (int, error) {
+	var count int
+	err := db.QueryRow("SELECT COUNT(*) FROM activities").Scan(&count)
+	return count, err
+}
+
 // convertToAPIStats converts full Stats to simplified APIStats
 func convertToAPIStats(s *Stats) *APIStats {
 	return &APIStats{
@@ -263,49 +363,3 @@ func convertComparisonResultToAPI(comp ComparisonResult) APIComparisonResult {
 		Trend: comp.Trend,
 	}
 }
-
-// loadActivitiesFromDB loads all activities from the database
-func loadActivitiesFromDB(db *sql.DB) ([]Activity, error) {
-	query := `
-		SELECT id, timestamp, lines, language, project,
-		       editor, file, COALESCE(branch, ''), is_write
-		FROM activities
-		ORDER BY timestamp ASC
-	`
-
-	rows, err := db.Query(query)
-	if err != nil {
-		return nil, err
-	}
-	defer rows.Close()
-
-	activities := []Activity{}
-	for rows.Next() {
-		var a Activity
-		var timestampUnix int64
-		var isWriteInt int
-
-		err := rows.Scan(
-			&a.ID,
-			&timestampUnix,
-			&a.Lines,
-			&a.Language,
-			&a.Project,
-			&a.Editor,
-			&a.File,
-			&a.Branch,
-			&isWriteInt,
-		)
-		if err != nil {
-			return nil, err
-		}
-
-		// Convert Unix timestamp to time.Time
-		a.Timestamp = time.Unix(timestampUnix, 0)
-		a.IsWrite = isWriteInt == 1
-
-		activities = append(activities, a)
-	}
-
-	return activities, rows.Err()
-}

core/stats/calculator.go

@@ -166,8 +166,7 @@ func (c *Calculator) calculatePeriod(
 			day.Time += a.Duration
 			day.Lines += a.Lines
 
-			// Add languages & projects to sets
-			if a.Language != "" {
+			if IsValidLanguage(a.Language) {
 				langSets[date].Add(a.Language)
 			}
 			if a.Project != "" {
@@ -198,18 +197,23 @@ func (c *Calculator) calculateLanguageStats(
 	periodActivities []Activity,
 	allActivities []Activity,
 ) []LanguageStats {
-	// Aggregate period data
 	periodData := make(map[string]*LanguageStats)
 	total := 0.0
 
 	for _, a := range periodActivities {
-		if periodData[a.Language] == nil {
-			periodData[a.Language] = &LanguageStats{Name: a.Language}
+		if !IsValidLanguage(a.Language) {
+			continue
 		}
-		periodData[a.Language].Time += a.Duration
-		periodData[a.Language].Lines += a.Lines
-		periodData[a.Language].Files++
+
+		lang := NormalizeLanguage(a.Language)
+		if periodData[lang] == nil {
+			periodData[lang] = &LanguageStats{Name: lang}
+		}
+		periodData[lang].Time += a.Duration
+		periodData[lang].Lines += a.Lines
+		periodData[lang].Files++
 		total += a.Duration
+
 	}
 
 	// Calculate lifetime hours for proficiency

core/stats/languages.go

@@ -211,3 +211,24 @@ func IsCodeLanguage(lang string) bool {
 	kind, ok := LanguageClassification[lower]
 	return ok && kind == "code"
 }
+
+var invalidLanguages = map[string]struct{}{
+	"":          {},
+	"n/a":       {},
+	"na":        {},
+	"unknown":   {},
+	"undefined": {},
+	"null":      {},
+	"none":      {},
+	"None":      {},
+}
+
+func IsValidLanguage(lang string) bool {
+	lang = strings.TrimSpace(strings.ToLower(lang))
+	_, invalid := invalidLanguages[lang]
+	return !invalid
+}
+
+func NormalizeLanguage(lang string) string {
+	return strings.TrimSpace(strings.ToLower(lang))
+}

core/stats/session.go

@@ -59,7 +59,9 @@ func (sm *SessionManager) GroupActivitiesIntoSessions(activities []Activity) []S
 		projects = util.NewStringSet()
 		languages = util.NewStringSet()
 		projects.Add(a.Project)
-		languages.Add(a.Language)
+		if IsValidLanguage(a.Language) {
+			languages.Add(NormalizeLanguage(a.Language))
+		}
 
 		current = &Session{
 			ID:         a.ID,
@@ -109,7 +111,9 @@ func (sm *SessionManager) GroupActivitiesIntoSessions(activities []Activity) []S
 		current.Activities = append(current.Activities, a)
 		current.EndTime = a.Timestamp
 		projects.Add(a.Project)
-		languages.Add(a.Language)
+		if IsValidLanguage(a.Language) {
+			languages.Add(NormalizeLanguage(a.Language))
+		}
 	}
 
 	// Finalize last session