optimize the query

Author: yilinxia

backend/app/services/gexf_node_service.py

@@ -61,52 +61,94 @@ def generate_gexf_nodes_for_topics(self, topics):
         topics_lower = [t.lower() for t in topics]
         placeholders = ",".join(["?"] * len(topics_lower))
 
-        # Debug: Check what columns actually exist in the repos table
-        schema_query = "DESCRIBE repos"
-        # try:
-        #     schema_result = self.con.execute(schema_query).fetchall()
-        #     # print("Repos table schema:")
-        #     # for col in schema_result:
-        #     #     print(f"  {col}")
-        # except Exception as e:
-        #     print(f"Could not get schema: {e}")
+        # Use consistent memory and thread settings with TopicService
+        available_memory = psutil.virtual_memory().available
+        memory_limit = min(available_memory * 0.3, 0.5 * 1024 * 1024 * 1024)  # Use 30% of available memory, max 0.5GB
+        self.con.execute(f"SET memory_limit TO '{int(memory_limit)}B'")
+        
+        # Set conservative thread count
+        cpu_count = psutil.cpu_count(logical=False) or 1
+        thread_count = max(1, min(cpu_count, 2))  # Use at most 2 threads
+        self.con.execute(f"SET threads TO {thread_count}")
+        
+        # Note: Cannot create indexes on read-only database
+        # The query will rely on DuckDB's built-in query optimization
 
+        # Optimized query with better structure and materialized CTEs
         query = f"""
-           WITH matching_repos AS (
+           WITH matching_repos AS MATERIALIZED (
                 SELECT DISTINCT r.nameWithOwner
                 FROM repos r
-                JOIN repo_topics t ON r.nameWithOwner = t.repo
+                INNER JOIN repo_topics t ON r.nameWithOwner = t.repo
                 WHERE LOWER(t.topic) IN ({placeholders})
             ),
-            repo_topics_agg AS (
+            repo_topics_agg AS MATERIALIZED (
                 SELECT 
                     r.nameWithOwner,
                     GROUP_CONCAT(t.topic, '|') AS topics
                 FROM repos r
-                JOIN repo_topics t ON r.nameWithOwner = t.repo
-                JOIN matching_repos mr ON r.nameWithOwner = mr.nameWithOwner
+                INNER JOIN repo_topics t ON r.nameWithOwner = t.repo
+                INNER JOIN matching_repos mr ON r.nameWithOwner = mr.nameWithOwner
                 GROUP BY r.nameWithOwner
             )
             SELECT 
                 r.nameWithOwner,
-                r.stars,
-                r.forks,
-                r.watchers,
-                r.isArchived,
-                r.languageCount,
-                r.pullRequests,
-                r.issues,
-                r.primaryLanguage,
+                COALESCE(r.stars, 0) as stars,
+                COALESCE(r.forks, 0) as forks,
+                COALESCE(r.watchers, 0) as watchers,
+                COALESCE(r.isArchived, false) as isArchived,
+                COALESCE(r.languageCount, 0) as languageCount,
+                COALESCE(r.pullRequests, 0) as pullRequests,
+                COALESCE(r.issues, 0) as issues,
+                COALESCE(r.primaryLanguage, '') as primaryLanguage,
                 r.createdAt,
-                r.license,
+                COALESCE(r.license, '') as license,
                 rt.topics,
                 r.bigquery_contributors,
                 r.bigquery_stargazers
             FROM repos r
-            JOIN matching_repos mr ON r.nameWithOwner = mr.nameWithOwner
-            JOIN repo_topics_agg rt ON r.nameWithOwner = rt.nameWithOwner;
+            INNER JOIN matching_repos mr ON r.nameWithOwner = mr.nameWithOwner
+            INNER JOIN repo_topics_agg rt ON r.nameWithOwner = rt.nameWithOwner
+            ORDER BY r.stars DESC, r.forks DESC
+            LIMIT 10000;  -- Limit results to prevent memory issues
         """
-        result = self.con.execute(query, topics_lower).fetchall()
+        
+        # Execute query with optimized settings
+        try:
+            result = self.con.execute(query, topics_lower).fetchall()
+        except Exception as e:
+            print(f"Query failed, falling back to simpler version: {e}")
+            # Fallback to simpler query if the optimized one fails
+            fallback_query = f"""
+                SELECT DISTINCT
+                    r.nameWithOwner,
+                    COALESCE(r.stars, 0) as stars,
+                    COALESCE(r.forks, 0) as forks,
+                    COALESCE(r.watchers, 0) as watchers,
+                    COALESCE(r.isArchived, false) as isArchived,
+                    COALESCE(r.languageCount, 0) as languageCount,
+                    COALESCE(r.pullRequests, 0) as pullRequests,
+                    COALESCE(r.issues, 0) as issues,
+                    COALESCE(r.primaryLanguage, '') as primaryLanguage,
+                    r.createdAt,
+                    COALESCE(r.license, '') as license,
+                    GROUP_CONCAT(t.topic, '|') as topics,
+                    r.bigquery_contributors,
+                    r.bigquery_stargazers
+                FROM repos r
+                INNER JOIN repo_topics t ON r.nameWithOwner = t.repo
+                WHERE LOWER(t.topic) IN ({placeholders})
+                GROUP BY r.nameWithOwner, r.stars, r.forks, r.watchers, r.isArchived, 
+                         r.languageCount, r.pullRequests, r.issues, r.primaryLanguage, 
+                         r.createdAt, r.license, r.bigquery_contributors, r.bigquery_stargazers
+                ORDER BY r.stars DESC, r.forks DESC
+                LIMIT 10000;
+            """
+            result = self.con.execute(fallback_query, topics_lower).fetchall()
+        
+        # Reset memory settings
+        self.con.execute("SET memory_limit TO DEFAULT")
+        self.con.execute("SET threads TO DEFAULT")
 
         # Debug: Print the first few rows to see what we're getting
         # if result: