Merge remote-tracking branch 'origin/main' into fix/projects-metadata

Author: joanagmaia

services/apps/git_integration/src/crowdgit/services/maintainer/bedrock.py

@@ -1,8 +1,11 @@
+import asyncio
 import json
+import random
 from typing import Generic, TypeVar
 
 import aioboto3
 from botocore.config import Config
+from botocore.exceptions import ClientError
 from pydantic import BaseModel, ValidationError
 
 from crowdgit.logger import logger
@@ -20,6 +23,10 @@ class BedrockResponse(BaseModel, Generic[T]):
     cost: float
 
 
+MAX_THROTTLE_RETRIES = 5
+THROTTLE_BASE_DELAY = 10  # seconds
+
+
 async def invoke_bedrock(
     instruction, pydantic_model: type[T], replacements=None, max_tokens=65000, temperature=0
 ) -> BedrockResponse[T]:
@@ -71,41 +78,54 @@ async def invoke_bedrock(
             }
         )
 
+        modelId = "us.anthropic.claude-sonnet-4-20250514-v1:0"
+        accept = "application/json"
+        contentType = "application/json"
+
+        for attempt in range(1, MAX_THROTTLE_RETRIES + 1):
+            try:
+                response = await bedrock_client.invoke_model(
+                    body=body, modelId=modelId, accept=accept, contentType=contentType
+                )
+                break
+            except ClientError as e:
+                if (
+                    e.response["Error"]["Code"] == "ThrottlingException"
+                    and attempt < MAX_THROTTLE_RETRIES
+                ):
+                    delay = THROTTLE_BASE_DELAY * (2 ** (attempt - 1)) + random.uniform(0, 2)
+                    logger.warning(
+                        f"Bedrock ThrottlingException (attempt {attempt}/{MAX_THROTTLE_RETRIES}), "
+                        f"retrying in {delay:.1f}s: {e}"
+                    )
+                    await asyncio.sleep(delay)
+                else:
+                    raise
+
         try:
-            modelId = "us.anthropic.claude-sonnet-4-20250514-v1:0"
-            accept = "application/json"
-            contentType = "application/json"
-            response = await bedrock_client.invoke_model(
-                body=body, modelId=modelId, accept=accept, contentType=contentType
-            )
+            body_bytes = await response["body"].read()
+            response_body = json.loads(body_bytes.decode("utf-8"))
+            raw_text = response_body["content"][0]["text"].replace('"""', "").strip()
+
+            # Expect pure JSON - no markdown handling
+            output = json.loads(raw_text)
 
+            # Calculate cost
+            input_tokens = response_body["usage"]["input_tokens"]
+            output_tokens = response_body["usage"]["output_tokens"]
+            input_cost = (input_tokens / 1000) * 0.003
+            output_cost = (output_tokens / 1000) * 0.015
+            total_cost = input_cost + output_cost
+
+            # Validate output with the provided model if it exists
             try:
-                body_bytes = await response["body"].read()
-                response_body = json.loads(body_bytes.decode("utf-8"))
-                raw_text = response_body["content"][0]["text"].replace('"""', "").strip()
-
-                # Expect pure JSON - no markdown handling
-                output = json.loads(raw_text)
-
-                # Calculate cost
-                input_tokens = response_body["usage"]["input_tokens"]
-                output_tokens = response_body["usage"]["output_tokens"]
-                input_cost = (input_tokens / 1000) * 0.003
-                output_cost = (output_tokens / 1000) * 0.015
-                total_cost = input_cost + output_cost
-
-                # Validate output with the provided model if it exists
-                try:
-                    validated_output = pydantic_model.model_validate(output, strict=True)
-                except ValidationError as ve:
-                    logger.error(f"Output validation failed: {ve}")
-                    raise ve
-
-                return BedrockResponse[T](output=validated_output, cost=total_cost)
-            except Exception as e:
-                logger.error("Failed to parse the response as JSON. Raw response:")
-                logger.error(response_body["content"][0]["text"])
-                raise e
+                validated_output = pydantic_model.model_validate(output, strict=True)
+            except ValidationError as ve:
+                logger.error(f"Output validation failed: {ve}")
+                raise ve
+
+            return BedrockResponse[T](output=validated_output, cost=total_cost)
         except Exception as e:
-            logger.error(f"Amazon Bedrock API error: {e}")
+            logger.error("Failed to parse the response as JSON. Raw response:")
+            logger.error(response_body["content"][0]["text"])
             raise e

services/libs/tinybird/datasources/project_insights_copy_ds.datasource

@@ -16,6 +16,10 @@ DESCRIPTION >
     - `organizationDependencyPercentage` column is the combined contribution percentage of dependent organizations.
     - `achievements` column is an array of tuples (leaderboardType, rank, totalCount) representing project achievements.
     - `healthScore` column is the overall health score for the project (0-100).
+    - `contributorHealthScore` column is the health score percentage for the contributors category (0-100).
+    - `popularityHealthScore` column is the health score percentage for the popularity category (0-100).
+    - `developmentHealthScore` column is the health score percentage for the development category (0-100).
+    - `securityHealthScore` column is the health score percentage for the security category (0-100).
     - `firstCommit` column is the timestamp of the first commit in the project.
     - `starsLast365Days` column is the count of stars in the last 365 days.
     - `forksLast365Days` column is the count of forks in the last 365 days.
@@ -43,6 +47,10 @@ SCHEMA >
     `organizationDependencyPercentage` Float64,
     `achievements` Array(Tuple(String, UInt64, UInt64)),
     `healthScore` Nullable(Float64),
+    `contributorHealthScore` Nullable(Float64),
+    `popularityHealthScore` Nullable(Float64),
+    `developmentHealthScore` Nullable(Float64),
+    `securityHealthScore` Nullable(Float64),
     `firstCommit` Nullable(DateTime64(3)),
     `starsLast365Days` UInt64,
     `forksLast365Days` UInt64,

services/libs/tinybird/pipes/project_insights.pipe

@@ -1,6 +1,6 @@
 DESCRIPTION >
     - `project_insights.pipe` serves project insights data for a specific project.
-    - Returns comprehensive metrics including project metadata (name, logoUrl, isLF), health score, contributor count, software value, contributor and organization dependency metrics, leaderboard achievements, and activity metrics for both current and previous 365-day periods.
+    - Returns comprehensive metrics including project metadata (name, logoUrl, isLF), health score with category breakdowns (contributors, popularity, development, security), contributor count, software value, contributor and organization dependency metrics, leaderboard achievements, and activity metrics for both current and previous 365-day periods.
     - Parameters:
     - `slug`: Optional string for a single project slug (e.g., 'kubernetes')
     - `slugs`: Optional array of project slugs for multi-project query (e.g., ['kubernetes', 'tensorflow'])
@@ -27,6 +27,10 @@ SQL >
         organizationDependencyPercentage,
         achievements,
         healthScore,
+        contributorHealthScore,
+        popularityHealthScore,
+        developmentHealthScore,
+        securityHealthScore,
         firstCommit,
         starsLast365Days,
         forksLast365Days,

services/libs/tinybird/pipes/project_insights_copy.pipe

@@ -39,7 +39,11 @@ SQL >
         contributorDependencyCount,
         contributorDependencyPercentage,
         organizationDependencyCount,
-        organizationDependencyPercentage
+        organizationDependencyPercentage,
+        contributorPercentage,
+        popularityPercentage,
+        developmentPercentage,
+        securityPercentage
     FROM health_score_copy_ds
 
 NODE project_insights_copy_achievements
@@ -106,6 +110,10 @@ SQL >
         COALESCE(dep.organizationDependencyPercentage, 0) AS organizationDependencyPercentage,
         COALESCE(ach.achievements, []) AS achievements,
         base.healthScore AS healthScore,
+        dep.contributorPercentage AS contributorHealthScore,
+        dep.popularityPercentage AS popularityHealthScore,
+        dep.developmentPercentage AS developmentHealthScore,
+        dep.securityPercentage AS securityHealthScore,
         base.firstCommit AS firstCommit,
         l365.starsLast365Days AS starsLast365Days,
         l365.forksLast365Days AS forksLast365Days,