Support configurable delay for SQL REST API polling interval

Author: aniket-s-kulkarni

src/dremioai/api/dremio/sql.py

@@ -185,10 +185,12 @@ async def get_results(
     if client is None:
         client = AsyncHttpClient()
 
+    delay = settings.instance().dremio.api.polling_interval
+
     endpoint = f"/v0/projects/{project_id}" if project_id else "/api/v3"
     job: Job = await client.get(f"{endpoint}/job/{qs.id}", deser=Job)
     while not job.done:
-        await asyncio.sleep(0.5)
+        await asyncio.sleep(delay)
         job = await client.get(f"{endpoint}/job/{qs.id}", deser=Job)
 
     if not job.succeeded:

src/dremioai/api/transport.py

@@ -43,7 +43,7 @@
 class RetryConfig:
     def __init__(self):
         if settings.instance() and settings.instance().dremio:
-            self.config = settings.instance().dremio.http_retry
+            self.config = settings.instance().dremio.api.http_retry
         else:
             self.config = settings.HttpRetry()
 

src/dremioai/config/settings.py

@@ -147,7 +147,7 @@ class HttpRetry(BaseModel):
     """Configuration for HTTP retry behavior with exponential backoff"""
 
     max_retries: Optional[int] = Field(
-        default=3,
+        default=20,
         description="Maximum number of retry attempts for rate-limited requests",
     )
     initial_delay: Optional[float] = Field(
@@ -162,6 +162,15 @@ class HttpRetry(BaseModel):
     model_config = ConfigDict(validate_assignment=True)
 
 
+class ApiSettings(BaseModel):
+    # HTTP retry configuration
+    http_retry: Optional[HttpRetry] = Field(default_factory=HttpRetry)
+    polling_interval: Optional[float] = Field(
+        default=1, description="Polling interval for REST api in seconds"
+    )
+    model_config = ConfigDict(validate_assignment=True)
+
+
 class Dremio(BaseModel):
     uri: Annotated[
         Union[str, HttpUrl, DremioCloudUri], AfterValidator(_resolve_dremio_uri)
@@ -177,10 +186,8 @@ class Dremio(BaseModel):
     allow_dml: Optional[bool] = False
     auth_issuer_uri_override: Optional[str] = None
     wlm: Optional[Wlm] = None
-    # Metrics server configuration
+    api: Optional[ApiSettings] = Field(default_factory=ApiSettings)
     metrics: Optional[Metrics] = None
-    # HTTP retry configuration
-    http_retry: Optional[HttpRetry] = Field(default_factory=HttpRetry)
     model_config = ConfigDict(validate_assignment=True)
 
     @field_serializer("raw_pat")

tests/api/test_transport_retry.py

@@ -300,7 +300,8 @@ def mock_settings_instance():
     )
 
     mock_dremio = MagicMock()
-    mock_dremio.http_retry = http_retry_config
+    mock_dremio.api = MagicMock()
+    mock_dremio.api.http_retry = http_retry_config
 
     # Create mock settings object
     mock_settings = MagicMock()