diff --git a/ansible_ai_connect/ai/api/model_pipelines/http/pipelines.py b/ansible_ai_connect/ai/api/model_pipelines/http/pipelines.py index 3ce05be77..41b021eb7 100644 --- a/ansible_ai_connect/ai/api/model_pipelines/http/pipelines.py +++ b/ansible_ai_connect/ai/api/model_pipelines/http/pipelines.py @@ -16,7 +16,9 @@ import logging from typing import Optional +import aiohttp import requests +from django.http import StreamingHttpResponse from health_check.exceptions import ServiceUnavailable from ansible_ai_connect.ai.api.exceptions import ( @@ -39,6 +41,8 @@ MetaData, ModelPipelineChatBot, ModelPipelineCompletions, + ModelPipelineStreamingChatBot, + StreamingChatBotParameters, ) from ansible_ai_connect.ai.api.model_pipelines.registry import Register from ansible_ai_connect.healthcheck.backends import ( @@ -120,13 +124,12 @@ def infer_from_parameters(self, api_key, model_id, context, prompt, suggestion_i raise NotImplementedError -@Register(api_type="http") -class HttpChatBotPipeline(HttpMetaData, ModelPipelineChatBot[HttpConfiguration]): +class HttpChatBotMetaData(HttpMetaData): def __init__(self, config: HttpConfiguration): super().__init__(config=config) - def invoke(self, params: ChatBotParameters) -> ChatBotResponse: + def prepare_data(self, params: ChatBotParameters): query = params.query conversation_id = params.conversation_id provider = params.provider @@ -142,11 +145,49 @@ def invoke(self, params: ChatBotParameters) -> ChatBotResponse: data["conversation_id"] = str(conversation_id) if system_prompt: data["system_prompt"] = str(system_prompt) + return data + + def self_test(self) -> Optional[HealthCheckSummary]: + summary: HealthCheckSummary = HealthCheckSummary( + { + MODEL_MESH_HEALTH_CHECK_PROVIDER: "http", + MODEL_MESH_HEALTH_CHECK_MODELS: "ok", + } + ) + try: + headers = {"Content-Type": "application/json"} + r = requests.get(self.config.inference_url + "/readiness", headers=headers) + r.raise_for_status() + + data = r.json() + ready = data.get("ready") + if not ready: + reason = data.get("reason") + summary.add_exception( + MODEL_MESH_HEALTH_CHECK_MODELS, + HealthCheckSummaryException(ServiceUnavailable(reason)), + ) + + except Exception as e: + logger.exception(str(e)) + summary.add_exception( + MODEL_MESH_HEALTH_CHECK_MODELS, + HealthCheckSummaryException(ServiceUnavailable(ERROR_MESSAGE), e), + ) + return summary + + +@Register(api_type="http") +class HttpChatBotPipeline(HttpChatBotMetaData, ModelPipelineChatBot[HttpConfiguration]): + + def __init__(self, config: HttpConfiguration): + super().__init__(config=config) + def invoke(self, params: ChatBotParameters) -> ChatBotResponse: response = requests.post( self.config.inference_url + "/v1/query", headers=self.headers, - json=data, + json=self.prepare_data(params), timeout=self.timeout(1), verify=self.config.verify_ssl, ) @@ -171,31 +212,44 @@ def invoke(self, params: ChatBotParameters) -> ChatBotResponse: detail = json.loads(response.text).get("detail", "") raise ChatbotInternalServerException(detail=detail) - def self_test(self) -> Optional[HealthCheckSummary]: - summary: HealthCheckSummary = HealthCheckSummary( - { - MODEL_MESH_HEALTH_CHECK_PROVIDER: "http", - MODEL_MESH_HEALTH_CHECK_MODELS: "ok", - } - ) - try: - headers = {"Content-Type": "application/json"} - r = requests.get(self.config.inference_url + "/readiness", headers=headers) - r.raise_for_status() - data = r.json() - ready = data.get("ready") - if not ready: - reason = data.get("reason") - summary.add_exception( - MODEL_MESH_HEALTH_CHECK_MODELS, - HealthCheckSummaryException(ServiceUnavailable(reason)), - ) +class HttpStreamingChatBotMetaData(HttpChatBotMetaData): - except Exception as e: - logger.exception(str(e)) - summary.add_exception( - MODEL_MESH_HEALTH_CHECK_MODELS, - HealthCheckSummaryException(ServiceUnavailable(ERROR_MESSAGE), e), - ) - return summary + def __init__(self, config: HttpConfiguration): + super().__init__(config=config) + + def prepare_data(self, params: StreamingChatBotParameters): + data = super().prepare_data(params) + + media_type = params.media_type + if media_type: + data["media_type"] = str(media_type) + + return data + + +@Register(api_type="http") +class HttpStreamingChatBotPipeline( + HttpStreamingChatBotMetaData, ModelPipelineStreamingChatBot[HttpConfiguration] +): + + def __init__(self, config: HttpConfiguration): + super().__init__(config=config) + + def invoke(self, params: StreamingChatBotParameters) -> StreamingHttpResponse: + raise NotImplementedError + + async def async_invoke(self, params: StreamingChatBotParameters) -> StreamingHttpResponse: + async with aiohttp.ClientSession(raise_for_status=True) as session: + headers = { + "Content-Type": "application/json", + "Accept": "application/json,text/event-stream", + } + async with session.post( + self.config.inference_url + "/v1/streaming_query", + json=self.prepare_data(params), + headers=headers, + ) as r: + async for chunk in r.content: + logger.debug(chunk) + yield chunk diff --git a/ansible_ai_connect/ai/api/model_pipelines/pipelines.py b/ansible_ai_connect/ai/api/model_pipelines/pipelines.py index f35f8f79c..8c66f1d53 100644 --- a/ansible_ai_connect/ai/api/model_pipelines/pipelines.py +++ b/ansible_ai_connect/ai/api/model_pipelines/pipelines.py @@ -242,6 +242,33 @@ def init( ChatBotResponse = Any +@define +class StreamingChatBotParameters(ChatBotParameters): + media_type: str + + @classmethod + def init( + cls, + query: str, + provider: Optional[str] = None, + model_id: Optional[str] = None, + conversation_id: Optional[str] = None, + system_prompt: Optional[str] = None, + media_type: Optional[str] = None, + ): + return cls( + query=query, + provider=provider, + model_id=model_id, + conversation_id=conversation_id, + system_prompt=system_prompt, + media_type=media_type, + ) + + +StreamingChatBotResponse = Any + + class MetaData(Generic[PIPELINE_CONFIGURATION], metaclass=ABCMeta): def __init__(self, config: PIPELINE_CONFIGURATION): @@ -274,6 +301,9 @@ def alias() -> str: def invoke(self, params: PIPELINE_PARAMETERS) -> PIPELINE_RETURN: raise NotImplementedError + async def async_invoke(self, params: PIPELINE_PARAMETERS) -> PIPELINE_RETURN: + raise NotImplementedError + @abstractmethod def self_test(self) -> Optional[HealthCheckSummary]: raise NotImplementedError @@ -381,3 +411,17 @@ def __init__(self, config: PIPELINE_CONFIGURATION): @staticmethod def alias(): return "chatbot-service" + + +class ModelPipelineStreamingChatBot( + ModelPipeline[PIPELINE_CONFIGURATION, ChatBotParameters, StreamingChatBotResponse], + Generic[PIPELINE_CONFIGURATION], + metaclass=ABCMeta, +): + + def __init__(self, config: PIPELINE_CONFIGURATION): + super().__init__(config=config) + + @staticmethod + def alias(): + return "streaming-chatbot-service" diff --git a/ansible_ai_connect/ai/api/model_pipelines/registry.py b/ansible_ai_connect/ai/api/model_pipelines/registry.py index f9ee4c88b..1426099ec 100644 --- a/ansible_ai_connect/ai/api/model_pipelines/registry.py +++ b/ansible_ai_connect/ai/api/model_pipelines/registry.py @@ -30,6 +30,7 @@ ModelPipelinePlaybookGeneration, ModelPipelineRoleExplanation, ModelPipelineRoleGeneration, + ModelPipelineStreamingChatBot, ) from ansible_ai_connect.main.settings.types import t_model_mesh_api_type @@ -45,6 +46,7 @@ ModelPipelinePlaybookExplanation, ModelPipelineRoleExplanation, ModelPipelineChatBot, + ModelPipelineStreamingChatBot, PipelineConfiguration, Serializer, ] diff --git a/ansible_ai_connect/ai/api/serializers.py b/ansible_ai_connect/ai/api/serializers.py index 329e4386b..f3bc7d5a0 100644 --- a/ansible_ai_connect/ai/api/serializers.py +++ b/ansible_ai_connect/ai/api/serializers.py @@ -352,6 +352,14 @@ class ChatRequestSerializer(serializers.Serializer): ) +class StreamingChatRequestSerializer(ChatRequestSerializer): + media_type = serializers.CharField( + required=False, + label="Media type", + help_text=("A media type to be used in the output from LLM."), + ) + + class ReferencedDocumentsSerializer(serializers.Serializer): docs_url = serializers.CharField() title = serializers.CharField() diff --git a/ansible_ai_connect/ai/api/versions/v1/ai/urls.py b/ansible_ai_connect/ai/api/versions/v1/ai/urls.py index 2921b3032..aad4bcdb8 100644 --- a/ansible_ai_connect/ai/api/versions/v1/ai/urls.py +++ b/ansible_ai_connect/ai/api/versions/v1/ai/urls.py @@ -25,4 +25,5 @@ path("generations/role/", views.GenerationRole.as_view(), name="generations/role"), path("feedback/", views.Feedback.as_view(), name="feedback"), path("chat/", views.Chat.as_view(), name="chat"), + path("streaming_chat/", views.StreamingChat.as_view(), name="streaming_chat"), ] diff --git a/ansible_ai_connect/ai/api/versions/v1/ai/views.py b/ansible_ai_connect/ai/api/versions/v1/ai/views.py index 1667003af..f47679e96 100644 --- a/ansible_ai_connect/ai/api/versions/v1/ai/views.py +++ b/ansible_ai_connect/ai/api/versions/v1/ai/views.py @@ -21,6 +21,7 @@ Feedback, GenerationPlaybook, GenerationRole, + StreamingChat, ) __all__ = [ @@ -32,4 +33,5 @@ "ExplanationRole", "Feedback", "Chat", + "StreamingChat", ] diff --git a/ansible_ai_connect/ai/api/views.py b/ansible_ai_connect/ai/api/views.py index 02bcc18e2..98716e006 100644 --- a/ansible_ai_connect/ai/api/views.py +++ b/ansible_ai_connect/ai/api/views.py @@ -21,6 +21,7 @@ from attr import asdict from django.apps import apps from django.conf import settings +from django.http import StreamingHttpResponse from django_prometheus.conf import NAMESPACE from drf_spectacular.utils import OpenApiResponse, extend_schema from oauth2_provider.contrib.rest_framework import IsAuthenticatedOrTokenHasScope @@ -77,10 +78,12 @@ ModelPipelinePlaybookGeneration, ModelPipelineRoleExplanation, ModelPipelineRoleGeneration, + ModelPipelineStreamingChatBot, PlaybookExplanationParameters, PlaybookGenerationParameters, RoleExplanationParameters, RoleGenerationParameters, + StreamingChatBotParameters, ) from ansible_ai_connect.ai.api.pipelines.completions import CompletionsPipeline from ansible_ai_connect.ai.api.telemetry import schema1 @@ -134,6 +137,7 @@ PlaybookGenerationAction, RoleGenerationAction, SentimentFeedback, + StreamingChatRequestSerializer, SuggestionQualityFeedback, ) from .telemetry.schema1 import ( @@ -1126,3 +1130,82 @@ def post(self, request) -> Response: status=rest_framework_status.HTTP_200_OK, headers=headers, ) + + +class StreamingChat(AACSAPIView): + """ + Send a message to the backend chatbot service and get a streaming reply. + """ + + class StreamingChatEndpointThrottle(EndpointRateThrottle): + scope = "chat" + + permission_classes = [ + permissions.IsAuthenticated, + IsAuthenticatedOrTokenHasScope, + IsRHInternalUser | IsTestUser, + ] + required_scopes = ["read", "write"] + schema1_event = schema1.ChatBotOperationalEvent # TODO + request_serializer_class = StreamingChatRequestSerializer + throttle_classes = [StreamingChatEndpointThrottle] + + llm: ModelPipelineStreamingChatBot + + def __init__(self): + super().__init__() + self.llm = apps.get_app_config("ai").get_model_pipeline(ModelPipelineStreamingChatBot) + + self.chatbot_enabled = ( + self.llm.config.inference_url + and self.llm.config.model_id + and settings.CHATBOT_DEFAULT_PROVIDER + ) + if self.chatbot_enabled: + logger.debug("Chatbot is enabled.") + else: + logger.debug("Chatbot is not enabled.") + + @extend_schema( + request=StreamingChatRequestSerializer, + responses={ + 200: ChatResponseSerializer, # TODO + 400: OpenApiResponse(description="Bad request"), + 403: OpenApiResponse(description="Forbidden"), + 413: OpenApiResponse(description="Prompt too long"), + 422: OpenApiResponse(description="Validation failed"), + 500: OpenApiResponse(description="Internal server error"), + 503: OpenApiResponse(description="Service unavailable"), + }, + summary="Streaming chat request", + ) + def post(self, request) -> Response: + if not self.chatbot_enabled: + raise ChatbotNotEnabledException() + + req_query = self.validated_data["query"] + req_system_prompt = self.validated_data.get("system_prompt") + req_provider = self.validated_data.get("provider", settings.CHATBOT_DEFAULT_PROVIDER) + conversation_id = self.validated_data.get("conversation_id") + media_type = self.validated_data.get("media_type") + + # Initialise Segment Event early, in case of exceptions + self.event.chat_prompt = anonymize_struct(req_query) + self.event.chat_system_prompt = req_system_prompt + self.event.provider_id = req_provider + self.event.conversation_id = conversation_id + self.event.modelName = self.req_model_id or self.llm.config.model_id + + return StreamingHttpResponse( + self.llm.async_invoke( + StreamingChatBotParameters.init( + query=req_query, + system_prompt=req_system_prompt, + model_id=self.req_model_id or self.llm.config.model_id, + provider=req_provider, + conversation_id=conversation_id, + media_type=media_type, + ) + ), + content_type="text/event-stream", + ) diff --git a/ansible_ai_connect/main/settings/base.py b/ansible_ai_connect/main/settings/base.py index fb2554d60..4b9e3ac86 100644 --- a/ansible_ai_connect/main/settings/base.py +++ b/ansible_ai_connect/main/settings/base.py @@ -55,6 +55,7 @@ # Application definition INSTALLED_APPS = [ + "daphne", "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", @@ -93,6 +94,11 @@ "csp.middleware.CSPMiddleware", ] +if os.environ.get("CSRF_TRUSTED_ORIGINS"): + CSRF_TRUSTED_ORIGINS = os.environ.get("CSRF_TRUSTED_ORIGINS").split(",") +else: + CSRF_TRUSTED_ORIGINS = ["http://localhost:8000"] + # Allow Prometheus to scrape metrics ALLOWED_CIDR_NETS = [os.environ.get("ALLOWED_CIDR_NETS", "10.0.0.0/8")] @@ -294,7 +300,7 @@ def is_ssl_enabled(value: str) -> bool: }, }, "handlers": { - "console": {"class": "logging.StreamHandler", "formatter": "simple", "level": "INFO"}, + "console": {"class": "logging.StreamHandler", "formatter": "simple", "level": "DEBUG"}, }, "loggers": { "django": { @@ -334,6 +340,11 @@ def is_ssl_enabled(value: str) -> bool: "level": "INFO", "propagate": False, }, + "ansible_ai_connect.ai.api.streaming_chat": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": False, + }, }, "root": { "handlers": ["console"], @@ -358,6 +369,7 @@ def is_ssl_enabled(value: str) -> bool: ] WSGI_APPLICATION = "ansible_ai_connect.main.wsgi.application" +ASGI_APPLICATION = "ansible_ai_connect.main.asgi.application" # Database # https://docs.djangoproject.com/en/4.1/ref/settings/#databases diff --git a/ansible_ai_connect/main/settings/legacy.py b/ansible_ai_connect/main/settings/legacy.py index 6543f43c8..cc7aef87c 100644 --- a/ansible_ai_connect/main/settings/legacy.py +++ b/ansible_ai_connect/main/settings/legacy.py @@ -192,6 +192,15 @@ def load_from_env_vars(): "stream": False, }, } + model_pipelines_config["ModelPipelineStreamingChatBot"] = { + "provider": "http", + "config": { + "inference_url": chatbot_service_url or "http://localhost:8000", + "model_id": chatbot_service_model_id or "granite3-8b", + "verify_ssl": model_service_verify_ssl, + "stream": False, + }, + } # Enable Health Checks where we have them implemented model_pipelines_config["ModelPipelineCompletions"]["config"][ diff --git a/ansible_ai_connect/main/views.py b/ansible_ai_connect/main/views.py index 60ca899bb..d81b6607b 100644 --- a/ansible_ai_connect/main/views.py +++ b/ansible_ai_connect/main/views.py @@ -28,7 +28,9 @@ from rest_framework.renderers import BaseRenderer from rest_framework.views import APIView -from ansible_ai_connect.ai.api.model_pipelines.pipelines import ModelPipelineChatBot +from ansible_ai_connect.ai.api.model_pipelines.pipelines import ( + ModelPipelineStreamingChatBot, +) from ansible_ai_connect.ai.api.permissions import ( IsOrganisationAdministrator, IsOrganisationLightspeedSubscriber, @@ -121,12 +123,12 @@ class ChatbotView(ProtectedTemplateView): IsRHInternalUser | IsTestUser, ] - llm: ModelPipelineChatBot + llm: ModelPipelineStreamingChatBot chatbot_enabled: bool def __init__(self): super().__init__() - self.llm = apps.get_app_config("ai").get_model_pipeline(ModelPipelineChatBot) + self.llm = apps.get_app_config("ai").get_model_pipeline(ModelPipelineStreamingChatBot) self.chatbot_enabled = ( self.llm.config.inference_url and self.llm.config.model_id diff --git a/requirements-aarch64.txt b/requirements-aarch64.txt index 588da2792..440177130 100644 --- a/requirements-aarch64.txt +++ b/requirements-aarch64.txt @@ -7,7 +7,9 @@ aiohappyeyeballs==2.3.5 # via aiohttp aiohttp==3.10.11 - # via langchain + # via + # -r requirements.in + # langchain aiosignal==1.3.1 # via aiohttp annotated-types==0.6.0 @@ -29,7 +31,9 @@ anyio==4.6.2.post1 argparse==1.4.0 # via uwsgi-readiness-check asgiref==3.8.1 - # via django + # via + # daphne + # django asttokens==2.4.1 # via stack-data attrs==23.2.0 @@ -37,6 +41,12 @@ attrs==23.2.0 # aiohttp # jsonschema # referencing + # service-identity + # twisted +autobahn==24.4.2 + # via daphne +automat==24.8.1 + # via twisted backcall==0.2.0 # via ipython backoff==2.2.1 @@ -68,13 +78,19 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via black +constantly==23.10.4 + # via twisted cryptography==43.0.1 # via # -r requirements.in # ansible-core + # autobahn # jwcrypto # pyopenssl + # service-identity # social-auth-core +daphne==4.1.2 + # via -r requirements.in decorator==5.1.1 # via ipython defusedxml==0.8.0rc2 @@ -167,13 +183,21 @@ httpx==0.27.2 # via # langsmith # ollama +hyperlink==21.0.0 + # via + # autobahn + # twisted idna==3.7 # via # -r requirements.in # anyio # httpx + # hyperlink # requests + # twisted # yarl +incremental==24.7.2 + # via twisted inflection==0.5.1 # via drf-spectacular ipython==8.10.0 @@ -307,10 +331,12 @@ pyasn1==0.6.0 # oauth2client # pyasn1-modules # rsa + # service-identity pyasn1-modules==0.4.0 # via # google-auth # oauth2client + # service-identity pycparser==2.21 # via cffi pydantic==2.9.2 @@ -336,6 +362,7 @@ pyopenssl==24.2.1 # via # -r requirements.in # pydrive2 + # twisted pyparsing==3.1.2 # via httplib2 pyrfc3339==1.1 @@ -410,6 +437,8 @@ segment-analytics-python==2.2.2 # via -r requirements.in semver==3.0.2 # via launchdarkly-server-sdk +service-identity==24.2.0 + # via twisted six==1.16.0 # via # asttokens @@ -455,6 +484,10 @@ traitlets==5.14.3 # via # ipython # matplotlib-inline +twisted[tls]==24.11.0 + # via daphne +txaio==23.1.1 + # via autobahn typing-extensions==4.11.0 # via # django-test-migrations @@ -464,6 +497,7 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # sqlalchemy + # twisted uritemplate==4.1.1 # via # drf-spectacular @@ -490,3 +524,8 @@ yamllint==1.35.1 # via ansible-lint yarl==1.17.2 # via aiohttp +zope-interface==7.2 + # via twisted + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements-x86_64.txt b/requirements-x86_64.txt index 4ca612d3d..284dcb588 100644 --- a/requirements-x86_64.txt +++ b/requirements-x86_64.txt @@ -7,7 +7,9 @@ aiohappyeyeballs==2.3.5 # via aiohttp aiohttp==3.10.11 - # via langchain + # via + # -r requirements.in + # langchain aiosignal==1.3.1 # via aiohttp annotated-types==0.6.0 @@ -29,7 +31,9 @@ anyio==4.6.2.post1 argparse==1.4.0 # via uwsgi-readiness-check asgiref==3.8.1 - # via django + # via + # daphne + # django asttokens==2.4.1 # via stack-data attrs==23.2.0 @@ -37,6 +41,12 @@ attrs==23.2.0 # aiohttp # jsonschema # referencing + # service-identity + # twisted +autobahn==24.4.2 + # via daphne +automat==24.8.1 + # via twisted backcall==0.2.0 # via ipython backoff==2.2.1 @@ -68,13 +78,19 @@ charset-normalizer==3.3.2 # via requests click==8.1.7 # via black +constantly==23.10.4 + # via twisted cryptography==43.0.1 # via # -r requirements.in # ansible-core + # autobahn # jwcrypto # pyopenssl + # service-identity # social-auth-core +daphne==4.1.2 + # via -r requirements.in decorator==5.1.1 # via ipython defusedxml==0.8.0rc2 @@ -167,13 +183,21 @@ httpx==0.27.2 # via # langsmith # ollama +hyperlink==21.0.0 + # via + # autobahn + # twisted idna==3.7 # via # -r requirements.in # anyio # httpx + # hyperlink # requests + # twisted # yarl +incremental==24.7.2 + # via twisted inflection==0.5.1 # via drf-spectacular ipython==8.10.0 @@ -307,10 +331,12 @@ pyasn1==0.6.0 # oauth2client # pyasn1-modules # rsa + # service-identity pyasn1-modules==0.4.0 # via # google-auth # oauth2client + # service-identity pycparser==2.21 # via cffi pydantic==2.9.2 @@ -336,6 +362,7 @@ pyopenssl==24.2.1 # via # -r requirements.in # pydrive2 + # twisted pyparsing==3.1.2 # via httplib2 pyrfc3339==1.1 @@ -410,6 +437,8 @@ segment-analytics-python==2.2.2 # via -r requirements.in semver==3.0.2 # via launchdarkly-server-sdk +service-identity==24.2.0 + # via twisted six==1.16.0 # via # asttokens @@ -455,6 +484,10 @@ traitlets==5.14.3 # via # ipython # matplotlib-inline +twisted[tls]==24.11.0 + # via daphne +txaio==23.1.1 + # via autobahn typing-extensions==4.11.0 # via # django-test-migrations @@ -464,6 +497,7 @@ typing-extensions==4.11.0 # pydantic # pydantic-core # sqlalchemy + # twisted uritemplate==4.1.1 # via # drf-spectacular @@ -490,3 +524,8 @@ yamllint==1.35.1 # via ansible-lint yarl==1.17.2 # via aiohttp +zope-interface==7.2 + # via twisted + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements.in b/requirements.in index 9f0f237a8..4c71fd087 100644 --- a/requirements.in +++ b/requirements.in @@ -9,6 +9,7 @@ # - https://peps.python.org/pep-0631 # - https://peps.python.org/pep-0508 # ====================================================================== +aiohttp==3.10.11 ansible-anonymizer==1.5.0 ansible-risk-insight==0.2.7 ansible-lint==24.2.2 @@ -17,6 +18,7 @@ boto3==1.26.84 black==24.3.0 certifi@git+https://github.com/ansible/system-certifi@5aa52ab91f9d579bfe52b5acf30ca799f1a563d9 cryptography==43.0.1 +daphne==4.1.2 Django==4.2.18 django-deprecate-fields==0.1.1 django-extensions==3.2.1 diff --git a/tools/configs/nginx-wisdom.conf b/tools/configs/nginx-wisdom.conf index 3f8327679..70c2bb3dd 100644 --- a/tools/configs/nginx-wisdom.conf +++ b/tools/configs/nginx-wisdom.conf @@ -2,6 +2,11 @@ upstream uwsgi { server unix:///var/run/uwsgi/ansible_wisdom.sock; } +upstream daphne { + server unix:///var/run/daphne/ansible_wisdom.sock; +} + + server { listen 8000 default_server; server_name _; @@ -14,4 +19,12 @@ server { uwsgi_pass uwsgi; include /etc/nginx/uwsgi_params; } + + location /api/v1/ai/streaming_chat/ { + proxy_pass http://daphne; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_redirect off; + } } diff --git a/tools/configs/supervisord.conf b/tools/configs/supervisord.conf index 05a897e5d..44e1826ea 100644 --- a/tools/configs/supervisord.conf +++ b/tools/configs/supervisord.conf @@ -28,6 +28,29 @@ stdout_logfile_maxbytes = 0 stderr_logfile = /dev/stderr stderr_logfile_maxbytes = 0 +[fcgi-program:daphne] +# TCP socket used by Nginx backend upstream +socket=tcp://localhost:9000 + +# When daphne is running in multiple processes, each needs to have a different socket. +# In such a case, it is recommended to include process # in the name of socket, but +# then those generated socket names cannot be specified in nginx config file... +# So use this with numprocs=1 for now. See https://github.com/django/daphne/issues/287 +# for more details. +numprocs=1 +command = /var/www/venv/bin/daphne -u /var/run/daphne/ansible_wisdom.sock --fd 0 --access-log - --proxy-headers ansible_ai_connect.main.asgi:application + +autostart = true +autorestart = true +stopwaitsecs = 1 +stopsignal = KILL +stopasgroup = true +killasgroup = true +stdout_logfile = /dev/stdout +stdout_logfile_maxbytes = 0 +stderr_logfile = /dev/stderr +stderr_logfile_maxbytes = 0 + ; [program:test] ; command = sleep infinity diff --git a/wisdom-service.Containerfile b/wisdom-service.Containerfile index 9d20acbf9..88cd80c44 100644 --- a/wisdom-service.Containerfile +++ b/wisdom-service.Containerfile @@ -50,7 +50,7 @@ RUN /var/www/venv/bin/python3.11 -m pip --no-cache-dir install --no-binary=all c RUN /var/www/venv/bin/python3.11 -m pip --no-cache-dir install -r/var/www/ansible-ai-connect-service/requirements.txt RUN /var/www/venv/bin/python3.11 -m pip --no-cache-dir install -e/var/www/ansible-ai-connect-service/ -RUN mkdir /var/run/uwsgi +RUN mkdir /var/run/uwsgi /var/run/daphne RUN echo -e "\ {\n\ @@ -99,6 +99,7 @@ RUN for dir in \ /var/log/supervisor \ /var/run/supervisor \ /var/run/uwsgi \ + /var/run/daphne \ /var/www/wisdom \ /var/log/nginx \ /etc/ari \