[CAI-299] Chatbot/fix website url and html2md converter

.changeset/twelve-avocados-marry.md

@@ -0,0 +1,5 @@
+---
+"chatbot": minor
+---
+
+use url from env var

apps/chatbot/src/modules/chatbot.py

@@ -41,10 +41,11 @@
     Response, StreamingResponse, AsyncStreamingResponse, PydanticResponse,
     AgentChatResponse, StreamingAgentChatResponse
 ] 
+WEBSITE_URL = os.getenv("CHB_WEBSITE_URL")
 SYSTEM_PROMPT = (
     "You are the virtual PagoPA S.p.A. assistant. Your name is Discovery.\n"
     "Your role is to provide accurate, professional, and helpful responses to users' queries regarding "
-    "the PagoPA DevPortal documentation available at: https://dev.developer.pagopa.it"
+    f"the PagoPA DevPortal documentation available at: {WEBSITE_URL}"
 )
 LANGFUSE_PUBLIC_KEY = get_ssm_parameter(os.getenv("CHB_LANGFUSE_PUBLIC_KEY"), os.getenv("LANGFUSE_INIT_PROJECT_PUBLIC_KEY"))
 LANGFUSE_SECRET_KEY = get_ssm_parameter(os.getenv("CHB_LANGFUSE_SECRET_KEY"), os.getenv("LANGFUSE_INIT_PROJECT_SECRET_KEY"))

apps/chatbot/src/modules/vector_database.py

@@ -123,7 +123,7 @@ def html2markdown(html):
 
     converter = html2text.HTML2Text()
     converter.ignore_images = True
-    converter.ignore_links = True
+    converter.ignore_links = False
     converter.ignore_mailto_links = True
 
     soup = BeautifulSoup(html, "html.parser")