How to configure local LLM by installing ollama?

[zd310807173]

[vinnitu]

I need this functionality too. And not only for local model. For example I need use proxy to calc costs etc. with LightLLM.
So I modify your code move tooling model from library to my code like this:

from langfuse.langchain import CallbackHandler

langfuse_handler = CallbackHandler()
invoke_config = {"callbacks": [langfuse_handler]}

from langchain.chat_models import init_chat_mode

query_writer = init_chat_model(
    getenv("TOOLING_MODEL_NAME", "gpt-4.1-nano"),
    model_provider=getenv("TOOLING_MODEL_PROVIDER", "openai"),
    base_url=getenv("OPENAI_API_BASE"),
    temperature=0,
)

analyze_datasource1 = initialize_simple_datasource_qa(
    query_writer, # <-- here I can pass any LLM
    domain=getenv("TABLEAU_DOMAIN"),
    site=getenv("TABLEAU_SITE"),
    jwt_client_id=getenv("TABLEAU_JWT_CLIENT_ID"),
    jwt_secret_id=getenv("TABLEAU_JWT_SECRET_ID"),
    jwt_secret=getenv("TABLEAU_JWT_SECRET"),
    tableau_api_version=getenv("TABLEAU_API_VERSION"),
    tableau_user=getenv("TABLEAU_USER"),
    datasource_luid=getenv("TABLEAU_DATASOURCE1"),
    # model_provider=getenv("MODEL_PROVIDER", "openai"),
    # tooling_llm_model=getenv("TOOLING_MODEL", "gpt-4.1-nano"),
    invoke_config=invoke_config, # <-- and here I need pass callback for invoke() to use with langfuse
)

What do you think about?

[heshaojian]

Could you try using LiteLLM with LangChain (https://python.langchain.com/docs/integrations/providers/litellm/)? LiteLLM acts like a Swiss Army knife for managing various LLM providers and models, including Ollama — it supports load balancing, fallback strategies, and timeout handling.

import os
from dotenv import load_dotenv

# langchain and langgraph package imports
from langchain_openai import ChatOpenAI
from langgraph.prebuilt import create_react_agent
from langchain_community.chat_models import ChatLiteLLM, ChatOllama
# langchain_tableau imports
from langchain_tableau.tools.simple_datasource_qa import initialize_simple_datasource_qa

# Load environment variables
load_dotenv()

# initialize an LLM
# llm = ChatOpenAI(model='gpt-4o-mini', temperature=0)#
llm = ChatLiteLLM(model='ollama/mistral-7B-Instruct-v0.1', temperature=0, base_url='http://localhost:11434')
# llm = ChatOllama(model='qwen2.5', temperature=0, base_url='http://localhost:11434')

# initalize `simple_datasource_qa` for querying a Tableau Published Datasource through VDS
analyze_datasource = initialize_simple_datasource_qa(
    domain=os.environ['TABLEAU_DOMAIN'],
    site=os.environ['TABLEAU_SITE'],
    jwt_client_id=os.environ['TABLEAU_JWT_CLIENT_ID'],
    jwt_secret_id=os.environ['TABLEAU_JWT_SECRET_ID'],
    jwt_secret=os.environ['TABLEAU_JWT_SECRET'],
    tableau_api_version=os.environ['TABLEAU_API_VERSION'],
    tableau_user=os.environ['TABLEAU_USER'],
    datasource_luid=os.environ['DATASOURCE_LUID'],
    tooling_llm_model='gpt-4o-mini'
)

# Add the tool to the array of tools used by the Agent
tools = [ analyze_datasource ]

# Build the Agent using the minimum components (LLM + Tools)
tableauAgent = create_react_agent(llm, tools)

# Run the Agent
messages = tableauAgent.invoke({"messages": [("human",'which states sell the most? Are those the same states with the most profits?')]})
print(messages)

[vinnitu]

I tried LiteLLM and use it. But in your code for tooling you can use other provider except openai and azure