Skip to content

Commit 3df80eb

Browse files
lesebleseb
authored
Merge pull request #315 from gyliu513/oaic
feat: Added openai client demos
2 parents d6497ec + c45e2b3 commit 3df80eb

7 files changed

Lines changed: 429 additions & 0 deletions

File tree

‎demos/06_openai_compatibility/01_chat_completion.py‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
"""
2+
Demo: OpenAI-Compatible Chat Completion
3+
4+
Description:
5+
This demo shows that existing OpenAI chat-completion code works against a
6+
Llama Stack server with only a base_url change.
7+
8+
Learning Objectives:
9+
- Use the OpenAI Python SDK to talk to Llama Stack
10+
- Perform non-streaming and streaming chat completions
11+
- Resolve an available model via the OpenAI-compatible models endpoint
12+
"""
13+
14+
# Copyright (c) Meta Platforms, Inc. and affiliates.
15+
# All rights reserved.
16+
#
17+
# This source code is licensed under the terms described in the LICENSE file in
18+
# the root directory of this source tree.
19+
20+
from __future__ import annotations
21+
22+
import os
23+
import sys
24+
25+
import fire
26+
from openai import OpenAI
27+
28+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
29+
from shared.utils import resolve_openai_model
30+
31+
try:
32+
from dotenv import load_dotenv
33+
except ImportError: # pragma: no cover - optional dependency
34+
load_dotenv = None
35+
36+
37+
def _maybe_load_dotenv() -> None:
38+
if load_dotenv is not None:
39+
load_dotenv()
40+
41+
42+
def _print_stream(stream) -> None:
43+
for chunk in stream:
44+
if not chunk.choices:
45+
continue
46+
delta = chunk.choices[0].delta
47+
if delta.content:
48+
print(delta.content, end="", flush=True)
49+
print()
50+
51+
52+
def main(
53+
host: str,
54+
port: int,
55+
model_id: str | None = None,
56+
prompt: str = "Give me a short summary of Llama Stack.",
57+
stream: bool = False,
58+
scheme: str = "http",
59+
) -> None:
60+
_maybe_load_dotenv()
61+
62+
if scheme not in {"http", "https"}:
63+
raise ValueError("scheme must be 'http' or 'https'")
64+
if host not in {"localhost", "127.0.0.1", "::1"} and scheme != "https":
65+
print("Warning: using HTTP for a non-local host. Consider --scheme https.")
66+
67+
client = OpenAI(
68+
base_url=f"{scheme}://{host}:{port}/v1",
69+
api_key=os.getenv("LLAMA_STACK_API_KEY", "fake"),
70+
)
71+
72+
resolved_model = resolve_openai_model(client, model_id)
73+
if resolved_model is None:
74+
return
75+
print(f"Using model: {resolved_model}")
76+
77+
messages = [{"role": "user", "content": prompt}]
78+
79+
if stream:
80+
response_stream = client.chat.completions.create(
81+
model=resolved_model,
82+
messages=messages,
83+
stream=True,
84+
)
85+
_print_stream(response_stream)
86+
response_stream.close()
87+
return
88+
89+
completion = client.chat.completions.create(
90+
model=resolved_model,
91+
messages=messages,
92+
)
93+
print(completion.choices[0].message.content)
94+
95+
96+
if __name__ == "__main__":
97+
fire.Fire(main)

‎demos/06_openai_compatibility/02_tool_calling.py‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
"""
2+
Demo: OpenAI-Compatible Tool Calling
3+
4+
Description:
5+
This demo shows the standard OpenAI function-calling flow against a
6+
Llama Stack server: define tools, let the model request a call, execute
7+
locally, and send results back.
8+
9+
Learning Objectives:
10+
- Define OpenAI-style function tools
11+
- Parse tool_calls from the model response
12+
- Execute a local function and return the result
13+
- Complete the full 3-step tool-calling loop
14+
"""
15+
16+
# Copyright (c) Meta Platforms, Inc. and affiliates.
17+
# All rights reserved.
18+
#
19+
# This source code is licensed under the terms described in the LICENSE file in
20+
# the root directory of this source tree.
21+
22+
from __future__ import annotations
23+
24+
import json
25+
import os
26+
import sys
27+
28+
import fire
29+
from openai import OpenAI
30+
from termcolor import colored
31+
32+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
33+
from shared.utils import resolve_openai_model
34+
35+
try:
36+
from dotenv import load_dotenv
37+
except ImportError: # pragma: no cover - optional dependency
38+
load_dotenv = None
39+
40+
41+
def _maybe_load_dotenv() -> None:
42+
if load_dotenv is not None:
43+
load_dotenv()
44+
45+
46+
# -- Simulated local tool --------------------------------------------------
47+
48+
WEATHER_DATA = {
49+
"san francisco": {"temperature": "62°F", "condition": "Foggy"},
50+
"new york": {"temperature": "45°F", "condition": "Cloudy"},
51+
"london": {"temperature": "50°F", "condition": "Rainy"},
52+
}
53+
54+
55+
def get_weather(location: str) -> str:
56+
"""Return simulated weather for a given location."""
57+
data = WEATHER_DATA.get(location.lower())
58+
if data is None:
59+
return json.dumps({"error": f"No weather data for {location}"})
60+
return json.dumps(data)
61+
62+
63+
TOOLS = [
64+
{
65+
"type": "function",
66+
"function": {
67+
"name": "get_weather",
68+
"description": "Get the current weather for a location.",
69+
"parameters": {
70+
"type": "object",
71+
"properties": {
72+
"location": {
73+
"type": "string",
74+
"description": "City name, e.g. 'San Francisco'",
75+
},
76+
},
77+
"required": ["location"],
78+
},
79+
},
80+
},
81+
]
82+
83+
TOOL_MAP = {
84+
"get_weather": get_weather,
85+
}
86+
87+
88+
def main(
89+
host: str,
90+
port: int,
91+
model_id: str | None = None,
92+
prompt: str = "What is the weather like in San Francisco?",
93+
scheme: str = "http",
94+
) -> None:
95+
_maybe_load_dotenv()
96+
97+
if scheme not in {"http", "https"}:
98+
raise ValueError("scheme must be 'http' or 'https'")
99+
if host not in {"localhost", "127.0.0.1", "::1"} and scheme != "https":
100+
print(colored("Warning: using HTTP for a non-local host. Consider --scheme https.", "yellow"))
101+
102+
client = OpenAI(
103+
base_url=f"{scheme}://{host}:{port}/v1",
104+
api_key=os.getenv("LLAMA_STACK_API_KEY", "fake"),
105+
)
106+
107+
resolved_model = resolve_openai_model(client, model_id)
108+
if resolved_model is None:
109+
return
110+
print(f"Using model: {resolved_model}")
111+
112+
# Step 1 — send the user message with tool definitions
113+
messages = [{"role": "user", "content": prompt}]
114+
print(colored(f"User> {prompt}", "blue"))
115+
116+
response = client.chat.completions.create(
117+
model=resolved_model,
118+
messages=messages,
119+
tools=TOOLS,
120+
tool_choice="auto",
121+
)
122+
123+
assistant_message = response.choices[0].message
124+
125+
# If the model replies directly without calling a tool, print and exit.
126+
if not assistant_message.tool_calls:
127+
print(colored(f"Assistant> {assistant_message.content}", "green"))
128+
return
129+
130+
# Step 2 — execute each requested tool call locally
131+
messages.append(assistant_message)
132+
for tool_call in assistant_message.tool_calls:
133+
fn_name = tool_call.function.name
134+
try:
135+
fn_args = json.loads(tool_call.function.arguments or "{}")
136+
if not isinstance(fn_args, dict):
137+
raise ValueError("arguments must be a JSON object")
138+
except (json.JSONDecodeError, ValueError) as exc:
139+
result = json.dumps({"error": f"Invalid arguments for {fn_name}: {exc}"})
140+
print(colored(f"Tool result: {result}", "yellow"))
141+
messages.append(
142+
{"role": "tool", "tool_call_id": tool_call.id, "content": result}
143+
)
144+
continue
145+
print(colored(f"Tool call: {fn_name}({fn_args})", "yellow"))
146+
147+
fn = TOOL_MAP.get(fn_name)
148+
if fn is None:
149+
result = json.dumps({"error": f"Unknown function: {fn_name}"})
150+
else:
151+
try:
152+
result = fn(**fn_args)
153+
except TypeError as exc:
154+
result = json.dumps({"error": f"Invalid parameters for {fn_name}: {exc}"})
155+
print(colored(f"Tool result: {result}", "yellow"))
156+
157+
messages.append(
158+
{
159+
"role": "tool",
160+
"tool_call_id": tool_call.id,
161+
"content": result,
162+
}
163+
)
164+
165+
# Step 3 — send tool results back and get the final answer
166+
final = client.chat.completions.create(
167+
model=resolved_model,
168+
messages=messages,
169+
tools=TOOLS,
170+
)
171+
print(colored(f"Assistant> {final.choices[0].message.content}", "green"))
172+
173+
174+
if __name__ == "__main__":
175+
fire.Fire(main)

‎demos/06_openai_compatibility/03_responses_api.py‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
"""
2+
Demo: OpenAI-Compatible Responses API
3+
4+
Description:
5+
This demo uses the OpenAI SDK's Responses API (client.responses.create)
6+
against a Llama Stack server, proving OpenResponses compatibility.
7+
8+
Learning Objectives:
9+
- Call the Responses API via the OpenAI Python SDK
10+
- Use both a plain string input and a structured message list
11+
- See how Llama Stack implements the OpenAI Responses API
12+
"""
13+
14+
# Copyright (c) Meta Platforms, Inc. and affiliates.
15+
# All rights reserved.
16+
#
17+
# This source code is licensed under the terms described in the LICENSE file in
18+
# the root directory of this source tree.
19+
20+
from __future__ import annotations
21+
22+
import os
23+
import sys
24+
25+
import fire
26+
from openai import OpenAI
27+
from termcolor import colored
28+
29+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
30+
from shared.utils import resolve_openai_model
31+
32+
try:
33+
from dotenv import load_dotenv
34+
except ImportError: # pragma: no cover - optional dependency
35+
load_dotenv = None
36+
37+
38+
def _maybe_load_dotenv() -> None:
39+
if load_dotenv is not None:
40+
load_dotenv()
41+
42+
43+
def main(
44+
host: str,
45+
port: int,
46+
model_id: str | None = None,
47+
scheme: str = "http",
48+
) -> None:
49+
_maybe_load_dotenv()
50+
51+
if scheme not in {"http", "https"}:
52+
raise ValueError("scheme must be 'http' or 'https'")
53+
if host not in {"localhost", "127.0.0.1", "::1"} and scheme != "https":
54+
print(colored("Warning: using HTTP for a non-local host. Consider --scheme https.", "yellow"))
55+
56+
client = OpenAI(
57+
base_url=f"{scheme}://{host}:{port}/v1",
58+
api_key=os.getenv("LLAMA_STACK_API_KEY", "fake"),
59+
)
60+
61+
resolved_model = resolve_openai_model(client, model_id)
62+
if resolved_model is None:
63+
return
64+
print(f"Using model: {resolved_model}")
65+
66+
# --- Example 1: plain string input ---
67+
print(colored("\n--- String input ---", "cyan"))
68+
response = client.responses.create(
69+
model=resolved_model,
70+
input="Give a one-sentence description of Llama Stack.",
71+
)
72+
print(response.output_text)
73+
74+
# --- Example 2: structured message-list input ---
75+
print(colored("\n--- Structured message-list input ---", "cyan"))
76+
response = client.responses.create(
77+
model=resolved_model,
78+
input=[
79+
{
80+
"role": "user",
81+
"content": "List three benefits of running open-source LLMs locally.",
82+
},
83+
],
84+
)
85+
print(response.output_text)
86+
87+
88+
if __name__ == "__main__":
89+
fire.Fire(main)

0 commit comments

Comments
 (0)