text-sglang.mdx

export const TextSglang = ({ modelId, toolCallParser, samplingParams }) => (

Install:

{`uv pip install "sglang>=0.5.10"`.split('\n').map((line, i) => {line}{'\n'})}

Launch server:

{(toolCallParser
  ? `sglang serve \\
    --model-path ${modelId} \\
    --host 0.0.0.0 \\
    --port 30000 \\
    --tool-call-parser ${toolCallParser}`
  : `sglang serve \\
    --model-path ${modelId} \\
    --host 0.0.0.0 \\
    --port 30000`
).split('\n').map((line, i) => {line}{'\n'})}

Query (OpenAI-compatible):

{`from openai import OpenAI
client = OpenAI(base_url="http://localhost:30000/v1", api_key="None")
response = client.chat.completions.create(
model="${modelId}",
messages=[{"role": "user", "content": "What is machine learning?"}],
${samplingParams || "temperature=0.3,"}
)
print(response.choices[0].message.content)`.split('\n').map((line, i) => {line}{'\n'})}

);