@@ -13,96 +13,77 @@ import (
1313 "strconv"
1414 "time"
1515
16- "nnctl/internal/flagx"
1716 "nnctl/internal/zig"
1817)
1918
20- func (a * App ) cmdChat (ctx context.Context , args []string ) error {
21- fs := newFlagSet (a .stderr (), "chat" )
22- mode := getenvDefault ("NNCTL_OPTIMIZE" , getenvDefault ("BUILD_MODE" , defaultBuildMode ))
23- host := "127.0.0.1"
24- port := 8090
25- apiHost := "127.0.0.1"
26- apiPort := 8080
27- modelPath := ""
28- modelName := "tiny-gpt-zig"
29- maxTokens := 120
30- temperature := 1.0
31- topK := 8
32- allowUntrained := false
33- readyTimeoutText := "2m"
34-
35- fs .StringVar (& mode , "mode" , mode , "Zig optimization mode" )
36- fs .StringVar (& mode , "optimize" , mode , "Zig optimization mode" )
37- fs .StringVar (& host , "host" , host , "chat app bind host" )
38- fs .IntVar (& port , "port" , port , "chat app bind port" )
39- fs .StringVar (& apiHost , "api-host" , apiHost , "TinyGPT inference bind host" )
40- fs .IntVar (& apiPort , "api-port" , apiPort , "TinyGPT inference bind port" )
41- fs .StringVar (& modelPath , "model" , modelPath , "TinyGPT checkpoint to serve" )
42- fs .StringVar (& modelName , "model-name" , modelName , "OpenAI-compatible model id" )
43- fs .IntVar (& maxTokens , "max-tokens" , maxTokens , "default generated tokens per reply" )
44- fs .Float64Var (& temperature , "temperature" , temperature , "default sampling temperature" )
45- fs .IntVar (& topK , "top-k" , topK , "default TinyGPT top-k sampler cutoff" )
46- fs .BoolVar (& allowUntrained , "allow-untrained" , allowUntrained , "serve a seeded untrained model when --model is omitted" )
47- fs .StringVar (& readyTimeoutText , "ready-timeout" , readyTimeoutText , "time to wait for inference server readiness" )
19+ type chatOptions struct {
20+ mode string
21+ host string
22+ port int
23+ apiHost string
24+ apiPort int
25+ modelPath string
26+ modelName string
27+ maxTokens int
28+ temperature float64
29+ topK int
30+ allowUntrained bool
31+ readyTimeoutText string
32+ }
4833
49- if err := flagx .ParseInterspersed (fs , args , map [string ]bool {
50- "mode" : true ,
51- "optimize" : true ,
52- "host" : true ,
53- "port" : true ,
54- "api-host" : true ,
55- "api-port" : true ,
56- "model" : true ,
57- "model-name" : true ,
58- "max-tokens" : true ,
59- "temperature" : true ,
60- "top-k" : true ,
61- "allow-untrained" : false ,
62- "ready-timeout" : true ,
63- }); err != nil {
64- return err
34+ func defaultChatOptions () chatOptions {
35+ return chatOptions {
36+ mode : getenvDefault ("NNCTL_OPTIMIZE" , getenvDefault ("BUILD_MODE" , defaultBuildMode )),
37+ host : "127.0.0.1" ,
38+ port : 8090 ,
39+ apiHost : "127.0.0.1" ,
40+ apiPort : 8080 ,
41+ modelName : "tiny-gpt-zig" ,
42+ maxTokens : 120 ,
43+ temperature : 1.0 ,
44+ topK : 8 ,
45+ readyTimeoutText : "2m" ,
6546 }
66- if fs . NArg () != 0 {
67- return fmt . Errorf ( "chat does not accept positional arguments" )
68- }
69- if modelPath == "" && ! allowUntrained {
47+ }
48+
49+ func ( a * App ) runChat ( ctx context. Context , opts chatOptions ) error {
50+ if opts . modelPath == "" && ! opts . allowUntrained {
7051 return fmt .Errorf ("chat requires --model <checkpoint> or --allow-untrained" )
7152 }
72- if port <= 0 || port > 65535 {
53+ if opts . port <= 0 || opts . port > 65535 {
7354 return fmt .Errorf ("--port must be between 1 and 65535" )
7455 }
75- if apiPort <= 0 || apiPort > 65535 {
56+ if opts . apiPort <= 0 || opts . apiPort > 65535 {
7657 return fmt .Errorf ("--api-port must be between 1 and 65535" )
7758 }
78- if maxTokens < 0 {
59+ if opts . maxTokens < 0 {
7960 return fmt .Errorf ("--max-tokens must be non-negative" )
8061 }
81- if topK < 0 {
62+ if opts . topK < 0 {
8263 return fmt .Errorf ("--top-k must be non-negative" )
8364 }
8465
85- readyTimeout , err := time .ParseDuration (readyTimeoutText )
66+ readyTimeout , err := time .ParseDuration (opts . readyTimeoutText )
8667 if err != nil {
8768 return fmt .Errorf ("parse --ready-timeout: %w" , err )
8869 }
8970
90- apiBaseURL := "http://" + net .JoinHostPort (apiHost , strconv .Itoa (apiPort ))
71+ apiBaseURL := "http://" + net .JoinHostPort (opts . apiHost , strconv .Itoa (opts . apiPort ))
9172 serverArgs := []string {
92- "--host" , apiHost ,
93- "--port" , strconv .Itoa (apiPort ),
94- "--model-name" , modelName ,
95- "--max-tokens" , strconv .Itoa (maxTokens ),
96- "--temperature" , strconv .FormatFloat (temperature , 'f' , - 1 , 64 ),
97- "--top-k" , strconv .Itoa (topK ),
73+ "--host" , opts . apiHost ,
74+ "--port" , strconv .Itoa (opts . apiPort ),
75+ "--model-name" , opts . modelName ,
76+ "--max-tokens" , strconv .Itoa (opts . maxTokens ),
77+ "--temperature" , strconv .FormatFloat (opts . temperature , 'f' , - 1 , 64 ),
78+ "--top-k" , strconv .Itoa (opts . topK ),
9879 }
99- if modelPath != "" {
100- serverArgs = append (serverArgs , "--model" , modelPath )
80+ if opts . modelPath != "" {
81+ serverArgs = append (serverArgs , "--model" , opts . modelPath )
10182 } else {
10283 serverArgs = append (serverArgs , "--allow-untrained" )
10384 }
10485
105- runArgs := zig .RunArgs ("run_tiny_gpt_openai" , zig.Options {Optimize : mode }, serverArgs )
86+ runArgs := zig .RunArgs ("run_tiny_gpt_openai" , zig.Options {Optimize : opts . mode }, serverArgs )
10687 fmt .Fprintf (a .stderr (), "==> %s\n " , zig .CommandString (a .zig , runArgs ))
10788 serverCmd := exec .CommandContext (ctx , a .zig , runArgs ... )
10889 serverCmd .Dir = a .repoRoot
@@ -131,13 +112,13 @@ func (a *App) cmdChat(ctx context.Context, args []string) error {
131112 mux .HandleFunc ("/" , serveChatApp )
132113 mux .Handle ("/api/chat" , chatProxy {
133114 apiBaseURL : apiBaseURL ,
134- modelName : modelName ,
135- maxTokens : maxTokens ,
136- temperature : temperature ,
115+ modelName : opts . modelName ,
116+ maxTokens : opts . maxTokens ,
117+ temperature : opts . temperature ,
137118 client : & http.Client {Timeout : 2 * time .Minute },
138119 })
139120
140- chatAddr := net .JoinHostPort (host , strconv .Itoa (port ))
121+ chatAddr := net .JoinHostPort (opts . host , strconv .Itoa (opts . port ))
141122 chatServer := & http.Server {
142123 Addr : chatAddr ,
143124 Handler : mux ,
0 commit comments