Skip to content

Commit 0a0fb06

Browse files
seakeeseakee
authored
Merge pull request #793 from seakee/feat/surface-runtime-models-awaiting-price-sync
✨ feat(pricing): surface newly discovered models awaiting price sync
2 parents cb3b09c + c87ac88 commit 0a0fb06

34 files changed

Lines changed: 2588 additions & 33 deletions

‎apps/manager-server/internal/http/controller/modelprice/handler.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,13 @@ func (h *Handler) Handle(w http.ResponseWriter, r *http.Request) {
2424

2525
path := strings.TrimRight(r.URL.Path, "/")
2626
switch {
27+
case path == "/v0/management/model-prices/runtime-models" && r.Method == http.MethodGet:
28+
status, err := h.App.ModelPriceService.RuntimeModelPricingStatus(r.Context())
29+
if err != nil {
30+
response.Error(w, http.StatusInternalServerError, err)
31+
return
32+
}
33+
response.JSON(w, http.StatusOK, status)
2734
case path == "/v0/management/model-prices/usage-summary" && r.Method == http.MethodGet:
2835
summary, err := h.App.ModelPriceService.UsageSummary(r.Context(), h.App.Config.QueryLimit)
2936
if err != nil {

‎apps/manager-server/internal/http/controller/modelprice/handler_test.go‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"github.com/seakee/cpa-manager-plus/apps/manager-server/internal/model"
1212
adminauthsvc "github.com/seakee/cpa-manager-plus/apps/manager-server/internal/service/adminauth"
1313
modelpricesvc "github.com/seakee/cpa-manager-plus/apps/manager-server/internal/service/modelprice"
14+
"github.com/seakee/cpa-manager-plus/apps/manager-server/internal/store"
1415
"github.com/seakee/cpa-manager-plus/apps/manager-server/internal/testutil"
1516
"github.com/seakee/cpa-manager-plus/apps/manager-server/internal/usage"
1617
)
@@ -57,3 +58,69 @@ func TestHandleUsageSummaryUsesQueryLimitAndPanelAuthorization(t *testing.T) {
5758
t.Fatalf("models = %#v", summary.Models)
5859
}
5960
}
61+
62+
type staticSetupResolver struct {
63+
setup store.Setup
64+
}
65+
66+
func (r staticSetupResolver) ResolveSetup(ctx context.Context) (store.Setup, bool, error) {
67+
return r.setup, true, nil
68+
}
69+
70+
func TestHandleRuntimeModels_AuthorizationAndResponse(t *testing.T) {
71+
cpaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
72+
switch r.URL.Path {
73+
case "/v0/management/api-keys":
74+
_ = json.NewEncoder(w).Encode(map[string]any{
75+
"api-keys": []string{"test-key"},
76+
})
77+
case "/v1/models":
78+
_ = json.NewEncoder(w).Encode(map[string]any{
79+
"data": []map[string]any{
80+
{"id": "m1"},
81+
{"id": "m2"},
82+
},
83+
})
84+
default:
85+
http.NotFound(w, r)
86+
}
87+
}))
88+
defer cpaServer.Close()
89+
90+
cfg := testutil.NewConfig(t)
91+
st := testutil.NewStore(t, cfg)
92+
resolver := staticSetupResolver{
93+
setup: store.Setup{
94+
CPAUpstreamURL: cpaServer.URL,
95+
ManagementKey: "mgmt-key",
96+
},
97+
}
98+
99+
handler := &Handler{App: &app.Context{
100+
Config: cfg,
101+
AdminAuthService: adminauthsvc.New(cfg, st),
102+
ModelPriceService: modelpricesvc.New(st, nil, resolver),
103+
}}
104+
105+
unauth := httptest.NewRecorder()
106+
handler.Handle(unauth, httptest.NewRequest(http.MethodGet, "/v0/management/model-prices/runtime-models", nil))
107+
if unauth.Code != http.StatusUnauthorized {
108+
t.Fatalf("expected 401, got %d", unauth.Code)
109+
}
110+
111+
req := httptest.NewRequest(http.MethodGet, "/v0/management/model-prices/runtime-models", nil)
112+
req.Header.Set("Authorization", "Bearer "+testutil.AdminKey)
113+
recorder := httptest.NewRecorder()
114+
handler.Handle(recorder, req)
115+
if recorder.Code != http.StatusOK {
116+
t.Fatalf("expected 200, got %d, body: %s", recorder.Code, recorder.Body.String())
117+
}
118+
119+
var status modelpricesvc.RuntimeModelPricingStatus
120+
if err := json.NewDecoder(recorder.Body).Decode(&status); err != nil {
121+
t.Fatalf("decode: %v", err)
122+
}
123+
if status.Count != 2 || status.UnpricedCount != 2 {
124+
t.Fatalf("expected count=2 and unpricedCount=2, got %d, %d", status.Count, status.UnpricedCount)
125+
}
126+
}

‎apps/manager-server/internal/service/modelprice/service.go‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -319,7 +319,7 @@ func (s *Service) Sync(ctx context.Context, req SyncRequest) (SyncResult, error)
319319
discoveryTimeout = defaultRuntimeModelDiscoveryTimeout
320320
}
321321
discoveryCtx, cancel := context.WithTimeout(ctx, discoveryTimeout)
322-
runtimeModels, err := s.discoverRuntimeModels(discoveryCtx)
322+
runtimeModels, err := s.DiscoverRuntimeModels(discoveryCtx)
323323
cancel()
324324

325325
if err != nil {
@@ -403,7 +403,56 @@ func (s *Service) Sync(ctx context.Context, req SyncRequest) (SyncResult, error)
403403
}, nil
404404
}
405405

406-
func (s *Service) discoverRuntimeModels(ctx context.Context) ([]string, error) {
406+
type RuntimeModelPricingStatus struct {
407+
Models []string `json:"models"`
408+
UnpricedModels []string `json:"unpricedModels"`
409+
Count int `json:"count"`
410+
UnpricedCount int `json:"unpricedCount"`
411+
}
412+
413+
func (s *Service) RuntimeModelPricingStatus(ctx context.Context) (RuntimeModelPricingStatus, error) {
414+
discoveryTimeout := s.runtimeModelDiscoveryTimeout
415+
if discoveryTimeout <= 0 {
416+
discoveryTimeout = defaultRuntimeModelDiscoveryTimeout
417+
}
418+
discoveryCtx, cancel := context.WithTimeout(ctx, discoveryTimeout)
419+
models, err := s.DiscoverRuntimeModels(discoveryCtx)
420+
cancel()
421+
if err != nil {
422+
return RuntimeModelPricingStatus{}, err
423+
}
424+
425+
prices, err := s.store.LoadModelPrices(ctx)
426+
if err != nil {
427+
return RuntimeModelPricingStatus{}, err
428+
}
429+
430+
normalizedModels := normalizedRequestedModels(models)
431+
sort.Strings(normalizedModels)
432+
433+
unpricedModels := make([]string, 0, len(normalizedModels))
434+
for _, m := range normalizedModels {
435+
if _, exists := prices[m]; !exists {
436+
unpricedModels = append(unpricedModels, m)
437+
}
438+
}
439+
440+
if normalizedModels == nil {
441+
normalizedModels = []string{}
442+
}
443+
if unpricedModels == nil {
444+
unpricedModels = []string{}
445+
}
446+
447+
return RuntimeModelPricingStatus{
448+
Models: normalizedModels,
449+
UnpricedModels: unpricedModels,
450+
Count: len(normalizedModels),
451+
UnpricedCount: len(unpricedModels),
452+
}, nil
453+
}
454+
455+
func (s *Service) DiscoverRuntimeModels(ctx context.Context) ([]string, error) {
407456
if s.setupResolver == nil {
408457
return nil, errors.New("runtime model discovery failed: missing setup resolver")
409458
}

‎apps/manager-server/internal/service/modelprice/service_test.go‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1844,3 +1844,212 @@ func TestSyncPreferredSourceFailurePreservationWithRuntimeModels(t *testing.T) {
18441844
t.Fatalf("expected models.dev price to be preserved, got %#v", p)
18451845
}
18461846
}
1847+
1848+
func TestRuntimeModelPricingStatus(t *testing.T) {
1849+
st := testutil.NewStore(t, testutil.NewConfig(t))
1850+
1851+
initialPrices := map[string]store.ModelPrice{
1852+
"synced-model": {
1853+
Prompt: 1.0,
1854+
Completion: 2.0,
1855+
Source: SyncSourceLiteLLM,
1856+
},
1857+
"manual-model": {
1858+
Prompt: 0.5,
1859+
Completion: 1.5,
1860+
Source: "manual",
1861+
},
1862+
}
1863+
if err := st.SaveModelPrices(context.Background(), initialPrices); err != nil {
1864+
t.Fatalf("save initial prices: %v", err)
1865+
}
1866+
1867+
remoteCalled := false
1868+
remoteServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1869+
remoteCalled = true
1870+
http.Error(w, "should not call remote price sources", http.StatusInternalServerError)
1871+
}))
1872+
defer remoteServer.Close()
1873+
1874+
cpaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1875+
switch r.URL.Path {
1876+
case "/v0/management/api-keys":
1877+
_ = json.NewEncoder(w).Encode(map[string]any{
1878+
"api-keys": []string{"secret-key-1", "secret-key-2"},
1879+
})
1880+
case "/v1/models":
1881+
if r.Header.Get("Authorization") != "Bearer secret-key-1" {
1882+
http.Error(w, "unauthorized", http.StatusUnauthorized)
1883+
return
1884+
}
1885+
_ = json.NewEncoder(w).Encode(map[string]any{
1886+
"data": []map[string]any{
1887+
{"id": "zebra-model"},
1888+
{"id": "synced-model"},
1889+
{"id": "alpha-model"},
1890+
{"id": "manual-model"},
1891+
{"id": "alpha-model"},
1892+
{"id": " "},
1893+
},
1894+
})
1895+
default:
1896+
http.NotFound(w, r)
1897+
}
1898+
}))
1899+
defer cpaServer.Close()
1900+
1901+
resolver := staticSetupResolver{
1902+
setup: store.Setup{
1903+
CPAUpstreamURL: cpaServer.URL,
1904+
ManagementKey: "mgmt-secret-key",
1905+
},
1906+
}
1907+
remoteURL := remoteServer.URL
1908+
svc := New(st, &remoteURL, resolver)
1909+
1910+
status, err := svc.RuntimeModelPricingStatus(context.Background())
1911+
if err != nil {
1912+
t.Fatalf("RuntimeModelPricingStatus failed: %v", err)
1913+
}
1914+
1915+
if remoteCalled {
1916+
t.Fatal("RuntimeModelPricingStatus must not call remote price sources")
1917+
}
1918+
1919+
expectedModels := []string{"alpha-model", "manual-model", "synced-model", "zebra-model"}
1920+
if len(status.Models) != len(expectedModels) {
1921+
t.Fatalf("expected %d models, got %d: %#v", len(expectedModels), len(status.Models), status.Models)
1922+
}
1923+
for i, m := range expectedModels {
1924+
if status.Models[i] != m {
1925+
t.Fatalf("expected model at %d to be %s, got %s", i, m, status.Models[i])
1926+
}
1927+
}
1928+
if status.Count != 4 {
1929+
t.Fatalf("expected count 4, got %d", status.Count)
1930+
}
1931+
1932+
expectedUnpriced := []string{"alpha-model", "zebra-model"}
1933+
if len(status.UnpricedModels) != len(expectedUnpriced) {
1934+
t.Fatalf("expected %d unpriced models, got %d: %#v", len(expectedUnpriced), len(status.UnpricedModels), status.UnpricedModels)
1935+
}
1936+
for i, m := range expectedUnpriced {
1937+
if status.UnpricedModels[i] != m {
1938+
t.Fatalf("expected unpriced model at %d to be %s, got %s", i, m, status.UnpricedModels[i])
1939+
}
1940+
}
1941+
if status.UnpricedCount != 2 {
1942+
t.Fatalf("expected unpricedCount 2, got %d", status.UnpricedCount)
1943+
}
1944+
1945+
storedPrices, err := st.LoadModelPrices(context.Background())
1946+
if err != nil {
1947+
t.Fatalf("load prices: %v", err)
1948+
}
1949+
if len(storedPrices) != 2 {
1950+
t.Fatalf("expected 2 stored prices, got %d", len(storedPrices))
1951+
}
1952+
}
1953+
1954+
func TestRuntimeModelPricingStatus_AllUnpricedWhenNoPrices(t *testing.T) {
1955+
st := testutil.NewStore(t, testutil.NewConfig(t))
1956+
1957+
cpaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1958+
switch r.URL.Path {
1959+
case "/v0/management/api-keys":
1960+
_ = json.NewEncoder(w).Encode(map[string]any{
1961+
"api-keys": []string{"test-key"},
1962+
})
1963+
case "/v1/models":
1964+
_ = json.NewEncoder(w).Encode(map[string]any{
1965+
"data": []map[string]any{
1966+
{"id": "m1"},
1967+
{"id": "m2"},
1968+
},
1969+
})
1970+
default:
1971+
http.NotFound(w, r)
1972+
}
1973+
}))
1974+
defer cpaServer.Close()
1975+
1976+
resolver := staticSetupResolver{
1977+
setup: store.Setup{
1978+
CPAUpstreamURL: cpaServer.URL,
1979+
ManagementKey: "mgmt-key",
1980+
},
1981+
}
1982+
svc := New(st, nil, resolver)
1983+
1984+
status, err := svc.RuntimeModelPricingStatus(context.Background())
1985+
if err != nil {
1986+
t.Fatalf("RuntimeModelPricingStatus failed: %v", err)
1987+
}
1988+
if status.Count != 2 || status.UnpricedCount != 2 {
1989+
t.Fatalf("expected count=2 and unpricedCount=2, got count=%d, unpriced=%d", status.Count, status.UnpricedCount)
1990+
}
1991+
if len(status.UnpricedModels) != 2 || status.UnpricedModels[0] != "m1" || status.UnpricedModels[1] != "m2" {
1992+
t.Fatalf("unexpected unpriced models: %#v", status.UnpricedModels)
1993+
}
1994+
}
1995+
1996+
func TestRuntimeModelPricingStatus_ZeroModels(t *testing.T) {
1997+
st := testutil.NewStore(t, testutil.NewConfig(t))
1998+
1999+
cpaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2000+
switch r.URL.Path {
2001+
case "/v0/management/api-keys":
2002+
_ = json.NewEncoder(w).Encode(map[string]any{
2003+
"api-keys": []string{"test-key"},
2004+
})
2005+
case "/v1/models":
2006+
_ = json.NewEncoder(w).Encode(map[string]any{
2007+
"data": []map[string]any{},
2008+
})
2009+
default:
2010+
http.NotFound(w, r)
2011+
}
2012+
}))
2013+
defer cpaServer.Close()
2014+
2015+
resolver := staticSetupResolver{
2016+
setup: store.Setup{
2017+
CPAUpstreamURL: cpaServer.URL,
2018+
ManagementKey: "mgmt-key",
2019+
},
2020+
}
2021+
svc := New(st, nil, resolver)
2022+
2023+
status, err := svc.RuntimeModelPricingStatus(context.Background())
2024+
if err != nil {
2025+
t.Fatalf("RuntimeModelPricingStatus failed: %v", err)
2026+
}
2027+
if status.Count != 0 || status.UnpricedCount != 0 {
2028+
t.Fatalf("expected count=0, unpricedCount=0, got %d, %d", status.Count, status.UnpricedCount)
2029+
}
2030+
if status.Models == nil || status.UnpricedModels == nil {
2031+
t.Fatal("expected models and unpricedModels to be non-nil empty slices")
2032+
}
2033+
}
2034+
2035+
func TestRuntimeModelPricingStatus_DiscoveryFailure(t *testing.T) {
2036+
st := testutil.NewStore(t, testutil.NewConfig(t))
2037+
2038+
cpaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
2039+
http.Error(w, "cpa internal error", http.StatusInternalServerError)
2040+
}))
2041+
defer cpaServer.Close()
2042+
2043+
resolver := staticSetupResolver{
2044+
setup: store.Setup{
2045+
CPAUpstreamURL: cpaServer.URL,
2046+
ManagementKey: "mgmt-key",
2047+
},
2048+
}
2049+
svc := New(st, nil, resolver)
2050+
2051+
_, err := svc.RuntimeModelPricingStatus(context.Background())
2052+
if err == nil {
2053+
t.Fatal("expected error on discovery failure, got nil")
2054+
}
2055+
}

‎apps/web/src/features/dashboard/components/UsageMetricsCard.module.scss‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,13 @@
5454
text-overflow: ellipsis;
5555
white-space: nowrap;
5656
}
57+
58+
.metricExtraAction {
59+
margin-left: auto;
60+
display: flex;
61+
align-items: center;
62+
z-index: 2;
63+
}
5764
}
5865

5966
.metricBody {

0 commit comments

Comments
 (0)