diff --git a/clients/ui/api/openapi/mod-arch.yaml b/clients/ui/api/openapi/mod-arch.yaml index 357ea8b08d..1e7d8c9824 100644 --- a/clients/ui/api/openapi/mod-arch.yaml +++ b/clients/ui/api/openapi/mod-arch.yaml @@ -2416,8 +2416,10 @@ components: default: metrics-artifact metricsType: type: string - example: “accuracy-metrics” - description: An artifact type representing a collection of related metrics, which will either be “performance-metrics” or “accuracy-metrics”. + enum: + - performance-metrics + - accuracy-metrics + - security-metrics customProperties: description: User provided custom properties which are not defined by its type. type: object diff --git a/clients/ui/bff/internal/mocks/model_catalog_client_mock.go b/clients/ui/bff/internal/mocks/model_catalog_client_mock.go index e281be93be..8f30c26edd 100644 --- a/clients/ui/bff/internal/mocks/model_catalog_client_mock.go +++ b/clients/ui/bff/internal/mocks/model_catalog_client_mock.go @@ -216,8 +216,10 @@ func (m *ModelCatalogClientMock) GetCatalogSourceModelArtifacts(client httpclien if sourceId == "sample-source" && (modelName == "repo1%2Fgranite-8b-code-instruct" || modelName == "repo1%2Fgranite-8b-code-instruct-quantized.w4a16") { performanceArtifacts := GetCatalogPerformanceMetricsArtifactListMock(4) accuracyArtifacts := GetCatalogAccuracyMetricsArtifactListMock() + securityArtifacts := GetCatalogSecurityMetricsArtifactListMock() modelArtifacts := GetCatalogModelArtifactListMock() combinedItems := append(performanceArtifacts.Items, accuracyArtifacts.Items...) + combinedItems = append(combinedItems, securityArtifacts.Items...) combinedItems = append(combinedItems, modelArtifacts.Items...) allMockModelArtifacts = models.CatalogModelArtifactList{ Items: combinedItems, @@ -227,8 +229,10 @@ func (m *ModelCatalogClientMock) GetCatalogSourceModelArtifacts(client httpclien } } else if sourceId == "sample-source" && modelName == "repo1%2Fgranite-7b-instruct" { accuracyArtifacts := GetCatalogAccuracyMetricsArtifactListMock() + securityArtifacts := GetCatalogSecurityMetricsArtifactListMock() modelArtifacts := GetCatalogModelArtifactListMock() - combinedItems := append(accuracyArtifacts.Items, modelArtifacts.Items...) + combinedItems := append(accuracyArtifacts.Items, securityArtifacts.Items...) + combinedItems = append(combinedItems, modelArtifacts.Items...) allMockModelArtifacts = models.CatalogModelArtifactList{ Items: combinedItems, Size: int32(len(combinedItems)), @@ -241,9 +245,48 @@ func (m *ModelCatalogClientMock) GetCatalogSourceModelArtifacts(client httpclien allMockModelArtifacts = GetCatalogModelArtifactListMock() } + if filterQuery := pageValues.Get("filterQuery"); filterQuery != "" { + allMockModelArtifacts = filterArtifactsByQuery(allMockModelArtifacts, filterQuery) + } + return &allMockModelArtifacts, nil } +func filterArtifactsByQuery(list models.CatalogModelArtifactList, filterQuery string) models.CatalogModelArtifactList { + metricsTypeFilter := extractMetricsTypeFilter(filterQuery) + if metricsTypeFilter == "" { + return list + } + + var filtered []models.CatalogArtifact + for _, item := range list.Items { + if item.MetricsType != nil && *item.MetricsType == metricsTypeFilter { + filtered = append(filtered, item) + } + } + + return models.CatalogModelArtifactList{ + Items: filtered, + Size: int32(len(filtered)), + PageSize: list.PageSize, + NextPageToken: "", + } +} + +func extractMetricsTypeFilter(filterQuery string) string { + prefix := "metricsType.string_value=" + idx := strings.Index(filterQuery, prefix) + if idx == -1 { + return "" + } + value := filterQuery[idx+len(prefix):] + value = strings.Trim(value, "\"") + if sepIdx := strings.IndexAny(value, "&, "); sepIdx != -1 { + value = value[:sepIdx] + } + return value +} + func (m *ModelCatalogClientMock) GetCatalogModelPerformanceArtifacts(client httpclient.HTTPClientInterface, sourceId string, modelName string, pageValues url.Values) (*models.CatalogModelArtifactList, error) { allMockModelPerformanceArtifacts := GetCatalogPerformanceMetricsArtifactListMock(4) return &allMockModelPerformanceArtifacts, nil diff --git a/clients/ui/bff/internal/mocks/static_data_mock.go b/clients/ui/bff/internal/mocks/static_data_mock.go index 6e75d58bb3..0b2d91ff2b 100644 --- a/clients/ui/bff/internal/mocks/static_data_mock.go +++ b/clients/ui/bff/internal/mocks/static_data_mock.go @@ -1715,6 +1715,130 @@ func GetCatalogAccuracyMetricsArtifactMock() []models.CatalogArtifact { }, } } + +func securityMetricsCustomProperties(id, benchmark, description, evaluation, providerID, resultMetric, modelID string, pass bool, result, threshold float64) *map[string]openapi.MetadataValue { + resultMap := map[string]openapi.MetadataValue{ + "id": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: id, + MetadataType: "MetadataStringValue", + }, + }, + "benchmark": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: benchmark, + MetadataType: "MetadataStringValue", + }, + }, + "category": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: "security", + MetadataType: "MetadataStringValue", + }, + }, + "description": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: description, + MetadataType: "MetadataStringValue", + }, + }, + "evaluation": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: evaluation, + MetadataType: "MetadataStringValue", + }, + }, + "model_id": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: modelID, + MetadataType: "MetadataStringValue", + }, + }, + "provider_id": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: providerID, + MetadataType: "MetadataStringValue", + }, + }, + "result_metric": { + MetadataStringValue: &openapi.MetadataStringValue{ + StringValue: resultMetric, + MetadataType: "MetadataStringValue", + }, + }, + "pass": { + MetadataBoolValue: &openapi.MetadataBoolValue{ + BoolValue: pass, + MetadataType: "MetadataBoolValue", + }, + }, + "lower_is_better": { + MetadataBoolValue: &openapi.MetadataBoolValue{ + BoolValue: true, + MetadataType: "MetadataBoolValue", + }, + }, + "result": { + MetadataDoubleValue: &openapi.MetadataDoubleValue{ + DoubleValue: result, + MetadataType: "MetadataDoubleValue", + }, + }, + "threshold": { + MetadataDoubleValue: &openapi.MetadataDoubleValue{ + DoubleValue: threshold, + MetadataType: "MetadataDoubleValue", + }, + }, + } + return &resultMap +} + +func GetCatalogSecurityMetricsArtifactMock() []models.CatalogArtifact { + return []models.CatalogArtifact{ + { + ArtifactType: *stringToPointer("metrics-artifact"), + MetricsType: stringToPointer("security-metrics"), + CreateTimeSinceEpoch: stringToPointer("1693526400000"), + LastUpdateTimeSinceEpoch: stringToPointer("1704067200000"), + CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000001", "intents", "Risk assessment with a context-aware custom intent typology and probes of increasing complexity. Runs as an AI Pipeline and requires a Data Science Pipelines setup.", "Context-aware vulnerability scan (Pipeline)", "garak-kfp", "attack_success_rate", "repo1/granite-8b-code-instruct", true, 0.12, 0.3), + }, + { + ArtifactType: *stringToPointer("metrics-artifact"), + MetricsType: stringToPointer("security-metrics"), + CreateTimeSinceEpoch: stringToPointer("1693526400000"), + LastUpdateTimeSinceEpoch: stringToPointer("1704067200000"), + CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000002", "intents", "Risk assessment with a context-aware custom intent typology and probes of increasing complexity. Standalone execution without pipeline dependencies.", "Context-aware vulnerability scan (Standalone)", "garak-standalone", "attack_success_rate", "repo1/granite-8b-code-instruct", false, 0.65, 0.3), + }, + { + ArtifactType: *stringToPointer("metrics-artifact"), + MetricsType: stringToPointer("security-metrics"), + CreateTimeSinceEpoch: stringToPointer("1695200000000"), + LastUpdateTimeSinceEpoch: stringToPointer("1706000000000"), + CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000003", "prompt-injection", "Evaluates model robustness against prompt injection attacks using adversarial input patterns.", "Prompt injection robustness test", "garak-standalone", "injection_success_rate", "repo1/granite-8b-code-instruct", true, 0.05, 0.1), + }, + { + ArtifactType: *stringToPointer("metrics-artifact"), + MetricsType: stringToPointer("security-metrics"), + CreateTimeSinceEpoch: stringToPointer("1696300000000"), + LastUpdateTimeSinceEpoch: stringToPointer("1707100000000"), + CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000004", "toxicity", "Scans model outputs for toxic, harmful, or offensive content across domain-specific scenarios.", "Domain-specific toxicity scan", "garak-kfp", "toxicity_rate", "repo1/granite-8b-code-instruct", true, 0.02, 0.05), + }, + { + ArtifactType: *stringToPointer("metrics-artifact"), + MetricsType: stringToPointer("security-metrics"), + CreateTimeSinceEpoch: stringToPointer("1697400000000"), + LastUpdateTimeSinceEpoch: stringToPointer("1708200000000"), + CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000005", "hallucination", "Detects factual hallucinations and unsupported claims in model responses. Runs as an AI Pipeline.", "Hallucination detection scan (Pipeline)", "garak-kfp", "hallucination_rate", "repo1/granite-8b-code-instruct", false, 0.42, 0.2), + }, + } +} + +func GetCatalogSecurityMetricsArtifactListMock() models.CatalogModelArtifactList { + allArtifactMock := GetCatalogSecurityMetricsArtifactMock() + return GetModelArtifactListMockWithItems(allArtifactMock, 10) +} + func GetModelArtifactListMockWithItems(items []models.CatalogArtifact, pageSize int32) models.CatalogModelArtifactList { return models.CatalogModelArtifactList{ Items: items, diff --git a/clients/ui/frontend/src/app/modelCatalogTypes.ts b/clients/ui/frontend/src/app/modelCatalogTypes.ts index 77f1cf8a24..a9686e362e 100644 --- a/clients/ui/frontend/src/app/modelCatalogTypes.ts +++ b/clients/ui/frontend/src/app/modelCatalogTypes.ts @@ -26,6 +26,7 @@ import { ModelRegistryCustomPropertyString, ModelRegistryCustomPropertyInt, ModelRegistryCustomPropertyDouble, + ModelRegistryCustomPropertyBool, } from './types'; import { McpServer, @@ -82,6 +83,7 @@ export enum CatalogArtifactType { export enum MetricsType { accuracyMetrics = 'accuracy-metrics', performanceMetrics = 'performance-metrics', + securityMetrics = 'security-metrics', } export enum CategoryName { @@ -139,6 +141,21 @@ export type AccuracyMetricsCustomProperties = { arc_v1?: ModelRegistryCustomPropertyDouble; } & Record; +export type SecurityMetricsCustomProperties = { + id?: ModelRegistryCustomPropertyString; + benchmark?: ModelRegistryCustomPropertyString; + category?: ModelRegistryCustomPropertyString; + description?: ModelRegistryCustomPropertyString; + evaluation?: ModelRegistryCustomPropertyString; + model_id?: ModelRegistryCustomPropertyString; + provider_id?: ModelRegistryCustomPropertyString; + result_metric?: ModelRegistryCustomPropertyString; + pass?: ModelRegistryCustomPropertyBool; + lower_is_better?: ModelRegistryCustomPropertyBool; + result?: ModelRegistryCustomPropertyDouble; + threshold?: ModelRegistryCustomPropertyDouble; +}; + export type CatalogPerformanceMetricsArtifact = Omit & { artifactType: CatalogArtifactType.metricsArtifact; metricsType: MetricsType.performanceMetrics; @@ -151,9 +168,16 @@ export type CatalogAccuracyMetricsArtifact = Omit & { + artifactType: CatalogArtifactType.metricsArtifact; + metricsType: MetricsType.securityMetrics; + customProperties?: SecurityMetricsCustomProperties; +}; + export type CatalogMetricsArtifact = | CatalogPerformanceMetricsArtifact - | CatalogAccuracyMetricsArtifact; + | CatalogAccuracyMetricsArtifact + | CatalogSecurityMetricsArtifact; export type CatalogArtifacts = CatalogModelArtifact | CatalogMetricsArtifact;