Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions clients/ui/api/openapi/mod-arch.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2416,8 +2416,10 @@ components:
default: metrics-artifact
metricsType:
type: string
example: “accuracy-metrics”
description: An artifact type representing a collection of related metrics, which will either be “performance-metrics” or “accuracy-metrics”.

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

small point here, I think is ok to keep the description

enum:
- performance-metrics
- accuracy-metrics
- security-metrics
customProperties:
description: User provided custom properties which are not defined by its type.
type: object
Expand Down
45 changes: 44 additions & 1 deletion clients/ui/bff/internal/mocks/model_catalog_client_mock.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,8 +216,10 @@ func (m *ModelCatalogClientMock) GetCatalogSourceModelArtifacts(client httpclien
if sourceId == "sample-source" && (modelName == "repo1%2Fgranite-8b-code-instruct" || modelName == "repo1%2Fgranite-8b-code-instruct-quantized.w4a16") {
performanceArtifacts := GetCatalogPerformanceMetricsArtifactListMock(4)
accuracyArtifacts := GetCatalogAccuracyMetricsArtifactListMock()
securityArtifacts := GetCatalogSecurityMetricsArtifactListMock()
modelArtifacts := GetCatalogModelArtifactListMock()
combinedItems := append(performanceArtifacts.Items, accuracyArtifacts.Items...)
combinedItems = append(combinedItems, securityArtifacts.Items...)
combinedItems = append(combinedItems, modelArtifacts.Items...)
allMockModelArtifacts = models.CatalogModelArtifactList{
Items: combinedItems,
Expand All @@ -227,8 +229,10 @@ func (m *ModelCatalogClientMock) GetCatalogSourceModelArtifacts(client httpclien
}
} else if sourceId == "sample-source" && modelName == "repo1%2Fgranite-7b-instruct" {
accuracyArtifacts := GetCatalogAccuracyMetricsArtifactListMock()
securityArtifacts := GetCatalogSecurityMetricsArtifactListMock()
modelArtifacts := GetCatalogModelArtifactListMock()
combinedItems := append(accuracyArtifacts.Items, modelArtifacts.Items...)
combinedItems := append(accuracyArtifacts.Items, securityArtifacts.Items...)
combinedItems = append(combinedItems, modelArtifacts.Items...)
allMockModelArtifacts = models.CatalogModelArtifactList{
Items: combinedItems,
Size: int32(len(combinedItems)),
Expand All @@ -241,9 +245,48 @@ func (m *ModelCatalogClientMock) GetCatalogSourceModelArtifacts(client httpclien
allMockModelArtifacts = GetCatalogModelArtifactListMock()
}

if filterQuery := pageValues.Get("filterQuery"); filterQuery != "" {
allMockModelArtifacts = filterArtifactsByQuery(allMockModelArtifacts, filterQuery)
}

return &allMockModelArtifacts, nil
}

func filterArtifactsByQuery(list models.CatalogModelArtifactList, filterQuery string) models.CatalogModelArtifactList {
metricsTypeFilter := extractMetricsTypeFilter(filterQuery)
if metricsTypeFilter == "" {
return list
}

var filtered []models.CatalogArtifact
for _, item := range list.Items {
if item.MetricsType != nil && *item.MetricsType == metricsTypeFilter {
filtered = append(filtered, item)
}
}

return models.CatalogModelArtifactList{
Items: filtered,
Size: int32(len(filtered)),
PageSize: list.PageSize,
NextPageToken: "",
}
}

func extractMetricsTypeFilter(filterQuery string) string {
prefix := "metricsType.string_value="
idx := strings.Index(filterQuery, prefix)
if idx == -1 {
return ""
}
value := filterQuery[idx+len(prefix):]
value = strings.Trim(value, "\"")
if sepIdx := strings.IndexAny(value, "&, "); sepIdx != -1 {
value = value[:sepIdx]
}
return value
}

func (m *ModelCatalogClientMock) GetCatalogModelPerformanceArtifacts(client httpclient.HTTPClientInterface, sourceId string, modelName string, pageValues url.Values) (*models.CatalogModelArtifactList, error) {
allMockModelPerformanceArtifacts := GetCatalogPerformanceMetricsArtifactListMock(4)
return &allMockModelPerformanceArtifacts, nil
Expand Down
124 changes: 124 additions & 0 deletions clients/ui/bff/internal/mocks/static_data_mock.go
Original file line number Diff line number Diff line change
Expand Up @@ -1715,6 +1715,130 @@ func GetCatalogAccuracyMetricsArtifactMock() []models.CatalogArtifact {
},
}
}

func securityMetricsCustomProperties(id, benchmark, description, evaluation, providerID, resultMetric, modelID string, pass bool, result, threshold float64) *map[string]openapi.MetadataValue {
resultMap := map[string]openapi.MetadataValue{
"id": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: id,
MetadataType: "MetadataStringValue",
},
},
"benchmark": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: benchmark,
MetadataType: "MetadataStringValue",
},
},
"category": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: "security",
MetadataType: "MetadataStringValue",
},
},
"description": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: description,
MetadataType: "MetadataStringValue",
},
},
"evaluation": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: evaluation,
MetadataType: "MetadataStringValue",
},
},
"model_id": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: modelID,
MetadataType: "MetadataStringValue",
},
},
"provider_id": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: providerID,
MetadataType: "MetadataStringValue",
},
},
"result_metric": {
MetadataStringValue: &openapi.MetadataStringValue{
StringValue: resultMetric,
MetadataType: "MetadataStringValue",
},
},
"pass": {
MetadataBoolValue: &openapi.MetadataBoolValue{
BoolValue: pass,
MetadataType: "MetadataBoolValue",
},
},
"lower_is_better": {
MetadataBoolValue: &openapi.MetadataBoolValue{
BoolValue: true,
MetadataType: "MetadataBoolValue",
},
},
"result": {
MetadataDoubleValue: &openapi.MetadataDoubleValue{
DoubleValue: result,
MetadataType: "MetadataDoubleValue",
},
},
"threshold": {
MetadataDoubleValue: &openapi.MetadataDoubleValue{
DoubleValue: threshold,
MetadataType: "MetadataDoubleValue",
},
},
}
return &resultMap
}

func GetCatalogSecurityMetricsArtifactMock() []models.CatalogArtifact {
return []models.CatalogArtifact{
{
ArtifactType: *stringToPointer("metrics-artifact"),
MetricsType: stringToPointer("security-metrics"),
CreateTimeSinceEpoch: stringToPointer("1693526400000"),
LastUpdateTimeSinceEpoch: stringToPointer("1704067200000"),
CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000001", "intents", "Risk assessment with a context-aware custom intent typology and probes of increasing complexity. Runs as an AI Pipeline and requires a Data Science Pipelines setup.", "Context-aware vulnerability scan (Pipeline)", "garak-kfp", "attack_success_rate", "repo1/granite-8b-code-instruct", true, 0.12, 0.3),
},
{
ArtifactType: *stringToPointer("metrics-artifact"),
MetricsType: stringToPointer("security-metrics"),
CreateTimeSinceEpoch: stringToPointer("1693526400000"),
LastUpdateTimeSinceEpoch: stringToPointer("1704067200000"),
CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000002", "intents", "Risk assessment with a context-aware custom intent typology and probes of increasing complexity. Standalone execution without pipeline dependencies.", "Context-aware vulnerability scan (Standalone)", "garak-standalone", "attack_success_rate", "repo1/granite-8b-code-instruct", false, 0.65, 0.3),
},
{
ArtifactType: *stringToPointer("metrics-artifact"),
MetricsType: stringToPointer("security-metrics"),
CreateTimeSinceEpoch: stringToPointer("1695200000000"),
LastUpdateTimeSinceEpoch: stringToPointer("1706000000000"),
CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000003", "prompt-injection", "Evaluates model robustness against prompt injection attacks using adversarial input patterns.", "Prompt injection robustness test", "garak-standalone", "injection_success_rate", "repo1/granite-8b-code-instruct", true, 0.05, 0.1),
},
{
ArtifactType: *stringToPointer("metrics-artifact"),
MetricsType: stringToPointer("security-metrics"),
CreateTimeSinceEpoch: stringToPointer("1696300000000"),
LastUpdateTimeSinceEpoch: stringToPointer("1707100000000"),
CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000004", "toxicity", "Scans model outputs for toxic, harmful, or offensive content across domain-specific scenarios.", "Domain-specific toxicity scan", "garak-kfp", "toxicity_rate", "repo1/granite-8b-code-instruct", true, 0.02, 0.05),
},
{
ArtifactType: *stringToPointer("metrics-artifact"),
MetricsType: stringToPointer("security-metrics"),
CreateTimeSinceEpoch: stringToPointer("1697400000000"),
LastUpdateTimeSinceEpoch: stringToPointer("1708200000000"),
CustomProperties: securityMetricsCustomProperties("a1b2c3d4-1001-4000-a000-000000000005", "hallucination", "Detects factual hallucinations and unsupported claims in model responses. Runs as an AI Pipeline.", "Hallucination detection scan (Pipeline)", "garak-kfp", "hallucination_rate", "repo1/granite-8b-code-instruct", false, 0.42, 0.2),
},
}
}

func GetCatalogSecurityMetricsArtifactListMock() models.CatalogModelArtifactList {
allArtifactMock := GetCatalogSecurityMetricsArtifactMock()
return GetModelArtifactListMockWithItems(allArtifactMock, 10)
}

func GetModelArtifactListMockWithItems(items []models.CatalogArtifact, pageSize int32) models.CatalogModelArtifactList {
return models.CatalogModelArtifactList{
Items: items,
Expand Down
26 changes: 25 additions & 1 deletion clients/ui/frontend/src/app/modelCatalogTypes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {
ModelRegistryCustomPropertyString,
ModelRegistryCustomPropertyInt,
ModelRegistryCustomPropertyDouble,
ModelRegistryCustomPropertyBool,
} from './types';
import {
McpServer,
Expand Down Expand Up @@ -82,6 +83,7 @@ export enum CatalogArtifactType {
export enum MetricsType {
accuracyMetrics = 'accuracy-metrics',
performanceMetrics = 'performance-metrics',
securityMetrics = 'security-metrics',
}

export enum CategoryName {
Expand Down Expand Up @@ -139,6 +141,21 @@ export type AccuracyMetricsCustomProperties = {
arc_v1?: ModelRegistryCustomPropertyDouble;
} & Record<string, ModelRegistryCustomPropertyDouble>;

export type SecurityMetricsCustomProperties = {
id?: ModelRegistryCustomPropertyString;
benchmark?: ModelRegistryCustomPropertyString;
category?: ModelRegistryCustomPropertyString;
description?: ModelRegistryCustomPropertyString;
evaluation?: ModelRegistryCustomPropertyString;
model_id?: ModelRegistryCustomPropertyString;
provider_id?: ModelRegistryCustomPropertyString;
result_metric?: ModelRegistryCustomPropertyString;
pass?: ModelRegistryCustomPropertyBool;
lower_is_better?: ModelRegistryCustomPropertyBool;
result?: ModelRegistryCustomPropertyDouble;
threshold?: ModelRegistryCustomPropertyDouble;
};

export type CatalogPerformanceMetricsArtifact = Omit<CatalogArtifactBase, 'customProperties'> & {
artifactType: CatalogArtifactType.metricsArtifact;
metricsType: MetricsType.performanceMetrics;
Expand All @@ -151,9 +168,16 @@ export type CatalogAccuracyMetricsArtifact = Omit<CatalogArtifactBase, 'customPr
customProperties?: AccuracyMetricsCustomProperties;
};

export type CatalogSecurityMetricsArtifact = Omit<CatalogArtifactBase, 'customProperties'> & {
artifactType: CatalogArtifactType.metricsArtifact;
metricsType: MetricsType.securityMetrics;
customProperties?: SecurityMetricsCustomProperties;
};

export type CatalogMetricsArtifact =
| CatalogPerformanceMetricsArtifact
| CatalogAccuracyMetricsArtifact;
| CatalogAccuracyMetricsArtifact
| CatalogSecurityMetricsArtifact;

export type CatalogArtifacts = CatalogModelArtifact | CatalogMetricsArtifact;

Expand Down
Loading