Skip to content

Commit 0b01f02

Browse files
tjohnson31415joerundenjhill
authored
feat: implement OpenVINO Model Server (OVMS) adapter (#18)
#### Motivation Adds another adapter server implementation to interface between the ModelMesh API and the OpenVINO Model Server (OVMS). From [the OVMS docs](https://docs.openvino.ai/latest/ovms_what_is_openvino_model_server.html): > OpenVINO Model Server (OVMS) is a high-performance system for serving machine learning models. It is based on C++ for high scalability and optimized for Intel solutions, so that you can take advantage of all the power of the Intel® Xeon® processor or Intel’s AI accelerators and expose it over a network interface. OVMS uses the same architecture and API as [TensorFlow Serving](https://github.com/tensorflow/serving), while applying OpenVINO for inference execution. Inference service is provided via gRPC or REST API, making it easy to deploy new algorithms and AI experiments. The adapter will be used to create a new OVMS `ServingRuntime` in KServe to enable use of this optimized runtime. OVMS supports running models in [OpenVINO's Intermediate Representation (IR) format](https://docs.openvino.ai/latest/openvino_docs_MO_DG_IR_and_opsets.html), as well as ONNX models. #### Modifications The model layout adaptations are very similar to Triton, but the multi-model management is significantly different. OVMS implements a subset of the TFS API, including multi-model management via a declarative configuration file. Reload of the configuration can be triggered with a call to OVMS's [`/v1/config/reload` REST API](https://github.com/openvinotoolkit/model_server/blob/main/docs/model_server_rest_api.md#config-reload). This is very different than the MLServer and Triton adapters. The approach used in this PR is to create a "ModelManager" that accepts a queue of load/unload requests, batches the changes to the config file, and manages the reload calls to OVMS. - adds the `model-mesh-ovms-adapter` directory with packages implementing the first pass at an adapter server for the OVMS runtime - small refactor to all adapters to use a single implementation of CalcMemCapcity from the utils package NB: Processing of the model schema is not included in this PR. Signed-off-by: Travis Johnson <[email protected]> Signed-off-by: Joe Runde <[email protected]> Signed-off-by: Nick Hill <[email protected]> Co-authored-by: Joe Runde <[email protected]> Co-authored-by: Nick Hill <[email protected]>
1 parent d3e6e42 commit 0b01f02

File tree

25 files changed

+2142
-55
lines changed

25 files changed

+2142
-55
lines changed

Dockerfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ COPY . ./
8585
RUN go build -o puller model-serving-puller/main.go
8686
RUN go build -o triton-adapter model-mesh-triton-adapter/main.go
8787
RUN go build -o mlserver-adapter model-mesh-mlserver-adapter/main.go
88+
RUN go build -o ovms-adapter model-mesh-ovms-adapter/main.go
8889

8990

9091
###############################################################################
@@ -119,6 +120,7 @@ COPY --from=build /opt/app/puller /opt/app/
119120
COPY --from=build /opt/app/triton-adapter /opt/app/
120121
COPY --from=build /opt/app/mlserver-adapter /opt/app/
121122
COPY --from=build /opt/app/model-mesh-triton-adapter/scripts/tf_pb.py /opt/scripts/
123+
COPY --from=build /opt/app/ovms-adapter /opt/app/
122124

123125
# Don't define an entrypoint. This is a multi-purpose image so the user should specify which binary they want to run (e.g. /opt/app/puller or /opt/app/triton-adapter)
124126
# ENTRYPOINT ["/opt/app/puller"]

internal/envconfig/envconfig.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ package envconfig
1616
import (
1717
"os"
1818
"strconv"
19+
"time"
1920

2021
"github.com/go-logr/logr"
2122
)
@@ -71,3 +72,18 @@ func GetEnvBool(key string, defaultValue bool, log logr.Logger) bool {
7172
}
7273
return defaultValue
7374
}
75+
76+
// Returns the duration value of environment variable "key" or a default value
77+
// Note if the environment variable cannot be parsed as a duration, including an
78+
// empty string, this will fail and exit.
79+
func GetEnvDuration(key string, defaultValue time.Duration, log logr.Logger) time.Duration {
80+
if strVal, found := os.LookupEnv(key); found {
81+
val, err := time.ParseDuration(strVal)
82+
if err != nil {
83+
log.Error(err, "Environment variable must be a duration", "env_var", key, "value", strVal)
84+
os.Exit(1)
85+
}
86+
return val
87+
}
88+
return defaultValue
89+
}

internal/util/loadmodel.go

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ import (
2222
)
2323

2424
const (
25-
modelTypeJSONKey string = "model_type"
26-
schemaPathJSONKey string = "schema_path"
25+
modelTypeJSONKey string = "model_type"
26+
schemaPathJSONKey string = "schema_path"
27+
diskSizeBytesJSONKey string = "disk_size_bytes"
2728
)
2829

2930
// GetModelType first tries to read the type from the LoadModelRequest.ModelKey json
@@ -68,3 +69,27 @@ func GetSchemaPath(req *mmesh.LoadModelRequest) (string, error) {
6869

6970
return schemaPath, nil
7071
}
72+
73+
func CalcMemCapacity(reqModelKey string, defaultSize int, multiplier float64, log logr.Logger) uint64 {
74+
// Try to calculate the model size from the disk size passed in the LoadModelRequest.ModelKey
75+
// but first set the default to fall back on if we cannot get the disk size.
76+
size := uint64(defaultSize)
77+
var modelKey map[string]interface{}
78+
err := json.Unmarshal([]byte(reqModelKey), &modelKey)
79+
if err != nil {
80+
log.Info("'SizeInBytes' will be defaulted as LoadModelRequest.ModelKey value is not valid JSON", "SizeInBytes", size, "model_key", reqModelKey, "error", err)
81+
} else {
82+
if modelKey[diskSizeBytesJSONKey] != nil {
83+
diskSize, ok := modelKey[diskSizeBytesJSONKey].(float64)
84+
if ok {
85+
size = uint64(diskSize * multiplier)
86+
log.Info("Setting 'SizeInBytes' to a multiple of model disk size", "SizeInBytes", size, "disk_size", diskSize, "multiplier", multiplier)
87+
} else {
88+
log.Info("'SizeInBytes' will be defaulted as LoadModelRequest.ModelKey 'disk_size_bytes' value is not a number", "SizeInBytes", size, "model_key", modelKey)
89+
}
90+
} else {
91+
log.Info("'SizeInBytes' will be defaulted as LoadModelRequest.ModelKey did not contain a value for 'disk_size_bytes'", "SizeInBytes", size, "model_key", modelKey)
92+
}
93+
}
94+
return size
95+
}

model-mesh-mlserver-adapter/server/server.go

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,6 @@ import (
4141

4242
const (
4343
mlserverServiceName string = "inference.GRPCInferenceService"
44-
diskSizeBytesJSONKey string = "disk_size_bytes"
4544
mlserverModelSubdir string = "_mlserver_models"
4645
mlserverRepositoryConfigFilename string = "model-settings.json"
4746
)
@@ -147,7 +146,7 @@ func (s *MLServerAdapterServer) LoadModel(ctx context.Context, req *mmesh.LoadMo
147146
return nil, status.Errorf(status.Code(mlserverErr), "Failed to load Model due to MLServer runtime error: %s", mlserverErr)
148147
}
149148

150-
size := calcMemCapacity(req.ModelKey, s.AdapterConfig, log)
149+
size := util.CalcMemCapacity(req.ModelKey, s.AdapterConfig.DefaultModelSizeInBytes, s.AdapterConfig.ModelSizeMultiplier, log)
151150

152151
log.Info("MLServer model loaded")
153152

@@ -426,31 +425,6 @@ func tensorMetadataToJson(tm modelschema.TensorMetadata) map[string]interface{}
426425
return json
427426
}
428427

429-
func calcMemCapacity(reqModelKey string, adapterConfig *AdapterConfiguration, log logr.Logger) uint64 {
430-
// Try to calculate the model size from the disk size passed in the LoadModelRequest.ModelKey
431-
// but first set the default to fall back on if we cannot get the disk size.
432-
size := uint64(adapterConfig.DefaultModelSizeInBytes)
433-
var modelKey map[string]interface{}
434-
err := json.Unmarshal([]byte(reqModelKey), &modelKey)
435-
if err != nil {
436-
log.Info("'SizeInBytes' will be defaulted as LoadModelRequest.ModelKey value is not valid JSON", "SizeInBytes", size, "LoadModelRequest.ModelKey", reqModelKey, "Error", err)
437-
} else {
438-
if modelKey[diskSizeBytesJSONKey] != nil {
439-
diskSize, ok := modelKey[diskSizeBytesJSONKey].(float64)
440-
if ok {
441-
size = uint64(diskSize * adapterConfig.ModelSizeMultiplier)
442-
log.Info("Setting 'SizeInBytes' to multiples of model disk size", "SizeInBytes", size, "ModelSizeMultiplier", adapterConfig.ModelSizeMultiplier)
443-
} else {
444-
log.Info("'SizeInBytes' will be defaulted as LoadModelRequest.ModelKey value is not a number", "SizeInBytes", size, "disc size bytes json key", diskSizeBytesJSONKey,
445-
"Model key", modelKey[diskSizeBytesJSONKey])
446-
}
447-
} else {
448-
log.Info("'SizeInBytes' will be defaulted as LoadModelRequest.ModelKey did not contain a value", "SizeInBytes", size, "diskSizeBytesJSONKey", diskSizeBytesJSONKey)
449-
}
450-
}
451-
return size
452-
}
453-
454428
func (s *MLServerAdapterServer) UnloadModel(ctx context.Context, req *mmesh.UnloadModelRequest) (*mmesh.UnloadModelResponse, error) {
455429
_, mlserverErr := s.ModelRepoClient.RepositoryModelUnload(ctx, &modelrepo.RepositoryModelUnloadRequest{
456430
ModelName: req.ModelId,

model-mesh-ovms-adapter/.gitignore

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# binaries created by test
2+
main
3+
4+
server/testdata/model_config_list.json
5+
server/testdata/generated

model-mesh-ovms-adapter/Makefile

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# Copyright 2022 IBM Corporation
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
test:
15+
./scripts/run_tests.sh
16+
17+
fmt:
18+
cd .. && make fmt
19+
20+
# Remove $(MAKECMDGOALS) if you don't intend make to just be a taskrunner
21+
.PHONY: $(MAKECMDGOALS)

model-mesh-ovms-adapter/README.md

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
# Model Mesh OpenVINO Model Server (OVMS)
2+
3+
This is an adapter which implements the internal model-mesh model management API for [OpenVINO Model Server](https://github.com/openvinotoolkit/model_server).
4+
5+
This adapter is different than the other adapters because OpenVINO is modeled after TensorflowServing and does not implement the KFSv2 API. OVMS also does not have a direct model management API; its multimodel support is implemented with an on-disk config file that can be reloaded with a REST API call.

model-mesh-ovms-adapter/main.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// Copyright 2022 IBM Corporation
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
package main
15+
16+
import (
17+
"fmt"
18+
"net"
19+
"os"
20+
21+
"github.com/kserve/modelmesh-runtime-adapter/internal/proto/mmesh"
22+
"github.com/kserve/modelmesh-runtime-adapter/model-mesh-ovms-adapter/server"
23+
"google.golang.org/grpc"
24+
"sigs.k8s.io/controller-runtime/pkg/log/zap"
25+
)
26+
27+
func main() {
28+
log := zap.New(zap.UseDevMode(true))
29+
log = log.WithName("OpenVINO Adapter")
30+
31+
adapterConfig, err := server.GetAdapterConfigurationFromEnv(log)
32+
if err != nil {
33+
log.Error(err, "Error reading configuration")
34+
os.Exit(1)
35+
}
36+
log.Info("Starting OpenVINO Adapter Server", "adapter_config", adapterConfig)
37+
38+
server := server.NewOvmsAdapterServer(adapterConfig.OvmsPort, adapterConfig, log)
39+
40+
lis, err := net.Listen("tcp", fmt.Sprintf("localhost:%d", adapterConfig.Port))
41+
if err != nil {
42+
log.Error(err, "*** Adapter failed to listen port")
43+
os.Exit(1)
44+
}
45+
log.Info("Adapter will run at port", "port", adapterConfig.Port, "OpenVINO port", adapterConfig.OvmsPort)
46+
47+
grpcServer := grpc.NewServer()
48+
mmesh.RegisterModelRuntimeServer(grpcServer, server)
49+
log.Info("Adapter gRPC Server Registered...")
50+
51+
err = grpcServer.Serve(lis)
52+
53+
if err != nil {
54+
log.Error(err, "*** Adapter terminated with error ")
55+
} else {
56+
log.Info("*** Adapter terminated")
57+
}
58+
}
Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#!/bin/sh
2+
# Copyright 2022 IBM Corporation
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.#
15+
16+
set -e
17+
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
18+
cd "${DIR}/.."
19+
20+
go build -o main main.go
21+
go test ./server -v
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
// Copyright 2022 IBM Corporation
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
package server
15+
16+
import (
17+
"context"
18+
"fmt"
19+
"io/ioutil"
20+
"os"
21+
"path/filepath"
22+
"strconv"
23+
"strings"
24+
25+
"github.com/go-logr/logr"
26+
27+
"github.com/kserve/modelmesh-runtime-adapter/internal/util"
28+
)
29+
30+
func adaptModelLayoutForRuntime(ctx context.Context, rootModelDir, modelID, modelType, modelPath, schemaPath string, log logr.Logger) error {
31+
// convert to lower case and remove anything after the :
32+
modelType = strings.ToLower(strings.Split(modelType, ":")[0])
33+
34+
ovmsModelIDDir, err := util.SecureJoin(rootModelDir, ovmsModelSubdir, modelID)
35+
if err != nil {
36+
log.Error(err, "Unable to securely join", "rootModelDir", rootModelDir, "ovmsModelSubdir", ovmsModelSubdir, "modelID", modelID)
37+
return err
38+
}
39+
// clean up and then create directory where the rewritten model repo will live
40+
if removeErr := os.RemoveAll(ovmsModelIDDir); removeErr != nil {
41+
log.Info("Ignoring error trying to remove dir", "Directory", ovmsModelIDDir, "Error", removeErr)
42+
}
43+
if mkdirErr := os.MkdirAll(ovmsModelIDDir, 0755); mkdirErr != nil {
44+
return fmt.Errorf("Error creating directories for path %s: %w", ovmsModelIDDir, mkdirErr)
45+
}
46+
47+
modelPathInfo, err := os.Stat(modelPath)
48+
if err != nil {
49+
return fmt.Errorf("Error calling stat on model file: %w", err)
50+
}
51+
52+
if !modelPathInfo.IsDir() {
53+
// simple case if ModelPath points to a file
54+
err = createOvmsModelRepositoryFromPath(modelPath, "1", schemaPath, modelType, ovmsModelIDDir, log)
55+
} else {
56+
files, err1 := ioutil.ReadDir(modelPath)
57+
if err1 != nil {
58+
return fmt.Errorf("Could not read files in dir %s: %w", modelPath, err1)
59+
}
60+
err = createOvmsModelRepositoryFromDirectory(files, modelPath, schemaPath, modelType, ovmsModelIDDir, log)
61+
}
62+
if err != nil {
63+
return fmt.Errorf("Error processing model/schema files for model %s: %w", modelID, err)
64+
}
65+
66+
return nil
67+
}
68+
69+
// Creates the ovms model structure /models/_ovms_models/model-id/1/<model files>
70+
// Within this path there will be a symlink back to the original /models/model-id directory tree.
71+
func createOvmsModelRepositoryFromDirectory(files []os.FileInfo, modelPath, schemaPath, modelType, ovmsModelIDDir string, log logr.Logger) error {
72+
var err error
73+
74+
// allow the directory to contain version directories
75+
// try to find the largest version directory
76+
versionNumber := largestNumberDir(files)
77+
if versionNumber != "" {
78+
// found a version directory so step into it
79+
if modelPath, err = util.SecureJoin(modelPath, versionNumber); err != nil {
80+
log.Error(err, "Unable to securely join", "modelPath", modelPath, "versionNumber", versionNumber)
81+
return err
82+
}
83+
} else {
84+
versionNumber = "1"
85+
}
86+
87+
return createOvmsModelRepositoryFromPath(modelPath, versionNumber, schemaPath, modelType, ovmsModelIDDir, log)
88+
}
89+
90+
func createOvmsModelRepositoryFromPath(modelPath, versionNumber, schemaPath, modelType, ovmsModelIDDir string, log logr.Logger) error {
91+
var err error
92+
93+
modelPathInfo, err := os.Stat(modelPath)
94+
if err != nil {
95+
return fmt.Errorf("Error calling stat on %s: %w", modelPath, err)
96+
}
97+
98+
linkPathComponents := []string{ovmsModelIDDir, versionNumber}
99+
if !modelPathInfo.IsDir() {
100+
// special case to rename the file for an ONNX model
101+
if modelType == "onnx" {
102+
linkPathComponents = append(linkPathComponents, onnxModelFilename)
103+
} else {
104+
linkPathComponents = append(linkPathComponents, modelPathInfo.Name())
105+
}
106+
}
107+
108+
linkPath, err := util.SecureJoin(linkPathComponents...)
109+
if err != nil {
110+
return fmt.Errorf("Error joining link path: %w", err)
111+
}
112+
113+
if err = os.MkdirAll(filepath.Dir(linkPath), 0755); err != nil {
114+
return fmt.Errorf("Error creating directories for path %s: %w", linkPath, err)
115+
}
116+
117+
if err = os.Symlink(modelPath, linkPath); err != nil {
118+
return fmt.Errorf("Error creating symlink: %w", err)
119+
}
120+
121+
if schemaPath == "" {
122+
return nil
123+
}
124+
125+
return nil
126+
}
127+
128+
// Returns the largest positive int dir as long as all fileInfo dirs are integers (files are ignored).
129+
// If fileInfos is empty or contains any any non-integer dirs, this will return the empty string.
130+
func largestNumberDir(fileInfos []os.FileInfo) string {
131+
largestInt := 0
132+
largestDir := ""
133+
for _, f := range fileInfos {
134+
if !f.IsDir() {
135+
continue
136+
}
137+
i, err := strconv.Atoi(f.Name())
138+
if err != nil {
139+
// must all be numbers
140+
return ""
141+
}
142+
if i > largestInt {
143+
largestInt = i
144+
largestDir = f.Name()
145+
}
146+
}
147+
return largestDir
148+
}

0 commit comments

Comments
 (0)