Skip to content

Commit ba64bd3

Browse files
neuromagek8s-ci-robot
neuromage
authored and
k8s-ci-robot
committed
Record TFX output artifacts in Metadata store (#884)
* WIP: ML Metadata in KFP * Move metadata tracking to its own package. * Clean up * Address review comments, update travis.yml * Add dependencies for building in Dockerfile * Log errors but continue to update run when metadata storing fails. * Update workspace to get latest ml-metadata version. * Update errors
1 parent dd1b11e commit ba64bd3

17 files changed

+453
-29
lines changed

‎.travis.yml

+8-2
Original file line numberDiff line numberDiff line change
@@ -31,8 +31,14 @@ matrix:
3131
- cd $TRAVIS_BUILD_DIR/backend/src
3232
- gimme -f 1.11.4
3333
- source ~/.gimme/envs/go1.11.4.env
34-
- go vet -all -shadow ./...
35-
- go test ./...
34+
- go vet -all -shadow ./agent/...
35+
- go vet -all -shadow ./cmd/...
36+
- go vet -all -shadow ./common/...
37+
- go vet -all -shadow ./crd/...
38+
- go test ./agent/...
39+
- go test ./cmd/...
40+
- go test ./common/...
41+
- go test ./crd/...
3642
- language: python
3743
python: "2.7"
3844
env: TOXENV=py27

‎BUILD.bazel

+4
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,8 @@ load("@bazel_gazelle//:def.bzl", "gazelle")
44
# gazelle:resolve proto protoc-gen-swagger/options/annotations.proto @com_github_grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:options_proto
55
# gazelle:resolve proto go protoc-gen-swagger/options/annotations.proto @com_github_grpc_ecosystem_grpc_gateway//protoc-gen-swagger/options:go_default_library
66
# gazelle:resolve go github.com/kubeflow/pipelines/backend/api/go_client //backend/api:go_default_library
7+
# gazelle:resolve go ml_metadata/metadata_store/mlmetadata @google_ml_metadata//ml_metadata/metadata_store:metadata_store_go
8+
# gazelle:resolve go ml_metadata/proto/metadata_store_go_proto @google_ml_metadata//ml_metadata/proto:metadata_store_go_proto
9+
# gazelle:resolve go ml_metadata/proto/metadata_store_service_go_proto @google_ml_metadata//ml_metadata/proto:metadata_store_service_go_proto
10+
# gazelle:exclude vendor/
711
gazelle(name = "gazelle")

‎WORKSPACE

+43-3
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,13 @@
11
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
2+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
23

34
http_archive(
45
name = "io_bazel_rules_go",
5-
sha256 = "7be7dc01f1e0afdba6c8eb2b43d2fa01c743be1b9273ab1eaf6c233df078d705",
6-
urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.16.5/rules_go-0.16.5.tar.gz"],
6+
sha256 = "492c3ac68ed9dcf527a07e6a1b2dcbf199c6bf8b35517951467ac32e421c06c1",
7+
urls = ["https://github.com/bazelbuild/rules_go/releases/download/0.17.0/rules_go-0.17.0.tar.gz"],
78
)
89

9-
load("@io_bazel_rules_go//go:def.bzl", "go_register_toolchains", "go_rules_dependencies")
10+
load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies")
1011

1112
go_rules_dependencies()
1213

@@ -22,6 +23,45 @@ load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository")
2223

2324
gazelle_dependencies()
2425

26+
http_archive(
27+
name = "org_tensorflow",
28+
sha256 = "24570d860d87dcfb936f53fb8dd30302452d0aa6b8b8537e4555c1bf839121a6",
29+
strip_prefix = "tensorflow-1.13.0-rc0",
30+
urls = [
31+
"https://github.com/tensorflow/tensorflow/archive/v1.13.0-rc0.tar.gz",
32+
],
33+
)
34+
35+
http_archive(
36+
name = "io_bazel_rules_closure",
37+
sha256 = "43c9b882fa921923bcba764453f4058d102bece35a37c9f6383c713004aacff1",
38+
strip_prefix = "rules_closure-9889e2348259a5aad7e805547c1a0cf311cfcd91",
39+
urls = [
40+
"https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/9889e2348259a5aad7e805547c1a0cf311cfcd91.tar.gz",
41+
"https://github.com/bazelbuild/rules_closure/archive/9889e2348259a5aad7e805547c1a0cf311cfcd91.tar.gz", # 2018-12-21
42+
],
43+
)
44+
45+
load("@org_tensorflow//tensorflow:workspace.bzl", "tf_workspace")
46+
47+
tf_workspace()
48+
49+
load("@bazel_tools//tools/build_defs/repo:git.bzl", "new_git_repository")
50+
51+
go_repository(
52+
name = "google_ml_metadata",
53+
commit = "becc26ab61f82bfe7c812894f56f597949ce0fdc",
54+
importpath = "github.com/google/ml-metadata",
55+
)
56+
57+
new_git_repository(
58+
name = "libmysqlclient",
59+
build_file = "@google_ml_metadata//ml_metadata:libmysqlclient.BUILD",
60+
remote = "https://github.com/MariaDB/mariadb-connector-c.git",
61+
tag = "v3.0.8-release",
62+
workspace_file = "@google_ml_metadata//ml_metadata:libmysqlclient.WORKSPACE",
63+
)
64+
2565
go_repository(
2666
name = "io_k8s_client_go",
2767
build_file_proto_mode = "disable_global",

‎backend/Dockerfile

+17-14
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,29 @@
1-
FROM golang:1.11-alpine3.7 as builder
1+
FROM l.gcr.io/google/bazel:0.21.0 as builder
22

3+
RUN apt-get update && \
4+
apt-get install -y cmake clang musl-dev openssl
35
WORKDIR /go/src/github.com/kubeflow/pipelines
4-
COPY . .
56

6-
# Needed musl-dev for github.com/mattn/go-sqlite3
7-
RUN apk update && apk upgrade && \
8-
apk add --no-cache bash git openssh gcc musl-dev
7+
COPY WORKSPACE WORKSPACE
8+
COPY backend/src backend/src
9+
COPY backend/api backend/api
910

10-
RUN GO111MODULE=on go build -o /bin/apiserver backend/src/apiserver/*.go
11+
RUN bazel build -c opt --action_env=PATH --define=grpc_no_ares=true backend/src/apiserver:apiserver
1112

13+
14+
# Compile
1215
FROM python:3.5 as compiler
1316

1417
RUN apt-get update -y && \
15-
apt-get install --no-install-recommends -y -q default-jdk wget
16-
18+
apt-get install --no-install-recommends -y -q default-jdk wget
1719
RUN pip3 install setuptools==40.5.0
1820

1921
RUN wget http://central.maven.org/maven2/io/swagger/swagger-codegen-cli/2.4.1/swagger-codegen-cli-2.4.1.jar -O /tmp/swagger-codegen-cli.jar
2022

23+
# WORKDIR /go/src/github.com/kubeflow/pipelines
2124
WORKDIR /go/src/github.com/kubeflow/pipelines
22-
COPY . .
25+
COPY backend/api backend/api
26+
COPY sdk sdk
2327
WORKDIR /go/src/github.com/kubeflow/pipelines/sdk/python
2428
RUN ./build.sh /kfp.tar.gz
2529
RUN pip3 install /kfp.tar.gz
@@ -36,22 +40,21 @@ COPY ./samples .
3640
#The "for" loop breaks on all whitespace, so we either need to override IFS or use the "read" command instead.
3741
RUN find . -maxdepth 2 -name '*.py' -type f | while read pipeline; do dsl-compile --py "$pipeline" --output "$pipeline.tar.gz"; done
3842

39-
40-
FROM alpine:3.8
43+
FROM debian:stretch
4144

4245
ARG COMMIT_SHA=unknown
4346
ENV COMMIT_SHA=${COMMIT_SHA}
4447

4548
WORKDIR /bin
4649

47-
COPY --from=builder /bin/apiserver /bin/apiserver
48-
COPY --from=builder /go/src/github.com/kubeflow/pipelines/third_party/license.txt /bin/license.txt
50+
COPY third_party/license.txt /bin/license.txt
51+
COPY --from=builder /go/src/github.com/kubeflow/pipelines/bazel-bin/backend/src/apiserver/linux_amd64_stripped/apiserver /bin/apiserver
4952
COPY backend/src/apiserver/config/ /config
5053

5154
COPY --from=compiler /samples/ /samples/
5255

5356
# Adding CA certificate so API server can download pipeline through URL
54-
RUN apk add ca-certificates
57+
RUN apt-get update && apt-get install -y ca-certificates
5558

5659
# Expose apiserver port
5760
EXPOSE 8888

‎backend/src/apiserver/BUILD.bazel

+5
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ go_library(
1313
deps = [
1414
"//backend/api:go_default_library",
1515
"//backend/src/apiserver/client:go_default_library",
16+
"//backend/src/apiserver/metadata:go_default_library",
1617
"//backend/src/apiserver/model:go_default_library",
1718
"//backend/src/apiserver/resource:go_default_library",
1819
"//backend/src/apiserver/server:go_default_library",
@@ -23,12 +24,16 @@ go_library(
2324
"@com_github_cenkalti_backoff//:go_default_library",
2425
"@com_github_fsnotify_fsnotify//:go_default_library",
2526
"@com_github_golang_glog//:go_default_library",
27+
"@com_github_golang_protobuf//proto:go_default_library",
2628
"@com_github_grpc_ecosystem_grpc_gateway//runtime:go_default_library",
2729
"@com_github_jinzhu_gorm//:go_default_library",
2830
"@com_github_jinzhu_gorm//dialects/sqlite:go_default_library",
2931
"@com_github_minio_minio_go//:go_default_library",
3032
"@com_github_pkg_errors//:go_default_library",
3133
"@com_github_spf13_viper//:go_default_library",
34+
"@google_ml_metadata//ml_metadata/metadata_store:metadata_store_go", # keep
35+
"@google_ml_metadata//ml_metadata/proto:metadata_store_go_proto", # keep
36+
"@google_ml_metadata//ml_metadata/proto:metadata_store_service_go_proto", # keep
3237
"@org_golang_google_grpc//:go_default_library",
3338
"@org_golang_google_grpc//reflection:go_default_library",
3439
],

‎backend/src/apiserver/client_manager.go

+37-1
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,25 @@ package main
1717
import (
1818
"database/sql"
1919
"fmt"
20+
"strconv"
2021
"time"
2122

2223
workflowclient "github.com/argoproj/argo/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
2324
"github.com/cenkalti/backoff"
2425
"github.com/golang/glog"
26+
"github.com/golang/protobuf/proto"
2527
"github.com/jinzhu/gorm"
2628
_ "github.com/jinzhu/gorm/dialects/sqlite"
2729
"github.com/kubeflow/pipelines/backend/src/apiserver/client"
30+
"github.com/kubeflow/pipelines/backend/src/apiserver/metadata"
2831
"github.com/kubeflow/pipelines/backend/src/apiserver/model"
2932
"github.com/kubeflow/pipelines/backend/src/apiserver/storage"
3033
"github.com/kubeflow/pipelines/backend/src/common/util"
3134
scheduledworkflowclient "github.com/kubeflow/pipelines/backend/src/crd/pkg/client/clientset/versioned/typed/scheduledworkflow/v1beta1"
3235
minio "github.com/minio/minio-go"
36+
37+
"ml_metadata/metadata_store/mlmetadata"
38+
mlpb "ml_metadata/proto/metadata_store_go_proto"
3339
)
3440

3541
const (
@@ -56,6 +62,8 @@ type ClientManager struct {
5662
swfClient scheduledworkflowclient.ScheduledWorkflowInterface
5763
time util.TimeInterface
5864
uuid util.UUIDGeneratorInterface
65+
66+
MetadataStore *mlmetadata.Store
5967
}
6068

6169
func (c *ClientManager) ExperimentStore() storage.ExperimentStoreInterface {
@@ -117,7 +125,6 @@ func (c *ClientManager) init() {
117125
c.experimentStore = storage.NewExperimentStore(db, c.time, c.uuid)
118126
c.pipelineStore = storage.NewPipelineStore(db, c.time, c.uuid)
119127
c.jobStore = storage.NewJobStore(db, c.time)
120-
c.runStore = storage.NewRunStore(db, c.time)
121128
c.resourceReferenceStore = storage.NewResourceReferenceStore(db)
122129
c.dBStatusStore = storage.NewDBStatusStore(db)
123130
c.objectStore = initMinioClient(getDurationConfig(initConnectionTimeout))
@@ -127,13 +134,42 @@ func (c *ClientManager) init() {
127134

128135
c.swfClient = client.CreateScheduledWorkflowClientOrFatal(
129136
getStringConfig(podNamespace), getDurationConfig(initConnectionTimeout))
137+
138+
metadataStore := initMetadataStore()
139+
runStore := storage.NewRunStore(db, c.time, metadataStore)
140+
c.runStore = runStore
141+
130142
glog.Infof("Client manager initialized successfully")
131143
}
132144

133145
func (c *ClientManager) Close() {
134146
c.db.Close()
135147
}
136148

149+
func initMetadataStore() *metadata.Store {
150+
port, err := strconv.Atoi(getStringConfig(mysqlServicePort))
151+
if err != nil {
152+
glog.Fatalf("Failed to parse valid MySQL service port from %q: %v", getStringConfig(mysqlServicePort), err)
153+
}
154+
155+
cfg := &mlpb.ConnectionConfig{
156+
Config: &mlpb.ConnectionConfig_Mysql{
157+
&mlpb.MySQLDatabaseConfig{
158+
Host: proto.String(getStringConfig(mysqlServiceHost)),
159+
Port: proto.Uint32(uint32(port)),
160+
Database: proto.String("mlmetadata"),
161+
User: proto.String("root"),
162+
},
163+
},
164+
}
165+
166+
mlmdStore, err := mlmetadata.NewStore(cfg)
167+
if err != nil {
168+
glog.Fatalf("Failed to create ML Metadata store: %v", err)
169+
}
170+
return metadata.NewStore(mlmdStore)
171+
}
172+
137173
func initDBClient(initConnectionTimeout time.Duration) *storage.DB {
138174
driverName := getStringConfig("DBConfig.DriverName")
139175
var arg string

‎backend/src/apiserver/main.go

+4-1
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ import (
3535
"github.com/pkg/errors"
3636
"google.golang.org/grpc"
3737
"google.golang.org/grpc/reflection"
38+
39+
_ "ml_metadata/metadata_store/mlmetadata"
40+
_ "ml_metadata/proto/metadata_store_go_proto"
41+
_ "ml_metadata/proto/metadata_store_service_go_proto"
3842
)
3943

4044
var (
@@ -48,7 +52,6 @@ type RegisterHttpHandlerFromEndpoint func(ctx context.Context, mux *runtime.Serv
4852

4953
func main() {
5054
flag.Parse()
51-
glog.Infof("starting API server")
5255

5356
initConfig()
5457
clientManager := newClientManager()

‎backend/src/apiserver/metadata/BUILD.bazel

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
2+
3+
go_library(
4+
name = "go_default_library",
5+
srcs = ["metadata_store.go"],
6+
importpath = "github.com/kubeflow/pipelines/backend/src/apiserver/metadata",
7+
visibility = ["//visibility:public"],
8+
deps = [
9+
"//backend/src/common/util:go_default_library",
10+
"@com_github_argoproj_argo//pkg/apis/workflow/v1alpha1:go_default_library",
11+
"@com_github_golang_protobuf//jsonpb:go_default_library_gen",
12+
"@com_github_golang_protobuf//proto:go_default_library",
13+
"@google_ml_metadata//ml_metadata/metadata_store:metadata_store_go",
14+
"@google_ml_metadata//ml_metadata/proto:metadata_store_go_proto",
15+
],
16+
)
17+
18+
go_test(
19+
name = "go_default_test",
20+
srcs = ["metadata_store_test.go"],
21+
embed = [":go_default_library"],
22+
deps = [
23+
"@com_github_golang_protobuf//proto:go_default_library",
24+
"@com_github_google_go_cmp//cmp:go_default_library",
25+
"@google_ml_metadata//ml_metadata/proto:metadata_store_go_proto",
26+
],
27+
)

0 commit comments

Comments
 (0)