feat: Support using structural simplified Ingress resource model in file storage (#34)

Author: CH3CHO

compose/.env

@@ -8,7 +8,7 @@ NACOS_PASSWORD=''
 NACOS_DATA_ENC_KEY='0123456789abcdef0123456789abcdef'
 NACOS_SERVER_TAG='v2.2.3'
 HIGRESS_RUNNER_TAG='0.0.3'
-HIGRESS_API_SERVER_TAG='0.0.8'
+HIGRESS_API_SERVER_TAG='0.0.9'
 HIGRESS_CONTROLLER_TAG='1.1.1'
 HIGRESS_PILOT_TAG='1.1.1'
 HIGRESS_GATEWAY_TAG='1.1.1'

compose/docker-compose.yml

@@ -108,7 +108,7 @@ services:
       - --kubeconfig=/home/higress/.kube/config
       - --gatewaySelectorKey=higress
       - --gatewaySelectorValue=higress-system-higress-gateway
-      - --ingressClass=higress
+      - --ingressClass=
     depends_on:
       apiserver:
         condition: service_healthy

src/apiserver/Makefile

@@ -1,6 +1,6 @@
 REGISTRY ?= higress-registry.cn-hangzhou.cr.aliyuncs.com/higress/
 IMAGE_NAME ?= api-server
-IMAGE_VERSION ?= 0.0.8
+IMAGE_VERSION ?= 0.0.9
 BUILD_TIME := $(shell date "+%Y%m%d-%H%M%S")
 COMMIT_ID := $(shell git rev-parse --short HEAD 2>/dev/null)
 IMAGE_TAG = $(if $(strip $(IMAGE_VERSION)),${IMAGE_VERSION},${BUILD_TIME}-${COMMIT_ID})

src/apiserver/pkg/apiserver/apiserver.go

@@ -18,6 +18,7 @@ package apiserver
 
 import (
 	"fmt"
+	"github.com/alibaba/higress/api-server/pkg/codec"
 	"github.com/alibaba/higress/api-server/pkg/options"
 	"github.com/alibaba/higress/api-server/pkg/registry"
 	"github.com/alibaba/higress/api-server/pkg/storage"
@@ -167,7 +168,7 @@ func (c completedConfig) New() (*HigressServer, error) {
 
 	storageCreateFunc := func(
 		groupResource schema.GroupResource,
-		codec runtime.Codec,
+		runtimeCodec runtime.Codec,
 		isNamespaced bool,
 		singularName string,
 		newFunc func() runtime.Object,
@@ -177,13 +178,14 @@ func (c completedConfig) New() (*HigressServer, error) {
 	) (rest.Storage, error) {
 		switch storageMode {
 		case options.Storage_File:
-			return registry.NewFileREST(groupResource, codec, storageOptions.FileOptions.RootDir, extension, isNamespaced, singularName, newFunc, newListFunc, attrFunc)
+			runtimeCodec = codec.NewFlatAwareCodec(groupResource, runtimeCodec)
+			return registry.NewFileREST(groupResource, runtimeCodec, storageOptions.FileOptions.RootDir, extension, isNamespaced, singularName, newFunc, newListFunc, attrFunc)
 		case options.Storage_Nacos:
 			var encryptionKey []byte = nil
 			if sensitive {
 				encryptionKey = storageOptions.NacosOptions.EncryptionKey
 			}
-			return registry.NewNacosREST(groupResource, codec, nacosConfigClient, isNamespaced, singularName, newFunc, newListFunc, attrFunc, encryptionKey), nil
+			return registry.NewNacosREST(groupResource, runtimeCodec, nacosConfigClient, isNamespaced, singularName, newFunc, newListFunc, attrFunc, encryptionKey), nil
 		default:
 			panic(fmt.Errorf("invalid storage mode: %s", storageMode))
 		}
@@ -326,19 +328,20 @@ func appendStorage(storages map[string]rest.Storage,
 	sensitive bool,
 ) {
 	groupResource := groupVersion.WithResource(pluralName).GroupResource()
-	codec, _, err := genericserverstorage.NewStorageCodec(genericserverstorage.StorageCodecConfig{
+	storageCodec, _, err := genericserverstorage.NewStorageCodec(genericserverstorage.StorageCodecConfig{
 		StorageMediaType:  contentType,
 		StorageSerializer: serializer.NewCodecFactory(Scheme),
 		StorageVersion:    Scheme.PrioritizedVersionsForGroup(groupResource.Group)[0],
 		MemoryVersion:     Scheme.PrioritizedVersionsForGroup(groupResource.Group)[0],
 		Config:            storagebackend.Config{}, // useless fields
 	})
 	if err != nil {
-		err = fmt.Errorf("unable to create REST storage for a resource due to %v, will die", err)
+		err = fmt.Errorf("unable to create storage codec for a resource due to %v, will die", err)
 		panic(err)
 	}
-	storage, err := storageCreatorFunc(groupResource, codec, isNamespaced, singularName, newFunc, newListFunc, attrFunc, sensitive)
+	storage, err := storageCreatorFunc(groupResource, storageCodec, isNamespaced, singularName, newFunc, newListFunc, attrFunc, sensitive)
 	if err != nil {
+		err = fmt.Errorf("unable to create REST storage for a resource due to %v, will die", err)
 		panic(err)
 	}
 	storages[pluralName] = storage

src/apiserver/pkg/codec/flat_codec.go

@@ -0,0 +1,111 @@
+package codec
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	networkingv1 "k8s.io/api/networking/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	kjson "sigs.k8s.io/json"
+	"sigs.k8s.io/yaml"
+)
+
+var (
+	v1IngressGvr        = networkingv1.SchemeGroupVersion.WithResource("ingresses")
+	v1IngressGvk        = networkingv1.SchemeGroupVersion.WithKind("Ingress")
+	v1IngressApiVersion = networkingv1.SchemeGroupVersion.Group + "/" + networkingv1.SchemeGroupVersion.Version
+
+	noApiVersionKindError = fmt.Errorf("the given data doesn't contain APIVersion and Kind data")
+	nonFlatResourceError  = fmt.Errorf("the given data doesn't represent a flat resource")
+)
+
+func NewFlatAwareCodec(groupResource schema.GroupResource, innerCodec runtime.Codec) runtime.Codec {
+	if v1IngressGvr.GroupResource() == groupResource {
+		return &flatIngressCodec{groupResource: groupResource, innerCodec: innerCodec}
+	}
+	return innerCodec
+}
+
+type flatIngressCodec struct {
+	groupResource schema.GroupResource
+	innerCodec    runtime.Codec
+}
+
+func (c *flatIngressCodec) Encode(obj runtime.Object, w io.Writer) error {
+	ingress, ok := obj.(*networkingv1.Ingress)
+	if !ok {
+		return c.innerCodec.Encode(obj, w)
+	}
+	fIngress := toFlatIngress(ingress)
+	return encodeFlatResource(fIngress, w)
+}
+
+func (c *flatIngressCodec) Identifier() runtime.Identifier {
+	return c.innerCodec.Identifier()
+}
+
+func (c *flatIngressCodec) Decode(data []byte, defaults *schema.GroupVersionKind, into runtime.Object) (runtime.Object, *schema.GroupVersionKind, error) {
+	fIngress := &flatIngress{}
+	err := decodeFlatResource(data, fIngress)
+	if err != nil {
+		if errors.Is(err, nonFlatResourceError) {
+			return c.innerCodec.Decode(data, defaults, into)
+		}
+		return nil, nil, err
+	}
+	ingress := fromFlatIngress(fIngress)
+	ingress.APIVersion = v1IngressApiVersion
+	ingress.Kind = v1IngressGvk.Kind
+	return ingress, defaults, err
+}
+
+func encodeFlatResource(obj interface{}, w io.Writer) error {
+	jsonData, err := json.Marshal(obj)
+	if err != nil {
+		return err
+	}
+	yamlData, err := yaml.JSONToYAML(jsonData)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(yamlData)
+	return err
+}
+
+func decodeFlatResource(data []byte, into interface{}) error {
+	jsonData, err := yaml.YAMLToJSON(data)
+	if err != nil {
+		return err
+	}
+	// Compatible with non-flat K8s YAMLs
+	_, err = tryFindApiVersionKind(jsonData)
+	if err == nil {
+		return nonFlatResourceError
+	}
+	if err := kjson.UnmarshalCaseSensitivePreserveInts(jsonData, into); err != nil {
+		return err
+	}
+	return nil
+}
+
+func tryFindApiVersionKind(data []byte) (*schema.GroupVersionKind, error) {
+	findKind := struct {
+		// +optional
+		APIVersion string `json:"apiVersion,omitempty"`
+		// +optional
+		Kind string `json:"kind,omitempty"`
+	}{}
+	if err := json.Unmarshal(data, &findKind); err != nil {
+		return nil, err
+	}
+	if findKind.APIVersion == "" || findKind.Kind == "" {
+		return nil, noApiVersionKindError
+	}
+	gv, err := schema.ParseGroupVersion(findKind.APIVersion)
+	if err != nil {
+		return nil, err
+	}
+	return &schema.GroupVersionKind{Group: gv.Group, Version: gv.Version, Kind: findKind.Kind}, nil
+}

src/apiserver/pkg/codec/flat_resources.go

@@ -0,0 +1,95 @@
+package codec
+
+import (
+	networkingv1 "k8s.io/api/networking/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+)
+
+type flatObjectMeta struct {
+	Labels      map[string]string `json:"labels,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+func (in *flatObjectMeta) DeepCopyInto(out *flatObjectMeta) {
+	if in.Labels != nil {
+		in, out := &in.Labels, &out.Labels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.Annotations != nil {
+		in, out := &in.Annotations, &out.Annotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+}
+
+type flatIngress struct {
+	metav1.TypeMeta `json:"-"`
+	flatObjectMeta  `json:",inline"`
+
+	// Spec exploded
+	DefaultBackend *networkingv1.IngressBackend `json:"defaultBackend,omitempty" protobuf:"bytes,1,opt,name=defaultBackend"`
+	TLS            []networkingv1.IngressTLS    `json:"tls,omitempty" protobuf:"bytes,2,rep,name=tls"`
+	Rules          []networkingv1.IngressRule   `json:"rules,omitempty" protobuf:"bytes,3,rep,name=rules"`
+}
+
+func (f *flatIngress) GetObjectKind() schema.ObjectKind {
+	return f.TypeMeta.GetObjectKind()
+}
+
+func (f *flatIngress) DeepCopyObject() runtime.Object {
+	out := flatIngress{}
+
+	out.TypeMeta = f.TypeMeta
+	f.flatObjectMeta.DeepCopyInto(&out.flatObjectMeta)
+
+	out.DefaultBackend = f.DefaultBackend
+	if f.TLS != nil {
+		in, out := &f.TLS, &out.TLS
+		*out = make([]networkingv1.IngressTLS, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if f.Rules != nil {
+		in, out := &f.Rules, &out.Rules
+		*out = make([]networkingv1.IngressRule, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+
+	return nil
+}
+
+func toFlatIngress(ingress *networkingv1.Ingress) *flatIngress {
+	return &flatIngress{
+		flatObjectMeta: flatObjectMeta{
+			Labels:      ingress.Labels,
+			Annotations: ingress.Annotations,
+		},
+		DefaultBackend: ingress.Spec.DefaultBackend,
+		TLS:            ingress.Spec.TLS,
+		Rules:          ingress.Spec.Rules,
+	}
+}
+
+func fromFlatIngress(flat *flatIngress) *networkingv1.Ingress {
+	return &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels:      flat.Labels,
+			Annotations: flat.Annotations,
+		},
+		Spec: networkingv1.IngressSpec{
+			DefaultBackend: flat.DefaultBackend,
+			TLS:            flat.TLS,
+			Rules:          flat.Rules,
+		},
+	}
+}

src/apiserver/pkg/registry/file_rest.go

@@ -36,6 +36,7 @@ var ErrFileNotExists = fmt.Errorf("file doesn't exist")
 
 const fileChangeProcessInterval = 100 * time.Millisecond
 const fileChangeProcessDelay = 250 * time.Millisecond
+const defaultNamespace = "higress-system"
 
 var _ rest.StandardStorage = &fileREST{}
 var _ rest.Scoper = &fileREST{}
@@ -496,6 +497,7 @@ func (f *fileREST) objectFileName(ctx context.Context, name string) string {
 }
 
 func (f *fileREST) write(encoder runtime.Encoder, filepath string, obj runtime.Object) error {
+	f.normalizeObjectMeta(obj, filepath)
 	buf := new(bytes.Buffer)
 	if err := encoder.Encode(obj, buf); err != nil {
 		return err
@@ -521,9 +523,19 @@ func (f *fileREST) read(decoder runtime.Decoder, path string, newFunc func() run
 	if err != nil {
 		return nil, err
 	}
+	f.normalizeObjectMeta(decodedObj, cleanedPath)
 	return decodedObj, nil
 }
 
+func (f *fileREST) normalizeObjectMeta(obj runtime.Object, path string) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		return
+	}
+	accessor.SetNamespace(defaultNamespace)
+	accessor.SetName(strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)))
+}
+
 func (f *fileREST) visitDir(dirname string, extension string, newFunc func() runtime.Object, codec runtime.Decoder, visitFunc func(string, runtime.Object)) error {
 	return filepath.Walk(dirname, func(path string, info os.FileInfo, err error) error {
 		if err != nil {