Added support for Proxy Enabled Application Profile in WCP env

Author: Rishabh-Baranwal

Diff for: ako-clean/cleanup.go

@@ -109,6 +109,7 @@ func (cfg *AKOCleanupConfig) Cleanup(ctx context.Context) error {
 		cleanupL4PolicySets,
 		cleanupPoolGroups,
 		cleanupPools,
+		cleanupAppProfiles,
 		func() error {
 			if VPCMode {
 				return nil
@@ -140,6 +141,7 @@ func (cfg *AKOCleanupConfig) Cleanup(ctx context.Context) error {
 	}
 	return nil
 }
+
 func getCloudInVPCMode() (string, error) {
 	uri := "/api/cloud/"
 	aviRestClientPool := avicache.SharedAVIClients(lib.GetTenant())
@@ -406,6 +408,19 @@ func cleanupPools() error {
 	return deleteAviResource("/api/pool", pools)
 }
 
+func cleanupAppProfiles() error {
+	aviObjCache := avicache.SharedAviObjCache()
+	appProfiles := make(map[string][]string)
+	for _, key := range aviObjCache.AppProfileCache.AviGetAllKeys() {
+		if _, ok := appProfiles[key.Namespace]; !ok {
+			appProfiles[key.Namespace] = []string{}
+		}
+		appProfCache, _ := aviObjCache.AppProfileCache.AviCacheGet(key)
+		appProfiles[key.Namespace] = append(appProfiles[key.Namespace], appProfCache.(*avicache.AviAppProfileCache).Uuid)
+	}
+	return deleteAviResource("/api/applicationprofile", appProfiles)
+}
+
 /*
 Below section is only applicable for T1 based Supervisor deployments
 */

Diff for: internal/cache/cache_utils.go

@@ -349,6 +349,16 @@ type AviVrfCache struct {
 	CloudConfigCksum uint32
 }
 
+type AviAppProfileCache struct {
+	Name                string
+	Tenant              string
+	Uuid                string
+	Type                string
+	CloudConfigCksum    uint32
+	EnableProxyProtocol bool
+	LastModified        string
+}
+
 func (v *AviVsCache) GetVSCopy() (*AviVsCache, bool) {
 	v.VSCacheLock.RLock()
 	defer v.VSCacheLock.RUnlock()

Diff for: internal/cache/controller_obj_cache.go

@@ -54,6 +54,7 @@ type AviObjCache struct {
 	VsCacheMeta        *AviCache
 	VsCacheLocal       *AviCache
 	ClusterStatusCache *AviCache
+	AppProfileCache    *AviCache
 }
 
 func NewAviObjCache() *AviObjCache {
@@ -71,6 +72,7 @@ func NewAviObjCache() *AviObjCache {
 	c.VrfCache = NewAviCache()
 	c.PKIProfileCache = NewAviCache()
 	c.ClusterStatusCache = NewAviCache()
+	c.AppProfileCache = NewAviCache()
 	return &c
 }
 
@@ -129,6 +131,12 @@ func (c *AviObjCache) AviObjCachePopulate(client []*clients.AviClient, version s
 	if err != nil {
 		return vsCacheCopy, allVsKeys, err
 	}
+	if utils.IsWCP() && tenant == lib.GetAdminTenant() {
+		err := c.AviObjAppProfileCachePopulate(client[0])
+		if err != nil {
+			return vsCacheCopy, allVsKeys, err
+		}
+	}
 	// Populate the VS cache
 	utils.AviLog.Infof("Refreshing all object cache")
 	c.AviRefreshObjectCache(client, cloud, tenant)
@@ -1917,6 +1925,51 @@ func (c *AviObjCache) AviObjVrfCachePopulate(client *clients.AviClient, cloud st
 	return nil
 }
 
+func (c *AviObjCache) AviObjAppProfileCachePopulate(client *clients.AviClient) error {
+	appName := lib.GetProxyEnabledApplicationProfileName()
+	uri := "/api/applicationprofile?name=" + appName + "&include_name=true" + "&created_by=" + lib.GetAKOUser()
+	result, err := lib.AviGetCollectionRaw(client, uri)
+	if err != nil {
+		utils.AviLog.Warnf("Get uri %v returned err for app profile %v", uri, err)
+		return err
+	}
+	elems := make([]json.RawMessage, result.Count)
+	err = json.Unmarshal(result.Results, &elems)
+	if err != nil {
+		utils.AviLog.Warnf("Failed to unmarshal app profile data, err: %v", err)
+		return err
+	}
+	for i := 0; i < result.Count; i++ {
+		appProf := models.ApplicationProfile{}
+		err = json.Unmarshal(elems[i], &appProf)
+		if err != nil {
+			utils.AviLog.Warnf("Failed to unmarshal app profile data, err: %v", err)
+			continue
+		}
+		if appProf.Name == nil || appProf.UUID == nil || appProf.TCPAppProfile == nil {
+			utils.AviLog.Warnf("Incomplete app profile data unmarshalled, %s", utils.Stringify(appProf))
+			continue
+		}
+		tenant := getTenantFromTenantRef(*appProf.TenantRef)
+		appType := *appProf.Type
+		ppe := *appProf.TCPAppProfile.ProxyProtocolEnabled
+		cksum := lib.AppProfileChecksum(appName, tenant, appType, ppe)
+		appProfileCacheObj := AviAppProfileCache{
+			Name:                appName,
+			Tenant:              tenant,
+			Uuid:                *appProf.UUID,
+			Type:                appType,
+			LastModified:        *appProf.LastModified,
+			CloudConfigCksum:    cksum,
+			EnableProxyProtocol: ppe,
+		}
+		k := NamespaceName{Namespace: tenant, Name: appName}
+		c.AppProfileCache.AviCacheAdd(k, &appProfileCacheObj)
+		utils.AviLog.Infof("Adding app profile to cache during refresh %s", utils.Stringify(appProfileCacheObj))
+	}
+	return nil
+}
+
 func (c *AviObjCache) AviObjVSCachePopulate(client *clients.AviClient, cloud string, vsCacheCopy *[]NamespaceName, overrideUri ...NextPage) error {
 	var rest_response interface{}
 	akoUser := lib.AKOUser

Diff for: internal/k8s/ako_init.go

@@ -1226,6 +1226,11 @@ func (c *AviController) FullSyncK8s(sync bool) error {
 			}
 		}
 
+		appProfileLabel := lib.GetModelName(lib.GetAdminTenant(), lib.GetProxyEnabledApplicationProfileName())
+		appKey := lib.ApplicationProfile + "/" + appProfileLabel
+		lib.IncrementQueueCounter(utils.ObjectIngestionLayer)
+		nodes.DequeueIngestion(appKey, true)
+
 		//Ingress Section
 		if utils.GetInformers().IngressInformer != nil {
 			for namespace := range acceptedNamespaces {

Diff for: internal/lib/constants.go

@@ -163,6 +163,7 @@ const (
 	PriorityLabel                              = "PriorityLabel"
 	SSLKeyCert                                 = "SSLKeyandCertificate"
 	PKIProfile                                 = "PKI Profile"
+	ApplicationProfile                         = "ApplicationProfile"
 	PassthroughPG                              = "Passthrough PG"
 	Passthroughpool                            = "Passthrough pool"
 	PassthroughVS                              = "Passthrough VirtualService"
@@ -261,6 +262,7 @@ const (
 	CalicoIPv6AddressAnnotation      = "projectcalico.org/IPv6Address"
 	AntreaTransportAddressAnnotation = "node.antrea.io/transport-addresses"
 	TenantAnnotation                 = "ako.vmware.com/tenant-name"
+	GwProxyProtocolEnableAnnotation  = "iaas.vmware.com/proxy-protocol-enabled"
 
 	// Specifies command used in namespace event handler
 	NsFilterAdd                    = "ADD"

Diff for: internal/lib/lib.go

@@ -689,6 +689,11 @@ func GetVrf() string {
 	return VRFContext
 }
 
+// One proxy enabled app profile per cluster
+func GetProxyEnabledApplicationProfileName() string {
+	return GetClusterName()
+}
+
 func GetVPCMode() bool {
 	if vpcMode, _ := strconv.ParseBool(os.Getenv(utils.VPC_MODE)); vpcMode {
 		return true
@@ -1450,6 +1455,10 @@ func L4PolicyChecksum(ports []int64, protocols []string, pools []string, ingesti
 	return checksum
 }
 
+func AppProfileChecksum(name, tenant, appType string, ppe bool) uint32 {
+	return utils.Hash(name + tenant + appType + strconv.FormatBool(ppe))
+}
+
 func IsNodePortMode() bool {
 	nodePortType := os.Getenv(SERVICE_TYPE)
 	if nodePortType == NODE_PORT {

Diff for: internal/nodes/avi_app_profile_translator.go

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2019-2020 VMware, Inc.
+ * All Rights Reserved.
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*   http://www.apache.org/licenses/LICENSE-2.0
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package nodes
+
+import (
+	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/lib"
+	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/pkg/utils"
+)
+
+func (o *AviObjectGraph) BuildAppProfileGraph(ns, name, key string) {
+	o.Lock.Lock()
+	defer o.Lock.Unlock()
+	aviAppProfNode := &AviAppProfileNode{
+		Name:                name,
+		Tenant:              ns,
+		Type:                lib.AllowedL4ApplicationProfile,
+		EnableProxyProtocol: true,
+	}
+	o.AddModelNode(aviAppProfNode)
+	aviAppProfNode.CalculateCheckSum()
+	utils.AviLog.Infof("key: %s, Added app profile node %s", key, name)
+	utils.AviLog.Debugf("key: %s, app profile node: [%v]", key, utils.Stringify(aviAppProfNode))
+}

Diff for: internal/nodes/avi_model_advl4_translator.go

@@ -16,6 +16,7 @@ package nodes
 
 import (
 	"fmt"
+	"strconv"
 	"strings"
 
 	"github.com/vmware/load-balancer-and-ingress-services-for-kubernetes/internal/lib"
@@ -138,6 +139,12 @@ func (o *AviObjectGraph) ConstructAdvL4VsNode(gatewayName, namespace, key string
 
 	avi_vs_meta.PortProto = portProtocols
 	avi_vs_meta.ApplicationProfile = utils.DEFAULT_L4_APP_PROFILE
+	if proxyProtoEnb, ok := gw.GetAnnotations()[lib.GwProxyProtocolEnableAnnotation]; ok {
+		proxyProtocolEnabled, err := strconv.ParseBool(proxyProtoEnb)
+		if err == nil && proxyProtocolEnabled {
+			avi_vs_meta.ApplicationProfile = lib.GetProxyEnabledApplicationProfileName()
+		}
+	}
 
 	avi_vs_meta.NetworkProfile = getNetworkProfile(isSCTP, isTCP, isUDP)
 

Diff for: internal/nodes/avi_model_nodes.go

@@ -357,6 +357,51 @@ func (o *AviObjectGraph) GetAviVRF() []*AviVrfNode {
 	}
 	return aviVrf
 }
+
+type AviAppProfileNode struct {
+	Name                string
+	Tenant              string
+	Type                string
+	EnableProxyProtocol bool
+	CloudConfigCksum    uint32
+}
+
+func (v *AviAppProfileNode) GetNodeType() string {
+	return "AppProfileNode"
+}
+
+func (v *AviAppProfileNode) CopyNode() AviModelNode {
+	newNode := AviAppProfileNode{}
+	bytes, err := json.Marshal(v)
+	if err != nil {
+		utils.AviLog.Warnf("Unable to marshal AviAppProfileNode: %s", err)
+	}
+	err = json.Unmarshal(bytes, &newNode)
+	if err != nil {
+		utils.AviLog.Warnf("Unable to unmarshal AviAppProfileNode: %s", err)
+	}
+	return &newNode
+}
+
+func (v *AviAppProfileNode) GetCheckSum() uint32 {
+	v.CalculateCheckSum()
+	return v.CloudConfigCksum
+}
+
+func (v *AviAppProfileNode) CalculateCheckSum() {
+	v.CloudConfigCksum = lib.AppProfileChecksum(v.Name, v.Tenant, v.Type, v.EnableProxyProtocol)
+}
+
+func (o *AviObjectGraph) GetAviAppProfileNode() *AviAppProfileNode {
+	for _, model := range o.modelNodes {
+		appProfNode, ok := model.(*AviAppProfileNode)
+		if ok {
+			return appProfNode
+		}
+	}
+	return nil
+}
+
 func (o *AviObjectGraph) GetIstioNodes() (*AviPkiProfileNode, *AviTLSKeyCertNode) {
 	var pkiNode *AviPkiProfileNode
 	var sslNode *AviTLSKeyCertNode

Diff for: internal/nodes/dequeue_ingestion.go

@@ -289,6 +289,15 @@ func DequeueIngestion(key string, fullsync bool) {
 			}
 		}
 	}
+	if objType == lib.ApplicationProfile && utils.IsWCP() {
+		modelName := lib.GetModelName(namespace, name)
+		aviModelGraph := NewAviObjectGraph()
+		aviModelGraph.BuildAppProfileGraph(namespace, name, key)
+		ok := saveAviModel(modelName, aviModelGraph, key)
+		if ok && !fullsync {
+			PublishKeyToRestLayer(modelName, key, sharedQueue)
+		}
+	}
 	if objType == utils.Namespace && utils.IsWCP() && isNamespaceDeleted(name) {
 		cache := avicache.SharedAviObjCache()
 		vsKeys := cache.VsCacheMeta.AviCacheGetAllParentVSKeys()