lazy loading customs now (#910)

* lazy loading customs now

* Moving to a lock free system for lazy loading

Author: i3149

pkg/api/api.go

@@ -44,6 +44,7 @@ const (
 	KT_INTERFACE_LOOKUP_TEXT_FILTER = "KT_INTERFACE_LOOKUP_TEXT_FILTER"
 	KT_NO_CUSTOM_COLUMNS            = "KT_NO_CUSTOM_COLUMNS"
 	KT_LOAD_CUSTOM_COLS_FOR_DEVS    = "KT_LOAD_CUSTOM_COLS_FOR_DEVS"
+	KT_API_LAZY_LOAD_CUSTOMS        = "KT_API_LAZY_LOAD_CUSTOMS"
 )
 
 var (
@@ -72,6 +73,7 @@ type KentikApi struct {
 	config          *ktranslate.Config
 	tagLookupClient tagging.EnumerationsAdminServiceClient
 	loadInterfaces  bool
+	lazyLoadCustoms bool
 }
 
 func NewKentikApi(ctx context.Context, log logger.ContextL, cfg *ktranslate.Config) (*KentikApi, error) {
@@ -81,7 +83,6 @@ func NewKentikApi(ctx context.Context, log logger.ContextL, cfg *ktranslate.Conf
 		intv, _ := strconv.Atoi(apiTimeoutStr)
 		apiTimeout = time.Duration(intv) * time.Second
 	}
-	log.Infof("Setting API timeout to %v", apiTimeout)
 
 	tr := &http.Transport{
 		DisableCompression: false,
@@ -94,14 +95,17 @@ func NewKentikApi(ctx context.Context, log logger.ContextL, cfg *ktranslate.Conf
 	client := &http.Client{Transport: tr, Timeout: apiTimeout}
 
 	kapi := &KentikApi{
-		ContextL:       log,
-		tr:             tr,
-		client:         client,
-		apiTimeout:     apiTimeout,
-		config:         cfg,
-		loadInterfaces: kt.LookupEnvBool(KT_API_LOAD_INTERFACES, false),
+		ContextL:        log,
+		tr:              tr,
+		client:          client,
+		apiTimeout:      apiTimeout,
+		config:          cfg,
+		loadInterfaces:  kt.LookupEnvBool(KT_API_LOAD_INTERFACES, false),
+		lazyLoadCustoms: kt.LookupEnvBool(KT_API_LAZY_LOAD_CUSTOMS, false),
 	}
 
+	log.Infof("Setting API timeout to %v, loadInterfaces=%v, lazyLoadCustoms=%v", apiTimeout, kapi.loadInterfaces, kapi.lazyLoadCustoms)
+
 	// Now, check to see if synthetics API works.
 	err := kapi.connectSynthAndLookup(ctx)
 	if err != nil {
@@ -235,12 +239,33 @@ func (api *KentikApi) GetDevicesAsMap(cid kt.Cid) map[string]*kt.Device {
 	return res
 }
 
-func (api *KentikApi) GetDevice(cid kt.Cid, did kt.DeviceID) *kt.Device {
+func (api *KentikApi) GetDevice(ctx context.Context, cid kt.Cid, did kt.DeviceID) *kt.Device {
 	if api == nil {
 		return nil
 	}
 	if c, ok := api.devices[cid]; ok {
-		return c[did]
+		dev := c[did]
+		if dev == nil {
+			return dev
+		}
+		if api.lazyLoadCustoms && !dev.LoadedCustoms() {
+			go func() {
+				for _, info := range api.config.KentikCreds {
+					md := metadata.New(map[string]string{
+						"X-CH-Auth-Email":     info.APIEmail,
+						"X-CH-Auth-API-Token": info.APIToken,
+					})
+					ctxo := metadata.NewOutgoingContext(ctx, md)
+					err := api.loadCustoms(ctxo, dev.IDStr, dev)
+					if err != nil {
+						api.Warnf("Cannot load customs for %s %s, %v", dev.IDStr, info.APIEmail, err)
+					} else {
+						return
+					}
+				}
+			}()
+		}
+		return dev
 	}
 	return nil
 }

pkg/cat/jchf.go

@@ -128,7 +128,7 @@ func (kc *KTranslate) flowToJCHF(ctx context.Context, dst *kt.JCHF, src *Flow, c
 
 	// Do we have info about this device?
 	custColNames := map[uint32]string{}
-	if d := kc.apic.GetDevice(dst.CompanyId, dst.DeviceId); d != nil {
+	if d := kc.apic.GetDevice(ctx, dst.CompanyId, dst.DeviceId); d != nil {
 		dst.DeviceName = d.Name
 		dst.CustomStr[UDR_TYPE] = d.DeviceSubtype
 		if len(d.SendingIps) > 0 {
@@ -279,7 +279,7 @@ func (kc *KTranslate) flowToJCHF(ctx context.Context, dst *kt.JCHF, src *Flow, c
 				}
 			case "ult_exit_device_id":
 				dst.CustomInt[name] = int32(v)
-				if d := kc.apic.GetDevice(dst.CompanyId, kt.DeviceID(v)); d != nil {
+				if d := kc.apic.GetDevice(ctx, dst.CompanyId, kt.DeviceID(v)); d != nil {
 					dst.CustomStr["ult_exit_device"] = d.Name
 					dst.CustomStr["ult_exit_site"] = d.Site.SiteName
 				}
@@ -408,7 +408,7 @@ func (kc *KTranslate) flowToJCHF(ctx context.Context, dst *kt.JCHF, src *Flow, c
 
 	// Check if there is ultimate exit data interface and pull this in also.
 	if udid, ok := dst.CustomInt["ult_exit_device_id"]; ok {
-		if d := kc.apic.GetDevice(dst.CompanyId, kt.DeviceID(udid)); d != nil {
+		if d := kc.apic.GetDevice(ctx, dst.CompanyId, kt.DeviceID(udid)); d != nil {
 			if ui, ok := dst.CustomInt["ult_exit_port"]; ok {
 				if i, ok := d.Interfaces[kt.IfaceID(ui)]; ok {
 					dst.CustomStr["ult_exit_port_alias"] = i.Alias

pkg/kt/device_types.go

@@ -42,6 +42,7 @@ type Device struct {
 	allUserTags   map[string]string
 	FullSite      *FullSite
 	IDStr         string
+	loadedCustoms bool
 }
 
 type DeviceLabel struct {
@@ -174,6 +175,13 @@ func (d *Device) AddCustoms(dd *devicepb.DeviceDetailed) {
 	customs := mapCustomColumns(dd.GetCustomColumnData())
 	d.Customs = customs
 	d.CustomStr = dd.GetCustomColumns()
+	d.loadedCustoms = true
+}
+
+func (d *Device) LoadedCustoms() bool {
+	lc := d.loadedCustoms
+	d.loadedCustoms = true
+	return lc
 }
 
 func (d *Device) AddInterface(p *interfacepb.Interface) {
@@ -303,6 +311,7 @@ func MapDeviceDetailedToDevice(dd *devicepb.DeviceDetailed) (*Device, error) {
 		SnmpV3:        snmpV3,
 		Labels:        labels,
 		Site:          site,
+		loadedCustoms: len(customs) > 0,
 	}, nil
 }