remove trash, tiny fixes

Author: hrissan

internal/aggregator/aggregator_handlers.go

@@ -646,10 +646,6 @@ func (a *Aggregator) handleSendSourceBucket(hctx *rpc.HandlerContext, args tlsta
 		[]int32{0, format.TagValueIDComponentAggregator, format.TagValueIDMappingCacheEventMiss},
 		float64(mappingMisses), hostTag, aera)
 
-	if !configR.EnableMappingAfterSampling {
-		a.addUnknownTags(unknownTags, aggBucket.time, hostTag, aera)
-	}
-
 	a.sh2.AddValueCounterHostAERA(args.Time, format.BuiltinMetricMetaAggSizeCompressed,
 		[]int32{0, 0, 0, 0, conveyor, spare},
 		float64(compressedSize), 1, hostTag, aera)
@@ -735,25 +731,25 @@ func (a *Aggregator) addUnknownTags(unknownTags map[string]data_model.CreateMapp
 	unknownMapRemove, unknownMapAdd, createMapAdd, avgRemovedHits, avgRemovedTotal := a.tagsMapper3.AddUnknownTags(unknownTags, time)
 
 	if unknownMapRemove != 0 {
-		a.sh2.AddCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueEvent,
+		a.sh2.AddValueCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueEvent,
 			[]int32{0, 0, format.TagValueIDMappingQueueEventUnknownMapRemove},
-			float64(unknownMapRemove), hostTag, aera)
-		a.sh2.AddCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueRemovedHitsAvg,
+			float64(unknownMapRemove), 1, hostTag, aera)
+		a.sh2.AddValueCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueRemovedHitsAvg,
 			[]int32{},
-			avgRemovedHits, hostTag, aera)
-		a.sh2.AddCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueRemovedTotalAvg,
+			avgRemovedHits, 1, hostTag, aera)
+		a.sh2.AddValueCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueRemovedTotalAvg,
 			[]int32{},
-			avgRemovedTotal, hostTag, aera)
+			avgRemovedTotal, 1, hostTag, aera)
 	}
 	if unknownMapAdd != 0 {
-		a.sh2.AddCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueEvent,
+		a.sh2.AddValueCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueEvent,
 			[]int32{0, 0, format.TagValueIDMappingQueueEventUnknownMapAdd},
-			float64(unknownMapAdd), hostTag, aera)
+			float64(unknownMapAdd), 1, hostTag, aera)
 	}
 	if createMapAdd != 0 {
-		a.sh2.AddCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueEvent,
+		a.sh2.AddValueCounterHostAERA(time, format.BuiltinMetricMetaMappingQueueEvent,
 			[]int32{0, 0, format.TagValueIDMappingQueueEventCreateMapAdd},
-			float64(createMapAdd), hostTag, aera)
+			float64(createMapAdd), 1, hostTag, aera)
 	}
 }
 

internal/aggregator/aggregator_insert.go

@@ -335,9 +335,6 @@ func (a *Aggregator) rowDataMarshalAppendPositions(buckets []*aggregatorBucket,
 
 	recentTs := buckets[0].time // by convention first bucket is recent, all others are historic
 	recentUnknownTags := buckets[0].unknownTags
-	if !configR.EnableMappingAfterSampling {
-		recentUnknownTags = nil // make [] extremely fast NOP in processUnknownTag
-	}
 	historicTag := int32(format.TagValueIDConveyorRecent)
 	if len(buckets) > 1 {
 		historicTag = format.TagValueIDConveyorHistoric
@@ -529,9 +526,7 @@ func (a *Aggregator) rowDataMarshalAppendPositions(buckets []*aggregatorBucket,
 		key = data_model.Key{Timestamp: t, Metric: format.BuiltinMetricIDContributorsLogRev, Tags: [format.MaxTags]int32{0, int32(insertTimeUnix)}}
 		res = appendSimpleValueStat(rnd, res, &key, float64(insertTimeUnix)-float64(t), 1, a.aggregatorHostTag.I, metricCache, unknownTags, recentUnknownTags)
 	}
-	if configR.EnableMappingAfterSampling {
-		a.addUnknownTags(unknownTags, recentTs, a.aggregatorHostTag, data_model.AgentEnvRouteArch{}) // no agent info here
-	}
+	a.addUnknownTags(unknownTags, recentTs, a.aggregatorHostTag, data_model.AgentEnvRouteArch{}) // no agent info here
 	return res, sampler.SamplerBuffers, stats, time.Since(startTime)
 }
 

internal/aggregator/config.go

@@ -46,7 +46,6 @@ type ConfigAggregatorRemote struct {
 	MigrationV3DisabledShards  map[int32]struct{}
 	MigrationDelaySec          int // delay in seconds between migration steps
 	ClusterShardsAddrs         []string
-	EnableMappingAfterSampling bool
 	DisableReceiveSampleBudget bool
 	OriginalSizeDecayHalfLife  time.Duration
 
@@ -211,13 +210,10 @@ func (c *ConfigAggregatorRemote) Bind(f *flag.FlagSet, d ConfigAggregatorRemote,
 		f.Func("migration-v3-disabled-shards", "List of disabled shards for migration v3", c.setMigrationV3DisabledShards)
 		f.IntVar(&c.MigrationDelaySec, "migration-delay-sec", d.MigrationDelaySec, "Delay in seconds between migration steps")
 
-		var enableMappingStorage bool
-		f.BoolVar(&enableMappingStorage, "enable-mapping-storage", false, "Enable full mapping inmemory&disk storage")
-		f.BoolVar(&c.EnableMappingAfterSampling, "enable-mapping-after-sampling", d.EnableMappingAfterSampling, "Enable mapping after sampling")
+		var enableMappingAfterSampling bool
+		f.BoolVar(&enableMappingAfterSampling, "enable-mapping-after-sampling", false, "Enable mapping after sampling")
 		f.IntVar(&c.MaxUnknownTagsInBucket, "mapping-queue-max-unknown-tags-in-bucket", d.MaxUnknownTagsInBucket, "Max unknown tags per bucket to add to mapping queue.")
 		f.IntVar(&c.MaxCreateTagsPerIteration, "mapping-queue-create-tags-per-iteration", d.MaxCreateTagsPerIteration, "Mapping queue will create no more tags per iteration (roughly second).")
-		var maxLoadTagsPerIteration int
-		f.IntVar(&maxLoadTagsPerIteration, "mapping-queue-load-tags-per-iteration", 0, "Mapping queue will load no more tags per iteration (roughly second).")
 		f.IntVar(&c.TagHitsToCreate, "mapping-queue-hits-to-create", d.TagHitsToCreate, "Tag mapping will be created if it is used in so many different seconds.")
 		f.IntVar(&c.TagTotalToCreate, "mapping-queue-total-to-create", d.TagTotalToCreate, "Tag mapping will be created if it is used so many times.")
 		f.IntVar(&c.MaxUnknownTagsToKeep, "mapping-queue-max-unknown-tags-to-keep", d.MaxUnknownTagsToKeep, "Mapping queue will remember and collect hits on so many different strings.")

internal/aggregator/tags-mapper3.go

@@ -24,12 +24,12 @@ type unknownTag struct { // fits well into cache line
 }
 
 type configTagsMapper3 struct {
-	MaxUnknownTagsInBucket    int // keep for low at first, then increase gradually
-	MaxCreateTagsPerIteration int // keep for low at first, then increase gradually
+	MaxUnknownTagsInBucket    int // limit # of allocations during aggregation
+	MaxCreateTagsPerIteration int // limit to protect against all kind of errors
 	TagHitsToCreate           int // if used in N different seconds, then create
-	TagTotalToCreate          int // if used K times, then create
+	TagTotalToCreate          int // if inserted in database K times, then create
 	MaxUnknownTagsToKeep      int
-	KeepTime                  int // if not used for the long time, it is removed
+	KeepTime                  int // if not used for the long time, it is removed from unknownTags
 	MaxSendTagsToAgent        int
 }