mask answlogged secrets

Digrinch · Digrinch · commit 61502c13fbb3 · 2025-05-19T11:37:40.000+03:00
commit_hash:2410713f037a9e1aec73931e26cb2c528e98c1e8
diff --git a/.mapping.json b/.mapping.json
@@ -56,6 +56,7 @@
   "cli/expvar.go":"load/projects/pandora/cli/expvar.go",
   "components/answ/filter/const.go":"load/projects/pandora/components/answ/filter/const.go",
   "components/answ/filter/status_code_filter.go":"load/projects/pandora/components/answ/filter/status_code_filter.go",
+  "components/answ/import.go":"load/projects/pandora/components/answ/import.go",
   "components/answ/sampler/config.go":"load/projects/pandora/components/answ/sampler/config.go",
   "components/answ/sampler/status_code_sampler.go":"load/projects/pandora/components/answ/sampler/status_code_sampler.go",
   "components/answ/utils.go":"load/projects/pandora/components/answ/utils.go",
@@ -485,6 +486,7 @@
   "lib/answlog/config.go":"load/projects/pandora/lib/answlog/config.go",
   "lib/answlog/filter.go":"load/projects/pandora/lib/answlog/filter.go",
   "lib/answlog/logger.go":"load/projects/pandora/lib/answlog/logger.go",
+  "lib/answlog/masker.go":"load/projects/pandora/lib/answlog/masker.go",
   "lib/answlog/sampler.go":"load/projects/pandora/lib/answlog/sampler.go",
   "lib/confutil/chosen_cases_filter.go":"load/projects/pandora/lib/confutil/chosen_cases_filter.go",
   "lib/confutil/chosen_cases_filter_test.go":"load/projects/pandora/lib/confutil/chosen_cases_filter_test.go",
diff --git a/components/answ/import.go b/components/answ/import.go
@@ -0,0 +1,12 @@
+package answ
+
+import (
+	"github.com/yandex/pandora/core/register"
+	"github.com/yandex/pandora/lib/answlog"
+)
+
+func Import() {
+	register.Plugin[answlog.LogMasker]("pass", func() *answlog.PassAllMasker {
+		return &answlog.PassAllMasker{}
+	})
+}
diff --git a/components/grpc/import/import.go b/components/grpc/import/import.go
@@ -10,11 +10,9 @@ import (
 )
 
 func Import(fs afero.Fs) {
-
 	register.Provider("grpc/json", func(conf grpcjson.Config) core.Provider {
 		return grpcjson.NewProvider(fs, conf)
 	})
-
 	register.Gun("grpc", grpc.NewGun, grpc.DefaultGunConfig)
 	register.Gun("grpc/scenario", scenario.NewGun, scenario.DefaultGunConfig)
 }
diff --git a/components/guns/grpc/core.go b/components/guns/grpc/core.go
@@ -85,6 +85,7 @@ func DefaultGunConfig() GunConfig {
 					Factor: 10,
 				},
 			},
+			Masking: &answlog.PassAllMasker{},
 		},
 	}
 }
@@ -162,7 +163,13 @@ func (g *Gun) prepareClientPool() (*clientpool.Pool[grpcdynamic.Stub], error) {
 }
 
 func NewGun(conf GunConfig) *Gun {
-	answLog := answlog.Init(conf.AnswLog.Path, conf.AnswLog.Enabled, answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)), answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 20), conf.AnswLog.Sampling.Enabled))
+	answLog := answlog.Init(
+		conf.AnswLog.Path,
+		conf.AnswLog.Enabled,
+		answlog.WithFilter(filter.NewGRPCStatusCodeFilter(conf.AnswLog.Filter)),
+		answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 20), conf.AnswLog.Sampling.Enabled),
+		answlog.WithMasker(conf.AnswLog.Masking),
+	)
 	return &Gun{Conf: conf, AnswLog: answLog}
 }
 
diff --git a/components/guns/grpc/scenario/core.go b/components/guns/grpc/scenario/core.go
@@ -50,12 +50,19 @@ func DefaultGunConfig() GunConfig {
 					Factor: 10,
 				},
 			},
+			Masking: &answlog.PassAllMasker{},
 		},
 	}
 }
 
 func NewGun(conf GunConfig) *Gun {
-	answLog := answlog.Init(conf.AnswLog.Path, conf.AnswLog.Enabled, answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)), answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 20), conf.AnswLog.Sampling.Enabled))
+	answLog := answlog.Init(
+		conf.AnswLog.Path,
+		conf.AnswLog.Enabled,
+		answlog.WithFilter(filter.NewGRPCStatusCodeFilter(conf.AnswLog.Filter)),
+		answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 20), conf.AnswLog.Sampling.Enabled),
+		answlog.WithMasker(conf.AnswLog.Masking),
+	)
 	r := rand.New(rand.NewSource(0)) //TODO: use real random
 	return &Gun{
 		templ: NewTextTemplater(),
@@ -74,6 +81,7 @@ func NewGun(conf GunConfig) *Gun {
 				Path:     conf.AnswLog.Path,
 				Filter:   conf.AnswLog.Filter,
 				Sampling: conf.AnswLog.Sampling,
+				Masking:  conf.AnswLog.Masking,
 			},
 		},
 			AnswLog: answLog},
diff --git a/components/guns/http/connect.go b/components/guns/http/connect.go
@@ -12,6 +12,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/yandex/pandora/components/answ/filter"
 	"github.com/yandex/pandora/components/answ/sampler"
+	"github.com/yandex/pandora/core"
 	"github.com/yandex/pandora/lib/answlog"
 	"github.com/yandex/pandora/lib/netutil"
 )
@@ -24,6 +25,21 @@ func NewConnectGun(cfg GunConfig, answLog *answlog.Logger) *BaseGun {
 	return NewBaseGun(newConnectClient, cfg, answLog)
 }
 
+func NewConnectGunFactory(conf GunConfig) func() core.Gun {
+	conf.Target, _ = PreResolveTargetAddr(&conf.Client, conf.Target)
+	conf.TargetResolved = conf.Target
+	answLog := answlog.Init(
+		conf.AnswLog.Path,
+		conf.AnswLog.Enabled,
+		answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)),
+		answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 600), conf.AnswLog.Sampling.Enabled),
+		answlog.WithMasker(conf.AnswLog.Masking),
+	)
+	return func() core.Gun {
+		return WrapGun(NewConnectGun(conf, answLog))
+	}
+}
+
 func DefaultConnectGunConfig() GunConfig {
 	return GunConfig{
 		SSL:    false,
@@ -43,6 +59,7 @@ func DefaultConnectGunConfig() GunConfig {
 					Factor: 10,
 				},
 			},
+			Masking: &answlog.PassAllMasker{},
 		},
 		HTTPTrace: HTTPTraceConfig{
 			DumpEnabled:  false,
diff --git a/components/guns/http/http.go b/components/guns/http/http.go
@@ -4,6 +4,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/yandex/pandora/components/answ/filter"
 	"github.com/yandex/pandora/components/answ/sampler"
+	"github.com/yandex/pandora/core"
 	"github.com/yandex/pandora/lib/answlog"
 )
 
@@ -26,6 +27,19 @@ func NewHTTP1Gun(cfg GunConfig, answLog *answlog.Logger) *BaseGun {
 	return NewBaseGun(HTTP1ClientConstructor, cfg, answLog)
 }
 
+func NewHTTP1GunFactory(conf GunConfig) func() core.Gun {
+	targetResolved, _ := PreResolveTargetAddr(&conf.Client, conf.Target)
+	conf.TargetResolved = targetResolved
+	answLog := answlog.Init(
+		conf.AnswLog.Path,
+		conf.AnswLog.Enabled,
+		answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)),
+		answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 600), conf.AnswLog.Sampling.Enabled),
+		answlog.WithMasker(conf.AnswLog.Masking),
+	)
+	return func() core.Gun { return WrapGun(NewHTTP1Gun(conf, answLog)) }
+}
+
 func HTTP1ClientConstructor(clientConfig ClientConfig, target string) Client {
 	transport := NewTransport(clientConfig.Transport, NewDialer(clientConfig.Dialer).DialContext, target)
 	client := NewRedirectingClient(transport, clientConfig.Redirect)
@@ -43,6 +57,22 @@ func NewHTTP2Gun(cfg GunConfig, answLog *answlog.Logger) (*BaseGun, error) {
 	return NewBaseGun(HTTP2ClientConstructor, cfg, answLog), nil
 }
 
+func NewHTTP2GunFactory(conf GunConfig) func() (core.Gun, error) {
+	targetResolved, _ := PreResolveTargetAddr(&conf.Client, conf.Target)
+	conf.TargetResolved = targetResolved
+	answLog := answlog.Init(
+		conf.AnswLog.Path,
+		conf.AnswLog.Enabled,
+		answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)),
+		answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 600), conf.AnswLog.Sampling.Enabled),
+		answlog.WithMasker(conf.AnswLog.Masking),
+	)
+	return func() (core.Gun, error) {
+		gun, err := NewHTTP2Gun(conf, answLog)
+		return WrapGun(gun), err
+	}
+}
+
 func HTTP2ClientConstructor(clientConfig ClientConfig, target string) Client {
 	transport := NewHTTP2Transport(clientConfig.Transport, NewDialer(clientConfig.Dialer).DialContext, target)
 	client := NewRedirectingClient(transport, clientConfig.Redirect)
@@ -71,6 +101,7 @@ func DefaultHTTPGunConfig() GunConfig {
 					Factor: 10,
 				},
 			},
+			Masking: &answlog.PassAllMasker{},
 		},
 		HTTPTrace: HTTPTraceConfig{
 			DumpEnabled:  false,
@@ -97,6 +128,7 @@ func DefaultHTTP2GunConfig() GunConfig {
 					Factor: 10,
 				},
 			},
+			Masking: &answlog.PassAllMasker{},
 		},
 		HTTPTrace: HTTPTraceConfig{
 			DumpEnabled:  false,
diff --git a/components/phttp/import/import.go b/components/phttp/import/import.go
@@ -2,45 +2,19 @@ package phttp
 
 import (
 	"github.com/spf13/afero"
-	"github.com/yandex/pandora/components/answ/filter"
-	"github.com/yandex/pandora/components/answ/sampler"
 	phttp "github.com/yandex/pandora/components/guns/http"
 	scenarioGun "github.com/yandex/pandora/components/guns/http_scenario"
 	httpProvider "github.com/yandex/pandora/components/providers/http"
 	scenarioProvider "github.com/yandex/pandora/components/providers/scenario/import"
-	"github.com/yandex/pandora/core"
 	"github.com/yandex/pandora/core/register"
-	"github.com/yandex/pandora/lib/answlog"
 )
 
 func Import(fs afero.Fs) {
 	httpProvider.Import(fs)
 	scenarioGun.Import(fs)
 	scenarioProvider.Import(fs)
 
-	register.Gun("http", func(conf phttp.GunConfig) func() core.Gun {
-		targetResolved, _ := phttp.PreResolveTargetAddr(&conf.Client, conf.Target)
-		conf.TargetResolved = targetResolved
-		answLog := answlog.Init(conf.AnswLog.Path, conf.AnswLog.Enabled, answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)), answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 600), conf.AnswLog.Sampling.Enabled))
-		return func() core.Gun { return phttp.WrapGun(phttp.NewHTTP1Gun(conf, answLog)) }
-	}, phttp.DefaultHTTPGunConfig)
-
-	register.Gun("http2", func(conf phttp.GunConfig) func() (core.Gun, error) {
-		targetResolved, _ := phttp.PreResolveTargetAddr(&conf.Client, conf.Target)
-		conf.TargetResolved = targetResolved
-		answLog := answlog.Init(conf.AnswLog.Path, conf.AnswLog.Enabled, answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)), answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 600), conf.AnswLog.Sampling.Enabled))
-		return func() (core.Gun, error) {
-			gun, err := phttp.NewHTTP2Gun(conf, answLog)
-			return phttp.WrapGun(gun), err
-		}
-	}, phttp.DefaultHTTP2GunConfig)
-
-	register.Gun("connect", func(conf phttp.GunConfig) func() core.Gun {
-		conf.Target, _ = phttp.PreResolveTargetAddr(&conf.Client, conf.Target)
-		conf.TargetResolved = conf.Target
-		answLog := answlog.Init(conf.AnswLog.Path, conf.AnswLog.Enabled, answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)), answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 600), conf.AnswLog.Sampling.Enabled))
-		return func() core.Gun {
-			return phttp.WrapGun(phttp.NewConnectGun(conf, answLog))
-		}
-	}, phttp.DefaultConnectGunConfig)
+	register.Gun("http", phttp.NewHTTP1GunFactory, phttp.DefaultHTTPGunConfig)
+	register.Gun("http2", phttp.NewHTTP2GunFactory, phttp.DefaultHTTP2GunConfig)
+	register.Gun("connect", phttp.NewConnectGunFactory, phttp.DefaultConnectGunConfig)
 }
diff --git a/core/register/register.go b/core/register/register.go
@@ -38,3 +38,8 @@ func DataSink(name string, newDataSink interface{}, defaultConfigOptional ...int
 	var ptr *core.DataSink
 	RegisterPtr(ptr, name, newDataSink, defaultConfigOptional...)
 }
+
+func Plugin[T interface{}](name string, newPlugin interface{}, defaultConfigOptional ...interface{}) {
+	var ptr *T
+	RegisterPtr(ptr, name, newPlugin, defaultConfigOptional...)
+}
diff --git a/lib/answlog/config.go b/lib/answlog/config.go
@@ -1,10 +1,11 @@
 package answlog
 
 type Config struct {
-	Enabled  bool     `config:"enabled"`
-	Path     string   `config:"path"`
-	Filter   string   `config:"filter" valid:"oneof=all warning error"`
-	Sampling Sampling `config:"sampling"`
+	Enabled  bool      `config:"enabled"`
+	Path     string    `config:"path"`
+	Filter   string    `config:"filter" valid:"oneof=all warning error"`
+	Sampling Sampling  `config:"sampling"`
+	Masking  LogMasker `config:"masking"`
 }
 
 type Sampling struct {
diff --git a/lib/answlog/logger.go b/lib/answlog/logger.go
@@ -1,6 +1,7 @@
 package answlog
 
 import (
+	"fmt"
 	"os"
 
 	"go.uber.org/zap"
@@ -11,15 +12,17 @@ type Logger struct {
 	logger *zap.Logger
 	LogSampler
 	LogFilter
+	LogMasker
 	loggingEnabled  bool
 	samplingEnabled bool
 }
 
-func NewLogger(logger *zap.Logger, sampler LogSampler, filter LogFilter, loggingEnabled bool, samplingEnabled bool) *Logger {
+func NewLogger(logger *zap.Logger, sampler LogSampler, filter LogFilter, masker LogMasker, loggingEnabled bool, samplingEnabled bool) *Logger {
 	return &Logger{
 		logger:          logger,
 		LogSampler:      sampler,
 		LogFilter:       filter,
+		LogMasker:       masker,
 		loggingEnabled:  loggingEnabled,
 		samplingEnabled: samplingEnabled,
 	}
@@ -30,6 +33,7 @@ func NewNop() *Logger {
 		logger:     zap.NewNop(),
 		LogSampler: &PassAllSampler{},
 		LogFilter:  &PassAllFilter{},
+		LogMasker:  &PassAllMasker{},
 	}
 }
 
@@ -43,6 +47,12 @@ func (l *Logger) Report(msg string, fields []zapcore.Field, lazyFields func() []
 		fields = append(fields, addFields...)
 	}
 
+	fields, err := l.LogMasker.Mask(fields)
+	if err != nil {
+		l.logger.Debug(fmt.Sprintf("Some error occurred while trying to mask fields: %s. Skipping log entry", err))
+		return
+	}
+
 	l.logger.Debug(msg, fields...)
 }
 
@@ -61,6 +71,12 @@ func WithFilter(filter LogFilter) LoggerOpt {
 	}
 }
 
+func WithMasker(masker LogMasker) LoggerOpt {
+	return func(l *Logger) {
+		l.LogMasker = masker
+	}
+}
+
 func Init(path string, enabled bool, opts ...LoggerOpt) *Logger {
 	zapLog := zap.NewNop()
 	if enabled {
@@ -72,7 +88,7 @@ func Init(path string, enabled bool, opts ...LoggerOpt) *Logger {
 		defer zapLog.Sync()
 	}
 
-	logger := NewLogger(zapLog, &PassAllSampler{}, &PassAllFilter{}, enabled, false)
+	logger := NewLogger(zapLog, &PassAllSampler{}, &PassAllFilter{}, &PassAllMasker{}, enabled, false)
 	for _, opt := range opts {
 		opt(logger)
 	}
diff --git a/lib/answlog/masker.go b/lib/answlog/masker.go
@@ -0,0 +1,11 @@
+package answlog
+
+import "go.uber.org/zap/zapcore"
+
+type LogMasker interface {
+	Mask(fields []zapcore.Field) ([]zapcore.Field, error)
+}
+
+type PassAllMasker struct{}
+
+func (m *PassAllMasker) Mask(fields []zapcore.Field) ([]zapcore.Field, error) { return fields, nil }
diff --git a/main.go b/main.go
@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/spf13/afero"
 	"github.com/yandex/pandora/cli"
+	"github.com/yandex/pandora/components/answ"
 	grpc "github.com/yandex/pandora/components/grpc/import"
 	"github.com/yandex/pandora/components/guns"
 	phttp "github.com/yandex/pandora/components/phttp/import"
@@ -17,6 +18,7 @@ func main() {
 	phttp.Import(fs)
 	grpc.Import(fs)
 	guns.Import(fs)
+	answ.Import()
 
 	cli.Run()
 }

Original file line number	Diff line number	Diff line change
`@@ -10,11 +10,9 @@ import (`
`10`	`10`	`)`
`11`	`11`
`12`	`12`	`func Import(fs afero.Fs) {`
`13`		`-`
`14`	`13`	`register.Provider("grpc/json", func(conf grpcjson.Config) core.Provider {`
`15`	`14`	`return grpcjson.NewProvider(fs, conf)`
`16`	`15`	`})`
`17`		`-`
`18`	`16`	`register.Gun("grpc", grpc.NewGun, grpc.DefaultGunConfig)`
`19`	`17`	`register.Gun("grpc/scenario", scenario.NewGun, scenario.DefaultGunConfig)`
`20`	`18`	`}`
Original file line number	Diff line number	Diff line change
`@@ -85,6 +85,7 @@ func DefaultGunConfig() GunConfig {`
`85`	`85`	`Factor: 10,`
`86`	`86`	`},`
`87`	`87`	`},`
	`88`	`+ Masking: &answlog.PassAllMasker{},`
`88`	`89`	`},`
`89`	`90`	`}`
`90`	`91`	`}`
`@@ -162,7 +163,13 @@ func (g Gun) prepareClientPool() (clientpool.Pool[grpcdynamic.Stub], error) {`
`162`	`163`	`}`
`163`	`164`
`164`	`165`	`func NewGun(conf GunConfig) *Gun {`
`165`		`- answLog := answlog.Init(conf.AnswLog.Path, conf.AnswLog.Enabled, answlog.WithFilter(filter.NewHTTPStatusCodeFilter(conf.AnswLog.Filter)), answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 20), conf.AnswLog.Sampling.Enabled))`
	`166`	`+ answLog := answlog.Init(`
	`167`	`+ conf.AnswLog.Path,`
	`168`	`+ conf.AnswLog.Enabled,`
	`169`	`+ answlog.WithFilter(filter.NewGRPCStatusCodeFilter(conf.AnswLog.Filter)),`
	`170`	`+ answlog.WithSampler(sampler.NewStatusCodeSampler(conf.AnswLog.Sampling.Pattern, 20), conf.AnswLog.Sampling.Enabled),`
	`171`	`+ answlog.WithMasker(conf.AnswLog.Masking),`
	`172`	`+ )`
`166`	`173`	`return &Gun{Conf: conf, AnswLog: answLog}`
`167`	`174`	`}`
`168`	`175`
Original file line number	Diff line number	Diff line change
`@@ -38,3 +38,8 @@ func DataSink(name string, newDataSink interface{}, defaultConfigOptional ...int`
`38`	`38`	`var ptr *core.DataSink`
`39`	`39`	`RegisterPtr(ptr, name, newDataSink, defaultConfigOptional...)`
`40`	`40`	`}`
	`41`	`+`
	`42`	`+func Plugin[T interface{}](name string, newPlugin interface{}, defaultConfigOptional ...interface{}) {`
	`43`	`+ var ptr *T`
	`44`	`+ RegisterPtr(ptr, name, newPlugin, defaultConfigOptional...)`
	`45`	`+}`