diff --git a/go.mod b/go.mod index 95bb88ec9..3c0ca46de 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,6 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Microsoft/hcsshim v0.11.4 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/containerd/continuity v0.4.2 // indirect github.com/containerd/fifo v1.1.0 // indirect @@ -62,7 +61,6 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/uuid v1.6.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -85,8 +83,6 @@ require ( github.com/stretchr/objx v0.5.2 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasthttp v1.49.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.45.0 // indirect @@ -99,7 +95,6 @@ require ( golang.org/x/sync v0.10.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 // indirect google.golang.org/grpc v1.65.0 // indirect diff --git a/go.sum b/go.sum index 9ece40c4b..c88688754 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,6 @@ github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7 github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go v1.44.69 h1:3A3DEizrCK6dAbBoRGh8KmoZij7She9snclG1ixY/xQ= @@ -86,7 +84,6 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= @@ -97,9 +94,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -115,8 +109,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= @@ -159,8 +151,6 @@ github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= -github.com/newrelic/go-agent/v3 v3.27.0 h1:Z3XB49d8FKjRcGzCyViCO9itBxiLPSpwjY1HlMvgamQ= -github.com/newrelic/go-agent/v3 v3.27.0/go.mod h1:TUzePinDc0BMH4Sui66rl4SBe6yOKJ5X/bRJekwuAtM= github.com/newrelic/go-agent/v3 v3.36.0 h1:PdvJZiUu45qg2qrOeia0gYr1vaZ2Ro6QIlqlgHObHXo= github.com/newrelic/go-agent/v3 v3.36.0/go.mod h1:GNTda53CohAhkgsc7/gqSsJhDZjj8vaky5u+vKz7wqM= github.com/newrelic/infra-identity-client-go v1.0.2 h1:RxkHAFOqeiBkW/fXvzkp3uORPl6crFt4I/zf4dCUFKM= @@ -216,10 +206,6 @@ github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFA github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.49.0 h1:9FdvCpmxB74LH4dPb7IJ1cOSsluR07XG3I1txXWwJpE= -github.com/valyala/fasthttp v1.49.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= @@ -263,7 +249,6 @@ golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -273,8 +258,6 @@ golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -309,7 +292,6 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= @@ -332,18 +314,14 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 h1:Z0hjGZePRE0ZBWotvtrwxFNrNE9CUAGtplaDK5NNI/g= google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98/go.mod h1:S7mY02OqCJTD0E1OiQy1F72PWFB4bZJ87cAtLPYgDR0= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 h1:FmF5cCW94Ij59cfpoLiwTgodWmm60eEV0CjlsVg2fuw= -google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 h1:bVf09lpb+OJbByTj913DRJioFFAjf/ZGxEz7MajTp2U= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98/go.mod h1:TUfxEVdsvPg18p6AslUXFoLdpED4oBnGwyqk3dV1XzM= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157 h1:7whR9kGa5LUwFtpLm2ArCEejtnxlGeLbAyjFY8sGNFw= +google.golang.org/genproto/googleapis/api v0.0.0-20240528184218-531527333157/go.mod h1:99sLkeliLXfdj2J75X3Ho+rrVCaJze0uwN7zDDkjPVU= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157 h1:Zy9XzmMEflZ/MAaA7vNcoebnRAld7FsPW1EeBB7V0m8= google.golang.org/genproto/googleapis/rpc v0.0.0-20240528184218-531527333157/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -352,8 +330,6 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= @@ -366,10 +342,7 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/agent/cmdchannel/fflag/ffhandler.go b/internal/agent/cmdchannel/fflag/ffhandler.go index 6dd610e7d..4974516b9 100644 --- a/internal/agent/cmdchannel/fflag/ffhandler.go +++ b/internal/agent/cmdchannel/fflag/ffhandler.go @@ -28,6 +28,7 @@ const ( FlagDmRegisterDeprecated = "dm_register_deprecated" FlagFluentBit19 = "fluent_bit_19_win" FlagFullInventoryDeletion = "full_inventory_deletion" + FlagFluentBitMetrics = "fluent_bit_metrics" // Config CfgYmlRegisterEnabled = "register_enabled" CfgYmlParallelizeInventory = "inventory_queue_len" @@ -185,7 +186,8 @@ func isBasicFeatureFlag(flag string) bool { return flag == FlagProtocolV4 || flag == FlagFullProcess || flag == FlagDmRegisterDeprecated || - flag == FlagFullInventoryDeletion + flag == FlagFullInventoryDeletion || + flag == FlagFluentBitMetrics } func (h *handler) setFFConfig(ff string, enabled bool) { diff --git a/pkg/integrations/v4/logs/cfg.go b/pkg/integrations/v4/logs/cfg.go index 3deb12536..9f00800a4 100644 --- a/pkg/integrations/v4/logs/cfg.go +++ b/pkg/integrations/v4/logs/cfg.go @@ -5,30 +5,47 @@ package logs import ( "bytes" "fmt" - "github.com/newrelic/infrastructure-agent/pkg/config" - "github.com/newrelic/infrastructure-agent/pkg/license" - "github.com/newrelic/infrastructure-agent/pkg/log" - "github.com/pkg/errors" "io/ioutil" "path/filepath" "regexp" + "runtime" "strconv" "strings" "text/template" + + "github.com/newrelic/infrastructure-agent/internal/agent/cmdchannel/fflag" + "github.com/newrelic/infrastructure-agent/internal/feature_flags" + "github.com/newrelic/infrastructure-agent/pkg/config" + "github.com/newrelic/infrastructure-agent/pkg/license" + "github.com/newrelic/infrastructure-agent/pkg/log" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) var cfgLogger = log.WithComponent("integrations.Supervisor.Config").WithField("process", "log-forwarder") // FluentBit default values. const ( - euEndpoint = "https://log-api.eu.newrelic.com/log/v1" - fedrampEndpoint = "https://gov-log-api.newrelic.com/log/v1" - stagingEndpoint = "https://staging-log-api.newrelic.com/log/v1" - logRecordModifierSource = "nri-agent" - defaultBufferMaxSize = 128 - memBufferLimit = 16384 - fbFileWatchLimit = 1024 - fluentBitDbName = "fb.db" + euEndpoint = "https://log-api.eu.newrelic.com/log/v1" + fedrampEndpoint = "https://gov-log-api.newrelic.com/log/v1" + stagingEndpoint = "https://staging-log-api.newrelic.com/log/v1" + stagingMetricsEndpoint = "staging-metric-api.newrelic.com" + productionMetricsEndpoint = "metric-api.newrelic.com" + productionEuMetricsEndpoint = "metric-api.eu.newrelic.com" + fedRampMetricsEndpoint = "gov-metric-api.newrelic.com" + logRecordModifierSource = "nri-agent" + defaultBufferMaxSize = 128 + memBufferLimit = 16384 + fbFileWatchLimit = 1024 + fluentBitDBName = "fb.db" + fbDefaultPort = 2020 + fbDefaultOutputPort = 443 + fbHostPort = "127.0.0.1" + fbMetricsPath = "/api/v2/metrics/prometheus" + fbScrapeInterval = "60s" + fbMetricsTag = "fb_metrics" + fbLabel = "fluent-bit" + fbhostLabel = "host" ) // FluentBit INPUT plugin types @@ -141,12 +158,23 @@ func (l *LogCfg) IsValid() bool { return l.Name != "" && (l.File != "" || l.Systemd != "" || l.Syslog != nil || l.Tcp != nil || l.Fluentbit != nil || l.Winlog != nil || l.Winevtlog != nil) } +type FBCfgService struct { + Flush int + LogLevel string + Daemon string + ParsersFile string + HTTPServer string + HTTPListen string + HTTPPort int +} + // FBCfg FluentBit automatically generated configuration. type FBCfg struct { Inputs []FBCfgInput Filters []FBCfgFilter ExternalCfg FBCfgExternal - Output FBCfgOutput + Service []FBCfgService + Output []FBCfgOutput } // Format will return the FBCfg in the fluent bit config file format. @@ -205,6 +233,11 @@ type FBCfgInput struct { TcpSeparator string // plugin: tcp TcpBufferSize int // plugin: tcp (note that the "tcp" plugin uses Buffer_Size (without "k"s!) instead of Buffer_Max_Size (with "k"s!)) UseANSI string // plugin: winlog and winevtlog + Alias string // plugin: prometheus + Host string + Port int + MetricsPath string + ScrapeInterval string } // FBCfgFilter FluentBit FILTER config block, only "grep" plugin supported. @@ -237,6 +270,14 @@ type FBCfgOutput struct { ValidateCerts bool Retry_Limit string SendMetrics bool + Alias string + Host string + Port int + URI string + Header string + TLS string + TLSVerify string + AddLabel map[string]string } type FBWinlogLuaScript struct { @@ -272,7 +313,7 @@ type FBOSConfig struct { } // NewFBConf creates a FluentBit config from several logging integration configs. -func NewFBConf(loggingCfgs LogsCfg, logFwdCfg *config.LogForward, entityGUID, hostname string) (fb FBCfg, e error) { +func NewFBConf(loggingCfgs LogsCfg, logFwdCfg *config.LogForward, entityGUID, hostname string, ff feature_flags.Retriever) (fb FBCfg, e error) { fb = FBCfg{ Inputs: []FBCfgInput{}, Filters: []FBCfgFilter{}, @@ -282,6 +323,9 @@ func NewFBConf(loggingCfgs LogsCfg, logFwdCfg *config.LogForward, entityGUID, ho var fbOSConfig FBOSConfig addOSDependantConfig(&fbOSConfig) + // enableMetrics := isMetricsEnabled(ff) + enableMetrics := true + totalFiles := 0 for i, block := range loggingCfgs { loggingCfgs[i].targetFilesCnt = getTotalTargetFilesForPath(block) @@ -336,10 +380,55 @@ func NewFBConf(loggingCfgs LogsCfg, logFwdCfg *config.LogForward, entityGUID, ho }, }) - // Newrelic OUTPUT plugin will send all the collected logs to Vortex - fb.Output = newNROutput(logFwdCfg) + // Including Prometheus scrapper input plugin by default to pull Fluent bit metrics based on ff + includePrometheusScrapperInputPlugin(&fb, enableMetrics) + + // Including service to expose port , Prometheus metric collection needs the HTTP server to be online at port 2020 + includeService(&fb, enableMetrics) + + // Newrelic OUTPUT plugin which will send all the collected logs to Vortex and with Prometheus output plugin which forwards FB metrics to NR + fb.Output = newNROutput(logFwdCfg, hostname, enableMetrics) + + return fb, nil +} + +func isMetricsEnabled(ff feature_flags.Retriever) bool { + if ff != nil { + enabled, exists := ff.GetFeatureFlag(fflag.FlagFluentBitMetrics) - return + return enabled && exists + } + + return false +} + +func includeService(fb *FBCfg, enableMetrics bool) { + if enableMetrics { + fb.Service = []FBCfgService{{ + Flush: 1, + LogLevel: "info", + Daemon: "off", + ParsersFile: "parsers.conf", + HTTPServer: "On", + HTTPListen: "0.0.0.0", + HTTPPort: fbDefaultPort, + }, + } + } +} + +func includePrometheusScrapperInputPlugin(fb *FBCfg, enableMetrics bool) { + if enableMetrics { + fb.Inputs = append(fb.Inputs, FBCfgInput{ //nolint:exhaustruct + Name: "prometheus_scrape", + Alias: "fb-metrics-collector", + Host: fbHostPort, + Port: fbDefaultPort, + Tag: fbMetricsTag, + MetricsPath: fbMetricsPath, + ScrapeInterval: fbScrapeInterval, + }) + } } func getTotalTargetFilesForPath(l LogCfg) int { @@ -361,7 +450,7 @@ func parseConfigBlock(l LogCfg, logsHomeDir string, fbOSConfig FBOSConfig) (inpu return } - dbPath := filepath.Join(logsHomeDir, fluentBitDbName) + dbPath := filepath.Join(logsHomeDir, fluentBitDBName) if l.File != "" { input, filters = parseFileInput(l, dbPath) @@ -722,33 +811,79 @@ func newModifyFilter(tag string) FBCfgFilter { } } -func newNROutput(cfg *config.LogForward) FBCfgOutput { - ret := FBCfgOutput{ - Name: "newrelic", - Match: "*", - LicenseKey: cfg.License, - IgnoreSystemProxy: cfg.ProxyCfg.IgnoreSystemProxy, - Proxy: cfg.ProxyCfg.Proxy, - CABundleFile: cfg.ProxyCfg.CABundleFile, - CABundleDir: cfg.ProxyCfg.CABundleDir, - ValidateCerts: cfg.ProxyCfg.ValidateCerts, - Retry_Limit: cfg.RetryLimit, - SendMetrics: cfg.FluentBitVerbose, +func getSystemInfo() (string, string, error) { + os := runtime.GOOS + arch := runtime.GOARCH + return os, arch, nil +} + +func newNROutput(cfg *config.LogForward, hostname string, enableMetrics bool) []FBCfgOutput { + os, arch, err := getSystemInfo() + if err != nil { + logrus.WithError(err).Error(fmt.Sprintf("Error retrieving system info: OS: %s, Hostname: %s, Error: %v\n", os, hostname, err)) + } + outputs := []FBCfgOutput{ + { + Name: "newrelic", + Match: "*", + LicenseKey: cfg.License, + IgnoreSystemProxy: cfg.ProxyCfg.IgnoreSystemProxy, + Proxy: cfg.ProxyCfg.Proxy, + CABundleFile: cfg.ProxyCfg.CABundleFile, + CABundleDir: cfg.ProxyCfg.CABundleDir, + ValidateCerts: cfg.ProxyCfg.ValidateCerts, + Retry_Limit: cfg.RetryLimit, + SendMetrics: cfg.FluentBitVerbose, + }, + } + + if enableMetrics { + outputs = append(outputs, FBCfgOutput{ + Name: "prometheus_remote_write", + Match: fbMetricsTag, + Alias: "fb-metrics-forwarder", + Port: fbDefaultOutputPort, + URI: fmt.Sprintf("/prometheus/v1/write?prometheus_server=%s", hostname), + Header: "Authorization Bearer ${NR_LICENSE_KEY_ENV_VAR}", + TLS: "On", + Host: productionMetricsEndpoint, + TLSVerify: "Off", + AddLabel: map[string]string{ + "app": fbLabel, + "source": fbhostLabel, + "os": os, + "hostname": hostname, + "arch": arch, + // TODO Update this with the actual host ID + "host.id": "12345", + }, + }) } if cfg.IsStaging { - ret.Endpoint = stagingEndpoint + outputs[0].Endpoint = stagingEndpoint + if enableMetrics { + outputs[1].Host = stagingMetricsEndpoint + } + } if cfg.IsFedramp { - ret.Endpoint = fedrampEndpoint + outputs[0].Endpoint = fedrampEndpoint + if enableMetrics { + outputs[1].Host = fedRampMetricsEndpoint + } } if license.IsRegionEU(cfg.License) { - ret.Endpoint = euEndpoint + outputs[0].Endpoint = euEndpoint + if enableMetrics { + outputs[1].Host = productionEuMetricsEndpoint + } + } - return ret + return outputs } func getBufferMaxSize(l LogCfg) int { diff --git a/pkg/integrations/v4/logs/cfg_template.go b/pkg/integrations/v4/logs/cfg_template.go index 86a3cede3..ca3ec30ef 100644 --- a/pkg/integrations/v4/logs/cfg_template.go +++ b/pkg/integrations/v4/logs/cfg_template.go @@ -74,6 +74,21 @@ var fbConfigFormat = `{{- range .Inputs }} {{- if .UseANSI }} Use_ANSI {{ .UseANSI }} {{- end }} + {{- if .Alias }} + Alias {{ .Alias }} + {{- end }} + {{- if .Port }} + Port {{ .Port }} + {{- end }} + {{- if .Host }} + Host {{ .Host }} + {{- end }} + {{- if .MetricsPath }} + Metrics_Path {{ .MetricsPath }} + {{- end }} + {{- if .ScrapeInterval }} + Scrape_Interval {{ .ScrapeInterval }} + {{- end }} {{ end -}} {{- range .Filters }} @@ -105,37 +120,91 @@ var fbConfigFormat = `{{- range .Inputs }} {{- end }} {{ end -}} -{{- if .Output }} +{{- range .Service }} +[SERVICE] + {{- if .Flush }} + Flush {{ .Flush }} + {{- end }} + {{- if .LogLevel }} + Log_Level {{ .LogLevel }} + {{- end }} + {{- if .Daemon }} + Daemon {{ .Daemon }} + {{- end }} + {{- if .ParsersFile }} + Parsers_File {{ .ParsersFile }} + {{- end }} + {{- if .HTTPServer }} + HTTP_Server {{ .HTTPServer }} + {{- end }} + {{- if .HTTPListen }} + HTTP_Listen {{ .HTTPListen }} + {{- end }} + {{- if .HTTPPort }} + HTTP_Port {{ .HTTPPort }} + {{- end }} +{{ end -}} + + +{{- range .Output }} [OUTPUT] - Name {{ .Output.Name }} - Match {{ .Output.Match }} - {{- if .Output.LicenseKey }} + Name {{ .Name }} + Match {{ .Match }} + {{- if .LicenseKey }} licenseKey ${NR_LICENSE_KEY_ENV_VAR} {{- end }} - {{- if .Output.Endpoint }} - endpoint {{ .Output.Endpoint }} + {{- if .Endpoint }} + endpoint {{ .Endpoint }} {{- end }} - {{- if .Output.Proxy }} - proxy {{ .Output.Proxy }} + {{- if .Proxy }} + proxy {{ .Proxy }} {{- end }} - {{- if .Output.IgnoreSystemProxy }} + {{- if .IgnoreSystemProxy }} ignoreSystemProxy true {{- end }} - {{- if .Output.CABundleFile }} - caBundleFile {{ .Output.CABundleFile }} + {{- if .CABundleFile }} + caBundleFile {{ .CABundleFile }} {{- end }} - {{- if .Output.CABundleDir }} - caBundleDir {{ .Output.CABundleDir }} + {{- if .CABundleDir }} + caBundleDir {{ .CABundleDir }} {{- end }} - {{- if not .Output.ValidateCerts }} + {{- if not .ValidateCerts }} + {{- if eq .Name "newrelic" }} validateProxyCerts false {{- end }} - {{- if .Output.Retry_Limit}} - Retry_Limit {{ .Output.Retry_Limit }} {{- end }} - {{- if .Output.SendMetrics}} - sendMetrics {{ .Output.SendMetrics}} + {{- if .Retry_Limit}} + Retry_Limit {{ .Retry_Limit }} + {{- end }} + {{- if .SendMetrics}} + sendMetrics {{ .SendMetrics}} {{- end}} + {{- if .Alias }} + Alias {{ .Alias }} + {{- end }} + {{- if .Host }} + Host {{ .Host }} + {{- end }} + {{- if .Port }} + Port {{ .Port }} + {{- end }} + {{- if .URI }} + Uri {{ .URI }} + {{- end }} + {{- if .Header }} + Header {{ .Header }} + {{- end }} + {{- if .TLS }} + Tls {{ .TLS }} + {{- end }} + {{- if .TLSVerify }} + Tls.verify {{ .TLSVerify }} + {{- end }} + {{- if .AddLabel }} + {{- range $key, $value := .AddLabel }} + add_label {{ $key }} {{ $value }} + {{- end }} + {{- end }} {{ end -}} {{- if .ExternalCfg.CfgFilePath }} diff --git a/pkg/integrations/v4/logs/cfg_test.go b/pkg/integrations/v4/logs/cfg_test.go index 73bd4214c..7097f86c4 100644 --- a/pkg/integrations/v4/logs/cfg_test.go +++ b/pkg/integrations/v4/logs/cfg_test.go @@ -11,10 +11,13 @@ import ( "strconv" "testing" + "github.com/newrelic/infrastructure-agent/internal/agent/cmdchannel/fflag" + "github.com/newrelic/infrastructure-agent/internal/feature_flags" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/shirou/gopsutil/v3/host" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) const windowsServer2016BuildNumber = 14393 @@ -62,7 +65,7 @@ func withFeedramp(cfg config.LogForward) config.LogForward { return cfg } -var outputBlock = FBCfgOutput{ +var outputBlock = []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", @@ -72,16 +75,40 @@ var outputBlock = FBCfgOutput{ CABundleDir: "/cabundles", ValidateCerts: true, Retry_Limit: "5", +}, +} + +var outputBlockFedramp = []FBCfgOutput{{ + Name: "newrelic", + Match: "*", + LicenseKey: "licenseKey", + IgnoreSystemProxy: true, + Proxy: "https://https-proxy:3129", + CABundleFile: "/cabundles/proxycert.pem", + CABundleDir: "/cabundles", + ValidateCerts: true, + Retry_Limit: "5", + Endpoint: fedrampEndpoint, +}, +} + +var outputBlockMultipleRetries = []FBCfgOutput{{ + Name: "newrelic", + Match: "*", + LicenseKey: "licenseKey", + IgnoreSystemProxy: true, + Proxy: "https://https-proxy:3129", + CABundleFile: "/cabundles/proxycert.pem", + CABundleDir: "/cabundles", + ValidateCerts: true, + Retry_Limit: "4", +}, } func TestNewFBConf(t *testing.T) { - outputBlockFedramp := outputBlock - outputBlockFedramp.Endpoint = fedrampEndpoint - outputBlockMultipleRetries := outputBlock logFwdCfgMultipleRetries := logFwdCfg logFwdCfgMultipleRetries.RetryLimit = "4" - outputBlockMultipleRetries.Retry_Limit = "4" tests := []struct { name string @@ -699,10 +726,15 @@ func TestNewFBConf(t *testing.T) { Output: outputBlock, }}, } + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, true).Times(len(tests)) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - fbConf, err := NewFBConf(tt.ohiCfg, &tt.logFwd, "0", "") + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, false) + + fbConf, err := NewFBConf(tt.ohiCfg, &tt.logFwd, "0", "", ffRetriever) assert.NoError(t, err) assert.Equal(t, tt.want, fbConf) }) @@ -861,6 +893,8 @@ func TestFBConfigForWinlog(t *testing.T) { if runtime.GOOS == "windows" && isWindowsBuildNumberLowerOrEqualsThan(windowsServer2016BuildNumber) { tests[2].expected.Inputs[0].UseANSI = "true" } + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, true).Times(len(tests)) for _, testItem := range tests { // Prevent the loop variable from being captured in the closure below @@ -868,7 +902,7 @@ func TestFBConfigForWinlog(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() - fbConf, err := NewFBConf(test.logsCfg, &logFwdCfg, "0", "") + fbConf, err := NewFBConf(test.logsCfg, &logFwdCfg, "0", "", ffRetriever) assert.NoError(t, err) assert.Equal(t, test.expected.Inputs, fbConf.Inputs) assert.Equal(t, test.expected.Filters[0], fbConf.Filters[0]) @@ -1032,6 +1066,8 @@ func TestFBConfigForWinevtlog(t *testing.T) { if runtime.GOOS == "windows" && isWindowsBuildNumberLowerOrEqualsThan(windowsServer2016BuildNumber) { tests[2].expected.Inputs[0].UseANSI = "true" } + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, true).Times(len(tests)) for _, testItem := range tests { // Prevent the loop variable from being captured in the closure below @@ -1039,7 +1075,7 @@ func TestFBConfigForWinevtlog(t *testing.T) { t.Run(testItem.name, func(t *testing.T) { t.Parallel() - fbConf, err := NewFBConf(test.logsCfg, &test.logFwd, "0", "") + fbConf, err := NewFBConf(test.logsCfg, &test.logFwd, "0", "", ffRetriever) assert.NoError(t, err) assert.Equal(t, test.expected.Inputs, fbConf.Inputs) assert.Equal(t, test.expected.Filters[0], fbConf.Filters[0]) @@ -1270,11 +1306,11 @@ func TestFBCfgFormat(t *testing.T) { }, }, }, - Output: FBCfgOutput{ + Output: []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", - }, + }}, ExternalCfg: FBCfgExternal{ CfgFilePath: "/path/to/fb/config", ParsersFilePath: "/path/to/fb/parsers", @@ -1366,18 +1402,19 @@ func TestFBCfgSendMetricFormat(t *testing.T) { }, }, }, - Output: FBCfgOutput{ + Output: []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", }, + }, } if noSendMetrics { return fbCfg } - fbCfg.Output.SendMetrics = sendMetrics + fbCfg.Output[0].SendMetrics = sendMetrics return fbCfg } @@ -1476,12 +1513,12 @@ func TestFBCfgSendMetricFalseFormat(t *testing.T) { }, }, }, - Output: FBCfgOutput{ + Output: []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", SendMetrics: false, - }, + }}, } result, extCfg, err := fbCfg.Format() @@ -1553,12 +1590,12 @@ func TestFBCfgSendMetricTrueFormat(t *testing.T) { }, }, }, - Output: FBCfgOutput{ + Output: []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", SendMetrics: true, - }, + }}, } result, extCfg, err := fbCfg.Format() @@ -1627,7 +1664,7 @@ func TestFBCfgFormatWithHostname(t *testing.T) { validateProxyCerts false ` - outputBlock := FBCfgOutput{ + outputBlock := []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", @@ -1639,7 +1676,7 @@ func TestFBCfgFormatWithHostname(t *testing.T) { CABundleFile: "/cabundles/proxycert.pem", CABundleDir: "/cabundles", ValidateCerts: false, - } + }} fbCfg := FBCfg{ Inputs: []FBCfgInput{ @@ -1917,6 +1954,8 @@ func TestNewFbConfForTargetFileCount(t *testing.T) { 0, }, } + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, false).Times(len(tests)) for _, testItem := range tests { // Prevent the loop variable from being captured in the closure below @@ -1924,7 +1963,7 @@ func TestNewFbConfForTargetFileCount(t *testing.T) { t.Run(test.name, func(t *testing.T) { t.Parallel() - NewFBConf(test.logsCfg, &test.logFwd, "0", "") + NewFBConf(test.logsCfg, &test.logFwd, "0", "", ffRetriever) assert.Equal(t, test.expectedCount, test.logsCfg[0].targetFilesCnt) }) } @@ -2008,11 +2047,11 @@ func TestMultilineParserFBCfgFormat(t *testing.T) { }, }, }, - Output: FBCfgOutput{ + Output: []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", - }, + }}, ExternalCfg: FBCfgExternal{ CfgFilePath: "/path/to/fb/config", ParsersFilePath: "/path/to/fb/parsers", @@ -2103,11 +2142,11 @@ func TestMlParserFBCfgWithMultipleParsers(t *testing.T) { }, }, }, - Output: FBCfgOutput{ + Output: []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "licenseKey", - }, + }}, ExternalCfg: FBCfgExternal{ CfgFilePath: "/path/to/fb/config", ParsersFilePath: "/path/to/fb/parsers", @@ -2119,3 +2158,123 @@ func TestMlParserFBCfgWithMultipleParsers(t *testing.T) { assert.Equal(t, "/path/to/fb/parsers", extCfg.ParsersFilePath) assert.Equal(t, expected, result) } + +func TestNewFBConfWithEnabledMetrics(t *testing.T) { + logFwdCfg := config.LogForward{ + HomeDir: "/var/db/newrelic-infra/newrelic-integrations/logging", + License: "licenseKey", + RetryLimit: "5", + ProxyCfg: config.LogForwardProxy{ + IgnoreSystemProxy: true, + Proxy: "https://https-proxy:3129", + CABundleFile: "/cabundles/proxycert.pem", + CABundleDir: "/cabundles", + ValidateCerts: true, + }, + } + + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(true, true) + test := []struct { + name string + logFwd config.LogForward + ohiCfg LogsCfg + want FBCfg + }{ + { + name: "single input", + logFwd: logFwdCfg, + ohiCfg: LogsCfg{ + { + Name: "log-file", + File: "file.path", + }, + }, + want: FBCfg{ + Inputs: []FBCfgInput{ + { + Name: "tail", + Tag: "log-file", + DB: dbDbPath, + Path: "file.path", + BufferMaxSize: "128k", + MemBufferLimit: "16384k", + SkipLongLines: "On", + PathKey: "filePath", + }, + { + Name: "prometheus_scrape", + Alias: "fb-metrics-collector", + Host: "127.0.0.1", + Port: 2020, + Tag: "fb_metrics", + MetricsPath: "/api/v2/metrics/prometheus", + ScrapeInterval: "60s", + }, + }, + Filters: []FBCfgFilter{ + inputRecordModifier("tail", "log-file"), + filterEntityBlockMetrics, + }, + Service: []FBCfgService{ + { + Flush: 1, + LogLevel: "info", + Daemon: "off", + ParsersFile: "parsers.conf", + HTTPServer: "On", + HTTPListen: "0.0.0.0", + HTTPPort: 2020, + }, + }, + Output: []FBCfgOutput{ + { + Name: "newrelic", + Match: "*", + LicenseKey: "licenseKey", + IgnoreSystemProxy: true, + Proxy: "https://https-proxy:3129", + CABundleFile: "/cabundles/proxycert.pem", + CABundleDir: "/cabundles", + ValidateCerts: true, + Retry_Limit: "5", + }, + { + Name: "prometheus_remote_write", + Match: "fb_metrics", + Alias: "fb-metrics-forwarder", + Port: 443, + URI: "/prometheus/v1/write?prometheus_server=hostname", + Header: "Authorization Bearer licenseKey", + Host: "metric-api.newrelic.com", + AddLabel: map[string]string{ + "app": "fluent-bit", + "source": "host", + "os": runtime.GOOS, // OS & Arch as per test system setup + "hostname": "hostname", + "arch": runtime.GOARCH, + }, + TLS: "On", + TLSVerify: "Off", + }, + }, + }, + }, + } + for _, tt := range test { + fbConf, err := NewFBConf(tt.ohiCfg, &tt.logFwd, "1234", "hostname", ffRetriever) + + require.NoError(t, err) + assert.Equal(t, tt.want, fbConf) + } +} + +var filterEntityBlockMetrics = FBCfgFilter{ + Name: "record_modifier", + Match: "*", + Records: map[string]string{ + "entity.guid.INFRA": "1234", + "plugin.type": "nri-agent", + "hostname": "hostname", + }, +} diff --git a/pkg/integrations/v4/logs/loader.go b/pkg/integrations/v4/logs/loader.go index 7de6b707c..1147ce230 100644 --- a/pkg/integrations/v4/logs/loader.go +++ b/pkg/integrations/v4/logs/loader.go @@ -10,6 +10,7 @@ import ( "github.com/newrelic/infrastructure-agent/pkg/log" "github.com/newrelic/infrastructure-agent/internal/agent/id" + "github.com/newrelic/infrastructure-agent/internal/feature_flags" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/newrelic/infrastructure-agent/pkg/integrations/v4/fs" "github.com/newrelic/infrastructure-agent/pkg/sysinfo/hostname" @@ -48,7 +49,7 @@ func (l *CfgLoader) GetLicenseKey() string { // LoadAll loads and parses the logging configuration. It returns ok=false in case an error occurred, which should block // the start of the log forwarding feature. -func (l *CfgLoader) LoadAll() (c FBCfg, ok bool) { +func (l *CfgLoader) LoadAll(ff feature_flags.Retriever) (c FBCfg, ok bool) { if l.config.ConfigsDir == "" && !l.config.Troubleshoot.Enabled { loaderLogger.Error("invalid config, lacking config folder or troubleshoot mode") return FBCfg{}, false @@ -75,7 +76,7 @@ func (l *CfgLoader) LoadAll() (c FBCfg, ok bool) { loaderLogger.Debug("Could not determine hostname.") } - c, err = NewFBConf(allFilesCfgs, &(l.config), agentGUID.String(), shortHostName) + c, err = NewFBConf(allFilesCfgs, &(l.config), agentGUID.String(), shortHostName, ff) if err != nil { loaderLogger.WithError(err).Error("could not process logging configurations") return FBCfg{}, false @@ -161,8 +162,8 @@ func (l *CfgLoader) loadTroubleshootCfg() *LogCfg { return nil } -func (l *CfgLoader) LoadAndFormat() (string, FBCfgExternal, error) { - fbConfig, ok := l.LoadAll() +func (l *CfgLoader) LoadAndFormat(ff feature_flags.Retriever) (string, FBCfgExternal, error) { + fbConfig, ok := l.LoadAll(ff) if !ok { return "", FBCfgExternal{}, errors.New("failed to load log configs") } diff --git a/pkg/integrations/v4/logs/loader_test.go b/pkg/integrations/v4/logs/loader_test.go index f8df27ba2..94f75cca5 100644 --- a/pkg/integrations/v4/logs/loader_test.go +++ b/pkg/integrations/v4/logs/loader_test.go @@ -3,11 +3,12 @@ package logs import ( - "io/ioutil" "os" "path/filepath" "testing" + "github.com/newrelic/infrastructure-agent/internal/agent/cmdchannel/fflag" + "github.com/newrelic/infrastructure-agent/internal/feature_flags" "github.com/newrelic/infrastructure-agent/pkg/config" "github.com/newrelic/infrastructure-agent/pkg/entity" "github.com/stretchr/testify/assert" @@ -23,11 +24,12 @@ var ( _, hostName, _ = hostnameProvider.Query() // Expected struct results - fbCfgOutput = FBCfgOutput{ + fbCfgOutput = []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "license", SendMetrics: false, + }, } fbCfgEntityDecoration = FBCfgFilter{ Name: "record_modifier", @@ -36,7 +38,7 @@ var ( "entity.guid.INFRA": "FOOBAR", "plugin.type": logRecordModifierSource, "hostname": hostName, - }, // see idnProvide below + }, } idnProvide = func() entity.Identity { return entity.Identity{ @@ -46,7 +48,9 @@ var ( } ) -func TestCfgLoader_LoadAll(t *testing.T) { +func TestCfgLoaderLoadAll(t *testing.T) { + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, false) validContent := ` logs: - name: foo @@ -79,33 +83,28 @@ logs: Output: fbCfgOutput, } - // Empty directory - emptyDir, err := ioutil.TempDir("", "test-load-empty") - defer os.RemoveAll(emptyDir) + emptyDir, err := os.MkdirTemp("", "test-load-empty") require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(emptyDir) }) - // Directory containing one empty file with yml extension - onlyEmptyCfg, err := ioutil.TempDir("", "test-load-non-empty") - defer os.RemoveAll(onlyEmptyCfg) + onlyEmptyCfg, err := os.MkdirTemp("", "test-load-non-empty") require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(onlyEmptyCfg) }) addFile(t, onlyEmptyCfg, "empty.yml", "") - // Directory containing a single valid configuration file - onlyValidCfg, err := ioutil.TempDir("", "test-load-content") - defer os.RemoveAll(onlyValidCfg) + onlyValidCfg, err := os.MkdirTemp("", "test-load-content") require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(onlyValidCfg) }) addFile(t, onlyValidCfg, "valid.yml", validContent) - // Directory containing only one example file - onlyExampleFile, err := ioutil.TempDir("", "test-load-content") - defer os.RemoveAll(onlyExampleFile) + onlyExampleFile, err := os.MkdirTemp("", "test-load-content") require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(onlyExampleFile) }) addFile(t, onlyExampleFile, "file.yml.example", validContent) - // Directory containing one example file and one valid configuration file - exampleFileAndValidCfg, err := ioutil.TempDir("", "test-load-content") - defer os.RemoveAll(onlyExampleFile) + exampleFileAndValidCfg, err := os.MkdirTemp("", "test-load-content") require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(exampleFileAndValidCfg) }) addFile(t, exampleFileAndValidCfg, "file.yml.example", validContent) addFile(t, exampleFileAndValidCfg, "valid.yml", validContent) @@ -116,18 +115,18 @@ logs: expectOK bool }{ {"empty folder", emptyDir, emptyCfg, false}, - {"non existing folder", "/some-non-existing-folder", emptyCfg, false}, - {"non empty folder with YML file but no configs", onlyEmptyCfg, emptyCfg, false}, + {"non-existing folder", "/some-non-existing-folder", emptyCfg, false}, + {"non-empty folder with YML file but no configs", onlyEmptyCfg, emptyCfg, false}, {"folder with valid file", onlyValidCfg, expectedCfg, true}, - {"folder with only example (non-yml) files", onlyExampleFile, emptyCfg, false}, - {"folder with a valid file and example (non-yml) files", exampleFileAndValidCfg, expectedCfg, true}, + {"folder with only example (non-YML) files", onlyExampleFile, emptyCfg, false}, + {"folder with a valid file and example (non-YML) files", exampleFileAndValidCfg, expectedCfg, true}, {"folder with valid file (verbose enabled)", onlyValidCfg, expectedCfg, true}, } + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // SUT conf := newTestConf(tt.folder, disabledTroubleshootCfg, false) - cfg, ok := NewFolderLoader(conf, idnProvide, hostnameProvider).LoadAll() + cfg, ok := NewFolderLoader(conf, idnProvide, hostnameProvider).LoadAll(ffRetriever) assert.Equal(t, tt.expectOK, ok) assert.Equal(t, tt.wantCfg, cfg) @@ -142,9 +141,9 @@ logs: file: /file/path ` - validFileFolder, err := ioutil.TempDir("", "test-load-content") - defer os.RemoveAll(validFileFolder) + validFileFolder, err := os.MkdirTemp("", "test-load-content") require.NoError(t, err) + t.Cleanup(func() { os.RemoveAll(validFileFolder) }) addFile(t, validFileFolder, "file.yml", validContent) expectedCfg := FBCfg{ @@ -170,30 +169,38 @@ logs: }, fbCfgEntityDecoration, }, - Output: FBCfgOutput{ + Output: []FBCfgOutput{{ Name: "newrelic", Match: "*", LicenseKey: "license", SendMetrics: true, - }, + }}, } conf := newTestConf(validFileFolder, disabledTroubleshootCfg, true) - cfg, ok := NewFolderLoader(conf, idnProvide, hostnameProvider).LoadAll() + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, false) + cfg, ok := NewFolderLoader(conf, idnProvide, hostnameProvider).LoadAll(ffRetriever) assert.Equal(t, true, ok) assert.Equal(t, expectedCfg, cfg) } -func TestCfgLoader_LoadAll_TroubleshootDisabed(t *testing.T) { +//nolint:exhaustruct +func TestCfgLoaderTroubleshootDisabled(t *testing.T) { disabledTroubleshootCfg := config.NewTroubleshootCfg(false, false, "") - _, ok := NewFolderLoader(newTestConf("", disabledTroubleshootCfg, false), idnProvide, hostnameProvider).LoadAll() + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, false) + _, ok := NewFolderLoader(newTestConf("", disabledTroubleshootCfg, false), idnProvide, hostnameProvider).LoadAll(ffRetriever) assert.False(t, ok, "should return ok=false when there is no logging configuration directory and troubleshoot is disabled") } -func TestCfgLoader_LoadAll_TroubleshootNoLogFile(t *testing.T) { +//nolint:exhaustruct +func TestCfgLoaderTroubleshootNoLogFile(t *testing.T) { troublesCfg := config.NewTroubleshootCfg(true, false, "") - cfg, ok := NewFolderLoader(newTestConf("", troublesCfg, false), idnProvide, hostnameProvider).LoadAll() - assert.Equal(t, ok, true, "Enabling troubleshoot with no logging configurations should start the log forwarder") + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, false) + cfg, ok := NewFolderLoader(newTestConf("", troublesCfg, false), idnProvide, hostnameProvider).LoadAll(ffRetriever) + assert.True(t, ok, "Enabling troubleshoot with no logging configurations should start the log forwarder") assert.Equal(t, FBCfg{ Inputs: []FBCfgInput{ { @@ -219,8 +226,10 @@ func TestCfgLoader_LoadAll_TroubleshootNoLogFile(t *testing.T) { func TestCfgLoader_LoadAll_TroubleshootLogFile(t *testing.T) { troublesCfg := config.NewTroubleshootCfg(true, true, "/agent_log_file") - cfg, ok := NewFolderLoader(newTestConf("", troublesCfg, false), idnProvide, hostnameProvider).LoadAll() - assert.Equal(t, ok, true, "Enabling troubleshoot with no logging configurations should start the log forwarder") + ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} + ffRetriever.On("GetFeatureFlag", fflag.FlagFluentBitMetrics).Return(false, false) + cfg, ok := NewFolderLoader(newTestConf("", troublesCfg, false), idnProvide, hostnameProvider).LoadAll(ffRetriever) + assert.True(t, ok, "Enabling troubleshoot with a log file should start the log forwarder") assert.Equal(t, FBCfg{ Inputs: []FBCfgInput{ { @@ -351,8 +360,7 @@ logs: { Name: "syslog-unix-tcp-test", Syslog: &LogSyslogCfg{ - URI: "unix_tcp:///var/tcp-socket-test", - Parser: "syslog-rfc3164", + URI: "unix_tcp:///var/tcp-socket-test", Parser: "syslog-rfc3164", UnixPermissions: "0644", }, }, @@ -469,5 +477,5 @@ func (m *mockHostnameResolver) Long() string { func addFile(t *testing.T, dir, name, contents string) { filePath := filepath.Join(dir, name) - require.NoError(t, ioutil.WriteFile(filePath, []byte(contents), 0666)) + require.NoError(t, os.WriteFile(filePath, []byte(contents), 0666)) } diff --git a/pkg/integrations/v4/supervisor_fb.go b/pkg/integrations/v4/supervisor_fb.go index de7d00f95..4b653a3fc 100644 --- a/pkg/integrations/v4/supervisor_fb.go +++ b/pkg/integrations/v4/supervisor_fb.go @@ -184,7 +184,7 @@ func buildFbExecutor(fbIntCfg fBSupervisorConfig, cfgLoader *logs.CfgLoader) fun return nil, errFbNotAvailable } - cfgContent, externalCfg, cErr := cfgLoader.LoadAndFormat() + cfgContent, externalCfg, cErr := cfgLoader.LoadAndFormat(fbIntCfg.ffRetriever) if cErr != nil { return nil, cErr } diff --git a/pkg/integrations/v4/supervisor_fb_test.go b/pkg/integrations/v4/supervisor_fb_test.go index 9d1e7b9c8..edc8f4ea5 100644 --- a/pkg/integrations/v4/supervisor_fb_test.go +++ b/pkg/integrations/v4/supervisor_fb_test.go @@ -415,6 +415,7 @@ func Test_buildFbExecutor(t *testing.T) { ffRetriever := &feature_flags.FeatureFlagRetrieverMock{} ffRetriever.ShouldNotGetFeatureFlag(fflag.FlagFluentBit19) + ffRetriever.ShouldNotGetFeatureFlag(fflag.FlagFluentBitMetrics) agentDir := "agentDir" integrationsDir := "agentDir"