Skip to content

Commit 30070cf

Browse files
authored
Add support for large NGINX config file sizes (#1053)
1 parent c25bb76 commit 30070cf

20 files changed

+51170
-47
lines changed

internal/config/config.go

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,13 +366,25 @@ func registerClientFlags(fs *flag.FlagSet) {
366366
fs.Int(
367367
ClientGRPCMaxMessageReceiveSizeKey,
368368
DefMaxMessageRecieveSize,
369-
"Updates the client grpc setting MaxRecvMsgSize with the specific value in MB.",
369+
"Updates the client grpc setting MaxRecvMsgSize with the specific value in bytes.",
370370
)
371371

372372
fs.Int(
373373
ClientGRPCMaxMessageSendSizeKey,
374374
DefMaxMessageSendSize,
375-
"Updates the client grpc setting MaxSendMsgSize with the specific value in MB.",
375+
"Updates the client grpc setting MaxSendMsgSize with the specific value in bytes.",
376+
)
377+
378+
fs.Uint32(
379+
ClientGRPCFileChunkSizeKey,
380+
DefFileChunkSize,
381+
"File chunk size in bytes.",
382+
)
383+
384+
fs.Uint32(
385+
ClientGRPCMaxFileSizeKey,
386+
DefMaxFileSize,
387+
"Max file size in bytes.",
376388
)
377389
}
378390

@@ -709,6 +721,8 @@ func resolveClient() *Client {
709721
MaxMessageSize: viperInstance.GetInt(ClientGRPCMaxMessageSizeKey),
710722
MaxMessageReceiveSize: viperInstance.GetInt(ClientGRPCMaxMessageReceiveSizeKey),
711723
MaxMessageSendSize: viperInstance.GetInt(ClientGRPCMaxMessageSendSizeKey),
724+
MaxFileSize: viperInstance.GetUint32(ClientGRPCMaxFileSizeKey),
725+
FileChunkSize: viperInstance.GetUint32(ClientGRPCFileChunkSizeKey),
712726
},
713727
Backoff: &BackOff{
714728
InitialInterval: viperInstance.GetDuration(ClientBackoffInitialIntervalKey),

internal/config/config_test.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ func checkDefaultsClientValues(t *testing.T, viperInstance *viper.Viper) {
104104
assert.Equal(t, DefMaxMessageSize, viperInstance.GetInt(ClientGRPCMaxMessageSizeKey))
105105
assert.Equal(t, DefMaxMessageRecieveSize, viperInstance.GetInt(ClientGRPCMaxMessageReceiveSizeKey))
106106
assert.Equal(t, DefMaxMessageSendSize, viperInstance.GetInt(ClientGRPCMaxMessageSendSizeKey))
107+
assert.Equal(t, DefFileChunkSize, viperInstance.GetUint32(ClientGRPCFileChunkSizeKey))
108+
assert.Equal(t, DefMaxFileSize, viperInstance.GetUint32(ClientGRPCMaxFileSizeKey))
107109
assert.Equal(t, make(map[string]string), viperInstance.GetStringMapString(LabelsRootKey))
108110
}
109111

@@ -781,6 +783,8 @@ func createConfig() *Config {
781783
MaxMessageSize: 1048575,
782784
MaxMessageReceiveSize: 1048575,
783785
MaxMessageSendSize: 1048575,
786+
MaxFileSize: 485753,
787+
FileChunkSize: 48575,
784788
},
785789
Backoff: &BackOff{
786790
InitialInterval: 200 * time.Millisecond,

internal/config/defaults.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
package config
66

77
import (
8-
"math"
98
"time"
109

1110
pkg "github.com/nginx/agent/v3/pkg/config"
@@ -28,9 +27,11 @@ const (
2827
DefCommandTLServerNameKey = ""
2928

3029
// Client GRPC Settings
31-
DefMaxMessageSize = 0 // 0 = unset
32-
DefMaxMessageRecieveSize = 4194304 // default 4 MB
33-
DefMaxMessageSendSize = math.MaxInt32
30+
DefMaxMessageSize = 0 // 0 = unset
31+
DefMaxMessageRecieveSize = 4194304 // default 4 MB
32+
DefMaxMessageSendSize = 4194304 // default 4 MB
33+
DefMaxFileSize uint32 = 1048576 // 1MB
34+
DefFileChunkSize uint32 = 524288 // 0.5MB
3435

3536
// Client HTTP Settings
3637
DefHTTPTimeout = 10 * time.Second

internal/config/flags.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ var (
3636
ClientGRPCMaxMessageSendSizeKey = pre(ClientRootKey) + "grpc_max_message_send_size"
3737
ClientGRPCMaxMessageReceiveSizeKey = pre(ClientRootKey) + "grpc_max_message_receive_size"
3838
ClientGRPCMaxMessageSizeKey = pre(ClientRootKey) + "grpc_max_message_size"
39+
ClientGRPCMaxFileSizeKey = pre(ClientRootKey) + "grpc_max_file_size"
40+
ClientGRPCFileChunkSizeKey = pre(ClientRootKey) + "grpc_file_chunk_size"
3941

4042
ClientBackoffInitialIntervalKey = pre(ClientRootKey) + "backoff_initial_interval"
4143
ClientBackoffMaxIntervalKey = pre(ClientRootKey) + "backoff_max_interval"

internal/config/testdata/nginx-agent.conf

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ client:
3535
max_message_size: 1048575
3636
max_message_receive_size: 1048575
3737
max_message_send_size: 1048575
38+
max_file_size: 485753
39+
file_chunk_size: 48575
3840
backoff:
3941
initial_interval: 200ms
4042
max_interval: 10s

internal/config/types.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -83,9 +83,11 @@ type (
8383
KeepAlive *KeepAlive `yaml:"keepalive" mapstructure:"keepalive"`
8484
// if MaxMessageSize is size set then we use that value,
8585
// otherwise MaxMessageRecieveSize and MaxMessageSendSize for individual settings
86-
MaxMessageSize int `yaml:"max_message_size" mapstructure:"max_message_size"`
87-
MaxMessageReceiveSize int `yaml:"max_message_receive_size" mapstructure:"max_message_receive_size"`
88-
MaxMessageSendSize int `yaml:"max_message_send_size" mapstructure:"max_message_send_size"`
86+
MaxMessageSize int `yaml:"max_message_size" mapstructure:"max_message_size"`
87+
MaxMessageReceiveSize int `yaml:"max_message_receive_size" mapstructure:"max_message_receive_size"`
88+
MaxMessageSendSize int `yaml:"max_message_send_size" mapstructure:"max_message_send_size"`
89+
MaxFileSize uint32 `yaml:"max_file_size" mapstructure:"max_file_size"`
90+
FileChunkSize uint32 `yaml:"file_chunk_size" mapstructure:"file_chunk_size"`
8991
}
9092

9193
KeepAlive struct {
Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
// Copyright (c) F5, Inc.
2+
//
3+
// This source code is licensed under the Apache License, Version 2.0 license found in the
4+
// LICENSE file in the root directory of this source tree.
5+
6+
package file
7+
8+
import (
9+
"context"
10+
"sync/atomic"
11+
12+
mpi "github.com/nginx/agent/v3/api/grpc/mpi/v1"
13+
"google.golang.org/grpc/metadata"
14+
"google.golang.org/protobuf/types/known/timestamppb"
15+
)
16+
17+
type FakeClientStreamingClient struct {
18+
sendCount atomic.Int32
19+
}
20+
21+
func (f *FakeClientStreamingClient) Send(req *mpi.FileDataChunk) error {
22+
f.sendCount.Add(1)
23+
return nil
24+
}
25+
26+
func (f *FakeClientStreamingClient) CloseAndRecv() (*mpi.UpdateFileResponse, error) {
27+
return &mpi.UpdateFileResponse{}, nil
28+
}
29+
30+
func (f *FakeClientStreamingClient) Header() (metadata.MD, error) {
31+
return metadata.MD{}, nil
32+
}
33+
34+
func (f *FakeClientStreamingClient) Trailer() metadata.MD {
35+
return nil
36+
}
37+
38+
func (f *FakeClientStreamingClient) CloseSend() error {
39+
return nil
40+
}
41+
42+
func (f *FakeClientStreamingClient) Context() context.Context {
43+
return context.Background()
44+
}
45+
46+
func (f *FakeClientStreamingClient) SendMsg(m any) error {
47+
return nil
48+
}
49+
50+
func (f *FakeClientStreamingClient) RecvMsg(m any) error {
51+
return nil
52+
}
53+
54+
type FakeServerStreamingClient struct {
55+
chunks map[uint32][]byte
56+
fileName string
57+
currentChunkID uint32
58+
}
59+
60+
func (f *FakeServerStreamingClient) Recv() (*mpi.FileDataChunk, error) {
61+
fileDataChunk := &mpi.FileDataChunk{
62+
Meta: &mpi.MessageMeta{
63+
MessageId: "123",
64+
CorrelationId: "1234",
65+
Timestamp: timestamppb.Now(),
66+
},
67+
}
68+
69+
if f.currentChunkID == 0 {
70+
fileDataChunk.Chunk = &mpi.FileDataChunk_Header{
71+
Header: &mpi.FileDataChunkHeader{
72+
FileMeta: &mpi.FileMeta{
73+
Name: f.fileName,
74+
Permissions: "666",
75+
},
76+
Chunks: 52,
77+
ChunkSize: 1,
78+
},
79+
}
80+
} else {
81+
fileDataChunk.Chunk = &mpi.FileDataChunk_Content{
82+
Content: &mpi.FileDataChunkContent{
83+
ChunkId: f.currentChunkID,
84+
Data: f.chunks[f.currentChunkID-1],
85+
},
86+
}
87+
}
88+
89+
f.currentChunkID++
90+
91+
return fileDataChunk, nil
92+
}
93+
94+
func (f *FakeServerStreamingClient) Header() (metadata.MD, error) {
95+
return metadata.MD{}, nil
96+
}
97+
98+
func (f *FakeServerStreamingClient) Trailer() metadata.MD {
99+
return metadata.MD{}
100+
}
101+
102+
func (f *FakeServerStreamingClient) CloseSend() error {
103+
return nil
104+
}
105+
106+
func (f *FakeServerStreamingClient) Context() context.Context {
107+
return context.Background()
108+
}
109+
110+
func (f *FakeServerStreamingClient) SendMsg(m any) error {
111+
return nil
112+
}
113+
114+
func (f *FakeServerStreamingClient) RecvMsg(m any) error {
115+
return nil
116+
}

0 commit comments

Comments
 (0)