Skip to content

Commit cde4938

Browse files
roybabayovroybabayov
committed
improve support for proxy protocol
* Add support for parsing tlv headers * Add support for parsing network id from common cloud providers. Note, currently only GCP is fully supported as I couldn't test the flow on AWS, Azure but it should be easy to add this support in the future. * Fix bug that we didn't account for the tlv when checking the header length * Fix bug that tlv would have been stored as extra data inside the address union Change-Id: Idb68144f34bfa302c478de54eea631e6f80883f9 Signed-off-by: Roy Babayov <[email protected]>
1 parent 499bd41 commit cde4938

File tree

6 files changed

+471
-212
lines changed

6 files changed

+471
-212
lines changed

‎CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,8 @@ endif (USE_GSS)
7979

8080
option(USE_MONITORING "Enable monitoring stack" ON)
8181

82+
option(PROXY_PROTOCOL_PARSE_COMMON_NETWORK_ID "Enable parsing of common network id from haproxy TLV" OFF)
83+
8284
option(USE_RPC_RDMA "platform supports RDMA" OFF)
8385
if (USE_RPC_RDMA)
8486
find_package(RDMA REQUIRED)
@@ -263,6 +265,7 @@ message(STATUS "USE_GSS = ${USE_GSS}")
263265
message(STATUS "USE_PROFILE = ${USE_PROFILE}")
264266
message(STATUS "USE_LTTNG_NTIRPC = ${USE_LTTNG_NTIRPC}")
265267
message(STATUS "USE_MONITORING = ${USE_MONITORING}")
268+
message(STATUS "PROXY_PROTOCOL_PARSE_COMMON_NETWORK_ID = ${PROXY_PROTOCOL_PARSE_COMMON_NETWORK_ID}")
266269

267270
#force command line options to be stored in cache
268271
set(_MSPAC_SUPPORT ${_MSPAC_SUPPORT}

‎src/haproxy.h renamed to ‎ntirpc/rpc/haproxy.h

Lines changed: 51 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -23,8 +23,16 @@
2323
* https://github.com/haproxy/haproxy.git
2424
*
2525
* Specifically, parts of this file: include/haproxy/connection-t.h
26+
*
27+
* NOTE: the definitions in this file are based on the haproxy spec in
28+
* https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
2629
*/
2730

31+
#include "config.h"
32+
33+
#ifndef _NTIRPC_RPC_HAPROXY_H
34+
#define _NTIRPC_RPC_HAPROXY_H
35+
2836
/* proxy protocol v2 definitions */
2937
#define PP2_SIGNATURE "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A"
3038
#define PP2_SIG_UINT32 0x0d0a0d0a
@@ -42,7 +50,7 @@
4250
#define PP2_VERSION_MASK 0xF0
4351

4452
#define PP2_VERSION2_CMD_LOCAL (PP2_CMD_LOCAL | PP2_VERSION)
45-
#define PP2_VERSIOB2_CMD_PROXY (PP2_CMD_PROXY | PP2_VERSION)
53+
#define PP2_VERSION2_CMD_PROXY (PP2_CMD_PROXY | PP2_VERSION)
4654

4755
/* fam byte */
4856
#define PP2_TRANS_UNSPEC 0x00
@@ -82,6 +90,17 @@
8290
#define PP2_SUBTYPE_SSL_KEY_ALG 0x25
8391
#define PP2_TYPE_NETNS 0x30
8492

93+
#define PP2_TYPE_UNSET 0x00
94+
95+
/* Define custom PP2_TYPE by common cloud provides */
96+
/* https://cloud.google.com/vpc/docs/about-vpc-hosted-services#proxy-protocol */
97+
#define PP2_TYPE_GCP 0xE0
98+
#define PP2_TYPE_GCP_EXPECTED_LENGTH 8
99+
/* https://docs.aws.amazon.com/elasticloadbalancing/latest/network/edit-target-group-attributes.html#proxy-protocol */
100+
#define PP2_TYPE_AWS 0xEA
101+
/* https://learn.microsoft.com/en-us/azure/private-link/private-link-service-overview#getting-connection-information-using-tcp-proxy-v2 */
102+
#define PP2_TYPE_AZURE 0xEE
103+
85104
#define PP2_CLIENT_SSL 0x01
86105
#define PP2_CLIENT_CERT_CONN 0x02
87106
#define PP2_CLIENT_CERT_SESS 0x04
@@ -97,29 +116,38 @@ struct proxy_header_part {
97116
uint16_t len; /* number of following bytes part of the header */
98117
};
99118

100-
union proxy_addr {
101-
struct { /* for TCP/UDP over IPv4, len = 12 */
102-
uint32_t src_addr;
103-
uint32_t dst_addr;
104-
uint16_t src_port;
105-
uint16_t dst_port;
106-
} ip4;
107-
struct { /* for TCP/UDP over IPv6, len = 36 */
108-
uint8_t src_addr[16];
109-
uint8_t dst_addr[16];
110-
uint16_t src_port;
111-
uint16_t dst_port;
112-
} ip6;
113-
struct { /* for AF_UNIX sockets, len = 216 */
114-
uint8_t src_addr[108];
115-
uint8_t dst_addr[108];
116-
} unx;
119+
struct proxy_header_addr_ip4_part { /* for TCP/UDP over IPv4, len = 12 */
120+
uint32_t src_addr;
121+
uint32_t dst_addr;
122+
uint16_t src_port;
123+
uint16_t dst_port;
124+
};
125+
_Static_assert(
126+
sizeof(struct proxy_header_addr_ip4_part) == PP2_ADDR_LEN_INET,
127+
"proxy_header_addr_ip4_part size is not equal to PP2_ADDR_LEN_INET");
128+
129+
struct proxy_header_addr_ip6_part { /* for TCP/UDP over IPv6, len = 36 */
130+
uint8_t src_addr[16];
131+
uint8_t dst_addr[16];
132+
uint16_t src_port;
133+
uint16_t dst_port;
134+
};
135+
_Static_assert(
136+
sizeof(struct proxy_header_addr_ip6_part) == PP2_ADDR_LEN_INET6,
137+
"proxy_header_addr_ip6_part size is not equal to PP2_ADDR_LEN_INET6");
138+
139+
typedef uint8_t proxy_protocol_tlv_type;
140+
typedef uint16_t proxy_protocol_tlv_length;
141+
struct proxy_protocol_tlv_header {
142+
proxy_protocol_tlv_type type;
143+
proxy_protocol_tlv_length length;
144+
/* value is stored in big-endian as received from socket */
145+
void *value;
117146
};
118147

119-
struct proxy_hdr_v2 {
120-
uint8_t sig[12]; /* hex 0D 0A 0D 0A 00 0D 0A 51 55 49 54 0A */
121-
struct proxy_header_part header_part;
122-
union proxy_addr addr;
148+
struct proxy_protocol_tlv_headers {
149+
uint16_t tlv_count;
150+
struct proxy_protocol_tlv_header *tlvs;
123151
};
124152

125-
enum xprt_stat recv_haproxy_header(SVCXPRT *xprt);
153+
#endif /* _NTIRPC_RPC_HAPROXY_H */

‎ntirpc/rpc/svc.h

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,11 @@ struct svc_xprt {
294294
struct rpc_address xp_local; /* local address, length, port */
295295
struct rpc_address xp_remote; /* remote address, length, port */
296296
struct rpc_address xp_proxy; /* proxy address, length, port */
297+
struct proxy_protocol_tlv_headers proxy_protocol_tlv_headers;
298+
/* Network id parsed from proxy protocol.
299+
* Populated only when PARSE_COMMON_NETWORK_ID flag is on
300+
*/
301+
struct network_id xp_remote_network_id;
297302

298303
#if defined(HAVE_BLKIN)
299304
/* blkin tracing */

‎ntirpc/rpc/types.h

Lines changed: 99 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -48,14 +48,16 @@
4848
#include <sys/types.h>
4949
#include <inttypes.h>
5050

51+
#include "haproxy.h"
52+
5153
#if defined(_WIN32)
5254

5355
#define __BEGIN_DECLS
5456
#define __END_DECLS
5557

5658
/* integral types */
5759
#ifndef _MSC_VER
58-
#include <_bsd_types.h> /* XXX mingw (defines u_long) */
60+
#include <_bsd_types.h> /* XXX mingw (defines u_long) */
5961
#endif
6062
typedef uint8_t u_char;
6163
typedef uint16_t u_int16_t;
@@ -77,7 +79,7 @@ struct iovec {
7779
};
7880

7981
#include <winsock2.h>
80-
#include <ws2tcpip.h> /* XXX mingw */
82+
#include <ws2tcpip.h> /* XXX mingw */
8183

8284
#endif
8385

@@ -109,107 +111,107 @@ typedef int32_t rpc_inline_t;
109111
* Package params support
110112
*/
111113

112-
#define TIRPC_GET_PARAMETERS 0
113-
#define TIRPC_PUT_PARAMETERS 1
114-
#define TIRPC_GET_DEBUG_FLAGS 2
115-
#define TIRPC_SET_DEBUG_FLAGS 3
116-
#define TIRPC_GET_OTHER_FLAGS 4
117-
#define TIRPC_SET_OTHER_FLAGS 5
114+
#define TIRPC_GET_PARAMETERS 0
115+
#define TIRPC_PUT_PARAMETERS 1
116+
#define TIRPC_GET_DEBUG_FLAGS 2
117+
#define TIRPC_SET_DEBUG_FLAGS 3
118+
#define TIRPC_GET_OTHER_FLAGS 4
119+
#define TIRPC_SET_OTHER_FLAGS 5
118120

119121
/*
120122
* Debug flags support
121123
*/
122124

123-
#define TIRPC_FLAG_NONE 0x0000000
124-
#define TIRPC_DEBUG_FLAG_NONE 0x0000000
125-
#define TIRPC_DEBUG_FLAG_ERROR 0x0000001
126-
#define TIRPC_DEBUG_FLAG_EVENT 0x0000002
127-
#define TIRPC_DEBUG_FLAG_WARN 0x0000004
128-
#define TIRPC_DEBUG_FLAG_LOCK 0x0000008
129-
#define TIRPC_DEBUG_FLAG_REFCNT 0x0000010
130-
#define TIRPC_DEBUG_FLAG_RBTREE 0x0000020
131-
#define TIRPC_DEBUG_FLAG_RPCSEC_GSS 0x0000040
132-
#define TIRPC_DEBUG_FLAG_AUTH 0x0000080
133-
#define TIRPC_DEBUG_FLAG_CLNT_DG 0x0000100
134-
#define TIRPC_DEBUG_FLAG_CLNT_RDMA 0x0000200
135-
#define TIRPC_DEBUG_FLAG_CLNT_SCTP 0x0000400
136-
#define TIRPC_DEBUG_FLAG_CLNT_VC 0x0000800
137-
#define TIRPC_DEBUG_FLAG_CLNT_BCAST 0x0001000
138-
#define TIRPC_DEBUG_FLAG_CLNT_RAW 0x0002000
139-
#define TIRPC_DEBUG_FLAG_CLNT_REQ 0x0004000
140-
#define TIRPC_DEBUG_FLAG_CLNT 0x0008000
141-
#define TIRPC_DEBUG_FLAG_SVC_DG 0x0010000
142-
#define TIRPC_DEBUG_FLAG_SVC_RDMA 0x0020000
143-
#define TIRPC_DEBUG_FLAG_SVC_SCTP 0x0040000
144-
#define TIRPC_DEBUG_FLAG_SVC_VC 0x0080000
145-
#define TIRPC_DEBUG_FLAG_SVC_RQST 0x0100000
146-
#define TIRPC_DEBUG_FLAG_SVC_XPRT 0x0200000
147-
#define TIRPC_DEBUG_FLAG_SVC 0x0400000
148-
#define TIRPC_DEBUG_FLAG_XDR 0x0800000
149-
#define TIRPC_DEBUG_FLAG_WORKER 0x1000000
150-
#define TIRPC_DEBUG_FLAG_RPC_MSG 0x2000000
151-
#define TIRPC_DEBUG_FLAG_RPC_RDMA 0x4000000
152-
#define TIRPC_DEBUG_FLAG_XDR_RDMA 0x8000000
125+
#define TIRPC_FLAG_NONE 0x0000000
126+
#define TIRPC_DEBUG_FLAG_NONE 0x0000000
127+
#define TIRPC_DEBUG_FLAG_ERROR 0x0000001
128+
#define TIRPC_DEBUG_FLAG_EVENT 0x0000002
129+
#define TIRPC_DEBUG_FLAG_WARN 0x0000004
130+
#define TIRPC_DEBUG_FLAG_LOCK 0x0000008
131+
#define TIRPC_DEBUG_FLAG_REFCNT 0x0000010
132+
#define TIRPC_DEBUG_FLAG_RBTREE 0x0000020
133+
#define TIRPC_DEBUG_FLAG_RPCSEC_GSS 0x0000040
134+
#define TIRPC_DEBUG_FLAG_AUTH 0x0000080
135+
#define TIRPC_DEBUG_FLAG_CLNT_DG 0x0000100
136+
#define TIRPC_DEBUG_FLAG_CLNT_RDMA 0x0000200
137+
#define TIRPC_DEBUG_FLAG_CLNT_SCTP 0x0000400
138+
#define TIRPC_DEBUG_FLAG_CLNT_VC 0x0000800
139+
#define TIRPC_DEBUG_FLAG_CLNT_BCAST 0x0001000
140+
#define TIRPC_DEBUG_FLAG_CLNT_RAW 0x0002000
141+
#define TIRPC_DEBUG_FLAG_CLNT_REQ 0x0004000
142+
#define TIRPC_DEBUG_FLAG_CLNT 0x0008000
143+
#define TIRPC_DEBUG_FLAG_SVC_DG 0x0010000
144+
#define TIRPC_DEBUG_FLAG_SVC_RDMA 0x0020000
145+
#define TIRPC_DEBUG_FLAG_SVC_SCTP 0x0040000
146+
#define TIRPC_DEBUG_FLAG_SVC_VC 0x0080000
147+
#define TIRPC_DEBUG_FLAG_SVC_RQST 0x0100000
148+
#define TIRPC_DEBUG_FLAG_SVC_XPRT 0x0200000
149+
#define TIRPC_DEBUG_FLAG_SVC 0x0400000
150+
#define TIRPC_DEBUG_FLAG_XDR 0x0800000
151+
#define TIRPC_DEBUG_FLAG_WORKER 0x1000000
152+
#define TIRPC_DEBUG_FLAG_RPC_MSG 0x2000000
153+
#define TIRPC_DEBUG_FLAG_RPC_RDMA 0x4000000
154+
#define TIRPC_DEBUG_FLAG_XDR_RDMA 0x8000000
153155

154156
/* or symbolic names for default */
155-
#define TIRPC_DEBUG_FLAG_DEFAULT \
156-
(TIRPC_DEBUG_FLAG_ERROR | \
157-
TIRPC_DEBUG_FLAG_EVENT | \
157+
#define TIRPC_DEBUG_FLAG_DEFAULT \
158+
(TIRPC_DEBUG_FLAG_ERROR | TIRPC_DEBUG_FLAG_EVENT | \
158159
TIRPC_DEBUG_FLAG_WARN)
159160

160-
#define TIRPC_DEBUG_FLAG_CLNT_RPCB (TIRPC_DEBUG_FLAG_CLNT)
161+
#define TIRPC_DEBUG_FLAG_CLNT_RPCB (TIRPC_DEBUG_FLAG_CLNT)
161162

162-
typedef void *(*mem_1_size_t) (size_t,
163-
const char *file, int line, const char *function);
164-
typedef void *(*mem_2_size_t) (size_t, size_t,
165-
const char *file, int line, const char *function);
166-
typedef void *(*mem_p_size_t) (void *, size_t,
167-
const char *file, int line, const char *function);
168-
typedef void (*mem_free_size_t) (void *, size_t);
169-
typedef void (*mem_format_t) (const char *fmt, ...);
170-
typedef void (*mem_char_t) (const char *);
163+
typedef void *(*mem_1_size_t)(size_t, const char *file, int line,
164+
const char *function);
165+
typedef void *(*mem_2_size_t)(size_t, size_t, const char *file, int line,
166+
const char *function);
167+
typedef void *(*mem_p_size_t)(void *, size_t, const char *file, int line,
168+
const char *function);
169+
typedef void (*mem_free_size_t)(void *, size_t);
170+
typedef void (*mem_format_t)(const char *fmt, ...);
171+
typedef void (*mem_char_t)(const char *);
171172

172173
/*
173174
* Package params support
174175
*/
175176
typedef struct tirpc_pkg_params {
176177
uint32_t debug_flags;
177178
uint32_t other_flags;
178-
mem_char_t thread_name_;
179-
mem_format_t warnx_;
180-
mem_free_size_t free_size_;
181-
mem_1_size_t malloc_;
182-
mem_2_size_t aligned_;
183-
mem_2_size_t calloc_;
184-
mem_p_size_t realloc_;
179+
mem_char_t thread_name_;
180+
mem_format_t warnx_;
181+
mem_free_size_t free_size_;
182+
mem_1_size_t malloc_;
183+
mem_2_size_t aligned_;
184+
mem_2_size_t calloc_;
185+
mem_p_size_t realloc_;
185186
} tirpc_pkg_params;
186187

187188
extern tirpc_pkg_params __ntirpc_pkg_params;
188189

189190
#include <misc/abstract_atomic.h>
190191

191-
#define __warnx(flags, ...) \
192-
do { \
193-
if (__ntirpc_pkg_params.debug_flags & (flags)) { \
194-
__ntirpc_pkg_params.warnx_(__VA_ARGS__); \
195-
} \
192+
#define __warnx(flags, ...) \
193+
do { \
194+
if (__ntirpc_pkg_params.debug_flags & (flags)) { \
195+
__ntirpc_pkg_params.warnx_(__VA_ARGS__); \
196+
} \
196197
} while (0)
197198

198199
#define __debug_flag(flags) (__ntirpc_pkg_params.debug_flags & (flags))
199200

200-
#define mem_alloc(size) __ntirpc_pkg_params.malloc_((size), \
201-
__FILE__, __LINE__, __func__)
202-
#define mem_aligned(align, size) __ntirpc_pkg_params.aligned_((align), (size), \
203-
__FILE__, __LINE__, __func__)
204-
#define mem_calloc(count, size) __ntirpc_pkg_params.calloc_((count), (size), \
205-
__FILE__, __LINE__, __func__)
206-
#define mem_realloc(p, size) __ntirpc_pkg_params.realloc_((p), (size), \
207-
__FILE__, __LINE__, __func__)
208-
#define mem_zalloc(size) __ntirpc_pkg_params.calloc_(1, (size), \
209-
__FILE__, __LINE__, __func__)
210-
211-
static inline void
212-
mem_free(void *p, size_t n)
201+
#define mem_alloc(size) \
202+
__ntirpc_pkg_params.malloc_((size), __FILE__, __LINE__, __func__)
203+
#define mem_aligned(align, size) \
204+
__ntirpc_pkg_params.aligned_((align), (size), __FILE__, __LINE__, \
205+
__func__)
206+
#define mem_calloc(count, size) \
207+
__ntirpc_pkg_params.calloc_((count), (size), __FILE__, __LINE__, \
208+
__func__)
209+
#define mem_realloc(p, size) \
210+
__ntirpc_pkg_params.realloc_((p), (size), __FILE__, __LINE__, __func__)
211+
#define mem_zalloc(size) \
212+
__ntirpc_pkg_params.calloc_(1, (size), __FILE__, __LINE__, __func__)
213+
214+
static inline void mem_free(void *p, size_t n)
213215
{
214216
__ntirpc_pkg_params.free_size_(p, n);
215217
}
@@ -220,8 +222,8 @@ mem_free(void *p, size_t n)
220222

221223
#include <string.h>
222224

223-
static inline void *
224-
mem_strdup_(const char *s, const char *file, int line, const char *function)
225+
static inline void *mem_strdup_(const char *s, const char *file, int line,
226+
const char *function)
225227
{
226228
size_t l = strlen(s) + 1;
227229
void *t = __ntirpc_pkg_params.malloc_(l, file, line, function);
@@ -256,7 +258,24 @@ struct netbuf {
256258

257259
struct rpc_address {
258260
struct netbuf nb;
259-
struct sockaddr_storage ss; /* address buffer */
261+
struct sockaddr_storage ss; /* address buffer */
262+
};
263+
264+
/*
265+
* The network id struct is used in combination with the client address
266+
* to identify the unique client.
267+
* Using proxy protocol, it's possible for clients from multiple networks to connect
268+
* to the server, although they have the same IP address.
269+
* This means that the client IP is not enough to identify the client and the network id
270+
* is needed as well.
271+
*/
272+
struct network_id {
273+
/* based on proxy protocol TLV header type values */
274+
uint32_t source;
275+
union {
276+
uint64_t gcp_psc_connection_id;
277+
/* TODO: add support for more common cloud providers */
278+
};
260279
};
261280

262281
/*
@@ -280,4 +299,4 @@ struct __rpc_sockinfo {
280299
int si_alen;
281300
};
282301

283-
#endif /* _TIRPC_TYPES_H */
302+
#endif /* _TIRPC_TYPES_H */

0 commit comments

Comments
 (0)