improve support for proxy protocol

* Add support for parsing tlv headers
* Add support for parsing network id from common
cloud providers. Note, currently only GCP is fully
supported as I couldn't test the flow on AWS, Azure
but it should be easy to add this support in the future.
* Fix bug that we didn't account for the tlv when
checking the header length
* Fix bug that tlv would have been stored as extra data
inside the address union

Change-Id: Idb68144f34bfa302c478de54eea631e6f80883f9
Signed-off-by: Roy Babayov <[email protected]>

Author: roybabayov

CMakeLists.txt

@@ -79,6 +79,8 @@ endif (USE_GSS)
 
 option(USE_MONITORING "Enable monitoring stack" ON)
 
+option(PROXY_PROTOCOL_PARSE_COMMON_NETWORK_ID "Enable parsing of common network id from haproxy TLV" OFF)
+
 option(USE_RPC_RDMA "platform supports RDMA" OFF)
 if (USE_RPC_RDMA)
   find_package(RDMA REQUIRED)
@@ -263,6 +265,7 @@ message(STATUS "USE_GSS = ${USE_GSS}")
 message(STATUS "USE_PROFILE = ${USE_PROFILE}")
 message(STATUS "USE_LTTNG_NTIRPC = ${USE_LTTNG_NTIRPC}")
 message(STATUS "USE_MONITORING = ${USE_MONITORING}")
+message(STATUS "PROXY_PROTOCOL_PARSE_COMMON_NETWORK_ID = ${PROXY_PROTOCOL_PARSE_COMMON_NETWORK_ID}")
 
 #force command line options to be stored in cache
 set(_MSPAC_SUPPORT ${_MSPAC_SUPPORT}

src/haproxy.h renamed to ntirpc/rpc/haproxy.h

@@ -23,8 +23,16 @@
  *       https://github.com/haproxy/haproxy.git
  *
  * Specifically, parts of this file: include/haproxy/connection-t.h
+ *
+ * NOTE: the definitions in this file are based on the haproxy spec in
+ * https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
  */
 
+#include "config.h"
+
+#ifndef _NTIRPC_RPC_HAPROXY_H
+#define _NTIRPC_RPC_HAPROXY_H
+
 /* proxy protocol v2 definitions */
 #define PP2_SIGNATURE        "\x0D\x0A\x0D\x0A\x00\x0D\x0A\x51\x55\x49\x54\x0A"
 #define PP2_SIG_UINT32       0x0d0a0d0a
@@ -42,7 +50,7 @@
 #define PP2_VERSION_MASK     0xF0
 
 #define PP2_VERSION2_CMD_LOCAL (PP2_CMD_LOCAL | PP2_VERSION)
-#define PP2_VERSIOB2_CMD_PROXY (PP2_CMD_PROXY | PP2_VERSION)
+#define PP2_VERSION2_CMD_PROXY (PP2_CMD_PROXY | PP2_VERSION)
 
 /* fam byte */
 #define PP2_TRANS_UNSPEC     0x00
@@ -82,6 +90,17 @@
 #define PP2_SUBTYPE_SSL_KEY_ALG 0x25
 #define PP2_TYPE_NETNS          0x30
 
+#define PP2_TYPE_UNSET 0x00
+
+/* Define custom PP2_TYPE by common cloud provides */
+/* https://cloud.google.com/vpc/docs/about-vpc-hosted-services#proxy-protocol */
+#define PP2_TYPE_GCP 0xE0
+#define PP2_TYPE_GCP_EXPECTED_LENGTH 8
+/* https://docs.aws.amazon.com/elasticloadbalancing/latest/network/edit-target-group-attributes.html#proxy-protocol */
+#define PP2_TYPE_AWS 0xEA
+/* https://learn.microsoft.com/en-us/azure/private-link/private-link-service-overview#getting-connection-information-using-tcp-proxy-v2 */
+#define PP2_TYPE_AZURE 0xEE
+
 #define PP2_CLIENT_SSL           0x01
 #define PP2_CLIENT_CERT_CONN     0x02
 #define PP2_CLIENT_CERT_SESS     0x04
@@ -97,29 +116,38 @@ struct proxy_header_part {
 	uint16_t len;      /* number of following bytes part of the header */
 };
 
-union proxy_addr { 
-	struct {   /* for TCP/UDP over IPv4, len = 12 */
-		uint32_t src_addr;
-		uint32_t dst_addr;
-		uint16_t src_port;
-		uint16_t dst_port;
-	} ip4;
-	struct {   /* for TCP/UDP over IPv6, len = 36 */
-		uint8_t  src_addr[16];
-		uint8_t  dst_addr[16];
-		uint16_t src_port;
-		uint16_t dst_port;
-	} ip6;
-	struct {   /* for AF_UNIX sockets, len = 216 */
-		uint8_t src_addr[108];
-		uint8_t dst_addr[108];
-	} unx;
+struct proxy_header_addr_ip4_part { /* for TCP/UDP over IPv4, len = 12 */
+	uint32_t src_addr;
+	uint32_t dst_addr;
+	uint16_t src_port;
+	uint16_t dst_port;
+};
+_Static_assert(
+	sizeof(struct proxy_header_addr_ip4_part) == PP2_ADDR_LEN_INET,
+	"proxy_header_addr_ip4_part size is not equal to PP2_ADDR_LEN_INET");
+
+struct proxy_header_addr_ip6_part { /* for TCP/UDP over IPv6, len = 36 */
+	uint8_t src_addr[16];
+	uint8_t dst_addr[16];
+	uint16_t src_port;
+	uint16_t dst_port;
+};
+_Static_assert(
+	sizeof(struct proxy_header_addr_ip6_part) == PP2_ADDR_LEN_INET6,
+	"proxy_header_addr_ip6_part size is not equal to PP2_ADDR_LEN_INET6");
+
+typedef uint8_t proxy_protocol_tlv_type;
+typedef uint16_t proxy_protocol_tlv_length;
+struct proxy_protocol_tlv_header {
+	proxy_protocol_tlv_type type;
+	proxy_protocol_tlv_length length;
+	/* value is stored in big-endian as received from socket */
+	void *value;
 };
 
-struct proxy_hdr_v2 {
-	uint8_t sig[12];   /* hex 0D 0A 0D 0A 00 0D 0A 51 55 49 54 0A */
-	struct proxy_header_part header_part;
-	union proxy_addr addr;
+struct proxy_protocol_tlv_headers {
+	uint16_t tlv_count;
+	struct proxy_protocol_tlv_header *tlvs;
 };
 
-enum xprt_stat recv_haproxy_header(SVCXPRT *xprt);
+#endif /* _NTIRPC_RPC_HAPROXY_H */

ntirpc/rpc/svc.h

@@ -294,6 +294,11 @@ struct svc_xprt {
 	struct rpc_address xp_local;	/* local address, length, port */
 	struct rpc_address xp_remote;	/* remote address, length, port */
 	struct rpc_address xp_proxy;	/* proxy address, length, port */
+	struct proxy_protocol_tlv_headers proxy_protocol_tlv_headers;
+	/* Network id parsed from proxy protocol.
+	 * Populated only when PARSE_COMMON_NETWORK_ID flag is on
+	 */
+	struct network_id xp_remote_network_id;
 
 #if defined(HAVE_BLKIN)
 	/* blkin tracing */

ntirpc/rpc/types.h

@@ -48,6 +48,8 @@
 #include <sys/types.h>
 #include <inttypes.h>
 
+#include "haproxy.h"
+
 #if defined(_WIN32)
 
 #define __BEGIN_DECLS
@@ -259,6 +261,23 @@ struct rpc_address {
 	struct sockaddr_storage ss;	/* address buffer */
 };
 
+/*
+ * The network id struct is used in combination with the client address
+ * to identify the unique client.
+ * Using proxy protocol, it's possible for clients from multiple networks to connect
+ * to the server, although they have the same IP address.
+ * This means that the client IP is not enough to identify the client and the network id
+ * is needed as well.
+ */
+struct network_id {
+	/* based on proxy protocol TLV header type values */
+	uint32_t source;
+	union {
+		uint64_t gcp_psc_connection_id;
+		/* TODO: add support for more common cloud providers */
+	};
+};
+
 /*
  * The format of the addres and options arguments of the XTI t_bind call.
  * Only provided for compatibility, it should not be used.

src/rpc_dplx_internal.h

@@ -33,6 +33,7 @@
 #include <rpc/svc.h>
 #include <rpc/xdr_ioq.h>
 #include <rpc/pool_queue.h>
+#include <rpc/haproxy.h>
 
 /* Svc event strategy */
 enum svc_event_type {
@@ -126,6 +127,12 @@ rpc_dplx_rec_init(struct rpc_dplx_rec *rec)
 	(void)clock_gettime(CLOCK_MONOTONIC_FAST, &(rec->recv.ts));
 
 	rec->xprt.xp_refcnt = 1;
+
+	// Init TLV headers, network id values in case they are not set later on
+	rec->xprt.proxy_protocol_tlv_headers.tlv_count = 0;
+	rec->xprt.proxy_protocol_tlv_headers.tlvs = NULL;
+	rec->xprt.xp_remote_network_id.source = PP2_TYPE_UNSET;
+	rec->xprt.xp_remote_network_id.gcp_psc_connection_id = 0;
 }
 
 static inline void
@@ -135,6 +142,14 @@ rpc_dplx_rec_destroy(struct rpc_dplx_rec *rec)
 	mutex_destroy(&rec->xprt.xp_lock);
 	mutex_destroy(&rec->writeq.qmutex);
 
+	if (rec->xprt.proxy_protocol_tlv_headers.tlv_count > 0) {
+		for (uint16_t i = 0; i < rec->xprt.proxy_protocol_tlv_headers.tlv_count; i++)
+			free(rec->xprt.proxy_protocol_tlv_headers.tlvs[i].value);
+		free(rec->xprt.proxy_protocol_tlv_headers.tlvs);
+		rec->xprt.proxy_protocol_tlv_headers.tlvs = NULL;
+		rec->xprt.proxy_protocol_tlv_headers.tlv_count = 0;
+	}
+
 #if defined(HAVE_BLKIN)
 	if (rec->xprt.blkin.svc_name)
 		mem_free(rec->xprt.blkin.svc_name, 2*INET6_ADDRSTRLEN);