Rework streams allocation

According to our ivsetigation typical streams count
opened by browser is about 15 - 25. We decide for
better memory locality to preallocate 2 pages
(8192 bytes, this is enough for 29 streams) and use
it as a streams memory. If is memory is exceeded we
allocate new 2 pages and use as a streams memory again.
We use 8 bytes at the end of the memory which is used
for streams as a pointer to the new page block.

Author: EvgeniiMekhanik

fw/http2.c

@@ -192,28 +192,135 @@ tfw_h2_apply_new_settings(TfwH2Ctx *ctx)
 	clear_bit(HTTP2_SETTINGS_NEED_TO_APPLY, ctx->settings_to_apply);
 }
 
-int
-tfw_h2_init(void)
+#define TFW_H2_PAGE_ORDER 1
+#define TFW_H2_BOUNDARY(base) 					\
+	(char *)base + PAGE_SIZE * (1 << TFW_H2_PAGE_ORDER);
+#define TFW_H2_NEXT_BLOCK(boundary)				\
+	(unsigned long *)(boundary - sizeof(unsigned long));
+
+TfwH2Ctx *
+tfw_h2_context_alloc(void)
 {
-	return tfw_h2_stream_cache_create();
+	struct page *pg;
+
+	pg = alloc_pages(GFP_ATOMIC, TFW_H2_PAGE_ORDER);
+	if (!pg)
+		return NULL;
+	return (TfwH2Ctx *)page_address(pg);
 }
 
 void
-tfw_h2_cleanup(void)
+tfw_h2_context_free(TfwH2Ctx *ctx)
 {
-	tfw_h2_stream_cache_destroy();
+	free_pages((unsigned long)ctx, TFW_H2_PAGE_ORDER);
 }
 
-TfwH2Ctx *
-tfw_h2_context_alloc(void)
+static inline void
+tfw_h2_context_init_stream_storage_impl(TfwH2Ctx *ctx, void *base,
+					unsigned long *next_block)
 {
-	return (TfwH2Ctx *)kzalloc(sizeof(TfwH2Ctx), GFP_ATOMIC);
+	TfwStream *stream = (TfwStream *)base;
+
+	while ((char *)stream <= (char *)next_block - sizeof(TfwStream)) {
+		stream->next = ctx->empty_list;
+		ctx->empty_list = stream;
+		stream++;
+	}
+}
+
+static inline int
+tfw_h2_conext_alloc_stream_storage_new_block(TfwH2Ctx *ctx)
+{
+	char *boundary;
+	unsigned long *next_block;
+	unsigned long *new_block, *next_new_block;
+	struct page *pg;
+
+	boundary = TFW_H2_BOUNDARY(ctx);
+	next_block = TFW_H2_NEXT_BLOCK(boundary);
+
+	pg = alloc_pages(GFP_ATOMIC, TFW_H2_PAGE_ORDER);
+	if (!unlikely(pg))
+		return -ENOMEM;
+
+	new_block = (unsigned long *)page_address(pg);
+	boundary = TFW_H2_BOUNDARY(new_block);
+	next_new_block = TFW_H2_NEXT_BLOCK(boundary); 
+
+	*next_new_block = *next_block;
+	*next_block = (unsigned long)new_block;
+
+	tfw_h2_context_init_stream_storage_impl(ctx, new_block,
+						next_new_block);
+
+	return 0;
+}
+
+static inline void
+tfw_h2_context_clear_stream_storage(TfwH2Ctx *ctx)
+{
+	char *boundary;
+	unsigned long *next_block;
+
+	boundary = TFW_H2_BOUNDARY(ctx);
+	next_block = TFW_H2_NEXT_BLOCK(boundary);
+
+	while (*next_block) {
+		unsigned long to_free;
+
+		to_free = *next_block;
+		boundary = TFW_H2_BOUNDARY(*next_block);
+		next_block = TFW_H2_NEXT_BLOCK(boundary);
+		free_pages(to_free, TFW_H2_PAGE_ORDER);
+	}
+}
+
+static inline void
+tfw_h2_context_init_stream_storage(TfwH2Ctx *ctx)
+{
+	char *boundary;
+	unsigned long *new_block;
+	unsigned long *next_block;
+
+	boundary = TFW_H2_BOUNDARY(ctx);
+	new_block = (unsigned long *)ctx + sizeof(TfwH2Ctx);
+	next_block = TFW_H2_NEXT_BLOCK(boundary);
+
+	/*
+	 * Pointer to the next page block for empty streams, will be allocated,
+	 * if count of preallocated streams exceeded.
+	 */
+	*next_block = 0;
+	tfw_h2_context_init_stream_storage_impl(ctx, new_block, next_block);
+}
+
+#undef TFW_H2_NEXT_BLOCK
+#undef TFW_H2_BOUNDARY
+#undef TFW_H2_PAGE_ORDER
+
+TfwStream *
+tfw_h2_conext_alloc_stream(TfwH2Ctx *ctx)
+{
+	TfwStream *stream;
+
+	if (!ctx->empty_list) {
+		if (tfw_h2_conext_alloc_stream_storage_new_block(ctx))
+			return NULL;
+	}
+
+	stream = ctx->empty_list;
+	ctx->empty_list = ctx->empty_list->next;
+	memset(stream, 0, sizeof(TfwStream));
+
+	return stream;
 }
 
 void
-tfw_h2_context_free(TfwH2Ctx *ctx)
+tfw_h2_conext_free_stream(TfwH2Ctx *ctx, TfwStream *stream)
 {
-	kfree(ctx);
+	BUG_ON(stream->xmit.resp || stream->xmit.skb_head);
+	stream->next = ctx->empty_list;
+	ctx->empty_list = stream;
 }
 
 int
@@ -236,6 +343,7 @@ tfw_h2_context_init(TfwH2Ctx *ctx, TfwH2Conn *conn)
 	INIT_LIST_HEAD(&idle_streams->list);
 
 	tfw_h2_init_stream_sched(&ctx->sched);
+	tfw_h2_context_init_stream_storage(ctx);
 
 	lset->hdr_tbl_sz = rset->hdr_tbl_sz = HPACK_TABLE_DEF_SIZE;
 	lset->push = rset->push = 1;
@@ -262,6 +370,7 @@ tfw_h2_context_clear(TfwH2Ctx *ctx)
 	 * postponed frames and connection closing initiated.
 	 */
 	ss_skb_queue_purge(&ctx->skb_head);
+	tfw_h2_context_clear_stream_storage(ctx);
 	tfw_hpack_clean(&ctx->hpack);
 }
 
@@ -321,7 +430,7 @@ tfw_h2_conn_streams_cleanup(TfwH2Ctx *ctx)
 		 * No further actions regarding streams dependencies/prio
 		 * is required at this stage.
 		 */
-		tfw_h2_delete_stream(cur);
+		tfw_h2_conext_free_stream(ctx, cur);
 		--ctx->streams_num;
 	}
 	sched->streams = RB_ROOT;

fw/http2.h

@@ -83,6 +83,7 @@ typedef struct tfw_conn_t TfwConn;
  *                        from _HTTP2_SETTINGS_MAX are used to save what
  *                        settings we sent to the client;
  * @conn		- pointer to h2 connection of this context;
+ * @empty_list		- pointer to the first empty stream;
  * @__off               - offset to reinitialize processing context;
  * @skb_head            - collected list of processed skbs containing HTTP/2
  *                        frames;
@@ -128,6 +129,7 @@ typedef struct tfw_h2_ctx_t {
 	unsigned int    new_settings[_HTTP2_SETTINGS_MAX - 1];
 	DECLARE_BITMAP  (settings_to_apply, 2 * _HTTP2_SETTINGS_MAX - 1);
 	TfwH2Conn	*conn;
+	TfwStream	*empty_list;
 	char            __off[0];
 	struct sk_buff  *skb_head;
 	TfwStream       *cur_stream;
@@ -142,8 +144,6 @@ typedef struct tfw_h2_ctx_t {
 	unsigned char   data_off;
 } TfwH2Ctx;
 
-int tfw_h2_init(void);
-void tfw_h2_cleanup(void);
 TfwH2Ctx *tfw_h2_context_alloc(void);
 void tfw_h2_context_free(TfwH2Ctx *ctx);
 int tfw_h2_context_init(TfwH2Ctx *ctx, TfwH2Conn *conn);
@@ -168,5 +168,7 @@ void tfw_h2_req_unlink_and_close_stream(TfwHttpReq *req);
 int tfw_h2_stream_xmit_prepare_resp(TfwStream *stream);
 int tfw_h2_entail_stream_skb(struct sock *sk, TfwH2Ctx *ctx, TfwStream *stream,
 			     unsigned int *len, bool should_split);
+TfwStream * tfw_h2_conext_alloc_stream(TfwH2Ctx *ctx);
+void tfw_h2_conext_free_stream(TfwH2Ctx *ctx, TfwStream *stream);
 
 #endif /* __HTTP2__ */

fw/http_stream.c

@@ -30,26 +30,6 @@
 
 #define HTTP2_DEF_WEIGHT	16
 
-static struct kmem_cache *stream_cache;
-
-int
-tfw_h2_stream_cache_create(void)
-{
-	stream_cache = kmem_cache_create("tfw_stream_cache", sizeof(TfwStream),
-					 0, 0, NULL);
-	if (!stream_cache)
-		return -ENOMEM;
-
-	return 0;
-}
-
-void
-tfw_h2_stream_cache_destroy(void)
-{
-	kmem_cache_destroy(stream_cache);
-}
-
-
 static inline void
 tfw_h2_conn_reset_stream_on_close(TfwH2Ctx *ctx, TfwStream *stream)
 {
@@ -102,10 +82,10 @@ tfw_h2_init_stream(TfwStream *stream, unsigned int id, unsigned short weight,
 }
 
 static TfwStream *
-tfw_h2_add_stream(TfwStreamSched *sched, unsigned int id, unsigned short weight,
-		  long int loc_wnd, long int rem_wnd)
+tfw_h2_add_stream(TfwH2Ctx *ctx, unsigned int id, unsigned short weight)
 {
 	TfwStream *new_stream;
+	TfwStreamSched *sched = &ctx->sched;
 	struct rb_node **new = &sched->streams.rb_node;
 	struct rb_node *parent = NULL;
 
@@ -123,11 +103,12 @@ tfw_h2_add_stream(TfwStreamSched *sched, unsigned int id, unsigned short weight,
 		}
 	}
 
-	new_stream = kmem_cache_alloc(stream_cache, GFP_ATOMIC | __GFP_ZERO);
+	new_stream = tfw_h2_conext_alloc_stream(ctx);
 	if (unlikely(!new_stream))
 		return NULL;
 
-	tfw_h2_init_stream(new_stream, id, weight, loc_wnd, rem_wnd);
+	tfw_h2_init_stream(new_stream, id, weight, ctx->lsettings.wnd_sz,
+			   ctx->rsettings.wnd_sz);
 
 	rb_link_node(&new_stream->node, parent, new);
 	rb_insert_color(&new_stream->node, &sched->streams);
@@ -225,9 +206,7 @@ tfw_h2_stream_create(TfwH2Ctx *ctx, unsigned int id)
 	       pri->exclusive, pri->stream_id, ctx, ctx->streams_num);
 
 	dep = tfw_h2_find_stream_dep(&ctx->sched, pri->stream_id);
-	stream = tfw_h2_add_stream(&ctx->sched, id, pri->weight,
-				   ctx->lsettings.wnd_sz,
-				   ctx->rsettings.wnd_sz);
+	stream = tfw_h2_add_stream(ctx, id, pri->weight);
 	if (!stream)
 		return NULL;
 
@@ -245,7 +224,7 @@ tfw_h2_stream_clean(TfwH2Ctx *ctx, TfwStream *stream)
 	       tfw_h2_get_stream_state(stream), __h2_strm_st_n(stream),
 	       stream->weight, ctx, ctx->streams_num);
 	tfw_h2_stop_stream(&ctx->sched, stream);
-	tfw_h2_delete_stream(stream);
+	tfw_h2_conext_free_stream(ctx, stream);
 	--ctx->streams_num;
 }
 
@@ -795,13 +774,6 @@ tfw_h2_find_stream(TfwStreamSched *sched, unsigned int id)
 	return NULL;
 }
 
-void
-tfw_h2_delete_stream(TfwStream *stream)
-{
-	BUG_ON(stream->xmit.resp || stream->xmit.skb_head);
-	kmem_cache_free(stream_cache, stream);
-}
-
 int
 tfw_h2_stream_init_for_xmit(TfwHttpResp *resp, TfwStreamXmitState state,
 			    unsigned long h_len, unsigned long b_len)
@@ -840,7 +812,7 @@ tfw_h2_stream_init_for_xmit(TfwHttpResp *resp, TfwStreamXmitState state,
 int
 tfw_h2_stream_init_t_len_for_xmit(TfwHttpResp *resp, unsigned long t_len)
 {
-	TfwH2Ctx *ctx = tfw_h2_context_unsafe(resp->req->conn);
+	TfwH2Ctx *ctx = tfw_h2_context(resp->req->conn);
 	TfwStream *stream;
 
 	spin_lock(&ctx->lock);

fw/http_stream.h

@@ -179,6 +179,8 @@ typedef enum {
  * @msg		- message that is currently being processed;
  * @parser	- the state of message processing;
  * @queue	- queue of half-closed or closed streams or NULL;
+ * @next	- pointer to the next free stream, if this stream is in
+ *		  empty list;
  * @xmit	- last http2 response info, used in `xmit` callbacks;
  */
 struct tfw_http_stream_t {
@@ -196,13 +198,12 @@ struct tfw_http_stream_t {
 	TfwMsg			*msg;
 	TfwHttpParser		parser;
 	TfwStreamQueue		*queue;
+	TfwStream		*next;
 	TfwHttpXmit		xmit;
 };
 
 typedef struct tfw_h2_ctx_t TfwH2Ctx;
 
-int tfw_h2_stream_cache_create(void);
-void tfw_h2_stream_cache_destroy(void);
 TfwStream *tfw_h2_stream_create(TfwH2Ctx *ctx, unsigned int id);
 void tfw_h2_stream_remove_idle(TfwH2Ctx *ctx, TfwStream *stream);
 void tfw_h2_stream_clean(TfwH2Ctx *ctx, TfwStream *stream);
@@ -212,7 +213,6 @@ TfwStreamFsmRes tfw_h2_stream_fsm(TfwH2Ctx *ctx, TfwStream *stream,
 				  unsigned char type, unsigned char flags,
 				  bool send, TfwH2Err *err);
 TfwStream *tfw_h2_find_stream(TfwStreamSched *sched, unsigned int id);
-void tfw_h2_delete_stream(TfwStream *stream);
 int tfw_h2_stream_init_for_xmit(TfwHttpResp *resp, TfwStreamXmitState state,
 				unsigned long h_len, unsigned long b_len);
 int tfw_h2_stream_init_t_len_for_xmit(TfwHttpResp *resp, unsigned long t_len);

fw/tls.c

@@ -961,18 +961,20 @@ static inline int
 tfw_tls_over(TlsCtx *tls, int state)
 {
 	int sk_proto = ((SsProto *)tls->sk->sk_user_data)->type;
-	TfwH2Conn *conn = (TfwH2Conn*)tls->sk->sk_user_data;
 
 	if (state == TTLS_HS_CB_FINISHED_NEW
 	    || state == TTLS_HS_CB_FINISHED_RESUMED)
 		TFW_INC_STAT_BH(serv.tls_hs_successful);
 
 	if (TFW_FSM_TYPE(sk_proto) == TFW_FSM_H2) {
+		TfwH2Conn *conn = (TfwH2Conn*)tls->sk->sk_user_data;
 		int r;
 
 		conn->h2 = tfw_h2_context_alloc();
-		if (!conn->h2)
+		if (!conn->h2) {
+			T_ERR("cannot allocate http2 connection context");
 			return -ENOMEM;
+		}
 	    	r = tfw_h2_context_init(conn->h2, conn); 
 	    	if (r) {
 			T_ERR("cannot establish a new h2 connection\n");
@@ -1244,26 +1246,17 @@ tfw_tls_init(void)
 				ttls_cli_id, tfw_tls_alpn_match,
 				tfw_ja5t_limit_conn, tfw_ja5t_limit_rec);
 
-	if ((r = tfw_h2_init()))
-		goto err_h2;
-
 	tfw_connection_hooks_register(&tls_conn_hooks, TFW_FSM_HTTPS);
 	tfw_connection_hooks_register(&tls_conn_hooks, TFW_FSM_H2);
 	tfw_mod_register(&tfw_tls_mod);
 
 	return 0;
-
-err_h2:
-	tfw_tls_do_cleanup();
-
-	return r;
 }
 
 void
 tfw_tls_exit(void)
 {
 	tfw_mod_unregister(&tfw_tls_mod);
 	tfw_connection_hooks_unregister(TFW_FSM_HTTPS);
-	tfw_h2_cleanup();
 	tfw_tls_do_cleanup();
 }