Implemented regex for locations and httptables

Author: supervisor

Makefile

@@ -146,7 +146,7 @@ KERNEL = /lib/modules/$(shell uname -r)/build
 
 export KERNEL TFW_CFLAGS AVX2 BMI2 ADX TFW_GCOV
 
-obj-m	+= lib/ db/core/ fw/ tls/
+obj-m	+= lib/ db/core/ regex/ fw/ tls/
 
 all: build
 

fw/cfg.c

@@ -109,6 +109,9 @@
  * them. Helpers below facilitate that.
  */
 
+unsigned short number_of_regex = 0;
+unsigned short number_of_db_regex = 0;
+
 static const char *
 __alloc_and_copy_literal(const char *src, size_t len, bool keep_bs)
 {

fw/cfg.h

@@ -336,6 +336,9 @@ struct TfwCfgSpec {
 	void (*cleanup)(TfwCfgSpec *self);
 };
 
+extern unsigned short number_of_regex;
+extern unsigned short number_of_db_regex;
+
 /**
  * Walks over a NULL-terminated array of TfwCfgSpec structures.
  */

fw/http_match.c

@@ -71,6 +71,7 @@
 #include "http_match.h"
 #include "http_msg.h"
 #include "cfg.h"
+#include "regex/kmod/rex.h"
 
 /**
  * Map an operator to that flags passed to tfw_str_eq_*() functions.
@@ -88,6 +89,35 @@ map_op_to_str_eq_flags(tfw_http_match_op_t op)
 	return flags_tbl[op];
 }
 
+extern int bpf_scan_bytes(const void *, __u32, struct rex_scan_attr *);
+
+bool
+tfw_match_regex(tfw_match_t op, const char *cstr, size_t len, const TfwStr *arg)
+{
+        char sss[100];
+        struct rex_scan_attr attr = {};
+        memcpy(&attr.database_id, cstr, sizeof(unsigned short));
+
+        tfw_str_to_cstr(arg, sss, 100);
+
+        pr_notice("attr.database_id=%u", attr.database_id);
+        pr_notice("arg=%s", sss);
+        int r;
+        bool result;
+        pr_notice("test_regex\n");
+        r = bpf_scan_bytes(arg->data, arg->len, &attr);
+
+	result = (!r && attr.nr_events && attr.last_event.expression);
+	if (result)
+		pr_notice("Expression found in pos=%llu\n number=%u",
+		          attr.last_event.to, attr.last_event.expression);
+	else
+		pr_notice("Expression not found!");
+
+	pr_notice("========================================================\n");
+	return result;
+}
+
 static bool
 tfw_rule_str_match(const TfwStr *str, const char *cstr,
 		   int cstr_len, tfw_str_eq_flags_t flags,
@@ -97,6 +127,9 @@ tfw_rule_str_match(const TfwStr *str, const char *cstr,
 		return tfw_str_eq_cstr_off(str, str->len - cstr_len,
 					   cstr, cstr_len, flags);
 
+	if (op == TFW_HTTP_MATCH_O_REGEX)
+		return tfw_match_regex(op, cstr, cstr_len, str);
+
 	return tfw_str_eq_cstr(str, cstr, cstr_len, flags);
 }
 
@@ -706,6 +739,83 @@ tfw_http_escape_pre_post(char *out , const char *str)
 	return len;
 }
 
+/*
+ * Here we create text file for every regex string which
+ * can be readed by hscollider.
+ * Next hscollider compile it and save to temporary DB.
+ * After it will be loaded to regex module DB.
+ * All operations after creating will be done in script start_regex.sh
+ *
+ * As it potentially possible situation then one DB conains several
+ * expressions, here are two variables:
+ * number_of_db_regex - nomber of databes which we will use to look for
+ * expression;
+ * number_of_regex - number of expression to know wich exactly expression
+ * was matched (parsing for it has not not implemented yet)
+ *
+ * After this function, number_of_db_regex will be written to start of arg,
+ * so the lenght of regex string must be longer then two bytes.
+ *
+ * Directory /tmp/tempesata is created from
+ * tempesta.sh script.
+ */
+int
+write_regex(const char *arg)
+{
+	struct file *fl;
+	loff_t off = 0;
+	int r;
+	char file_name[25];
+	char reg_number[6];
+	int len = strlen(arg) - 1;
+	int len1;
+
+	if (len < sizeof(unsigned short)) {
+		T_ERR_NL("String of regex too short\n");
+		return -EINVAL;
+	}
+
+	++number_of_db_regex;
+	sprintf(file_name, "/tmp/tempesta/%u.txt", number_of_db_regex);
+
+	fl = filp_open(file_name, O_CREAT | O_WRONLY, 0600);
+	if (IS_ERR(fl)) {
+		T_ERR_NL("Cannot create regex file %s\n",
+		          file_name);
+		return -EINVAL;
+	}
+	BUG_ON(!fl || !fl->f_path.dentry);
+
+	if (!fl->f_op->fallocate) {
+		T_ERR_NL("File requires filesystem with fallocate support\n");
+		filp_close(fl, NULL);
+		return -EINVAL;
+	}
+
+	++number_of_regex;
+	sprintf(reg_number, "%i:", number_of_regex);
+	len1 = strlen(reg_number);
+	r = kernel_write(fl, (void *)reg_number, len1, &off);
+	if (r != len1)
+		goto err;
+
+	r = kernel_write(fl, (void *)&arg[1], len, &off);
+	if (r != len)
+		goto err;
+
+
+	r = kernel_write(fl, "\n", 1, &off);
+	if (r != 1)
+		goto err;
+
+	filp_close(fl, NULL);
+	return 0;
+err:
+	T_ERR_NL("Cannot write regex\n");
+	filp_close(fl, NULL);
+	return r;
+}
+
 const char *
 tfw_http_arg_adjust(const char *arg, tfw_http_match_fld_t field,
 		    const char *raw_hdr_name, size_t *size_out,
@@ -751,6 +861,11 @@ tfw_http_arg_adjust(const char *arg, tfw_http_match_fld_t field,
 	if (wc_arg || (len > 1 && arg[len - 1] == '*' && arg[len - 2] != '\\'))
 		*op_out = TFW_HTTP_MATCH_O_PREFIX;
 
+	if (!wc_arg && arg[0] == '^') {
+		*op_out = TFW_HTTP_MATCH_O_REGEX;
+		write_regex(arg);
+	}
+
 	/*
 	 * For argument started with wildcard, the suffix matching
 	 * pattern should be applied.
@@ -779,6 +894,12 @@ tfw_http_arg_adjust(const char *arg, tfw_http_match_fld_t field,
 	len = tfw_http_escape_pre_post(pos, arg);
 	*size_out += full_name_len + len + 1;
 
+	/*
+	 * Save number_of_db_regex to use it in tfw_match_regex
+	 */
+	if (*op_out == TFW_HTTP_MATCH_O_REGEX)
+		memcpy(arg_out, &number_of_db_regex, sizeof(number_of_db_regex));
+
 	return arg_out;
 }
 

fw/http_match.h

@@ -52,6 +52,7 @@ typedef enum {
 	TFW_HTTP_MATCH_O_EQ,
 	TFW_HTTP_MATCH_O_PREFIX,
 	TFW_HTTP_MATCH_O_SUFFIX,
+	TFW_HTTP_MATCH_O_REGEX,
 	_TFW_HTTP_MATCH_O_COUNT
 } tfw_http_match_op_t;
 
@@ -170,6 +171,11 @@ int tfw_http_search_cookie(const char *cstr, unsigned long clen,
 			   TfwStr **pos, TfwStr *end, TfwStr *val,
 			   tfw_http_match_op_t op, bool is_resp_hdr);
 
+int write_regex(const char *arg);
+
+bool tfw_match_regex(tfw_match_t op, const char *cstr, size_t len,
+                     const TfwStr *arg);
+
 #define tfw_http_chain_rules_for_each(chain, func)			\
 ({									\
 	int r = 0;							\

fw/vhost.c

@@ -39,6 +39,7 @@
 #include "http_sess.h"
 #include "client.h"
 #include "tls_conf.h"
+#include "regex/kmod/rex.h"
 
 /*
  * The hash table entry for mapping @sni to @vhost for SAN certificates handling.
@@ -75,6 +76,7 @@ static const TfwCfgEnum tfw_match_enum[] = {
 	{ "eq",		TFW_HTTP_MATCH_O_EQ },
 	{ "prefix",	TFW_HTTP_MATCH_O_PREFIX },
 	{ "suffix",	TFW_HTTP_MATCH_O_SUFFIX },
+        { "regex",	TFW_HTTP_MATCH_O_REGEX },
 	{ 0 }
 };
 
@@ -177,6 +179,14 @@ __tfw_match_prefix(tfw_match_t op, const char *cstr, size_t len, TfwStr *arg)
 	return tfw_str_eq_cstr(arg, cstr, len, flags);
 }
 
+extern int bpf_scan_bytes(const void *, __u32, struct rex_scan_attr *);
+
+static bool
+__tfw_match_regex(tfw_match_t op, const char *cstr, size_t len, TfwStr *arg)
+{
+	return tfw_match_regex(op, cstr, len, arg);
+}
+
 typedef bool (*__tfw_match_fn)(tfw_match_t, const char *, size_t, TfwStr *);
 
 static const __tfw_match_fn __tfw_match_fn_tbl[] = {
@@ -185,6 +195,7 @@ static const __tfw_match_fn __tfw_match_fn_tbl[] = {
 	[TFW_HTTP_MATCH_O_EQ]		= __tfw_match_eq,
 	[TFW_HTTP_MATCH_O_PREFIX]	= __tfw_match_prefix,
 	[TFW_HTTP_MATCH_O_SUFFIX]	= __tfw_match_suffix,
+        [TFW_HTTP_MATCH_O_REGEX]	= __tfw_match_regex,
 };
 
 /*
@@ -1290,8 +1301,15 @@ tfw_location_init(TfwLocation *loc, tfw_match_t op, const char *arg,
 		    + sizeof(TfwHdrModsDesc) * TFW_USRHDRS_ARRAY_SZ * 2
 		    + sizeof(TfwHdrModsDesc *) * TFW_HTTP_HDR_RAW * 2;
 
-	if ((argmem = kmalloc(len + 1, GFP_KERNEL)) == NULL)
-		return -ENOMEM;
+	if (op != TFW_HTTP_MATCH_O_REGEX) {
+		if ((argmem = kmalloc(len + 1, GFP_KERNEL)) == NULL)
+			return -ENOMEM;
+	}
+	else {/*If it is a regex we need only number of DB*/
+		if ((argmem = kmalloc(2 + 1, GFP_KERNEL)) == NULL)
+			return -ENOMEM;
+	}
+
 	if ((data = kzalloc(size, GFP_KERNEL)) == NULL) {
 		kfree(argmem);
 		return -ENOMEM;
@@ -1325,7 +1343,16 @@ tfw_location_init(TfwLocation *loc, tfw_match_t op, const char *arg,
 		(TfwHdrModsDesc **)(loc->mod_hdrs[TFW_VHOST_HDRMOD_RESP].hdrs
 				    + TFW_USRHDRS_ARRAY_SZ);
 
-	memcpy((void *)loc->arg, (void *)arg, len + 1);
+	if (op != TFW_HTTP_MATCH_O_REGEX)
+		memcpy((void *)loc->arg, (void *)arg, len + 1);
+	else {
+		write_regex(arg);
+		/*
+		 * Save number_of_db_regex to use it in tfw_match_regex
+		 */
+		memcpy((void *)loc->arg, (void *)&number_of_db_regex,
+		                sizeof(number_of_db_regex));
+	}
 
 	return 0;
 }

install.txt

@@ -0,0 +1,28 @@
+Colm (Colm Programming Language)
+git clone https://github.com/adrian-thurston/colm.git
+
+$ ./autogen.sh
+$ ./configure
+$ make
+$ make install
+
+add LD_LIBRARY_PATH="/usr/local/lib" to /etc/environment
+
+
+
+Regal
+git clone https://github.com/adrian-thurston/ragel.git
+
+$ ./autogen.sh
+$ ./configure --with-colm=/usr/local/
+$ make
+$ make install
+
+
+
+https://github.com/tempesta-tech/linux-regex-module.git
+./cmake ./
+
+after compilation
+copy hscollider from /linux-regex-module/bin/ to /tempesta/scripts/
+

regex/Makefile

@@ -0,0 +1,82 @@
+obj-m := xdp_rex.o
+
+CC_FLAGS_HSRUNTIME	:= -isystem $(shell $(CC) -print-file-name=include)
+CC_FLAGS_REMOVE_SIMD	:= -mno-80387 -mno-fp-ret-in-387 -mno-sse -mno-mmx -mno-sse2 -mno-3dnow -mno-avx
+CC_FLAGS_HSRUNTIME	+= -DHAVE_SSE2
+CC_FLAGS_SIMD		:= -msse4.2 -msse4.1
+CC_FLAGS_HSRUNTIME	+= -DHAVE_SSE41 -DHAVE_SSE42
+CC_FLAGS_SIMD		+= -mavx -mavx2
+CC_FLAGS_HSRUNTIME	+= -DHAVE_AVX -DHAVE_AVX2
+#CC_FLAGS_SIMD		+= -mavx512f -mavx512cd -mavx512bw -mavx512vl -mavx512vnni
+#CC_FLAGS_HSRUNTIME	+= -DHAVE_AVX512
+#CC_FLAGS_SIMD		+= -mavx512vbmi -mavx512vbmi2 -mavx512vnni
+#CC_FLAGS_HSRUNTIME	+= -DHAVE_AVX512VBMI
+
+CC_FLAGS_HSRUNTIME		+= $(CC_FLAGS_SIMD)
+CC_FLAGS_REMOVE_HSRUNTIME	:= $(CC_FLAGS_REMOVE_SIMD)
+CC_FLAGS_REMOVE_HSRUNTIME	+= -Wdeclaration-after-statement
+CC_FLAGS_HSRUNTIME		+= -Wframe-larger-than=2048
+CC_FLAGS_HSRUNTIME		+= -std=gnu11
+CC_FLAGS_REMOVE_HSRUNTIME	+= -std=gnu99
+
+ccflags-y			+= -std=c99
+ccflags-y			+= -I$(src) -I$(src)/kmod
+ccflags-y			+= $(CC_FLAGS_HSRUNTIME)
+ccflags-remove-y		+= $(CC_FLAGS_REMOVE_HSRUNTIME)
+
+CFLAGS_kmod/rex.o		:= $(CC_FLAGS_REMOVE_HSRUNTIME)
+CFLAGS_REMOVE_kmod/rex.o	:= $(CC_FLAGS_HSRUNTIME)
+CFLAGS_alloc.o			:= $(CC_FLAGS_REMOVE_SIMD)
+CFLAGS_REMOVE_alloc.o		:= $(CC_FLAGS_SIMD)
+CFLAGS_scratch.o		:= $(CC_FLAGS_REMOVE_SIMD)
+CFLAGS_REMOVE_scratch.o		:= $(CC_FLAGS_SIMD)
+CFLAGS_database.o		:= $(CC_FLAGS_REMOVE_SIMD)
+CFLAGS_REMOVE_database.o	:= $(CC_FLAGS_SIMD)
+
+xdp_rex-m	:= kmod/rex.o               \
+		   alloc.o                  \
+		   scratch.o                \
+		   runtime.o                \
+		   database.o               \
+		   hs_version.o             \
+		   stream_compress.o        \
+		   fdr/fdr.o                \
+		   fdr/teddy_avx2.o         \
+		   fdr/teddy.o              \
+		   hwlm/hwlm.o              \
+		   hwlm/noodle_engine.o     \
+		   nfa/accel.o              \
+		   nfa/castle.o             \
+		   nfa/gough.o              \
+		   nfa/lbr.o                \
+		   nfa/limex_64.o           \
+		   nfa/limex_accel.o        \
+		   nfa/limex_native.o       \
+		   nfa/limex_simd128.o      \
+		   nfa/limex_simd256.o      \
+		   nfa/limex_simd384.o      \
+		   nfa/limex_simd512.o      \
+		   nfa/mcclellan.o          \
+		   nfa/mcsheng.o            \
+		   nfa/mcsheng_data.o       \
+		   nfa/mpv.o                \
+		   nfa/nfa_api_dispatch.o   \
+		   nfa/repeat.o             \
+		   nfa/sheng.o              \
+		   nfa/shufti.o             \
+		   nfa/tamarama.o           \
+		   nfa/truffle.o            \
+		   rose/block.o             \
+		   rose/catchup.o           \
+		   rose/init.o              \
+		   rose/match.o             \
+		   rose/program_runtime.o   \
+		   rose/stream.o            \
+		   som/som_runtime.o        \
+		   som/som_stream.o         \
+		   util/cpuid_flags.o       \
+		   util/masked_move.o       \
+		   util/multibit.o          \
+		   util/simd_utils.o        \
+		   util/state_compress.o    \
+#