feat: 优化echart图表展示

feat: 优化管理数据文件列表 feat: 支持统计服务级别WAF数据汇总 feat: 优化代码注释 feat:事务日程调整
tl-open-source · Mar 18, 2023 · 6c30482 · 6c30482
1 parent ca64066
commit 6c30482
Show file tree

Hide file tree

Showing 24 changed files with 351 additions and 333 deletions.
diff --git a/api/router/tl_ops_api_get_state.lua b/api/router/tl_ops_api_get_state.lua
@@ -36,7 +36,6 @@ local Router = function()
     end
     local service_list = cjson.decode(list_str)
 
-
     for service_name, nodes in pairs(service_list) do
         -- service级别cache
         local health_lock_cache = shared:get(tl_ops_utils_func:gen_node_key(tl_ops_constant_health.cache_key.lock, service_name))
@@ -62,12 +61,49 @@ local Router = function()
             limit_version_cache = 0 --"version cache nil"
         end
 
+        -- waf统计， service级别是包含了service下所有waf规则下的汇总统计
+        local waf_api_count = cache_waf_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf_count.cache_key.service_counting_list, service_name, "api"))
+        if not waf_api_count then
+            waf_api_count = "{}"
+        end
+
+        local waf_ip_count = cache_waf_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf_count.cache_key.service_counting_list, service_name, "ip"))
+        if not waf_ip_count then
+            waf_ip_count = "{}"
+        end
+
+        local waf_cc_count = cache_waf_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf_count.cache_key.service_counting_list, service_name, "cc"))
+        if not waf_cc_count then
+            waf_cc_count = "{}"
+        end
+
+        local waf_cookie_count = cache_waf_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf_count.cache_key.service_counting_list, service_name, "cookie"))
+        if not waf_cookie_count then
+            waf_cookie_count = "{}"
+        end
+
+        local waf_header_count = cache_waf_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf_count.cache_key.service_counting_list, service_name, "header"))
+        if not waf_header_count then
+            waf_header_count = "{}"
+        end
+
+        local waf_param_count = cache_waf_count:get001(tl_ops_utils_func:gen_node_key(tl_ops_constant_waf_count.cache_key.service_counting_list, service_name, "param"))
+        if not waf_param_count then
+            waf_param_count = "{}"
+        end
+
         cache_state.service[service_name] = {
             health_lock = health_lock_cache,
             health_version = health_version_cache,
             health_uncheck = health_uncheck_cache,
             limit_state = limit_state_cache,
             limit_version = limit_version_cache,
+            waf_api_count = cjson.decode(waf_api_count),
+            waf_ip_count = cjson.decode(waf_ip_count),
+            waf_cc_count = cjson.decode(waf_cc_count),
+            waf_cookie_count = cjson.decode(waf_cookie_count),
+            waf_header_count = cjson.decode(waf_header_count),
+            waf_param_count = cjson.decode(waf_param_count),
         }
         cache_state.service[service_name].nodes = { }
 
@@ -226,7 +262,8 @@ local Router = function()
     cache_state.balance['count_interval'] = tl_ops_constant_balance_count.interval
 
     -- waf相关
-
+
+
     -- 其他
     -- cache_state.other['dict_keys'] = shared:get_keys(1024)
 

diff --git a/api/router/tl_ops_api_get_store.lua b/api/router/tl_ops_api_get_store.lua
@@ -11,6 +11,14 @@ local tl_ops_manage_env = require("tl_ops_manage_env")
 local cjson             = require("cjson.safe");
 cjson.encode_empty_table_as_object(false)
 
+local store_list = {
+    "tl-ops-balance-api","tl-ops-balance-body","tl-ops-balance-cookie",
+    "tl-ops-balance-header","tl-ops-balance-param","tl-ops-service",
+    "tl-ops-health","tl-ops-plugins-manage","tl-ops-limit",
+    "tl-ops-balance","tl-ops-waf","tl-ops-waf-api", "tl-ops-waf-ip",
+    "tl-ops-waf-header","tl-ops-waf-param","tl-ops-waf-cookie","tl-ops-waf-cc",
+    "tl-ops-auth","tl-ops-time-alert","tl-ops-ssl","tl-ops-page-proxy"
+}
 
 -- 读取文件
 local read = function( filename )
@@ -40,225 +48,49 @@ local read = function( filename )
 end
 
 
-local Router = function() 
-    local api_content, api_size = read("tl-ops-balance-api.tlstore");
-    if not api_content then
-        api_content = {}
-    end
-    local body_content, body_size = read("tl-ops-balance-body.tlstore");
-    if not body_content then
-        body_content = {}
-    end
-    local cookie_content, cookie_size = read("tl-ops-balance-cookie.tlstore");
-    if not cookie_content then
-        cookie_content = {}
-    end
-    local header_content, header_size = read("tl-ops-balance-header.tlstore");
-    if not header_content then
-        header_content = {}
-    end
-    local param_content, param_size = read("tl-ops-balance-param.tlstore");
-    if not param_content then
-        param_content = {}
-    end
-    local service_content, service_size = read("tl-ops-service.tlstore");
-    if not service_content then
-        service_content = {}
-    end
-    local health_content, health_size = read("tl-ops-health.tlstore");
-    if not health_content then
-        health_content = {}
-    end
-    local plugins_manage_content, plugins_manage_size = read("tl-ops-plugins-manage.tlstore");
-    if not plugins_manage_content then
-        plugins_manage_content = {}
-    end
-    local limit_content, limit_size = read("tl-ops-limit.tlstore");
-    if not limit_content then
-        limit_content = {}
-    end
-    local balance_content, balance_size = read("tl-ops-balance.tlstore");
-    if not balance_content then
-        balance_content = {}
-    end
-    local waf_content, waf_size = read("tl-ops-waf.tlstore");
-    if not waf_content then
-        waf_content = {}
-    end
-    local waf_api_content, waf_api_size = read("tl-ops-waf-api.tlstore");
-    if not waf_api_content then
-        waf_api_content = {}
-    end
-    local waf_ip_content, waf_ip_size = read("tl-ops-waf-ip.tlstore");
-    if not waf_ip_content then
-        waf_ip_content = {}
-    end
-    local waf_cookie_content, waf_cookie_size = read("tl-ops-waf-cookie.tlstore");
-    if not waf_cookie_content then
-        waf_cookie_content = {}
-    end
-    local waf_param_content, waf_param_size = read("tl-ops-waf-param.tlstore");
-    if not waf_param_content then
-        waf_param_content = {}
-    end
-    local waf_header_content, waf_header_size = read("tl-ops-waf-header.tlstore");
-    if not waf_header_content then
-        waf_header_content = {}
-    end
-    local waf_cc_content, waf_cc_size = read("tl-ops-waf-cc.tlstore");
-    if not waf_cc_content then
-        waf_cc_content = {}
-    end
-    local auth, auth_size = read("tl-ops-auth.tlstore");
-    if not auth then
-        auth = {}
-    end
-    local time_alert, time_alert_size = read("tl-ops-time-alert.tlstore");
-    if not time_alert then
-        time_alert = {}
+local Router = function()
+    ngx.req.read_body()
+    local args = ngx.req.get_post_args()
+
+    local page = args.page
+    if not page then
+        page = 1
     end
-        local time_alert, time_alert_size = read("tl-ops-time-alert.tlstore");
-    if not time_alert then
-        time_alert = {}
+    page = tonumber(page)
+
+    local limit = args.limit
+    if not limit then
+        limit = #store_list
     end
+    limit = tonumber(limit)
+
+    local start = (page - 1) * limit + 1
+    local tail = page * limit
 
-    local res_data = {
-        api = {
-            id = 1,
-            name = "tl-ops-balance-api.tlstore",
-            size = api_size,
-            version = #api_content/2,
-            list = api_content,
-        },
-        body = {
-            id = 1,
-            name = "tl-ops-balance-body.tlstore",
-            size = body_size,
-            version = #body_content/2,
-            list = body_content,
-        },
-        cookie = {
-            id = 2,
-            name = "tl-ops-balance-cookie.tlstore",
-            size = cookie_size,
-            version = #cookie_content/2,
-            list = cookie_content,
-        },
-        header = {
-            id = 3,
-            name = "tl-ops-balance-header.tlstore",
-            size = header_size,
-            version = #header_content/2,
-            list = header_content,
-        },
-        param = {
-            id = 4,
-            name = "tl-ops-balance-param.tlstore",
-            size = param_size,
-            version = #param_content/2,
-            list = param_content,
-        },
-        service = {
-            id = 5,
-            name = "tl-ops-service.tlstore",
-            size = service_size,
-            version = #service_content/2,
-            list = service_content,
-        },
-        health = {
-            id = 6,
-            name = "tl-ops-health.tlstore",
-            size = health_size,
-            version = #health_content,
-            list = health_content,
-        },
-        limit = {
-            id = 7,
-            name = "tl-ops-limit.tlstore",
-            size = limit_size,
-            version = #limit_content/3,
-            list = limit_content,
-        },
-        plugins_manage = {
-            id = 8,
-            name = "tl-ops-plugins-manage.tlstore",
-            size = plugins_manage_size,
-            version = #plugins_manage_content,
-            list = plugins_manage_content,
-        },
-        balance = {
-            id = 8,
-            name = "tl-ops-balance.tlstore",
-            size = balance_size,
-            version = #balance_content,
-            list = balance_content,
-        },
-        waf = {
-            id = 9,
-            name = "tl-ops-waf.tlstore",
-            size = waf_size,
-            version = #waf_content,
-            list = waf_content,
-        },
-        waf_api = {
-            id = 10,
-            name = "tl-ops-waf-api.tlstore",
-            size = waf_api_size,
-            version = #waf_api_content,
-            list = waf_api_content,
-        },
-        waf_ip = {
-            id = 11,
-            name = "tl-ops-waf-ip.tlstore",
-            size = waf_ip_size,
-            version = #waf_ip_content,
-            list = waf_ip_content,
-        },
-        waf_cookie = {
-            id = 12,
-            name = "tl-ops-waf-cookie.tlstore",
-            size = waf_cookie_size,
-            version = #waf_cookie_content,
-            list = waf_cookie_content,
-        },
-        waf_param = {
-            id = 13,
-            name = "tl-ops-waf-param.tlstore",
-            size = waf_param_size,
-            version = #waf_param_content,
-            list = waf_param_content,
-        },
-        waf_header = {
-            id = 14,
-            name = "tl-ops-waf-header.tlstore",
-            size = waf_header_size,
-            version = #waf_header_content,
-            list = waf_header_content,
-        },
-        waf_cc = {
-            id = 15,
-            name = "tl-ops-waf-cc.tlstore",
-            size = waf_cc_size,
-            version = #waf_cc_content,
-            list = waf_cc_content,
-        },
-        auth = {
-            id = 16,
-            name = "tl-ops-auth.tlstore",
-            size = auth_size,
-            version = #auth,
-            list = auth,
-        },
-        time_alert = {
-            id = 17,
-            name = "tl-ops-time-alert.tlstore",
-            size = time_alert_size,
-            version = #time_alert,
-            list = time_alert,
-        }
+    local res_data = { 
+        count = #store_list,
+        list = {}
     }
 
+    for index, value in ipairs(store_list) do
+        if index >= start and index <= tail then
+            local name = value .. ".tlstore";
+            local content, size = read(name);
+            if not content then
+                content = {}
+            end
+            table.insert(res_data.list, {
+                id = index,
+                name = value .. ".tlstore",
+                size = size,
+                version = "暂不统计",
+                list = content,
+            })
+        end
+    end
+
     tl_ops_utils_func:set_ngx_req_return_ok(tl_ops_rt.ok, "success", res_data);
  end
-
+
+
 return Router
diff --git a/bin/install_centeros.sh b/bin/install_centeros.sh
@@ -6,7 +6,7 @@ TL_OPS_PATH="/usr/local/tl-ops-manage/"
 TL_OPS_CONF_PATH="/usr/local/tl-ops-manage/conf/tl_ops_manage.conf"
 TL_OPS_LUA_PATH="/usr/local/openresty/lualib/?.lua;;/usr/local/tl-ops-manage/?.lua;;"
 TL_OPS_LUAC_PATH="/usr/local/openresty/lualib/?.so;;"
-TL_OPS_VER="v3.4.0"
+TL_OPS_VER="v3.4.1"
 
 echo_msg(){
     cur_time=$(date "+%Y-%m-%d %H:%M:%S")

diff --git a/bin/install_ubuntu.sh b/bin/install_ubuntu.sh
@@ -6,7 +6,7 @@ TL_OPS_PATH="/usr/local/tl-ops-manage/"
 TL_OPS_CONF_PATH="/usr/local/tl-ops-manage/conf/tl_ops_manage.conf"
 TL_OPS_LUA_PATH="/usr/local/openresty/lualib/?.lua;;/usr/local/tl-ops-manage/?.lua;;"
 TL_OPS_LUAC_PATH="/usr/local/openresty/lualib/?.so;;"
-TL_OPS_VER="v3.4.0"
+TL_OPS_VER="v3.4.1"
 
 echo_msg(){
     cur_time=$(date "+%Y-%m-%d %H:%M:%S")

diff --git a/constant/tl_ops_constant_waf_count.lua b/constant/tl_ops_constant_waf_count.lua
@@ -3,7 +3,7 @@ local tl_ops_constant_waf_count = {
     cache_key = {
         -- 临时字段
         lock = "tl_ops_waf_count_lock",
-        node_req_succ = "tl_ops_waf_node_req_succ",                -- 以节点单位命中次数次数 int
+        service_req_succ = "tl_ops_waf_service_req_succ",          -- 以服务为单位命中次数次数 int
         ip_req_succ = "tl_ops_waf_ip_req_succ",                    -- 以ip规则命中次数次数 int
         api_req_succ = "tl_ops_waf_api_req_succ",                  -- 以api规则命中次数次数 int
         cc_req_succ = "tl_ops_waf_cc_req_succ",                    -- 以cc规则命中次数次数 int
@@ -12,7 +12,7 @@ local tl_ops_constant_waf_count = {
         param_req_succ = "tl_ops_waf_param_req_succ",              -- 以param规则命中次数次数 int
 
         -- 持久化字段
-        node_counting_list = "tl_ops_waf_node_counting_list",          -- 以ip规则单位，周期内统计次数集合 list
+        service_counting_list = "tl_ops_waf_service_counting_list",    -- 以服务单位，周期内统计次数集合 list
         ip_counting_list = "tl_ops_waf_ip_counting_list",              -- 以ip规则单位，周期内统计次数集合 list
         api_counting_list = "tl_ops_waf_api_counting_list",            -- 以api规则单位，周期内统计次数集合 list
         cc_counting_list = "tl_ops_waf_cc_counting_list",              -- 以cc规则为单位，周期内统计次数集合 list