From faa4755e2a44736b9cab2278ff1747411aec8832 Mon Sep 17 00:00:00 2001 From: 6unyoung Date: Wed, 5 Nov 2025 10:09:16 +0900 Subject: [PATCH] FEATURE: Add cmdlog optional filtering --- cmdlog.c | 251 +++++++++++++++++- cmdlog.h | 6 +- .../ch13-command-administration.md | 39 +++ memcached.c | 133 ++++++++-- 4 files changed, 409 insertions(+), 20 deletions(-) diff --git a/cmdlog.c b/cmdlog.c index 40d8ad99f..171321ded 100644 --- a/cmdlog.c +++ b/cmdlog.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "memcached/util.h" @@ -38,6 +39,10 @@ #define CMDLOG_FILENAME_LENGTH CMDLOG_DIRPATH_LENGTH + 128 #define CMDLOG_FILENAME_FORMAT "%s/command_%d_%d_%d_%d.log" +#define CMDLOG_FILTER_MAXNUM 10 +#define CMDLOG_FILTER_CMD_MAXLEN 15 +#define CMDLOG_FILTER_KEY_MAXLEN 16000 + /* cmdlog state */ #define CMDLOG_NOT_STARTED 0 /* not started */ #define CMDLOG_OVERFLOW_STOP 1 /* stop by command log overflow */ @@ -59,6 +64,12 @@ struct cmd_log_buffer { uint32_t last; }; +/* command log filter structure */ +struct cmd_log_filter { + char command[CMDLOG_FILTER_CMD_MAXLEN + 1]; + char key[CMDLOG_FILTER_KEY_MAXLEN + 1]; +}; + /*command log flush structure */ struct cmd_log_flush { pthread_t tid; /* flush thread id */ @@ -85,6 +96,8 @@ struct cmd_log_global { struct cmd_log_buffer buffer; struct cmd_log_flush flush; struct cmd_log_stats stats; + struct cmd_log_filter filters[CMDLOG_FILTER_MAXNUM]; + int nfilters; volatile bool reqstop; }; struct cmd_log_global cmdlog; @@ -397,25 +410,42 @@ char *cmdlog_stats(void) return str; } -void cmdlog_write(char *client_ip, char *command) +void cmdlog_write(char *client_ip, char *command, int cmdlen) { struct tm *ptm; struct timeval val; struct cmd_log_buffer *buffer = &cmdlog.buffer; char inputstr[CMDLOG_INPUT_SIZE]; + char *ptr = command; int inputlen; int nwritten; + int seglen; gettimeofday(&val, NULL); ptm = localtime(&val.tv_sec); - nwritten = snprintf(inputstr, CMDLOG_INPUT_SIZE, "%02d:%02d:%02d.%06ld %s %s\n", - ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (long)val.tv_usec, client_ip, command); + nwritten = snprintf(inputstr, CMDLOG_INPUT_SIZE, "%02d:%02d:%02d.%06ld %s ", + ptm->tm_hour, ptm->tm_min, ptm->tm_sec, (long)val.tv_usec, client_ip); + + while (ptr < command + cmdlen && nwritten < CMDLOG_INPUT_SIZE) { + seglen = snprintf(inputstr + nwritten, CMDLOG_INPUT_SIZE - nwritten, "%s", ptr); + nwritten += seglen; + ptr += seglen; + + if (ptr < command + cmdlen && nwritten < CMDLOG_INPUT_SIZE) { + inputstr[nwritten++] = ' '; + ptr++; + } + } + /* truncated ? */ - if (nwritten > CMDLOG_INPUT_SIZE) { + if (nwritten >= CMDLOG_INPUT_SIZE) { inputstr[CMDLOG_INPUT_SIZE-4] = '.'; inputstr[CMDLOG_INPUT_SIZE-3] = '.'; inputstr[CMDLOG_INPUT_SIZE-2] = '\n'; + } else { + inputstr[nwritten++] = '\n'; + inputstr[nwritten] = '\0'; } inputlen = strlen(inputstr); @@ -447,3 +477,216 @@ void cmdlog_write(char *client_ip, char *command) } pthread_mutex_unlock(&buffer->lock); } + +static int get_key_info_by_command(const char *command, bool *multi_key) +{ + assert(command); + int cmdlen = strlen(command); + *multi_key = false; + + if (cmdlen < 3) { + return -1; + } + + if ((cmdlen == 3 && memcmp(command, "gat", 3) == 0) || + (cmdlen == 4 && memcmp(command, "gats", 4) == 0) || + (cmdlen == 3 && memcmp(command, "bop", 3) == 0) || + (cmdlen == 3 && memcmp(command, "mop", 3) == 0) || + (cmdlen == 3 && memcmp(command, "sop", 3) == 0) || + (cmdlen == 3 && memcmp(command, "lop", 3) == 0)) { + + if ((cmdlen == 8 && memcmp(command, "bop_mget", 8) == 0) || + (cmdlen == 9 && memcmp(command, "bop_smget", 9) == 0)) { + + return -1; + } + return 2; + } else if ((cmdlen == 3 && memcmp(command, "add", 3) == 0) || + (cmdlen == 3 && memcmp(command, "set", 3) == 0) || + (cmdlen == 7 && memcmp(command, "replace", 7) == 0) || + (cmdlen == 7 && memcmp(command, "prepend", 7) == 0) || + (cmdlen == 6 && memcmp(command, "append", 6) == 0) || + (cmdlen == 3 && memcmp(command, "cas", 3) == 0) || + (cmdlen == 4 && memcmp(command, "incr", 4) == 0) || + (cmdlen == 4 && memcmp(command, "decr", 4) == 0) || + (cmdlen == 6 && memcmp(command, "delete", 6) == 0) || + (cmdlen == 7 && memcmp(command, "getattr", 7) == 0) || + (cmdlen == 7 && memcmp(command, "setattr", 7) == 0) || + (cmdlen == 5 && memcmp(command, "touch", 5) == 0)) { + + return 1; + } else if ((cmdlen == 3 && memcmp(command, "get", 3) == 0) || + (cmdlen == 4 && memcmp(command, "gets", 4) == 0)) { + + *multi_key = true; + return 1; + } + return -1; +} + +bool is_cmdlog_filter_match(token_t *tokens, size_t ntokens) +{ + char *cmd = ""; + char *subcmd = ""; + char *key = ""; + char *fcmd, *fkey; + int cmd_len = 0; + int subcmd_len = 0; + int key_len = 0; + int key_idx = -1; + int fcmd_len; + bool multi_key; + bool matched = false; + + if (cmdlog.nfilters == 0) { + return true; + } + + if (ntokens < 2) { + return false; + } + + cmd = tokens[0].value; + cmd_len = tokens[0].length; + + if (strcmp(cmd, "bop") == 0 || + strcmp(cmd, "mop") == 0 || + strcmp(cmd, "sop") == 0 || + strcmp(cmd, "lop") == 0 || + strcmp(cmd, "sasl") == 0 || + strcmp(cmd, "reload") == 0 || + strcmp(cmd, "scan") == 0 || + strcmp(cmd, "cmdlog") == 0 || + strcmp(cmd, "lqdetect") == 0 || + strcmp(cmd, "config") == 0 || + strcmp(cmd, "zkensemble") == 0 || + strcmp(cmd, "dump") == 0) { + + subcmd = tokens[1].value; + subcmd_len = tokens[1].length; + } + + key_idx = get_key_info_by_command(cmd, &multi_key); + if ((key_idx == 2 && ntokens >= 4) || (key_idx == 1 && ntokens >= 3)) { + key = tokens[key_idx].value; + key_len = tokens[key_idx].length; + } + + pthread_mutex_lock(&cmdlog.lock); + for (int i = 0; i < cmdlog.nfilters; ++i) { + fcmd = cmdlog.filters[i].command; + fcmd_len = strlen(fcmd); + fkey = cmdlog.filters[i].key; + + if (fcmd[0] == '\0') { + matched = true; + } else if (cmd_len + subcmd_len + 1 == fcmd_len) { + matched = memcmp(cmd, fcmd, cmd_len) + memcmp(subcmd, fcmd + cmd_len + 1, subcmd_len) == 0 ? true : false; + } else if (cmd_len + subcmd_len + 1 > fcmd_len) { + matched = memcmp(cmd, fcmd, cmd_len) == 0 ? true : false; + } else { + continue; + } + + if (matched) { + if (fkey[0] == '\0') { + break; + } + + do { + matched = string_pattern_match(key, key_len, fkey, strlen(fkey)); + if (matched) { + break; + } + + key = tokens[++key_idx].value; + key_len = tokens[key_idx].length; + } while (multi_key && key_idx < ntokens - 1); + + if (matched) { + break; + } + } + } + pthread_mutex_unlock(&cmdlog.lock); + return matched; +} + +int cmdlog_filter_add(const char *command, int command_len, const char *subcommand, const char *key, int key_len) +{ + char cmd_buf[CMDLOG_FILTER_CMD_MAXLEN]; + int success; + + if (command_len > CMDLOG_FILTER_CMD_MAXLEN || key_len > CMDLOG_FILTER_KEY_MAXLEN) { + return -1; + } + + if (key_len > 0) { + for (int i = 0; key[i] != '\0' && i < CMDLOG_FILTER_KEY_MAXLEN; ++i) { + if (!isgraph(key[i])) { + return -1; + } + } + } + + if (strcmp(subcommand, "") == 0) { + strcpy(cmd_buf, command); + } else { + sprintf(cmd_buf, "%s %s", command, subcommand); + } + + success = 0; + pthread_mutex_lock(&cmdlog.lock); + if (cmdlog.nfilters < CMDLOG_FILTER_MAXNUM) { + strcpy(cmdlog.filters[cmdlog.nfilters].command, cmd_buf); + strcpy(cmdlog.filters[cmdlog.nfilters].key, key); + cmdlog.nfilters++; + success++; + } + pthread_mutex_unlock(&cmdlog.lock); + return success; +} + +int cmdlog_filter_remove(int idx, bool remove_all) +{ + int nremove; + + if (remove_all == false && (idx < 0 || idx >= cmdlog.nfilters)) { + return -1; + } + + nremove = 0; + pthread_mutex_lock(&cmdlog.lock); + if (remove_all) { + nremove += cmdlog.nfilters; + cmdlog.nfilters = 0; + } else { + for (int i = idx; i < cmdlog.nfilters - 1; ++i) { + cmdlog.filters[i] = cmdlog.filters[i + 1]; + } + cmdlog.nfilters--; + nremove++; + } + pthread_mutex_unlock(&cmdlog.lock); + return nremove; +} + +char *cmdlog_filter_list(void) +{ + char *buf = (char *)malloc((cmdlog.nfilters + 1) * CMDLOG_INPUT_SIZE); + int nwritten; + + if (!buf) { + return NULL; + } + + nwritten = 0; + pthread_mutex_lock(&cmdlog.lock); + nwritten = snprintf(buf, CMDLOG_INPUT_SIZE, "\t(%d / %d)\n", cmdlog.nfilters, CMDLOG_FILTER_MAXNUM); + for (int i = 0; i < cmdlog.nfilters; ++i) { + nwritten += snprintf(buf + nwritten, CMDLOG_INPUT_SIZE, "\t%d. command = %s, key = %s\n", + i, cmdlog.filters[i].command, cmdlog.filters[i].key); + } + pthread_mutex_unlock(&cmdlog.lock); + return buf; +} diff --git a/cmdlog.h b/cmdlog.h index ef3aacbc8..a42310fd5 100644 --- a/cmdlog.h +++ b/cmdlog.h @@ -30,5 +30,9 @@ void cmdlog_final(void); int cmdlog_start(char *file_path, bool *already_started); void cmdlog_stop(bool *already_stopped); char *cmdlog_stats(void); -void cmdlog_write(char *client_ip, char *command); +void cmdlog_write(char *client_ip, char *command, int cmdlen); +bool is_cmdlog_filter_match(token_t *tokens, size_t ntokens); +int cmdlog_filter_add(const char *command, int command_len, const char *subcommand, const char *key, int key_len); +int cmdlog_filter_remove(int idx, bool remove_all); +char *cmdlog_filter_list(void); #endif diff --git a/docs/ascii-protocol/ch13-command-administration.md b/docs/ascii-protocol/ch13-command-administration.md index f6b94489f..6af041bcc 100644 --- a/docs/ascii-protocol/ch13-command-administration.md +++ b/docs/ascii-protocol/ch13-command-administration.md @@ -869,6 +869,7 @@ start 명령을 시작으로 logging이 종료될 때 까지의 모든 command cmdlog start []\r\n cmdlog stop\r\n cmdlog stats\r\n +cmdlog filter \r\n ``` \는 logging 정보를 저장할 file의 path이다. @@ -910,6 +911,44 @@ The number of log files : 1 //file_coun The log file name: /Users/temp/command_11211_20160126_192729_{n}.log //path/file_name ``` +filter 명령은 특정 command와 key만 logging 하도록 필터링할 수 있다. +command는 exact matching, key는 glob matching 방식으로 필터링 한다. + +| | filter 명령의 동작 | +| ----------------------------------------------------- | --------------------- | +| add (command |key |command key ) | 새로운 필터 추가 | +| remove (all|) | 일치하는 필터 삭제 | +| list | 현재 필터 목록을 출력 | + +filter add 명령은 필터링할 명령어와 키 조합을 필터 목록에 추가한다. +`mget`, `mgets`, `bop mget`, `bop smget` 명령은 key를 필터로 사용하지 않는다. +`get`, `gets` 명령은 다수의 key에 대해 필터와 일치하는지 전부 검사한다. +response string은 아래와 같다. + +| Response String | 설명 | +| -------------------------------- | --------------------------------------------------------------------- | +| OK | 성공 | +| SERVER_ERROR filter list is full | 존재하는 필터 수가 이미 최대치이므로 더 추가할 수 없음 | +| CLIENT_ERROR invalid parameters | 인자로 입력한 command나 key가 유효하지 않은 문자거나 최대 길이를 넘음 | + +filter remove 명령은 필터 리스트에서 특정 인덱스의 필터를 삭제한다. +인덱스 대신 all을 입력하면 존재하던 모든 필터를 삭제한다. response string은 아래와 같다. + +| Response String | 설명 | +| -------------------------------- | ---------------------------------------------------- | +| OK | 성공 | +| SERVER_ERROR filter list is empty | 존재하는 필터가 없음 | +| CLIENT_ERROR invalid parameters | 인자로 입력한 인덱스가 유효하지 않거나 존재하지 않음 | + +filter list 명령은 존재하는 필터 목록을 출력한다. + +``` +(3 / 10) +0. command = add, key = 123 +1. command = , key = kvkey:12 +2. command = bop_insert, key = +``` + ## Long Query Detect 명령 diff --git a/memcached.c b/memcached.c index 5aac4b92d..95efc15d9 100644 --- a/memcached.c +++ b/memcached.c @@ -10639,13 +10639,115 @@ static void process_scan_command(conn *c, token_t *tokens, const size_t ntokens) #endif #ifdef COMMAND_LOGGING +static void process_cmdlog_filter(conn *c, token_t *tokens, const size_t ntokens) +{ + const char *subcommand; + int ret = 0; + + CHECK_NTOKENS(ntokens, 4, 9); + + subcommand = tokens[2].value; + + if (strcmp(subcommand, "add") == 0) { + char *cmd = ""; + char *subcmd = ""; + char *key = ""; + int cmd_len = 0; + int key_len = 0; + int read_ntokens = 3; + + CHECK_NTOKENS(ntokens, 6, 9); + + if (strcmp(tokens[read_ntokens].value, "command") == 0) { + cmd = tokens[++read_ntokens].value; + cmd_len += tokens[read_ntokens++].length; + + if ((ntokens == 7 || ntokens == 9) && + (strcmp(cmd, "bop") == 0 || + strcmp(cmd, "mop") == 0 || + strcmp(cmd, "sop") == 0 || + strcmp(cmd, "lop") == 0 || + strcmp(cmd, "sasl") == 0 || + strcmp(cmd, "reload") == 0 || + strcmp(cmd, "scan") == 0 || + strcmp(cmd, "cmdlog") == 0 || + strcmp(cmd, "lqdetect") == 0 || + strcmp(cmd, "config") == 0 || + strcmp(cmd, "zkensemble") == 0 || + strcmp(cmd, "dump") == 0)) { + + subcmd = tokens[read_ntokens].value; + cmd_len += tokens[read_ntokens++].length; + } + } + + if (read_ntokens < ntokens-1 && strcmp(tokens[read_ntokens].value, "key") == 0) { + key = tokens[++read_ntokens].value; + key_len = tokens[read_ntokens++].length; + } + + if ((read_ntokens != ntokens-1) || (cmd_len == 0 && key_len == 0)) { + print_invalid_command(c, tokens, ntokens); + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + + ret = cmdlog_filter_add(cmd, cmd_len, subcmd, key, key_len); + + if (ret > 0) { + out_string(c, "OK"); + } else if (ret == 0) { + out_string(c, "SERVER_ERROR filter list is full"); + } else { + out_string(c, "CLIENT_ERROR invalid parameters"); + } + } + else if (strcmp(subcommand, "remove") == 0) { + int idx = 0; + bool remove_all = false; + + CHECK_NTOKENS_EQ(ntokens, 5); + + if (strcmp(tokens[3].value, "all") == 0) { + remove_all = true; + } else if (!safe_strtol(tokens[3].value, &idx)) { + idx = -1; + } + + ret = cmdlog_filter_remove(idx, remove_all); + + if (ret > 0) { + out_string(c, "OK"); + } else if (ret == 0) { + out_string(c, "SERVER_ERROR filter list is empty"); + } else { + out_string(c, "CLIENT_ERROR invalid parameters"); + } + } + else if (strcmp(subcommand, "list") == 0) { + char *ret_str; + + CHECK_NTOKENS_EQ(ntokens, 4); + + ret_str = cmdlog_filter_list(); + + if (ret_str) { + write_and_free(c, ret_str, strlen(ret_str)); + } else { + out_string(c, "SERVER_ERROR out of memory"); + } + } else { + out_string(c, "ERROR unknown command"); + } +} + static void process_cmdlog_command(conn *c, token_t *tokens, const size_t ntokens) { char *subcommand; bool already_check = false; - if (ntokens < 3 || ntokens > 4) { - out_string(c, "\t* Usage: cmdlog [start [path] | stop | stats]\n"); + if (ntokens < 3) { + out_string(c, "\t* Usage: cmdlog {start [path] | stop | stats | filter {add | remove | list} }\n"); return; } @@ -10658,23 +10760,21 @@ static void process_cmdlog_command(conn *c, token_t *tokens, const size_t ntoken subcommand = tokens[SUBCOMMAND_TOKEN].value; - if (strcmp(subcommand, "start") == 0) { - char *fpath = NULL; - if (ntokens == 4) { - fpath = tokens[SUBCOMMAND_TOKEN+1].value; - } + + if (ntokens <= 4 && strcmp(subcommand, "start") == 0) { + char *fpath = ntokens == 4 ? tokens[SUBCOMMAND_TOKEN+1].value : NULL; int ret = cmdlog_start(fpath, &already_check); + if (already_check) { out_string(c, "\tcommand logging already started.\n"); - return; - } - if (ret == 0) { + } else if (ret == 0) { out_string(c, "\tcommand logging started.\n"); } else { out_string(c, "\tcommand logging failed to start.\n"); } } else if (ntokens == 3 && strcmp(subcommand, "stop") == 0) { cmdlog_stop(&already_check); + if (already_check) { out_string(c, "\tcommand logging already stopped.\n"); } else { @@ -10682,13 +10782,16 @@ static void process_cmdlog_command(conn *c, token_t *tokens, const size_t ntoken } } else if (ntokens == 3 && strcmp(subcommand, "stats") == 0) { char *str = cmdlog_stats(); + if (str) { write_and_free(c, str, strlen(str)); } else { out_string(c, "\tcommand logging failed to get stats memory.\n"); } + } else if (ntokens > 3 && strcmp(subcommand, "filter") == 0) { + process_cmdlog_filter(c, tokens, ntokens); } else { - out_string(c, "\t* Usage: cmdlog [start [path] | stop | stats]\n"); + out_string(c, "\t* Usage: cmdlog {start [path] | stop | stats | filter {add | remove | list} }\n"); } } #endif @@ -14018,14 +14121,14 @@ static void process_command_ascii(conn *c, char *command, int cmdlen) return; } + ntokens = tokenize_command(command, cmdlen, tokens, MAX_TOKENS); + #ifdef COMMAND_LOGGING - if (cmdlog_in_use) { - cmdlog_write(c->client_ip, command); + if (cmdlog_in_use && is_cmdlog_filter_match(tokens, ntokens)) { + cmdlog_write(c->client_ip, command, cmdlen); } #endif - ntokens = tokenize_command(command, cmdlen, tokens, MAX_TOKENS); - if (ntokens < 2 || tokens[COMMAND_TOKEN].length < 3) { out_string(c, "ERROR unknown command"); return;