From be69b0c60d8e3e15487b72326bf45aae06345b60 Mon Sep 17 00:00:00 2001 From: bspark Date: Wed, 31 Jul 2019 17:14:00 +0900 Subject: [PATCH 01/10] FEATURE: bulk command logging for bop insert --- Makefile.am | 5 +- cmd_in_second.c | 247 ++++++++++++++++++++++++++++++++++++++++++++++++ cmd_in_second.h | 14 +++ memcached.c | 107 ++++++++++++++++++++- stats.c | 4 +- stats.h | 2 +- 6 files changed, 372 insertions(+), 7 deletions(-) create mode 100644 cmd_in_second.c create mode 100644 cmd_in_second.h diff --git a/Makefile.am b/Makefile.am index ca103440b..27074c96e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -85,7 +85,10 @@ memcached_SOURCES = \ cmdlog.h \ lqdetect.c \ lqdetect.h \ - trace.h + trace.h \ + cmd_in_second.c \ + cmd_in_second.h + memcached_LDFLAGS =-R '$(libdir)' memcached_CFLAGS = @PROFILER_FLAGS@ ${AM_CFLAGS} memcached_DEPENDENCIES = libmcd_util.la diff --git a/cmd_in_second.c b/cmd_in_second.c new file mode 100644 index 000000000..f17083489 --- /dev/null +++ b/cmd_in_second.c @@ -0,0 +1,247 @@ +#include "cmd_in_second.h" +#include "include/memcached/extension.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define LOG_PER_TIMER 500 +#define LOG_LENGTH 400 +#define IP_LENGTH 16 +#define KEY_LENGTH 256 + +typedef enum cmd_in_second_state{ + NOT_STARTED, + ON_LOGGING, + ON_FLUSHING +}state; + +typedef struct cmd_in_second_log{ + char key[256]; + char client_ip[16]; + int32_t timer_idx; +}logtype; + +typedef struct cmd_in_second_buffer { + logtype *ring; + int32_t front; + int32_t rear; + int32_t capacity; +}buffertype; + +typedef struct cmd_in_second_timer{ + struct timeval* times; + int32_t size; + int32_t counter; +}timertype; + +struct cmd_in_second { + char cmd[10]; + char collection_name[4]; + struct cmd_in_second_buffer buffer; + int32_t bulk_limit; + timertype timer; + state cur_state; +}; + +static EXTENSION_LOGGER_DESCRIPTOR *mc_logger; +static struct cmd_in_second this; + +static bool is_bulk_cmd() +{ + const logtype* front = &this.buffer.ring[this.buffer.front]; + const logtype* rear = &this.buffer.ring[this.buffer.rear]; + + struct timeval front_time = this.timer.times[front->timer_idx]; + struct timeval rear_time = this.timer.times[rear->timer_idx]; + + return rear_time.tv_sec - front_time.tv_sec <= 1; +} + +static void get_whole_cmd(char* whole_cmd) +{ + if (this.collection_name) { + sprintf(whole_cmd, "%s %s", this.collection_name, this.cmd); + return; + } + sprintf(whole_cmd, "%s", this.cmd); +} + +static bool buffer_empty() +{ + return this.buffer.front == this.buffer.rear; +} + +static void* buffer_flush_thread() +{ + int32_t fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); + + if (fd < 0) { + mc_logger->log(EXTENSION_LOG_WARNING, NULL, + "Can't open cmd_in_second log file: %s\n", "cmd_in_second.log"); + return NULL; + } + + char whole_cmd[20] = {0}; + get_whole_cmd(whole_cmd); + + buffertype* buffer = &this.buffer; + timertype* timer = &this.timer; + + int32_t timer_idx = -1; + + char* log_str = (char*)malloc(LOG_LENGTH * this.bulk_limit * sizeof(char)); + + if (log_str == NULL) { + mc_logger->log(EXTENSION_LOG_WARNING, NULL, "Can't allocate memory"); + return NULL; + } + + while (!buffer_empty()) { + + logtype front = buffer->ring[buffer->front++]; + + char time_str[50] = ""; + + if (front.timer_idx != timer_idx) { + + const struct timeval* front_time = &timer->times[front.timer_idx]; + const struct tm *lt = localtime((time_t*)&front_time->tv_sec); + + sprintf(time_str, "%04d-%02d-%02d %02d:%02d:%02d.%06d\n", lt ->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec, (int32_t)front_time->tv_usec); + timer_idx = front.timer_idx; + } + char log[LOG_LENGTH] = ""; + sprintf(log, "%s%s %s %s\n", time_str, whole_cmd, front.key, front.client_ip); + strncat(log_str, log, LOG_LENGTH); + + } + + write(fd, log_str, strlen(log_str)); + + close(fd); + + free(log_str); + + free(this.timer.times); + this.timer.times = NULL; + + free(this.buffer.ring); + this.buffer.ring = NULL; + + this.cur_state = NOT_STARTED; + + return NULL; +} + +static int32_t buffer_flush() +{ + + this.cur_state = ON_FLUSHING; + + int32_t ret = 0; + pthread_t tid; + pthread_attr_t attr; + + if (pthread_attr_init(&attr) != 0 || + pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0 || + (ret = pthread_create(&tid, &attr, buffer_flush_thread, NULL)) != 0) + { + mc_logger->log(EXTENSION_LOG_WARNING, NULL, + "Can't create buffer flush thread: %s\n", strerror(ret)); + } + + + return ret; +} + +static void buffer_add(const logtype* log) +{ + + struct cmd_in_second_buffer* buffer = &this.buffer; + + const bool buffer_full = (buffer->rear+1) % buffer->capacity == buffer->front; + + if (buffer_full) { + if (is_bulk_cmd()) { + buffer_flush(); + return; + } + buffer->front = (buffer->front+1) % buffer->capacity; + } + + buffer->ring[buffer->rear] = *log; + buffer->rear = (buffer->rear+1) % buffer->capacity; + +} + +static bool is_cmd_to_log(const char* collection_name, const char* cmd) +{ + return strcmp(this.collection_name, collection_name) == 0 && strcmp(this.cmd, cmd) == 0; +} + +bool cmd_in_second_write(const char* collection_name, const char* cmd, const char* key, const char* client_ip) +{ + if (this.cur_state != ON_LOGGING || !is_cmd_to_log(collection_name, cmd)) { + return false; + } + + timertype *timer = &this.timer; + + logtype log = { {0}, {0} }; + + snprintf(log.client_ip, IP_LENGTH, "%s", client_ip); + snprintf(log.key, KEY_LENGTH, "%s", key); + + if (timer->counter == 0) { + timer->size++; + gettimeofday(&timer->times[timer->size-1], NULL); + } + + log.timer_idx = timer->size-1; + + buffer_add(&log); + timer->counter = (timer->counter+1) % LOG_PER_TIMER; + + return true; +} + +int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const int32_t bulk_limit) +{ + + if (this.cur_state != NOT_STARTED) { + return CMD_IN_SECOND_STARTED_ALREADY; + } + + this.bulk_limit = bulk_limit; + + this.buffer.capacity = bulk_limit+1; + this.buffer.front = 0; + this.buffer.rear = 0; + this.buffer.ring = (logtype*)malloc(this.buffer.capacity * sizeof(logtype)); + + if (this.buffer.ring == NULL) { + return CMD_IN_SECOND_NO_MEM; + } + + this.timer.size = 0; + this.timer.counter = 0; + this.timer.times = (struct timeval*)malloc(bulk_limit * sizeof(struct timeval)); + + if (this.timer.times == NULL) { + free(this.buffer.ring); + return CMD_IN_SECOND_NO_MEM; + } + + sprintf(this.collection_name, "%s", collection_name); + sprintf(this.cmd, "%s", cmd); + + this.cur_state = ON_LOGGING; + + return CMD_IN_SECOND_START; +} diff --git a/cmd_in_second.h b/cmd_in_second.h new file mode 100644 index 000000000..c1a25106e --- /dev/null +++ b/cmd_in_second.h @@ -0,0 +1,14 @@ +#ifndef __CMD_IN_SECOND_LOG__ +#define __CMD_IN_SECOND_LOG__ +#endif + +#include +#include +#include + +#define CMD_IN_SECOND_START 0 +#define CMD_IN_SECOND_STARTED_ALREADY 1 +#define CMD_IN_SECOND_NO_MEM 2 + +int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const int32_t bulk_limit); +bool cmd_in_second_write(const char*collection_name, const char* cmd, const char* key, const char* client_ip); diff --git a/memcached.c b/memcached.c index 22b72262e..f234c6e31 100644 --- a/memcached.c +++ b/memcached.c @@ -36,6 +36,7 @@ #ifdef ENABLE_ZK_INTEGRATION #include "arcus_zk.h" #endif +#include "cmd_in_second.h" #if defined(ENABLE_SASL) || defined(ENABLE_ISASL) #define SASL_ENABLED @@ -2288,7 +2289,7 @@ static void process_bop_insert_complete(conn *c) { c->coll_eitem = NULL; if (settings.detail_enabled) { - stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -5490,7 +5491,7 @@ static void process_bin_bop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { - stats_prefix_record_bop_insert(key, nkey, false); + stats_prefix_record_bop_insert(key, nkey, c->client_ip, false); } switch (ret) { @@ -5569,7 +5570,7 @@ static void process_bin_bop_insert_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -11250,7 +11251,7 @@ static void process_bop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { - stats_prefix_record_bop_insert(key, nkey, false); + stats_prefix_record_bop_insert(key, nkey, c->client_ip, false); } switch (ret) { @@ -13030,6 +13031,100 @@ static void process_setattr_command(conn *c, token_t *tokens, const size_t ntoke } } +static void process_second_command(conn *c, token_t *tokens, const size_t ntokens) { + + assert(c != NULL); + + bool is_collection_cmd = true; + + if(strlen(tokens[SUBCOMMAND_TOKEN].value) > 3){ + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + + if (strcmp("lop", tokens[SUBCOMMAND_TOKEN].value) == 0) { + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value)) { + + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + } else if (strcmp("sop", tokens[SUBCOMMAND_TOKEN].value) == 0) { + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("exist", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value) != 0) { + + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + } else if (strcmp("mop", tokens[SUBCOMMAND_TOKEN].value) == 0) { + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("get", tokens[SUBCOMMAND_TOKEN].value) != 0) { + + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + } else if (strcmp("bop", tokens[SUBCOMMAND_TOKEN].value) == 0) { + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("upsert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) != 0) { + + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + } else { /* key value command */ + if (strcmp("set", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("add", tokens[SUBCOMMAND_TOKEN].value) != 0 && + strcmp("replace", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("append", tokens[SUBCOMMAND_TOKEN].value) != 0 && + strcmp("prepend", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("cas", tokens[SUBCOMMAND_TOKEN].value) != 0 && + strcmp("get", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("gets", tokens[SUBCOMMAND_TOKEN].value) != 0 && + strcmp("delete", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("incr", tokens[SUBCOMMAND_TOKEN].value) != 0 && + strcmp("decr", tokens[SUBCOMMAND_TOKEN].value) != 0) { + + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + is_collection_cmd = false; + } + + int32_t cmd_idx = SUBCOMMAND_TOKEN; + char collection_name[20] = ""; + char cmd[20] = ""; + + if (is_collection_cmd) { + sprintf(collection_name, "%s", tokens[cmd_idx++].value); + } + + sprintf(cmd, "%s", tokens[cmd_idx].value); + + int32_t cnt_to_log= 0; + + if (!safe_strtol(tokens[cmd_idx+1].value, &cnt_to_log)) { + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + + STATS_LOCK(); + const int32_t start_code = cmd_in_second_start(collection_name, cmd, cnt_to_log); + STATS_UNLOCK(); + + char response[100] = ""; + + switch (start_code) { + case CMD_IN_SECOND_STARTED_ALREADY: + sprintf(response, "cmd in second already started"); + break; + case CMD_IN_SECOND_NO_MEM: + sprintf(response, "SERVER_ERROR out of memory"); + break; + case CMD_IN_SECOND_START: + settings.detail_enabled = true; + sprintf(response, "%s %s will be logged", collection_name, cmd); + break; + } + out_string(c, response); +} + static void process_command(conn *c, char *command, int cmdlen) { /* One more token is reserved in tokens strucure @@ -13151,6 +13246,10 @@ static void process_command(conn *c, char *command, int cmdlen) { process_config_command(c, tokens, ntokens); } + else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "cmd_in_second") == 0)) + { + process_second_command(c, tokens, ntokens); + } #ifdef ENABLE_ZK_INTEGRATION else if ((ntokens >= 3) && (strcmp(tokens[COMMAND_TOKEN].value, "zkensemble") == 0)) { process_zkensemble_command(c, tokens, ntokens); diff --git a/stats.c b/stats.c index 6986c4261..7217ed8d3 100644 --- a/stats.c +++ b/stats.c @@ -25,6 +25,7 @@ * Steven Grimm */ #include "config.h" +#include "cmd_in_second.h" #include "memcached.h" #include #include @@ -618,7 +619,7 @@ void stats_prefix_record_bop_create(const char *key, const size_t nkey) { STATS_UNLOCK(); } -void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -629,6 +630,7 @@ void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const bo pfs->num_bop_insert_hits++; } } + cmd_in_second_write("bop", "insert", key, client_ip); STATS_UNLOCK(); } diff --git a/stats.h b/stats.h index 9bfafddc5..083dfed1a 100644 --- a/stats.h +++ b/stats.h @@ -43,7 +43,7 @@ void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bo void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bool is_hit); void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool is_hit); void stats_prefix_record_bop_create(const char *key, const size_t nkey); -void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bool is_hit); void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bool is_hit); void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool is_hit); From 499bc4b2b647ccbeee1b6bb1aee0184f6437548e Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Wed, 31 Jul 2019 18:03:44 +0900 Subject: [PATCH 02/10] FEATURE: bulk command logging for all commmand --- cmd_in_second.c | 3 -- memcached.c | 132 ++++++++++++++++++++++++------------------------ stats.c | 91 +++++++++++++++++++++------------ stats.h | 62 +++++++++++------------ t/bulk.t | 70 +++++++++++++++++++++++++ 5 files changed, 227 insertions(+), 131 deletions(-) create mode 100644 t/bulk.t diff --git a/cmd_in_second.c b/cmd_in_second.c index f17083489..6bf5e4efb 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -156,7 +156,6 @@ static int32_t buffer_flush() "Can't create buffer flush thread: %s\n", strerror(ret)); } - return ret; } @@ -177,7 +176,6 @@ static void buffer_add(const logtype* log) buffer->ring[buffer->rear] = *log; buffer->rear = (buffer->rear+1) % buffer->capacity; - } static bool is_cmd_to_log(const char* collection_name, const char* cmd) @@ -194,7 +192,6 @@ bool cmd_in_second_write(const char* collection_name, const char* cmd, const cha timertype *timer = &this.timer; logtype log = { {0}, {0} }; - snprintf(log.client_ip, IP_LENGTH, "%s", client_ip); snprintf(log.key, KEY_LENGTH, "%s", key); diff --git a/memcached.c b/memcached.c index f234c6e31..c73804625 100644 --- a/memcached.c +++ b/memcached.c @@ -1628,7 +1628,7 @@ static void process_lop_insert_complete(conn *c) { ret = ENGINE_SUCCESS; } if (settings.detail_enabled) { - stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } #ifdef DETECT_LONG_QUERY @@ -1690,7 +1690,7 @@ static void process_sop_insert_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -1743,7 +1743,7 @@ static void process_sop_delete_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, + stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -1791,7 +1791,7 @@ static void process_sop_exist_complete(conn *c) { c->coll_key, c->coll_nkey, value->ptr, value->len, &exist, 0); if (settings.detail_enabled) { - stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, + stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } @@ -1863,7 +1863,7 @@ static void process_mop_insert_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_mop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_mop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -1913,7 +1913,7 @@ static void process_mop_update_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_mop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_mop_update(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -1995,7 +1995,7 @@ static void process_mop_delete_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_mop_delete(c->coll_key, c->coll_nkey, + stats_prefix_record_mop_delete(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -2115,7 +2115,7 @@ static void process_mop_get_complete(conn *c) } if (settings.detail_enabled) { - stats_prefix_record_mop_get(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_mop_get(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } #ifdef DETECT_LONG_QUERY @@ -2391,7 +2391,7 @@ static void process_bop_update_complete(conn *c) } if (settings.detail_enabled) { - stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, + stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -2484,7 +2484,7 @@ static void process_bop_mget_complete(conn *c) { &cur_access_count, &flags, &trimmed, 0); if (settings.detail_enabled) { - stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, + stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -3071,7 +3071,7 @@ static void process_mget_complete(conn *c) it = NULL; } if (settings.detail_enabled) { - stats_prefix_record_get(key, nkey, NULL != it); + stats_prefix_record_get(key, nkey, c->client_ip, NULL != it); } if (it) { /* get_item_info() always returns true. */ @@ -3660,9 +3660,9 @@ static void complete_incr_bin(conn *c) { if (settings.detail_enabled) { if (incr) { - stats_prefix_record_incr(key, nkey); + stats_prefix_record_incr(key, nkey, c->client_ip); } else { - stats_prefix_record_decr(key, nkey); + stats_prefix_record_decr(key, nkey, c->client_ip); } } @@ -3916,7 +3916,7 @@ static void process_bin_get(conn *c) { } if (settings.detail_enabled && ret != ENGINE_EWOULDBLOCK) { - stats_prefix_record_get(key, nkey, ret == ENGINE_SUCCESS); + stats_prefix_record_get(key, nkey, c->client_ip, ret == ENGINE_SUCCESS); } } @@ -4494,7 +4494,7 @@ static void process_bin_lop_create(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_lop_create(key, nkey); + stats_prefix_record_lop_create(key, nkey, c->client_ip); } switch (ret) { @@ -4562,7 +4562,7 @@ static void process_bin_lop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { - stats_prefix_record_lop_insert(key, nkey, false); + stats_prefix_record_lop_insert(key, nkey, c->client_ip, false); } switch (ret) { @@ -4629,7 +4629,7 @@ static void process_bin_lop_insert_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -4708,7 +4708,7 @@ static void process_bin_lop_delete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_lop_delete(key, nkey, + stats_prefix_record_lop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -4797,7 +4797,7 @@ static void process_bin_lop_get(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_lop_get(key, nkey, + stats_prefix_record_lop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -4914,7 +4914,7 @@ static void process_bin_sop_create(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_sop_create(key, nkey); + stats_prefix_record_sop_create(key, nkey, c->client_ip); } switch (ret) { @@ -4987,11 +4987,11 @@ static void process_bin_sop_prepare_nread(conn *c) { if (settings.detail_enabled && ret != ENGINE_SUCCESS) { if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) - stats_prefix_record_sop_insert(key, nkey, false); + stats_prefix_record_sop_insert(key, nkey, c->client_ip, false); else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) - stats_prefix_record_sop_delete(key, nkey, false); + stats_prefix_record_sop_delete(key, nkey, c->client_ip, false); else - stats_prefix_record_sop_exist(key, nkey, false); + stats_prefix_record_sop_exist(key, nkey, c->client_ip, false); } switch (ret) { @@ -5080,7 +5080,7 @@ static void process_bin_sop_insert_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -5139,7 +5139,7 @@ static void process_bin_sop_delete_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, + stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -5189,7 +5189,7 @@ static void process_bin_sop_exist_complete(conn *c) { &exist, c->binary_header.request.vbucket); if (settings.detail_enabled) { - stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -5284,7 +5284,7 @@ static void process_bin_sop_get(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_sop_get(key, nkey, + stats_prefix_record_sop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -5411,7 +5411,7 @@ static void process_bin_bop_create(conn *c) { ret = ENGINE_SUCCESS; } if (settings.detail_enabled) { - stats_prefix_record_bop_create(key, nkey); + stats_prefix_record_bop_create(key, nkey, c->client_ip); } switch (ret) { @@ -5648,7 +5648,7 @@ static void process_bin_bop_update_complete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, + stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -5762,7 +5762,7 @@ static void process_bin_bop_update_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { - stats_prefix_record_bop_update(key, nkey, false); + stats_prefix_record_bop_update(key, nkey, c->client_ip, false); } switch (ret) { @@ -5839,7 +5839,7 @@ static void process_bin_bop_delete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_bop_delete(key, nkey, + stats_prefix_record_bop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -5944,7 +5944,7 @@ static void process_bin_bop_get(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_bop_get(key, nkey, + stats_prefix_record_bop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } @@ -6077,7 +6077,7 @@ static void process_bin_bop_count(conn *c) { c->binary_header.request.vbucket); if (settings.detail_enabled) { - stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_bop_count(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } switch (ret) { @@ -6720,7 +6720,7 @@ static void process_bin_getattr(conn *c) { c->binary_header.request.vbucket); if (settings.detail_enabled) { - stats_prefix_record_getattr(key, nkey); + stats_prefix_record_getattr(key, nkey, c->client_ip); } switch (ret) { @@ -6857,7 +6857,7 @@ static void process_bin_setattr(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_setattr(key, nkey); + stats_prefix_record_setattr(key, nkey, c->client_ip); } switch (ret) { @@ -7342,7 +7342,7 @@ static void process_bin_update(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_set(key, nkey); + stats_prefix_record_set(key, nkey, c->client_ip); } ENGINE_ERROR_CODE ret; @@ -7427,7 +7427,7 @@ static void process_bin_append_prepend(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_set(key, nkey); + stats_prefix_record_set(key, nkey, c->client_ip); } ENGINE_ERROR_CODE ret; @@ -7598,7 +7598,7 @@ static void process_bin_delete(conn *c) { } if (settings.detail_enabled) { - stats_prefix_record_delete(key, nkey); + stats_prefix_record_delete(key, nkey, c->client_ip); } switch (ret) { @@ -8518,7 +8518,7 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, it = NULL; } if (settings.detail_enabled) { - stats_prefix_record_get(key, nkey, NULL != it); + stats_prefix_record_get(key, nkey, c->client_ip, NULL != it); } if (it) { @@ -8732,7 +8732,7 @@ static void process_update_command(conn *c, token_t *tokens, const size_t ntoken } if (settings.detail_enabled) { - stats_prefix_record_set(key, nkey); + stats_prefix_record_set(key, nkey, c->client_ip); } ENGINE_ERROR_CODE ret; @@ -8821,9 +8821,9 @@ static void process_arithmetic_command(conn *c, token_t *tokens, const size_t nt if (settings.detail_enabled) { if (incr) { - stats_prefix_record_incr(key, nkey); + stats_prefix_record_incr(key, nkey, c->client_ip); } else { - stats_prefix_record_decr(key, nkey); + stats_prefix_record_decr(key, nkey, c->client_ip); } } @@ -8916,7 +8916,7 @@ static void process_delete_command(conn *c, token_t *tokens, const size_t ntoken } if (settings.detail_enabled) { - stats_prefix_record_delete(key, nkey); + stats_prefix_record_delete(key, nkey, c->client_ip); } ENGINE_ERROR_CODE ret; @@ -9950,7 +9950,7 @@ static void process_lop_get(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { - stats_prefix_record_lop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_lop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } #ifdef DETECT_LONG_QUERY @@ -10058,7 +10058,7 @@ static void process_lop_prepare_nread(conn *c, int cmd, size_t vlen, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { - stats_prefix_record_lop_insert(key, nkey, false); + stats_prefix_record_lop_insert(key, nkey, c->client_ip, false); } switch (ret) { @@ -10101,7 +10101,7 @@ static void process_lop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { - stats_prefix_record_lop_create(key, nkey); + stats_prefix_record_lop_create(key, nkey, c->client_ip); } switch (ret) { @@ -10140,7 +10140,7 @@ static void process_lop_delete(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { - stats_prefix_record_lop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_lop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } #ifdef DETECT_LONG_QUERY @@ -10368,7 +10368,7 @@ static void process_sop_get(conn *c, char *key, size_t nkey, uint32_t count, } if (settings.detail_enabled) { - stats_prefix_record_sop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_sop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } #ifdef DETECT_LONG_QUERY @@ -10483,11 +10483,11 @@ static void process_sop_prepare_nread(conn *c, int cmd, size_t vlen, char *key, if (settings.detail_enabled && ret != ENGINE_SUCCESS) { if (cmd == (int)OPERATION_SOP_INSERT) - stats_prefix_record_sop_insert(key, nkey, false); + stats_prefix_record_sop_insert(key, nkey, c->client_ip, false); else if (cmd == (int)OPERATION_SOP_DELETE) - stats_prefix_record_sop_delete(key, nkey, false); + stats_prefix_record_sop_delete(key, nkey, c->client_ip, false); else - stats_prefix_record_sop_exist(key, nkey, false); + stats_prefix_record_sop_exist(key, nkey, c->client_ip, false); } switch (ret) { @@ -10540,7 +10540,7 @@ static void process_sop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { - stats_prefix_record_sop_create(key, nkey); + stats_prefix_record_sop_create(key, nkey, c->client_ip); } switch (ret) { @@ -10761,7 +10761,7 @@ static void process_bop_get(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { - stats_prefix_record_bop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_bop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } #ifdef DETECT_LONG_QUERY @@ -10877,7 +10877,7 @@ static void process_bop_count(conn *c, char *key, size_t nkey, &elem_count, &access_count, 0); if (settings.detail_enabled) { - stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); + stats_prefix_record_bop_count(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS)); } #ifdef DETECT_LONG_QUERY @@ -10927,7 +10927,7 @@ static void process_bop_position(conn *c, char *key, size_t nkey, bkrange, order, &position, 0); if (settings.detail_enabled) { - stats_prefix_record_bop_position(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_bop_position(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } switch (ret) { @@ -10986,7 +10986,7 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b &flags, 0); if (settings.detail_enabled) { - stats_prefix_record_bop_pwg(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_bop_pwg(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } switch (ret) { @@ -11099,7 +11099,7 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER elem_array, &elem_count, &flags, 0); if (settings.detail_enabled) { - stats_prefix_record_bop_gbp(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_bop_gbp(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } #ifdef DETECT_LONG_QUERY @@ -11209,7 +11209,7 @@ static void process_bop_update_prepare_nread(conn *c, int cmd, char *key, size_t } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { - stats_prefix_record_bop_update(key, nkey, false); + stats_prefix_record_bop_update(key, nkey, c->client_ip, false); } switch (ret) { @@ -11393,7 +11393,7 @@ static void process_bop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { - stats_prefix_record_bop_create(key, nkey); + stats_prefix_record_bop_create(key, nkey, c->client_ip); } switch (ret) { @@ -11434,7 +11434,7 @@ static void process_bop_delete(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { - stats_prefix_record_bop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_bop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } #ifdef DETECT_LONG_QUERY @@ -11492,9 +11492,9 @@ static void process_bop_arithmetic(conn *c, char *key, size_t nkey, bkey_range * if (settings.detail_enabled) { if (incr) { - stats_prefix_record_bop_incr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_bop_incr(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } else { - stats_prefix_record_bop_decr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + stats_prefix_record_bop_decr(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } } @@ -11743,7 +11743,7 @@ static void process_mop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { - stats_prefix_record_mop_insert(key, nkey, false); + stats_prefix_record_mop_insert(key, nkey, c->client_ip, false); } switch (ret) { @@ -11833,7 +11833,7 @@ static void process_mop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { - stats_prefix_record_mop_create(key, nkey); + stats_prefix_record_mop_create(key, nkey, c->client_ip); } switch (ret) { @@ -12850,7 +12850,7 @@ static void process_getattr_command(conn *c, token_t *tokens, const size_t ntoke attr_ids, attr_count, &attr_data, 0); if (settings.detail_enabled) { - stats_prefix_record_getattr(key, nkey); + stats_prefix_record_getattr(key, nkey, c->client_ip); } } @@ -13006,7 +13006,7 @@ static void process_setattr_command(conn *c, token_t *tokens, const size_t ntoke } if (settings.detail_enabled) { - stats_prefix_record_setattr(key, nkey); + stats_prefix_record_setattr(key, nkey, c->client_ip); } } diff --git a/stats.c b/stats.c index 7217ed8d3..3d160adbe 100644 --- a/stats.c +++ b/stats.c @@ -339,7 +339,7 @@ static PREFIX_STATS *stats_prefix_find(const char *key, const size_t nkey) { /* * Records a "get" of a key. */ -void stats_prefix_record_get(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -356,7 +356,7 @@ void stats_prefix_record_get(const char *key, const size_t nkey, const bool is_h /* * Records a "delete" of a key. */ -void stats_prefix_record_delete(const char *key, const size_t nkey) { +void stats_prefix_record_delete(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -370,7 +370,7 @@ void stats_prefix_record_delete(const char *key, const size_t nkey) { /* * Records a "set" of a key. */ -void stats_prefix_record_set(const char *key, const size_t nkey) { +void stats_prefix_record_set(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -378,13 +378,14 @@ void stats_prefix_record_set(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_sets++; } + cmd_in_second_write("", "set", key, client_ip); STATS_UNLOCK(); } /* * Records a "incr" of a key. */ -void stats_prefix_record_incr(const char *key, const size_t nkey) { +void stats_prefix_record_incr(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -392,13 +393,14 @@ void stats_prefix_record_incr(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_incrs++; } + cmd_in_second_write("", "incr", key, client_ip); STATS_UNLOCK(); } /* * Records a "decr" of a key. */ -void stats_prefix_record_decr(const char *key, const size_t nkey) { +void stats_prefix_record_decr(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -406,13 +408,14 @@ void stats_prefix_record_decr(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_decrs++; } + cmd_in_second_write("", "decr", key, client_ip); STATS_UNLOCK(); } /* * LIST stats */ -void stats_prefix_record_lop_create(const char *key, const size_t nkey) { +void stats_prefix_record_lop_create(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -420,10 +423,11 @@ void stats_prefix_record_lop_create(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_lop_creates++; } + cmd_in_second_write("lop", "create", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -434,10 +438,11 @@ void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const bo pfs->num_lop_insert_hits++; } } + cmd_in_second_write("lop", "insert", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -448,10 +453,11 @@ void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const bo pfs->num_lop_delete_hits++; } } + cmd_in_second_write("lop", "delete", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_lop_get(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_lop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -462,13 +468,14 @@ void stats_prefix_record_lop_get(const char *key, const size_t nkey, const bool pfs->num_lop_get_hits++; } } + cmd_in_second_write("lop", "get", key, client_ip); STATS_UNLOCK(); } /* * SET stats */ -void stats_prefix_record_sop_create(const char *key, const size_t nkey) { +void stats_prefix_record_sop_create(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -476,10 +483,11 @@ void stats_prefix_record_sop_create(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_sop_creates++; } + cmd_in_second_write("sop", "create", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -490,10 +498,11 @@ void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const bo pfs->num_sop_insert_hits++; } } + cmd_in_second_write("sop", "insert", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -504,10 +513,11 @@ void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const bo pfs->num_sop_delete_hits++; } } + cmd_in_second_write("sop", "delete", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_get(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_sop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -518,10 +528,11 @@ void stats_prefix_record_sop_get(const char *key, const size_t nkey, const bool pfs->num_sop_get_hits++; } } + cmd_in_second_write("sop", "get", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -532,13 +543,14 @@ void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const boo pfs->num_sop_exist_hits++; } } + cmd_in_second_write("sop", "exist", key, client_ip); STATS_UNLOCK(); } /* * MAP stats */ -void stats_prefix_record_mop_create(const char *key, const size_t nkey) { +void stats_prefix_record_mop_create(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -546,10 +558,11 @@ void stats_prefix_record_mop_create(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_mop_creates++; } + cmd_in_second_write("mop", "create", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -560,10 +573,11 @@ void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const bo pfs->num_mop_insert_hits++; } } + cmd_in_second_write("mop", "insert", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_mop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -574,10 +588,11 @@ void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bo pfs->num_mop_update_hits++; } } + cmd_in_second_write("mop", "update", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -588,10 +603,11 @@ void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bo pfs->num_mop_delete_hits++; } } + cmd_in_second_write("mop", "delete", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_mop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -602,13 +618,14 @@ void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool pfs->num_mop_get_hits++; } } + cmd_in_second_write("mop", "get", key, client_ip); STATS_UNLOCK(); } /* * B+TREE stats */ -void stats_prefix_record_bop_create(const char *key, const size_t nkey) { +void stats_prefix_record_bop_create(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -616,6 +633,7 @@ void stats_prefix_record_bop_create(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_bop_creates++; } + cmd_in_second_write("bop", "create", key, client_ip); STATS_UNLOCK(); } @@ -634,7 +652,7 @@ void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const ch STATS_UNLOCK(); } -void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -645,10 +663,11 @@ void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bo pfs->num_bop_update_hits++; } } + cmd_in_second_write("bop", "update", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -659,10 +678,11 @@ void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bo pfs->num_bop_delete_hits++; } } + cmd_in_second_write("bop", "delete", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -673,10 +693,11 @@ void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool pfs->num_bop_incr_hits++; } } + cmd_in_second_write("bop", "incr", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -687,10 +708,11 @@ void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const bool pfs->num_bop_decr_hits++; } } + cmd_in_second_write("bop", "decr", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_get(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -701,10 +723,11 @@ void stats_prefix_record_bop_get(const char *key, const size_t nkey, const bool pfs->num_bop_get_hits++; } } + cmd_in_second_write("bop", "get", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_count(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_count(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -715,10 +738,11 @@ void stats_prefix_record_bop_count(const char *key, const size_t nkey, const boo pfs->num_bop_count_hits++; } } + cmd_in_second_write("bop", "count", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_position(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_position(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -729,10 +753,11 @@ void stats_prefix_record_bop_position(const char *key, const size_t nkey, const pfs->num_bop_position_hits++; } } + cmd_in_second_write("bop", "position", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -743,10 +768,11 @@ void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool pfs->num_bop_pwg_hits++; } } + cmd_in_second_write("bop", "pwg", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool is_hit) { +void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -757,13 +783,14 @@ void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool pfs->num_bop_gbp_hits++; } } + cmd_in_second_write("bop", "gbp", key, client_ip); STATS_UNLOCK(); } /* * ATTR stats */ -void stats_prefix_record_getattr(const char *key, const size_t nkey) { +void stats_prefix_record_getattr(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -771,10 +798,11 @@ void stats_prefix_record_getattr(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_getattrs++; } + cmd_in_second_write("", "getattr", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_setattr(const char *key, const size_t nkey) { +void stats_prefix_record_setattr(const char *key, const size_t nkey, const char* client_ip) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -782,6 +810,7 @@ void stats_prefix_record_setattr(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_setattrs++; } + cmd_in_second_write("", "setattr", key, client_ip); STATS_UNLOCK(); } diff --git a/stats.h b/stats.h index 083dfed1a..bab0852b5 100644 --- a/stats.h +++ b/stats.h @@ -23,37 +23,37 @@ int stats_prefix_count(void); int stats_prefix_insert(const char *prefix, const size_t nprefix); #endif int stats_prefix_delete(const char *prefix, const size_t nprefix); -void stats_prefix_record_get(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_delete(const char *key, const size_t nkey); -void stats_prefix_record_set(const char *key, const size_t nkey); -void stats_prefix_record_incr(const char *key, const size_t nkey); -void stats_prefix_record_decr(const char *key, const size_t nkey); -void stats_prefix_record_lop_create(const char *key, const size_t nkey); -void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_lop_get(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_sop_create(const char *key, const size_t nkey); -void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_sop_get(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_mop_create(const char *key, const size_t nkey); -void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_create(const char *key, const size_t nkey); +void stats_prefix_record_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_delete(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_set(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_incr(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_decr(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_lop_create(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_lop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_sop_create(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_sop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_mop_create(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_mop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_mop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_create(const char *key, const size_t nkey, const char* client_ip); void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_get(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_count(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_position(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool is_hit); -void stats_prefix_record_getattr(const char *key, const size_t nkey); -void stats_prefix_record_setattr(const char *key, const size_t nkey); +void stats_prefix_record_bop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_count(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_position(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); +void stats_prefix_record_getattr(const char *key, const size_t nkey, const char* client_ip); +void stats_prefix_record_setattr(const char *key, const size_t nkey, const char* client_ip); /*@null@*/ char *stats_prefix_dump(int *length); diff --git a/t/bulk.t b/t/bulk.t new file mode 100644 index 000000000..13b57eccb --- /dev/null +++ b/t/bulk.t @@ -0,0 +1,70 @@ +#!/usr/bin/perl + +use strict; +use Test::More tests => 243; +use Time::HiRes qw(gettimeofday time); +use FindBin qw($Bin); +use lib "$Bin/lib"; +use MemcachedTest; + +my $engine = shift; +my $server = get_memcached($engine); +my $sock = $server->sock; + +#mem_cmd_is($sock, $cmd, $val, $rst); +# +sub request_log{ + my ($whole_cmd, $cnt) = @_; + my $rst = "$whole_cmd will be logged"; + mem_cmd_is($sock, "cmd_in_second $whole_cmd $cnt", "", $rst); +} + +sub do_bulk_insert { + + my ($collection, $cmd, $count) = @_; + my $start_time = time; + + my $key = "mykey"; + + for (my $index=0; $index < $count; $index++) { + my $val = "datum$index"; + my $vleng = length($val); + + my $whole_cmd = "$collection $cmd $key $vleng"; + + if ($collection eq "bop") { + $whole_cmd = "$collection $cmd $key $index $vleng"; + } + + my $rst = "STORED"; + + if ($index == 0) { + my $create = "create 13 0 0"; + + $whole_cmd = "$collection $cmd $key $vleng $create"; + + if($collection eq "bop") { + $whole_cmd = "$collection $cmd $key $index $vleng $create"; + } + + $rst = "CREATED_STORED"; + } + + mem_cmd_is($sock, $whole_cmd, $val, $rst); + + #sleep(0.001); + } + + + my $end_time = time; + + cmp_ok($end_time - $start_time, "<=", 1000, "1 second"); +} + +unlink "../cmd_in_second.log"; +request_log("bop insert", 2000); +do_bulk_bop_insert("bop", "insert", 2100); + +open(my $file_handle, "cmd_in_second.log") or die "log file not exist\n"; + +release_memcached($engine, $server); From 42d796a14b46755560357a20f08536bc4abb5c01 Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Wed, 31 Jul 2019 19:05:29 +0900 Subject: [PATCH 03/10] fix warning --- cmd_in_second.c | 19 ++++++++++++------ cmd_in_second.h | 2 +- t/bulk.t | 51 ++++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 16 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index 6bf5e4efb..f0b6d340a 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -22,8 +22,8 @@ typedef enum cmd_in_second_state{ }state; typedef struct cmd_in_second_log{ - char key[256]; - char client_ip[16]; + char key[KEY_LENGTH]; + char client_ip[IP_LENGTH]; int32_t timer_idx; }logtype; @@ -65,7 +65,7 @@ static bool is_bulk_cmd() static void get_whole_cmd(char* whole_cmd) { - if (this.collection_name) { + if (strlen(this.collection_name)) { sprintf(whole_cmd, "%s %s", this.collection_name, this.cmd); return; } @@ -87,7 +87,7 @@ static void* buffer_flush_thread() return NULL; } - char whole_cmd[20] = {0}; + char whole_cmd[20] = ""; get_whole_cmd(whole_cmd); buffertype* buffer = &this.buffer; @@ -102,6 +102,8 @@ static void* buffer_flush_thread() return NULL; } + int32_t expected_write_length = 0; + while (!buffer_empty()) { logtype front = buffer->ring[buffer->front++]; @@ -115,14 +117,19 @@ static void* buffer_flush_thread() sprintf(time_str, "%04d-%02d-%02d %02d:%02d:%02d.%06d\n", lt ->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec, (int32_t)front_time->tv_usec); timer_idx = front.timer_idx; + + expected_write_length += 27; } char log[LOG_LENGTH] = ""; sprintf(log, "%s%s %s %s\n", time_str, whole_cmd, front.key, front.client_ip); strncat(log_str, log, LOG_LENGTH); + expected_write_length += LOG_LENGTH; } - write(fd, log_str, strlen(log_str)); + if (write(fd, log_str, expected_write_length) != expected_write_length) { + mc_logger->log(EXTENSION_LOG_WARNING, NULL, "write length is difference to expectation."); + } close(fd); @@ -191,7 +198,7 @@ bool cmd_in_second_write(const char* collection_name, const char* cmd, const cha timertype *timer = &this.timer; - logtype log = { {0}, {0} }; + logtype log = {"", "", 0}; snprintf(log.client_ip, IP_LENGTH, "%s", client_ip); snprintf(log.key, KEY_LENGTH, "%s", key); diff --git a/cmd_in_second.h b/cmd_in_second.h index c1a25106e..34a63fd36 100644 --- a/cmd_in_second.h +++ b/cmd_in_second.h @@ -11,4 +11,4 @@ #define CMD_IN_SECOND_NO_MEM 2 int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const int32_t bulk_limit); -bool cmd_in_second_write(const char*collection_name, const char* cmd, const char* key, const char* client_ip); +bool cmd_in_second_write(const char* collection_name, const char* cmd, const char* key, const char* client_ip); diff --git a/t/bulk.t b/t/bulk.t index 13b57eccb..270dcee0c 100644 --- a/t/bulk.t +++ b/t/bulk.t @@ -19,34 +19,52 @@ sub request_log{ mem_cmd_is($sock, "cmd_in_second $whole_cmd $cnt", "", $rst); } -sub do_bulk_insert { +sub do_bulk_coll_insert { - my ($collection, $cmd, $count) = @_; + my ($collection, $count) = @_; my $start_time = time; + my $cmd = "insert"; my $key = "mykey"; for (my $index=0; $index < $count; $index++) { my $val = "datum$index"; my $vleng = length($val); - my $whole_cmd = "$collection $cmd $key $vleng"; + my $whole_cmd; - if ($collection eq "bop") { + if ($collection eq "sop") { + $whole_cmd = "$collection $cmd $key $vleng"; + } + + elsif ($collection eq "bop" or $collection eq "lop") { $whole_cmd = "$collection $cmd $key $index $vleng"; } + elsif ($collection eq "mop") { + my $field = "f$index"; + $whole_cmd = "$collection $cmd $key $field $vleng"; + } + + my $rst = "STORED"; if ($index == 0) { my $create = "create 13 0 0"; - $whole_cmd = "$collection $cmd $key $vleng $create"; + if ($collection eq "sop") { + $whole_cmd = "$collection $cmd $key $vleng $create"; + } - if($collection eq "bop") { + elsif ($collection eq "bop" or $collection eq "lop") { $whole_cmd = "$collection $cmd $key $index $vleng $create"; } + elsif ($collection eq "mop") { + my $field = "f$index"; + $whole_cmd = "$collection $cmd $key $field $vleng $create"; + } + $rst = "CREATED_STORED"; } @@ -62,9 +80,24 @@ sub do_bulk_insert { } unlink "../cmd_in_second.log"; -request_log("bop insert", 2000); -do_bulk_bop_insert("bop", "insert", 2100); +request_log("bop insert", 50); +do_bulk_coll_insert("bop", 51); + +my $file_handle; +open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; +close($file_handle); + +sleep(1); +mem_cmd_is($sock, "bop insert mykey 1000 9", "datum1000", "STORED"); +=pod + +unlink "../cmd_in_second.log"; +request_log("mop insert", 10); +do_bulk_coll_insert("mop", 11); -open(my $file_handle, "cmd_in_second.log") or die "log file not exist\n"; +open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; +close($file_handle); +=cut release_memcached($engine, $server); + From 638e828587242690ca0e89e4c5f7b891daf9fd63 Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Thu, 1 Aug 2019 16:29:44 +0900 Subject: [PATCH 04/10] code tag --- cmd_in_second.c | 178 ++++++++++++--------- include/memcached/types.h | 2 +- memcached.c | 314 +++++++++++++++++++++++++++++++++++++- stats.c | 106 +++++++++++++ stats.h | 37 +++++ t/bulk.t | 121 ++++++++++++--- 6 files changed, 655 insertions(+), 103 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index f0b6d340a..b1b1e4681 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -10,66 +10,69 @@ #include #include -#define LOG_PER_TIMER 500 #define LOG_LENGTH 400 #define IP_LENGTH 16 #define KEY_LENGTH 256 -typedef enum cmd_in_second_state{ +typedef enum cmd_in_second_state { NOT_STARTED, ON_LOGGING, ON_FLUSHING -}state; +} state; -typedef struct cmd_in_second_log{ +typedef struct cmd_in_second_log { char key[KEY_LENGTH]; char client_ip[IP_LENGTH]; - int32_t timer_idx; -}logtype; +} logtype; typedef struct cmd_in_second_buffer { - logtype *ring; + logtype* ring; int32_t front; int32_t rear; int32_t capacity; -}buffertype; +} buffertype; -typedef struct cmd_in_second_timer{ - struct timeval* times; - int32_t size; - int32_t counter; -}timertype; +typedef struct cmd_in_second_timer { + struct timeval* ring; + int32_t front; + int32_t rear; + int32_t capacity; + int32_t circular_counter; +} timertype; struct cmd_in_second { - char cmd[10]; - char collection_name[4]; + char cmd[20]; + char collection_name[10]; struct cmd_in_second_buffer buffer; - int32_t bulk_limit; timertype timer; + int32_t bulk_limit; + int32_t log_per_timer; state cur_state; }; -static EXTENSION_LOGGER_DESCRIPTOR *mc_logger; static struct cmd_in_second this; static bool is_bulk_cmd() { - const logtype* front = &this.buffer.ring[this.buffer.front]; - const logtype* rear = &this.buffer.ring[this.buffer.rear]; + const bool timer_empty = this.timer.front == this.timer.rear; - struct timeval front_time = this.timer.times[front->timer_idx]; - struct timeval rear_time = this.timer.times[rear->timer_idx]; + if (timer_empty) { + return false; + } + + const struct timeval* front_time = &this.timer.ring[this.timer.front]; + const struct timeval* rear_time = &this.timer.ring[this.timer.rear]; - return rear_time.tv_sec - front_time.tv_sec <= 1; + return rear_time->tv_sec - front_time->tv_sec <= 1; } static void get_whole_cmd(char* whole_cmd) { if (strlen(this.collection_name)) { - sprintf(whole_cmd, "%s %s", this.collection_name, this.cmd); + snprintf(whole_cmd, 20, "%s %s", this.collection_name, this.cmd); return; } - sprintf(whole_cmd, "%s", this.cmd); + snprintf(whole_cmd, 15, "%s", this.cmd); } static bool buffer_empty() @@ -79,64 +82,74 @@ static bool buffer_empty() static void* buffer_flush_thread() { - int32_t fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); + const int32_t fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); if (fd < 0) { - mc_logger->log(EXTENSION_LOG_WARNING, NULL, - "Can't open cmd_in_second log file: %s\n", "cmd_in_second.log"); + perror("Can't open cmd_in_second log file: cmd_in_second.log"); return NULL; } - char whole_cmd[20] = ""; - get_whole_cmd(whole_cmd); - - buffertype* buffer = &this.buffer; - timertype* timer = &this.timer; - - int32_t timer_idx = -1; + buffertype* const buffer = &this.buffer; + timertype* const timer = &this.timer; char* log_str = (char*)malloc(LOG_LENGTH * this.bulk_limit * sizeof(char)); if (log_str == NULL) { - mc_logger->log(EXTENSION_LOG_WARNING, NULL, "Can't allocate memory"); + perror("Can't allocate memory"); return NULL; } - int32_t expected_write_length = 0; + char whole_cmd[20] = ""; + get_whole_cmd(whole_cmd); + + const size_t whole_cmd_len = strlen(whole_cmd); + const int32_t whitespaces = 3; + + size_t expected_write_length = 0; + int32_t circular_log_counter = 0; while (!buffer_empty()) { - logtype front = buffer->ring[buffer->front++]; + const logtype front = buffer->ring[buffer->front]; + buffer->front = (buffer->front+1) % buffer->capacity; char time_str[50] = ""; - if (front.timer_idx != timer_idx) { + if (circular_log_counter == 0) { + + const struct timeval* front_time = &timer->ring[timer->front]; + const struct tm* lt = localtime((time_t*)&front_time->tv_sec); - const struct timeval* front_time = &timer->times[front.timer_idx]; - const struct tm *lt = localtime((time_t*)&front_time->tv_sec); + timer->front = (timer->front+1) % timer->capacity; - sprintf(time_str, "%04d-%02d-%02d %02d:%02d:%02d.%06d\n", lt ->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday, lt->tm_hour, lt->tm_min, lt->tm_sec, (int32_t)front_time->tv_usec); - timer_idx = front.timer_idx; + if (lt == NULL) { + perror("localtime failed"); + continue; + } + sprintf(time_str, "%04d-%02d-%02d %02d:%02d:%02d.%06d\n", lt ->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday, + lt->tm_hour, lt->tm_min, lt->tm_sec, (int32_t)front_time->tv_usec); expected_write_length += 27; } + char log[LOG_LENGTH] = ""; - sprintf(log, "%s%s %s %s\n", time_str, whole_cmd, front.key, front.client_ip); + snprintf(log, LOG_LENGTH, "%s%s %s %s\n", time_str, whole_cmd, front.key, front.client_ip); strncat(log_str, log, LOG_LENGTH); - expected_write_length += LOG_LENGTH; + expected_write_length += whole_cmd_len + strlen(front.key) + strlen(front.client_ip) + whitespaces; + circular_log_counter = (circular_log_counter+1) % this.log_per_timer; } if (write(fd, log_str, expected_write_length) != expected_write_length) { - mc_logger->log(EXTENSION_LOG_WARNING, NULL, "write length is difference to expectation."); + perror("write length is difference to expectation."); } close(fd); free(log_str); - free(this.timer.times); - this.timer.times = NULL; + free(this.timer.ring); + this.timer.ring = NULL; free(this.buffer.ring); this.buffer.ring = NULL; @@ -148,19 +161,19 @@ static void* buffer_flush_thread() static int32_t buffer_flush() { - this.cur_state = ON_FLUSHING; - int32_t ret = 0; pthread_t tid; pthread_attr_t attr; + int32_t ret = 0; + if (pthread_attr_init(&attr) != 0 || pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0 || (ret = pthread_create(&tid, &attr, buffer_flush_thread, NULL)) != 0) { - mc_logger->log(EXTENSION_LOG_WARNING, NULL, - "Can't create buffer flush thread: %s\n", strerror(ret)); + perror("Can't create buffer flush thread"); + return ret; } return ret; @@ -169,7 +182,10 @@ static int32_t buffer_flush() static void buffer_add(const logtype* log) { - struct cmd_in_second_buffer* buffer = &this.buffer; + struct cmd_in_second_buffer* const buffer = &this.buffer; + + buffer->ring[buffer->rear] = *log; + buffer->rear = (buffer->rear+1) % buffer->capacity; const bool buffer_full = (buffer->rear+1) % buffer->capacity == buffer->front; @@ -181,41 +197,55 @@ static void buffer_add(const logtype* log) buffer->front = (buffer->front+1) % buffer->capacity; } - buffer->ring[buffer->rear] = *log; - buffer->rear = (buffer->rear+1) % buffer->capacity; +} + +static void timer_add() +{ + struct cmd_in_second_timer* const timer = &this.timer; + + const bool timer_full = (timer->rear+1) % timer->capacity == timer->front; + + if (timer_full) { + timer->front = (timer->front+1) % timer->capacity; + } + + if (gettimeofday(&timer->ring[timer->rear], NULL) == -1) { + perror("gettimeofday failed"); + return; + }; + + timer->rear = (timer->rear+1) % timer->capacity; } static bool is_cmd_to_log(const char* collection_name, const char* cmd) { - return strcmp(this.collection_name, collection_name) == 0 && strcmp(this.cmd, cmd) == 0; + return strcmp(this.collection_name, collection_name) == 0 && + strcmp(this.cmd, cmd) == 0; } -bool cmd_in_second_write(const char* collection_name, const char* cmd, const char* key, const char* client_ip) +bool cmd_in_second_write(const char* collection_name, const char* cmd, + const char* key, const char* client_ip) { if (this.cur_state != ON_LOGGING || !is_cmd_to_log(collection_name, cmd)) { return false; } - timertype *timer = &this.timer; - - logtype log = {"", "", 0}; + logtype log = {"", ""}; snprintf(log.client_ip, IP_LENGTH, "%s", client_ip); snprintf(log.key, KEY_LENGTH, "%s", key); - if (timer->counter == 0) { - timer->size++; - gettimeofday(&timer->times[timer->size-1], NULL); + if (this.timer.circular_counter == 0) { + timer_add(); } - log.timer_idx = timer->size-1; - buffer_add(&log); - timer->counter = (timer->counter+1) % LOG_PER_TIMER; + this.timer.circular_counter = (this.timer.circular_counter+1) % this.log_per_timer; return true; } -int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const int32_t bulk_limit) +int32_t cmd_in_second_start(const char* collection_name, const char* cmd, + const int32_t bulk_limit) { if (this.cur_state != NOT_STARTED) { @@ -233,17 +263,21 @@ int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const return CMD_IN_SECOND_NO_MEM; } - this.timer.size = 0; - this.timer.counter = 0; - this.timer.times = (struct timeval*)malloc(bulk_limit * sizeof(struct timeval)); + this.log_per_timer = bulk_limit / 10 + (bulk_limit % 10 != 0); + this.timer.capacity = this.log_per_timer + 1; + this.timer.front = 0; + this.timer.rear = 0; + this.timer.circular_counter = 0; + + this.timer.ring = (struct timeval*)malloc(this.timer.capacity * sizeof(struct timeval)); - if (this.timer.times == NULL) { + if (this.timer.ring == NULL) { free(this.buffer.ring); return CMD_IN_SECOND_NO_MEM; } - sprintf(this.collection_name, "%s", collection_name); - sprintf(this.cmd, "%s", cmd); + snprintf(this.collection_name, 5, "%s", collection_name); + snprintf(this.cmd, 15, "%s", cmd); this.cur_state = ON_LOGGING; diff --git a/include/memcached/types.h b/include/memcached/types.h index 5e12291ff..a0587bf34 100644 --- a/include/memcached/types.h +++ b/include/memcached/types.h @@ -38,7 +38,7 @@ struct iovec { #define SUPPORT_BOP_SMGET #define JHPARK_OLD_SMGET_INTERFACE #define MAX_EFLAG_COMPARE_COUNT 100 - +#define CMD_IN_SECOND 1 #ifdef __cplusplus extern "C" { diff --git a/memcached.c b/memcached.c index c73804625..c8f470456 100644 --- a/memcached.c +++ b/memcached.c @@ -1628,7 +1628,11 @@ static void process_lop_insert_complete(conn *c) { ret = ENGINE_SUCCESS; } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } #ifdef DETECT_LONG_QUERY @@ -1690,7 +1694,11 @@ static void process_sop_insert_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -1743,8 +1751,13 @@ static void process_sop_delete_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -1791,8 +1804,13 @@ static void process_sop_exist_complete(conn *c) { c->coll_key, c->coll_nkey, value->ptr, value->len, &exist, 0); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, + (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -1863,7 +1881,11 @@ static void process_mop_insert_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_mop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_mop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -1913,7 +1935,11 @@ static void process_mop_update_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_mop_update(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_mop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -1995,8 +2021,13 @@ static void process_mop_delete_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_mop_delete(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_mop_delete(c->coll_key, c->coll_nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -2115,7 +2146,11 @@ static void process_mop_get_complete(conn *c) } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_mop_get(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_mop_get(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -2289,7 +2324,11 @@ static void process_bop_insert_complete(conn *c) { c->coll_eitem = NULL; if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -2391,8 +2430,13 @@ static void process_bop_update_complete(conn *c) } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -2484,8 +2528,13 @@ static void process_bop_mget_complete(conn *c) { &cur_access_count, &flags, &trimmed, 0); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } if (ret == ENGINE_SUCCESS) { @@ -3071,7 +3120,11 @@ static void process_mget_complete(conn *c) it = NULL; } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_get(key, nkey, c->client_ip, NULL != it); +#else + stats_prefix_record_get(key, nkey, NULL != it); +#endif } if (it) { /* get_item_info() always returns true. */ @@ -3659,11 +3712,19 @@ static void complete_incr_bin(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND if (incr) { stats_prefix_record_incr(key, nkey, c->client_ip); } else { stats_prefix_record_decr(key, nkey, c->client_ip); } +#else + if (incr) { + stats_prefix_record_incr(key, nkey); + } else { + stats_prefix_record_decr(key, nkey); + } +#endif } switch (ret) { @@ -3916,7 +3977,11 @@ static void process_bin_get(conn *c) { } if (settings.detail_enabled && ret != ENGINE_EWOULDBLOCK) { +#ifdef CMD_IN_SECOND stats_prefix_record_get(key, nkey, c->client_ip, ret == ENGINE_SUCCESS); +#else + stats_prefix_record_get(key, nkey, ret == ENGINE_SUCCESS); +#endif } } @@ -4494,7 +4559,11 @@ static void process_bin_lop_create(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_create(key, nkey, c->client_ip); +#else + stats_prefix_record_lop_create(key, nkey); +#endif } switch (ret) { @@ -4562,7 +4631,11 @@ static void process_bin_lop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_insert(key, nkey, c->client_ip, false); +#else + stats_prefix_record_lop_insert(key, nkey, false); +#endif } switch (ret) { @@ -4629,7 +4702,11 @@ static void process_bin_lop_insert_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -4708,8 +4785,13 @@ static void process_bin_lop_delete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_lop_delete(key, nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -4797,8 +4879,13 @@ static void process_bin_lop_get(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_lop_get(key, nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -4914,7 +5001,11 @@ static void process_bin_sop_create(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_create(key, nkey, c->client_ip); +#else + stats_prefix_record_sop_create(key, nkey); +#endif } switch (ret) { @@ -4986,12 +5077,22 @@ static void process_bin_sop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) stats_prefix_record_sop_insert(key, nkey, c->client_ip, false); else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) stats_prefix_record_sop_delete(key, nkey, c->client_ip, false); else stats_prefix_record_sop_exist(key, nkey, c->client_ip, false); +#else + if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) + stats_prefix_record_sop_insert(key, nkey, false); + else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) + stats_prefix_record_sop_delete(key, nkey, false); + else + stats_prefix_record_sop_exist(key, nkey, false); +#endif + } switch (ret) { @@ -5080,7 +5181,11 @@ static void process_bin_sop_insert_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -5139,8 +5244,13 @@ static void process_bin_sop_delete_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -5189,7 +5299,11 @@ static void process_bin_sop_exist_complete(conn *c) { &exist, c->binary_header.request.vbucket); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -5284,8 +5398,13 @@ static void process_bin_sop_get(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_sop_get(key, nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -5411,7 +5530,11 @@ static void process_bin_bop_create(conn *c) { ret = ENGINE_SUCCESS; } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_create(key, nkey, c->client_ip); +#else + stats_prefix_record_bop_create(key, nkey); +#endif } switch (ret) { @@ -5491,7 +5614,11 @@ static void process_bin_bop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_insert(key, nkey, c->client_ip, false); +#else + stats_prefix_record_bop_insert(key, nkey, false); +#endif } switch (ret) { @@ -5570,7 +5697,11 @@ static void process_bin_bop_insert_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -5648,8 +5779,13 @@ static void process_bin_bop_update_complete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -5762,7 +5898,11 @@ static void process_bin_bop_update_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_update(key, nkey, c->client_ip, false); +#else + stats_prefix_record_bop_update(key, nkey, false); +#endif } switch (ret) { @@ -5839,8 +5979,13 @@ static void process_bin_bop_delete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_delete(key, nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -5944,8 +6089,13 @@ static void process_bin_bop_get(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_get(key, nkey, + (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -6077,7 +6227,11 @@ static void process_bin_bop_count(conn *c) { c->binary_header.request.vbucket); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_count(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); +#endif } switch (ret) { @@ -6720,7 +6874,11 @@ static void process_bin_getattr(conn *c) { c->binary_header.request.vbucket); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_getattr(key, nkey, c->client_ip); +#else + stats_prefix_record_getattr(key, nkey); +#endif } switch (ret) { @@ -6857,7 +7015,11 @@ static void process_bin_setattr(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_setattr(key, nkey, c->client_ip); +#else + stats_prefix_record_setattr(key, nkey); +#endif } switch (ret) { @@ -7342,7 +7504,11 @@ static void process_bin_update(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_set(key, nkey, c->client_ip); +#else + stats_prefix_record_set(key, nkey); +#endif } ENGINE_ERROR_CODE ret; @@ -7427,7 +7593,11 @@ static void process_bin_append_prepend(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_set(key, nkey, c->client_ip); +#else + stats_prefix_record_set(key, nkey); +#endif } ENGINE_ERROR_CODE ret; @@ -7598,7 +7768,11 @@ static void process_bin_delete(conn *c) { } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_delete(key, nkey, c->client_ip); +#else + stats_prefix_record_delete(key, nkey); +#endif } switch (ret) { @@ -8518,7 +8692,11 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, it = NULL; } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_get(key, nkey, c->client_ip, NULL != it); +#else + stats_prefix_record_get(key, nkey, NULL != it); +#endif } if (it) { @@ -8732,7 +8910,11 @@ static void process_update_command(conn *c, token_t *tokens, const size_t ntoken } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_set(key, nkey, c->client_ip); +#else + stats_prefix_record_set(key, nkey); +#endif } ENGINE_ERROR_CODE ret; @@ -8820,11 +9002,19 @@ static void process_arithmetic_command(conn *c, token_t *tokens, const size_t nt } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND if (incr) { stats_prefix_record_incr(key, nkey, c->client_ip); } else { stats_prefix_record_decr(key, nkey, c->client_ip); } +#else + if (incr) { + stats_prefix_record_incr(key, nkey); + } else { + stats_prefix_record_decr(key, nkey); + } +#endif } ENGINE_ERROR_CODE ret; @@ -8916,7 +9106,11 @@ static void process_delete_command(conn *c, token_t *tokens, const size_t ntoken } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_delete(key, nkey, c->client_ip); +#else + stats_prefix_record_delete(key, nkey); +#endif } ENGINE_ERROR_CODE ret; @@ -9950,7 +10144,11 @@ static void process_lop_get(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_lop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -10058,7 +10256,11 @@ static void process_lop_prepare_nread(conn *c, int cmd, size_t vlen, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_insert(key, nkey, c->client_ip, false); +#else + stats_prefix_record_lop_insert(key, nkey, false); +#endif } switch (ret) { @@ -10101,7 +10303,11 @@ static void process_lop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_create(key, nkey, c->client_ip); +#else + stats_prefix_record_lop_create(key, nkey); +#endif } switch (ret) { @@ -10140,7 +10346,11 @@ static void process_lop_delete(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_lop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_lop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -10368,7 +10578,11 @@ static void process_sop_get(conn *c, char *key, size_t nkey, uint32_t count, } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_sop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -10482,12 +10696,21 @@ static void process_sop_prepare_nread(conn *c, int cmd, size_t vlen, char *key, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND if (cmd == (int)OPERATION_SOP_INSERT) stats_prefix_record_sop_insert(key, nkey, c->client_ip, false); else if (cmd == (int)OPERATION_SOP_DELETE) stats_prefix_record_sop_delete(key, nkey, c->client_ip, false); else stats_prefix_record_sop_exist(key, nkey, c->client_ip, false); +#else + if (cmd == (int)OPERATION_SOP_INSERT) + stats_prefix_record_sop_insert(key, nkey, false); + else if (cmd == (int)OPERATION_SOP_DELETE) + stats_prefix_record_sop_delete(key, nkey, false); + else + stats_prefix_record_sop_exist(key, nkey, false); +#endif } switch (ret) { @@ -10540,7 +10763,11 @@ static void process_sop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_sop_create(key, nkey, c->client_ip); +#else + stats_prefix_record_sop_create(key, nkey); +#endif } switch (ret) { @@ -10761,7 +10988,11 @@ static void process_bop_get(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -10877,7 +11108,11 @@ static void process_bop_count(conn *c, char *key, size_t nkey, &elem_count, &access_count, 0); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_count(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS)); +#else + stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); +#endif } #ifdef DETECT_LONG_QUERY @@ -10927,7 +11162,11 @@ static void process_bop_position(conn *c, char *key, size_t nkey, bkrange, order, &position, 0); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_position(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_position(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -10986,7 +11225,11 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b &flags, 0); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_pwg(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_pwg(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -11099,7 +11342,11 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER elem_array, &elem_count, &flags, 0); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_gbp(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_gbp(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -11209,7 +11456,11 @@ static void process_bop_update_prepare_nread(conn *c, int cmd, char *key, size_t } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_update(key, nkey, c->client_ip, false); +#else + stats_prefix_record_bop_update(key, nkey, false); +#endif } switch (ret) { @@ -11251,7 +11502,11 @@ static void process_bop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_insert(key, nkey, c->client_ip, false); +#else + stats_prefix_record_bop_insert(key, nkey, false); +#endif } switch (ret) { @@ -11393,7 +11648,11 @@ static void process_bop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_create(key, nkey, c->client_ip); +#else + stats_prefix_record_bop_create(key, nkey); +#endif } switch (ret) { @@ -11434,7 +11693,11 @@ static void process_bop_delete(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_bop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#else + stats_prefix_record_bop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } #ifdef DETECT_LONG_QUERY @@ -11491,11 +11754,18 @@ static void process_bop_arithmetic(conn *c, char *key, size_t nkey, bkey_range * } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND if (incr) { stats_prefix_record_bop_incr(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } else { stats_prefix_record_bop_decr(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } +#else + if (incr) { + stats_prefix_record_bop_incr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } else { + stats_prefix_record_bop_decr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#endif } switch (ret) { @@ -11743,7 +12013,11 @@ static void process_mop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { +#ifdef CMD_IN_SECOND stats_prefix_record_mop_insert(key, nkey, c->client_ip, false); +#else + stats_prefix_record_mop_insert(key, nkey, false); +#endif } switch (ret) { @@ -11833,7 +12107,11 @@ static void process_mop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_mop_create(key, nkey, c->client_ip); +#else + stats_prefix_record_mop_create(key, nkey); +#endif } switch (ret) { @@ -12850,7 +13128,11 @@ static void process_getattr_command(conn *c, token_t *tokens, const size_t ntoke attr_ids, attr_count, &attr_data, 0); if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_getattr(key, nkey, c->client_ip); +#else + stats_prefix_record_getattr(key, nkey); +#endif } } @@ -13006,7 +13288,11 @@ static void process_setattr_command(conn *c, token_t *tokens, const size_t ntoke } if (settings.detail_enabled) { +#ifdef CMD_IN_SECOND stats_prefix_record_setattr(key, nkey, c->client_ip); +#else + stats_prefix_record_setattr(key, nkey); +#endif } } @@ -13031,6 +13317,7 @@ static void process_setattr_command(conn *c, token_t *tokens, const size_t ntoke } } +#ifdef CMD_IN_SECOND static void process_second_command(conn *c, token_t *tokens, const size_t ntokens) { assert(c != NULL); @@ -13068,7 +13355,12 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken } else if (strcmp("bop", tokens[SUBCOMMAND_TOKEN].value) == 0) { if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("upsert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) != 0) { + strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("count", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("incr", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("decr", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("mget", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("smget", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("position", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && + strcmp("gbp", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("pwg", tokens[SUBCOMMAND_TOKEN+1].value) != 0) { + out_string(c, "CLIENT_ERROR bad command line format"); return; @@ -13076,10 +13368,10 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken } else { /* key value command */ if (strcmp("set", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("add", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("replace", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("append", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("prepend", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("cas", tokens[SUBCOMMAND_TOKEN].value) != 0 && + strcmp("prepend", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("delete", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("get", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("gets", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("delete", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("incr", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("decr", tokens[SUBCOMMAND_TOKEN].value) != 0) { + strcmp("mget", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("cas", tokens[SUBCOMMAND_TOKEN].value) != 0 && + strcmp("incr", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("decr", tokens[SUBCOMMAND_TOKEN].value) != 0) { out_string(c, "CLIENT_ERROR bad command line format"); return; @@ -13087,9 +13379,16 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken is_collection_cmd = false; } - int32_t cmd_idx = SUBCOMMAND_TOKEN; + if ((is_collection_cmd && ntokens != 5) || + (!is_collection_cmd && ntokens != 4)) { + + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } + char collection_name[20] = ""; char cmd[20] = ""; + int32_t cmd_idx = SUBCOMMAND_TOKEN; if (is_collection_cmd) { sprintf(collection_name, "%s", tokens[cmd_idx++].value); @@ -13099,7 +13398,7 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken int32_t cnt_to_log= 0; - if (!safe_strtol(tokens[cmd_idx+1].value, &cnt_to_log)) { + if (!safe_strtol(tokens[cmd_idx+1].value, &cnt_to_log) || cnt_to_log <= 0) { out_string(c, "CLIENT_ERROR bad command line format"); return; } @@ -13124,6 +13423,7 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken } out_string(c, response); } +#endif static void process_command(conn *c, char *command, int cmdlen) { @@ -13246,10 +13546,12 @@ static void process_command(conn *c, char *command, int cmdlen) { process_config_command(c, tokens, ntokens); } +#ifdef CMD_IN_SECOND else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "cmd_in_second") == 0)) { process_second_command(c, tokens, ntokens); } +#endif #ifdef ENABLE_ZK_INTEGRATION else if ((ntokens >= 3) && (strcmp(tokens[COMMAND_TOKEN].value, "zkensemble") == 0)) { process_zkensemble_command(c, tokens, ntokens); diff --git a/stats.c b/stats.c index 3d160adbe..c4fc2f407 100644 --- a/stats.c +++ b/stats.c @@ -550,7 +550,11 @@ void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const cha /* * MAP stats */ +#ifdef CMD_IN_SECOND void stats_prefix_record_mop_create(const char *key, const size_t nkey, const char* client_ip) { +#else +void stats_prefix_record_mop_create(const char *key, const size_t nkey) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -558,11 +562,17 @@ void stats_prefix_record_mop_create(const char *key, const size_t nkey, const ch if (NULL != pfs) { pfs->num_mop_creates++; } +#ifdef CMD_IN_SECOND cmd_in_second_write("mop", "create", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -573,11 +583,17 @@ void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const ch pfs->num_mop_insert_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("mop", "insert", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_mop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -588,11 +604,17 @@ void stats_prefix_record_mop_update(const char *key, const size_t nkey, const ch pfs->num_mop_update_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("mop", "update", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -603,11 +625,17 @@ void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const ch pfs->num_mop_delete_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("mop", "delete", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_mop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -618,14 +646,20 @@ void stats_prefix_record_mop_get(const char *key, const size_t nkey, const char* pfs->num_mop_get_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("mop", "get", key, client_ip); +#endif STATS_UNLOCK(); } /* * B+TREE stats */ +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_create(const char *key, const size_t nkey, const char* client_ip) { +#else +void stats_prefix_record_bop_create(const char *key, const size_t nkey) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -633,11 +667,17 @@ void stats_prefix_record_bop_create(const char *key, const size_t nkey, const ch if (NULL != pfs) { pfs->num_bop_creates++; } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "create", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -648,11 +688,17 @@ void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const ch pfs->num_bop_insert_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "insert", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -663,11 +709,17 @@ void stats_prefix_record_bop_update(const char *key, const size_t nkey, const ch pfs->num_bop_update_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "update", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -682,7 +734,11 @@ void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const ch STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -693,11 +749,17 @@ void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const char pfs->num_bop_incr_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "incr", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -708,11 +770,17 @@ void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const char pfs->num_bop_decr_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "decr", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_get(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -723,11 +791,17 @@ void stats_prefix_record_bop_get(const char *key, const size_t nkey, const char* pfs->num_bop_get_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "get", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_count(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_count(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -738,11 +812,17 @@ void stats_prefix_record_bop_count(const char *key, const size_t nkey, const cha pfs->num_bop_count_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "count", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_position(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_position(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -753,11 +833,17 @@ void stats_prefix_record_bop_position(const char *key, const size_t nkey, const pfs->num_bop_position_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "position", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -768,11 +854,17 @@ void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const char* pfs->num_bop_pwg_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "pwg", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +#else +void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool is_hit) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -783,14 +875,20 @@ void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const char* pfs->num_bop_gbp_hits++; } } +#ifdef CMD_IN_SECOND cmd_in_second_write("bop", "gbp", key, client_ip); +#endif STATS_UNLOCK(); } /* * ATTR stats */ +#ifdef CMD_IN_SECOND void stats_prefix_record_getattr(const char *key, const size_t nkey, const char* client_ip) { +#else +void stats_prefix_record_getattr(const char *key, const size_t nkey) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -798,11 +896,17 @@ void stats_prefix_record_getattr(const char *key, const size_t nkey, const char* if (NULL != pfs) { pfs->num_getattrs++; } +#ifdef CMD_IN_SECOND cmd_in_second_write("", "getattr", key, client_ip); +#endif STATS_UNLOCK(); } +#ifdef CMD_IN_SECOND void stats_prefix_record_setattr(const char *key, const size_t nkey, const char* client_ip) { +#else +void stats_prefix_record_setattr(const char *key, const size_t nkey) { +#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -810,7 +914,9 @@ void stats_prefix_record_setattr(const char *key, const size_t nkey, const char* if (NULL != pfs) { pfs->num_setattrs++; } +#ifdef CMD_IN_SECOND cmd_in_second_write("", "setattr", key, client_ip); +#endif STATS_UNLOCK(); } diff --git a/stats.h b/stats.h index bab0852b5..c5f41082b 100644 --- a/stats.h +++ b/stats.h @@ -22,7 +22,9 @@ int stats_prefix_count(void); #if 1 // NEW_PREFIX_STATS_MANAGEMENT int stats_prefix_insert(const char *prefix, const size_t nprefix); #endif + int stats_prefix_delete(const char *prefix, const size_t nprefix); +#ifdef CMD_IN_SECOND void stats_prefix_record_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); void stats_prefix_record_delete(const char *key, const size_t nkey, const char* client_ip); void stats_prefix_record_set(const char *key, const size_t nkey, const char* client_ip); @@ -55,5 +57,40 @@ void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const char* void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); void stats_prefix_record_getattr(const char *key, const size_t nkey, const char* client_ip); void stats_prefix_record_setattr(const char *key, const size_t nkey, const char* client_ip); +#else +void stats_prefix_record_get(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_delete(const char *key, const size_t nkey); +void stats_prefix_record_set(const char *key, const size_t nkey); +void stats_prefix_record_incr(const char *key, const size_t nkey); +void stats_prefix_record_decr(const char *key, const size_t nkey); +void stats_prefix_record_lop_create(const char *key, const size_t nkey); +void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_lop_get(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_sop_create(const char *key, const size_t nkey); +void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_sop_get(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_mop_create(const char *key, const size_t nkey); +void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_create(const char *key, const size_t nkey); +void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_get(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_count(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_position(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool is_hit); +void stats_prefix_record_getattr(const char *key, const size_t nkey); +void stats_prefix_record_setattr(const char *key, const size_t nkey); +#endif + /*@null@*/ char *stats_prefix_dump(int *length); diff --git a/t/bulk.t b/t/bulk.t index 270dcee0c..717fc6c62 100644 --- a/t/bulk.t +++ b/t/bulk.t @@ -1,31 +1,55 @@ #!/usr/bin/perl use strict; -use Test::More tests => 243; +use Test::More tests => 20781; use Time::HiRes qw(gettimeofday time); use FindBin qw($Bin); use lib "$Bin/lib"; use MemcachedTest; +use Cwd; my $engine = shift; my $server = get_memcached($engine); my $sock = $server->sock; +my $start = 0; +my $on_logging = 1; + +my $bulk_size = 3461; + #mem_cmd_is($sock, $cmd, $val, $rst); # sub request_log{ - my ($whole_cmd, $cnt) = @_; - my $rst = "$whole_cmd will be logged"; - mem_cmd_is($sock, "cmd_in_second $whole_cmd $cnt", "", $rst); + my ($whole_cmd, $cnt, $state) = @_; + + if ($cnt <= 0) { + my $rst= "CLIENT_ERROR bad command line format"; + mem_cmd_is($sock, "cmd_in_second $whole_cmd $cnt", "", $rst); + return; + } + + if ($state == $start){ + my $rst = "$whole_cmd will be logged"; + mem_cmd_is($sock, "cmd_in_second $whole_cmd $cnt", "", $rst); + return; + } + + if ($state == $on_logging) { + my $rst = "cmd in second already started"; + mem_cmd_is($sock, "cmd_in_second $whole_cmd $cnt", "", $rst); + } } sub do_bulk_coll_insert { - my ($collection, $count) = @_; + if (-e "cmd_in_second.log") { + unlink "cmd_in_second.log" or die "can't remove old file\n"; + } + + my ($collection, $key, $count) = @_; my $start_time = time; my $cmd = "insert"; - my $key = "mykey"; for (my $index=0; $index < $count; $index++) { my $val = "datum$index"; @@ -68,36 +92,85 @@ sub do_bulk_coll_insert { $rst = "CREATED_STORED"; } - mem_cmd_is($sock, $whole_cmd, $val, $rst); + if ($count >= 10) { + if ($index % (int($count/10)) == 0) { + request_log("mop insert", 1000, $on_logging); + } + } + + mem_cmd_is($sock, $whole_cmd, $val, $rst) or die; - #sleep(0.001); } my $end_time = time; - cmp_ok($end_time - $start_time, "<=", 1000, "1 second"); + cmp_ok($end_time - $start_time, "<=", 1000, "all commands are done in a second"); + + my $file_handle; + open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; + close($file_handle); } -unlink "../cmd_in_second.log"; -request_log("bop insert", 50); -do_bulk_coll_insert("bop", 51); -my $file_handle; -open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; -close($file_handle); +sub wrong_cmd_test{ -sleep(1); -mem_cmd_is($sock, "bop insert mykey 1000 9", "datum1000", "STORED"); -=pod + request_log("sop insert", 0, $start); + request_log("sop insert", -1, $start); -unlink "../cmd_in_second.log"; -request_log("mop insert", 10); -do_bulk_coll_insert("mop", 11); + my $bad_format = "CLIENT_ERROR bad command line format"; + mem_cmd_is($sock, "cmd_in_second lop upsert 1000", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second sop upsert 1000", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second mop upsert 1000", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second bop cas 1000", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second bop insert", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second bop insert blahblah", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second set 1000 blahblah", "", $bad_format); -open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; -close($file_handle); + my $unknown = "ERROR unknown command"; + mem_cmd_is($sock, "cmd_in_second bop", "", $unknown); + mem_cmd_is($sock, "cmd_in_second bop insert 1000 blahblah", "", $unknown); + mem_cmd_is($sock, "cmd_in_second bop insert 1000 1000", "", $unknown); + +} + +sub non_bulk_cmd { + + my $request_cnt = 10000; + request_log("sop exist", $request_cnt, $start); + unlink "cmd_in_second.log" or die "can't remove old file\n"; + + for (my $i=0; $i < 5; $i++) { + for (my $j = 0; $j < $bulk_size; $j++) { + my $val = "datum$j"; + my $vleng = length($val); + + mem_cmd_is($sock, "sop exist skey $vleng", $val, "EXIST"); + sleep(0.001); + } + } + + if (-e "cmd_in_second.log") { + die "log made by non-bulk command"; + } +} + + +wrong_cmd_test(); +sleep(3); +#extremely small cases +=pod +request_log("bop insert", 1, $start); +do_bulk_coll_insert("bop", "bkey", 1); +request_log("bop insert", 9, $start); +do_bulk_coll_insert("bop", "bkey2", 9); =cut -release_memcached($engine, $server); +request_log("sop insert", $bulk_size, $start); +do_bulk_coll_insert("sop", "skey", $bulk_size+1); +sleep(1); + +#non_bulk_cmd(); + +release_memcached($engine, $server); From 964e935b322edabdc994f33fe86f7f124a8edce0 Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Tue, 6 Aug 2019 12:13:54 +0900 Subject: [PATCH 05/10] implement own lock --- cmd_in_second.c | 46 ++++-- cmd_in_second.h | 5 +- memcached.c | 398 ++++++++++++++++++++---------------------------- stats.c | 165 ++------------------ stats.h | 37 ----- t/bulk.t | 25 ++- 6 files changed, 237 insertions(+), 439 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index b1b1e4681..4c341698d 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -37,6 +37,7 @@ typedef struct cmd_in_second_timer { int32_t front; int32_t rear; int32_t capacity; + int32_t last_elem_idx; int32_t circular_counter; } timertype; @@ -48,6 +49,7 @@ struct cmd_in_second { int32_t bulk_limit; int32_t log_per_timer; state cur_state; + pthread_mutex_t lock; }; static struct cmd_in_second this; @@ -61,9 +63,12 @@ static bool is_bulk_cmd() } const struct timeval* front_time = &this.timer.ring[this.timer.front]; - const struct timeval* rear_time = &this.timer.ring[this.timer.rear]; + const struct timeval* last_time = &this.timer.ring[this.timer.last_elem_idx]; - return rear_time->tv_sec - front_time->tv_sec <= 1; + //printf("%d\n", this.timer.last_elem_idx); + //assert(0); + + return last_time->tv_sec - front_time->tv_sec <= 1; } static void get_whole_cmd(char* whole_cmd) @@ -161,7 +166,6 @@ static void* buffer_flush_thread() static int32_t buffer_flush() { - this.cur_state = ON_FLUSHING; pthread_t tid; pthread_attr_t attr; @@ -191,12 +195,12 @@ static void buffer_add(const logtype* log) if (buffer_full) { if (is_bulk_cmd()) { + this.cur_state = ON_FLUSHING; buffer_flush(); return; } buffer->front = (buffer->front+1) % buffer->capacity; } - } static void timer_add() @@ -214,6 +218,7 @@ static void timer_add() return; }; + timer->last_elem_idx = timer->rear; timer->rear = (timer->rear+1) % timer->capacity; } @@ -226,7 +231,9 @@ static bool is_cmd_to_log(const char* collection_name, const char* cmd) bool cmd_in_second_write(const char* collection_name, const char* cmd, const char* key, const char* client_ip) { + pthread_mutex_lock(&this.lock); if (this.cur_state != ON_LOGGING || !is_cmd_to_log(collection_name, cmd)) { + pthread_mutex_unlock(&this.lock); return false; } @@ -241,38 +248,57 @@ bool cmd_in_second_write(const char* collection_name, const char* cmd, buffer_add(&log); this.timer.circular_counter = (this.timer.circular_counter+1) % this.log_per_timer; + pthread_mutex_unlock(&this.lock); return true; } +void cmd_in_second_init() +{ + assert("test"); + this.cur_state = NOT_STARTED; + pthread_mutex_init(&this.lock, NULL); + + this.buffer.front = 0; + this.buffer.rear = 0; + this.buffer.ring = NULL; + + this.timer.front = 0; + this.timer.rear = 0; + this.timer.capacity = 0; + this.timer.circular_counter = 0; + this.timer.last_elem_idx = 0; + this.timer.ring = NULL; +} + int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const int32_t bulk_limit) { + pthread_mutex_lock(&this.lock); + if (this.cur_state != NOT_STARTED) { + pthread_mutex_unlock(&this.lock); return CMD_IN_SECOND_STARTED_ALREADY; } this.bulk_limit = bulk_limit; this.buffer.capacity = bulk_limit+1; - this.buffer.front = 0; - this.buffer.rear = 0; this.buffer.ring = (logtype*)malloc(this.buffer.capacity * sizeof(logtype)); if (this.buffer.ring == NULL) { + pthread_mutex_unlock(&this.lock); return CMD_IN_SECOND_NO_MEM; } this.log_per_timer = bulk_limit / 10 + (bulk_limit % 10 != 0); this.timer.capacity = this.log_per_timer + 1; - this.timer.front = 0; - this.timer.rear = 0; - this.timer.circular_counter = 0; this.timer.ring = (struct timeval*)malloc(this.timer.capacity * sizeof(struct timeval)); if (this.timer.ring == NULL) { free(this.buffer.ring); + pthread_mutex_unlock(&this.lock); return CMD_IN_SECOND_NO_MEM; } @@ -281,5 +307,7 @@ int32_t cmd_in_second_start(const char* collection_name, const char* cmd, this.cur_state = ON_LOGGING; + pthread_mutex_unlock(&this.lock); + return CMD_IN_SECOND_START; } diff --git a/cmd_in_second.h b/cmd_in_second.h index 34a63fd36..406b5b012 100644 --- a/cmd_in_second.h +++ b/cmd_in_second.h @@ -1,5 +1,5 @@ -#ifndef __CMD_IN_SECOND_LOG__ -#define __CMD_IN_SECOND_LOG__ +#ifndef __CMD_IN_SECOND_ +#define __CMD_IN_SECOND_ #endif #include @@ -10,5 +10,6 @@ #define CMD_IN_SECOND_STARTED_ALREADY 1 #define CMD_IN_SECOND_NO_MEM 2 +void cmd_in_second_init(void); int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const int32_t bulk_limit); bool cmd_in_second_write(const char* collection_name, const char* cmd, const char* key, const char* client_ip); diff --git a/memcached.c b/memcached.c index c8f470456..26447836a 100644 --- a/memcached.c +++ b/memcached.c @@ -1628,10 +1628,9 @@ static void process_lop_insert_complete(conn *c) { ret = ENGINE_SUCCESS; } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("lop", "insert", c->coll_key, c->client_ip); #endif } @@ -1694,10 +1693,9 @@ static void process_sop_insert_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "insert", c->coll_key, c->client_ip); #endif } @@ -1751,12 +1749,10 @@ static void process_sop_delete_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "delete", c->coll_key, c->client_ip); #endif } @@ -1804,12 +1800,10 @@ static void process_sop_exist_complete(conn *c) { c->coll_key, c->coll_nkey, value->ptr, value->len, &exist, 0); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, c->client_ip, - (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "exist", c->coll_key, c->client_ip); #endif } @@ -1881,10 +1875,9 @@ static void process_mop_insert_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_mop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_mop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("mop", "insert", c->coll_key, c->client_ip); #endif } @@ -1935,10 +1928,9 @@ static void process_mop_update_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_mop_update(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_mop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("mop", "update", c->coll_key, c->client_ip); #endif } @@ -2021,12 +2013,10 @@ static void process_mop_delete_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_mop_delete(c->coll_key, c->coll_nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_mop_delete(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("mop", "delete", c->coll_key, c->client_ip); #endif } @@ -2146,10 +2136,9 @@ static void process_mop_get_complete(conn *c) } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_mop_get(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_mop_get(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("mop", "get", c->coll_key, c->client_ip); #endif } @@ -2324,10 +2313,9 @@ static void process_bop_insert_complete(conn *c) { c->coll_eitem = NULL; if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "insert", c->coll_key, c->client_ip); #endif } @@ -2430,12 +2418,10 @@ static void process_bop_update_complete(conn *c) } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "update", c->coll_key, c->client_ip); #endif } @@ -2528,12 +2514,10 @@ static void process_bop_mget_complete(conn *c) { &cur_access_count, &flags, &trimmed, 0); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "get", c->coll_key, c->client_ip); #endif } @@ -3120,10 +3104,9 @@ static void process_mget_complete(conn *c) it = NULL; } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_get(key, nkey, c->client_ip, NULL != it); -#else stats_prefix_record_get(key, nkey, NULL != it); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "get", key, c->client_ip); #endif } if (it) { @@ -3712,19 +3695,17 @@ static void complete_incr_bin(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - if (incr) { - stats_prefix_record_incr(key, nkey, c->client_ip); - } else { - stats_prefix_record_decr(key, nkey, c->client_ip); - } -#else - if (incr) { + if (incr) { stats_prefix_record_incr(key, nkey); - } else { +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "incr", key, c->client_ip); +#endif + } else { stats_prefix_record_decr(key, nkey); - } +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "decr", key, c->client_ip); #endif + } } switch (ret) { @@ -3977,10 +3958,9 @@ static void process_bin_get(conn *c) { } if (settings.detail_enabled && ret != ENGINE_EWOULDBLOCK) { -#ifdef CMD_IN_SECOND - stats_prefix_record_get(key, nkey, c->client_ip, ret == ENGINE_SUCCESS); -#else stats_prefix_record_get(key, nkey, ret == ENGINE_SUCCESS); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "decr", key, c->client_ip); #endif } } @@ -4559,10 +4539,9 @@ static void process_bin_lop_create(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_lop_create(key, nkey, c->client_ip); -#else stats_prefix_record_lop_create(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("lop", "create", key, c->client_ip); #endif } @@ -4631,10 +4610,9 @@ static void process_bin_lop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - stats_prefix_record_lop_insert(key, nkey, c->client_ip, false); -#else stats_prefix_record_lop_insert(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("lop", "insert", key, c->client_ip); #endif } @@ -4702,10 +4680,9 @@ static void process_bin_lop_insert_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("lop", "insert", c->coll_key, c->client_ip); #endif } @@ -4785,12 +4762,10 @@ static void process_bin_lop_delete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_lop_delete(key, nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_lop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("lop", "delete", key, c->client_ip); #endif } @@ -4879,12 +4854,10 @@ static void process_bin_lop_get(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_lop_get(key, nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_lop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("lop", "get", key, c->client_ip); #endif } @@ -5001,10 +4974,9 @@ static void process_bin_sop_create(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_create(key, nkey, c->client_ip); -#else stats_prefix_record_sop_create(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "create", key, c->client_ip); #endif } @@ -5077,22 +5049,22 @@ static void process_bin_sop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) - stats_prefix_record_sop_insert(key, nkey, c->client_ip, false); - else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) - stats_prefix_record_sop_delete(key, nkey, c->client_ip, false); - else - stats_prefix_record_sop_exist(key, nkey, c->client_ip, false); -#else - if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) + if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) { stats_prefix_record_sop_insert(key, nkey, false); - else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "insert", key, c->client_ip); +#endif + } else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) { stats_prefix_record_sop_delete(key, nkey, false); - else +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "delete", key, c->client_ip); +#endif + } else { stats_prefix_record_sop_exist(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "exist", key, c->client_ip); #endif - + } } switch (ret) { @@ -5181,10 +5153,9 @@ static void process_bin_sop_insert_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "insert", c->coll_key, c->client_ip); #endif } @@ -5244,12 +5215,10 @@ static void process_bin_sop_delete_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "delete", c->coll_key, c->client_ip); #endif } @@ -5299,10 +5268,9 @@ static void process_bin_sop_exist_complete(conn *c) { &exist, c->binary_header.request.vbucket); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "exist", c->coll_key, c->client_ip); #endif } @@ -5398,12 +5366,10 @@ static void process_bin_sop_get(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_get(key, nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_sop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "get", key, c->client_ip); #endif } @@ -5530,10 +5496,9 @@ static void process_bin_bop_create(conn *c) { ret = ENGINE_SUCCESS; } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_create(key, nkey, c->client_ip); -#else stats_prefix_record_bop_create(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "create", key, c->client_ip); #endif } @@ -5614,10 +5579,9 @@ static void process_bin_bop_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_insert(key, nkey, c->client_ip, false); -#else stats_prefix_record_bop_insert(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "insert", key, c->client_ip); #endif } @@ -5697,10 +5661,9 @@ static void process_bin_bop_insert_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "insert", c->coll_key, c->client_ip); #endif } @@ -5779,12 +5742,10 @@ static void process_bin_bop_update_complete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "insert", c->coll_key, c->client_ip); #endif } @@ -5898,10 +5859,9 @@ static void process_bin_bop_update_prepare_nread(conn *c) { } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_update(key, nkey, c->client_ip, false); -#else stats_prefix_record_bop_update(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "update", key, c->client_ip); #endif } @@ -5979,12 +5939,10 @@ static void process_bin_bop_delete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_delete(key, nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "delete", key, c->client_ip); #endif } @@ -6089,12 +6047,10 @@ static void process_bin_bop_get(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_get(key, nkey, c->client_ip, - (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "get", key, c->client_ip); #endif } @@ -6227,10 +6183,9 @@ static void process_bin_bop_count(conn *c) { c->binary_header.request.vbucket); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_count(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "count", key, c->client_ip); #endif } @@ -6874,10 +6829,9 @@ static void process_bin_getattr(conn *c) { c->binary_header.request.vbucket); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_getattr(key, nkey, c->client_ip); -#else stats_prefix_record_getattr(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "getattr", key, c->client_ip); #endif } @@ -7015,10 +6969,9 @@ static void process_bin_setattr(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_setattr(key, nkey, c->client_ip); -#else stats_prefix_record_setattr(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "setattr", key, c->client_ip); #endif } @@ -7504,10 +7457,9 @@ static void process_bin_update(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_set(key, nkey, c->client_ip); -#else stats_prefix_record_set(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "set", key, c->client_ip); #endif } @@ -7593,10 +7545,9 @@ static void process_bin_append_prepend(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_set(key, nkey, c->client_ip); -#else stats_prefix_record_set(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "set", key, c->client_ip); #endif } @@ -7768,10 +7719,9 @@ static void process_bin_delete(conn *c) { } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_delete(key, nkey, c->client_ip); -#else stats_prefix_record_delete(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "delete", key, c->client_ip); #endif } @@ -8692,10 +8642,9 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, it = NULL; } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_get(key, nkey, c->client_ip, NULL != it); -#else stats_prefix_record_get(key, nkey, NULL != it); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "get", key, c->client_ip); #endif } @@ -8910,10 +8859,9 @@ static void process_update_command(conn *c, token_t *tokens, const size_t ntoken } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_set(key, nkey, c->client_ip); -#else stats_prefix_record_set(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "set", key, c->client_ip); #endif } @@ -9002,19 +8950,17 @@ static void process_arithmetic_command(conn *c, token_t *tokens, const size_t nt } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - if (incr) { - stats_prefix_record_incr(key, nkey, c->client_ip); - } else { - stats_prefix_record_decr(key, nkey, c->client_ip); - } -#else if (incr) { stats_prefix_record_incr(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "incr", key, c->client_ip); +#endif } else { stats_prefix_record_decr(key, nkey); - } +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "decr", key, c->client_ip); #endif + } } ENGINE_ERROR_CODE ret; @@ -9106,10 +9052,9 @@ static void process_delete_command(conn *c, token_t *tokens, const size_t ntoken } if (settings.detail_enabled) { + stats_prefix_record_delete(key, nkey); #ifdef CMD_IN_SECOND - stats_prefix_record_delete(key, nkey, c->client_ip); -#else - stats_prefix_record_delete(key, nkey); + cmd_in_second_write("", "delete", key, c->client_ip); #endif } @@ -10144,10 +10089,9 @@ static void process_lop_get(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { + stats_prefix_record_lop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); #ifdef CMD_IN_SECOND - stats_prefix_record_lop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else - stats_prefix_record_lop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + cmd_in_second_write("", "get", key, c->client_ip); #endif } @@ -10256,10 +10200,9 @@ static void process_lop_prepare_nread(conn *c, int cmd, size_t vlen, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { + stats_prefix_record_lop_insert(key, nkey, false); #ifdef CMD_IN_SECOND - stats_prefix_record_lop_insert(key, nkey, c->client_ip, false); -#else - stats_prefix_record_lop_insert(key, nkey, false); + cmd_in_second_write("lop", "insert", key, c->client_ip); #endif } @@ -10303,10 +10246,9 @@ static void process_lop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { + stats_prefix_record_lop_create(key, nkey); #ifdef CMD_IN_SECOND - stats_prefix_record_lop_create(key, nkey, c->client_ip); -#else - stats_prefix_record_lop_create(key, nkey); + cmd_in_second_write("lop", "create", key, c->client_ip); #endif } @@ -10346,10 +10288,9 @@ static void process_lop_delete(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { + stats_prefix_record_lop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); #ifdef CMD_IN_SECOND - stats_prefix_record_lop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else - stats_prefix_record_lop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + cmd_in_second_write("lop", "delete", key, c->client_ip); #endif } @@ -10578,10 +10519,9 @@ static void process_sop_get(conn *c, char *key, size_t nkey, uint32_t count, } if (settings.detail_enabled) { + stats_prefix_record_sop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); #ifdef CMD_IN_SECOND - stats_prefix_record_sop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else - stats_prefix_record_sop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + cmd_in_second_write("sop", "get", key, c->client_ip); #endif } @@ -10696,21 +10636,22 @@ static void process_sop_prepare_nread(conn *c, int cmd, size_t vlen, char *key, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - if (cmd == (int)OPERATION_SOP_INSERT) - stats_prefix_record_sop_insert(key, nkey, c->client_ip, false); - else if (cmd == (int)OPERATION_SOP_DELETE) - stats_prefix_record_sop_delete(key, nkey, c->client_ip, false); - else - stats_prefix_record_sop_exist(key, nkey, c->client_ip, false); -#else - if (cmd == (int)OPERATION_SOP_INSERT) + if (cmd == (int)OPERATION_SOP_INSERT) { stats_prefix_record_sop_insert(key, nkey, false); - else if (cmd == (int)OPERATION_SOP_DELETE) +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "insert", key, c->client_ip); +#endif + } else if (cmd == (int)OPERATION_SOP_DELETE) { stats_prefix_record_sop_delete(key, nkey, false); - else +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "delete", key, c->client_ip); +#endif + } else { stats_prefix_record_sop_exist(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "exist", key, c->client_ip); #endif + } } switch (ret) { @@ -10763,10 +10704,9 @@ static void process_sop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_sop_create(key, nkey, c->client_ip); -#else stats_prefix_record_sop_create(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("sop", "create", key, c->client_ip); #endif } @@ -10988,10 +10928,9 @@ static void process_bop_get(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_get(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "get", key, c->client_ip); #endif } @@ -11108,10 +11047,9 @@ static void process_bop_count(conn *c, char *key, size_t nkey, &elem_count, &access_count, 0); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_count(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS)); -#else stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "get", key, c->client_ip); #endif } @@ -11162,10 +11100,9 @@ static void process_bop_position(conn *c, char *key, size_t nkey, bkrange, order, &position, 0); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_position(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_position(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "position", key, c->client_ip); #endif } @@ -11225,10 +11162,9 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b &flags, 0); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_pwg(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_pwg(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "pwg", key, c->client_ip); #endif } @@ -11342,10 +11278,9 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER elem_array, &elem_count, &flags, 0); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_gbp(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_gbp(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "gbp", key, c->client_ip); #endif } @@ -11456,10 +11391,9 @@ static void process_bop_update_prepare_nread(conn *c, int cmd, char *key, size_t } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_update(key, nkey, c->client_ip, false); -#else stats_prefix_record_bop_update(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "update", key, c->client_ip); #endif } @@ -11502,10 +11436,9 @@ static void process_bop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_insert(key, nkey, c->client_ip, false); -#else stats_prefix_record_bop_insert(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "insert", key, c->client_ip); #endif } @@ -11648,10 +11581,9 @@ static void process_bop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_create(key, nkey, c->client_ip); -#else stats_prefix_record_bop_create(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "create", key, c->client_ip); #endif } @@ -11693,10 +11625,9 @@ static void process_bop_delete(conn *c, char *key, size_t nkey, } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_bop_delete(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#else stats_prefix_record_bop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "delete", key, c->client_ip); #endif } @@ -11754,18 +11685,17 @@ static void process_bop_arithmetic(conn *c, char *key, size_t nkey, bkey_range * } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - if (incr) { - stats_prefix_record_bop_incr(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); - } else { - stats_prefix_record_bop_decr(key, nkey, c->client_ip, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); - } -#else if (incr) { stats_prefix_record_bop_incr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "incr", key, c->client_ip); +#endif } else { stats_prefix_record_bop_decr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); +#ifdef CMD_IN_SECOND + cmd_in_second_write("bop", "decr", key, c->client_ip); #endif + } } switch (ret) { @@ -12013,10 +11943,9 @@ static void process_mop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, } if (settings.detail_enabled && ret != ENGINE_SUCCESS) { -#ifdef CMD_IN_SECOND - stats_prefix_record_mop_insert(key, nkey, c->client_ip, false); -#else stats_prefix_record_mop_insert(key, nkey, false); +#ifdef CMD_IN_SECOND + cmd_in_second_write("mop", "insert", key, c->client_ip); #endif } @@ -12107,10 +12036,9 @@ static void process_mop_create(conn *c, char *key, size_t nkey, item_attr *attrp } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_mop_create(key, nkey, c->client_ip); -#else stats_prefix_record_mop_create(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("mop", "create", key, c->client_ip); #endif } @@ -13128,10 +13056,9 @@ static void process_getattr_command(conn *c, token_t *tokens, const size_t ntoke attr_ids, attr_count, &attr_data, 0); if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_getattr(key, nkey, c->client_ip); -#else stats_prefix_record_getattr(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "getattr", key, c->client_ip); #endif } } @@ -13288,10 +13215,9 @@ static void process_setattr_command(conn *c, token_t *tokens, const size_t ntoke } if (settings.detail_enabled) { -#ifdef CMD_IN_SECOND - stats_prefix_record_setattr(key, nkey, c->client_ip); -#else stats_prefix_record_setattr(key, nkey); +#ifdef CMD_IN_SECOND + cmd_in_second_write("", "setattr", key, c->client_ip); #endif } } @@ -16119,6 +16045,10 @@ int main (int argc, char **argv) { cmdlog_init(settings.port, mc_logger); #endif +#ifdef CMD_IN_SECOND + cmd_in_second_init(); +#endif + #ifdef DETECT_LONG_QUERY /* initialise long query detection */ if (lqdetect_init() == -1) { diff --git a/stats.c b/stats.c index c4fc2f407..6986c4261 100644 --- a/stats.c +++ b/stats.c @@ -25,7 +25,6 @@ * Steven Grimm */ #include "config.h" -#include "cmd_in_second.h" #include "memcached.h" #include #include @@ -339,7 +338,7 @@ static PREFIX_STATS *stats_prefix_find(const char *key, const size_t nkey) { /* * Records a "get" of a key. */ -void stats_prefix_record_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_get(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -356,7 +355,7 @@ void stats_prefix_record_get(const char *key, const size_t nkey, const char* cli /* * Records a "delete" of a key. */ -void stats_prefix_record_delete(const char *key, const size_t nkey, const char* client_ip) { +void stats_prefix_record_delete(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -370,7 +369,7 @@ void stats_prefix_record_delete(const char *key, const size_t nkey, const char* /* * Records a "set" of a key. */ -void stats_prefix_record_set(const char *key, const size_t nkey, const char* client_ip) { +void stats_prefix_record_set(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -378,14 +377,13 @@ void stats_prefix_record_set(const char *key, const size_t nkey, const char* cli if (NULL != pfs) { pfs->num_sets++; } - cmd_in_second_write("", "set", key, client_ip); STATS_UNLOCK(); } /* * Records a "incr" of a key. */ -void stats_prefix_record_incr(const char *key, const size_t nkey, const char* client_ip) { +void stats_prefix_record_incr(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -393,14 +391,13 @@ void stats_prefix_record_incr(const char *key, const size_t nkey, const char* cl if (NULL != pfs) { pfs->num_incrs++; } - cmd_in_second_write("", "incr", key, client_ip); STATS_UNLOCK(); } /* * Records a "decr" of a key. */ -void stats_prefix_record_decr(const char *key, const size_t nkey, const char* client_ip) { +void stats_prefix_record_decr(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -408,14 +405,13 @@ void stats_prefix_record_decr(const char *key, const size_t nkey, const char* cl if (NULL != pfs) { pfs->num_decrs++; } - cmd_in_second_write("", "decr", key, client_ip); STATS_UNLOCK(); } /* * LIST stats */ -void stats_prefix_record_lop_create(const char *key, const size_t nkey, const char* client_ip) { +void stats_prefix_record_lop_create(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -423,11 +419,10 @@ void stats_prefix_record_lop_create(const char *key, const size_t nkey, const ch if (NULL != pfs) { pfs->num_lop_creates++; } - cmd_in_second_write("lop", "create", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -438,11 +433,10 @@ void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const ch pfs->num_lop_insert_hits++; } } - cmd_in_second_write("lop", "insert", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -453,11 +447,10 @@ void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const ch pfs->num_lop_delete_hits++; } } - cmd_in_second_write("lop", "delete", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_lop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_lop_get(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -468,14 +461,13 @@ void stats_prefix_record_lop_get(const char *key, const size_t nkey, const char* pfs->num_lop_get_hits++; } } - cmd_in_second_write("lop", "get", key, client_ip); STATS_UNLOCK(); } /* * SET stats */ -void stats_prefix_record_sop_create(const char *key, const size_t nkey, const char* client_ip) { +void stats_prefix_record_sop_create(const char *key, const size_t nkey) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -483,11 +475,10 @@ void stats_prefix_record_sop_create(const char *key, const size_t nkey, const ch if (NULL != pfs) { pfs->num_sop_creates++; } - cmd_in_second_write("sop", "create", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -498,11 +489,10 @@ void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const ch pfs->num_sop_insert_hits++; } } - cmd_in_second_write("sop", "insert", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -513,11 +503,10 @@ void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const ch pfs->num_sop_delete_hits++; } } - cmd_in_second_write("sop", "delete", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_sop_get(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -528,11 +517,10 @@ void stats_prefix_record_sop_get(const char *key, const size_t nkey, const char* pfs->num_sop_get_hits++; } } - cmd_in_second_write("sop", "get", key, client_ip); STATS_UNLOCK(); } -void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { +void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const bool is_hit) { PREFIX_STATS *pfs; STATS_LOCK(); @@ -543,18 +531,13 @@ void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const cha pfs->num_sop_exist_hits++; } } - cmd_in_second_write("sop", "exist", key, client_ip); STATS_UNLOCK(); } /* * MAP stats */ -#ifdef CMD_IN_SECOND -void stats_prefix_record_mop_create(const char *key, const size_t nkey, const char* client_ip) { -#else void stats_prefix_record_mop_create(const char *key, const size_t nkey) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -562,17 +545,10 @@ void stats_prefix_record_mop_create(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_mop_creates++; } -#ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "create", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -583,17 +559,10 @@ void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const bo pfs->num_mop_insert_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "insert", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_mop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -604,17 +573,10 @@ void stats_prefix_record_mop_update(const char *key, const size_t nkey, const bo pfs->num_mop_update_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "update", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -625,17 +587,10 @@ void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const bo pfs->num_mop_delete_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "delete", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_mop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -646,20 +601,13 @@ void stats_prefix_record_mop_get(const char *key, const size_t nkey, const bool pfs->num_mop_get_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "get", key, client_ip); -#endif STATS_UNLOCK(); } /* * B+TREE stats */ -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_create(const char *key, const size_t nkey, const char* client_ip) { -#else void stats_prefix_record_bop_create(const char *key, const size_t nkey) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -667,17 +615,10 @@ void stats_prefix_record_bop_create(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_bop_creates++; } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "create", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -688,17 +629,10 @@ void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const bo pfs->num_bop_insert_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "insert", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -709,17 +643,10 @@ void stats_prefix_record_bop_update(const char *key, const size_t nkey, const bo pfs->num_bop_update_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "update", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -730,15 +657,10 @@ void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const bo pfs->num_bop_delete_hits++; } } - cmd_in_second_write("bop", "delete", key, client_ip); STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -749,17 +671,10 @@ void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const bool pfs->num_bop_incr_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "incr", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -770,17 +685,10 @@ void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const bool pfs->num_bop_decr_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "decr", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_get(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -791,17 +699,10 @@ void stats_prefix_record_bop_get(const char *key, const size_t nkey, const bool pfs->num_bop_get_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "get", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_count(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_count(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -812,17 +713,10 @@ void stats_prefix_record_bop_count(const char *key, const size_t nkey, const boo pfs->num_bop_count_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "count", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_position(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_position(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -833,17 +727,10 @@ void stats_prefix_record_bop_position(const char *key, const size_t nkey, const pfs->num_bop_position_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "position", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -854,17 +741,10 @@ void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool pfs->num_bop_pwg_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "pwg", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const char* client_ip, const bool is_hit) { -#else void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool is_hit) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -875,20 +755,13 @@ void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool pfs->num_bop_gbp_hits++; } } -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "gbp", key, client_ip); -#endif STATS_UNLOCK(); } /* * ATTR stats */ -#ifdef CMD_IN_SECOND -void stats_prefix_record_getattr(const char *key, const size_t nkey, const char* client_ip) { -#else void stats_prefix_record_getattr(const char *key, const size_t nkey) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -896,17 +769,10 @@ void stats_prefix_record_getattr(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_getattrs++; } -#ifdef CMD_IN_SECOND - cmd_in_second_write("", "getattr", key, client_ip); -#endif STATS_UNLOCK(); } -#ifdef CMD_IN_SECOND -void stats_prefix_record_setattr(const char *key, const size_t nkey, const char* client_ip) { -#else void stats_prefix_record_setattr(const char *key, const size_t nkey) { -#endif PREFIX_STATS *pfs; STATS_LOCK(); @@ -914,9 +780,6 @@ void stats_prefix_record_setattr(const char *key, const size_t nkey) { if (NULL != pfs) { pfs->num_setattrs++; } -#ifdef CMD_IN_SECOND - cmd_in_second_write("", "setattr", key, client_ip); -#endif STATS_UNLOCK(); } diff --git a/stats.h b/stats.h index c5f41082b..9bfafddc5 100644 --- a/stats.h +++ b/stats.h @@ -22,42 +22,7 @@ int stats_prefix_count(void); #if 1 // NEW_PREFIX_STATS_MANAGEMENT int stats_prefix_insert(const char *prefix, const size_t nprefix); #endif - int stats_prefix_delete(const char *prefix, const size_t nprefix); -#ifdef CMD_IN_SECOND -void stats_prefix_record_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_delete(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_set(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_incr(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_decr(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_lop_create(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_lop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_lop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_lop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_sop_create(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_sop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_sop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_sop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_sop_exist(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_mop_create(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_mop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_mop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_mop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_mop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_create(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_bop_insert(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_update(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_delete(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_incr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_decr(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_get(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_count(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_position(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const char* client_ip, const bool is_hit); -void stats_prefix_record_getattr(const char *key, const size_t nkey, const char* client_ip); -void stats_prefix_record_setattr(const char *key, const size_t nkey, const char* client_ip); -#else void stats_prefix_record_get(const char *key, const size_t nkey, const bool is_hit); void stats_prefix_record_delete(const char *key, const size_t nkey); void stats_prefix_record_set(const char *key, const size_t nkey); @@ -90,7 +55,5 @@ void stats_prefix_record_bop_pwg(const char *key, const size_t nkey, const bool void stats_prefix_record_bop_gbp(const char *key, const size_t nkey, const bool is_hit); void stats_prefix_record_getattr(const char *key, const size_t nkey); void stats_prefix_record_setattr(const char *key, const size_t nkey); -#endif - /*@null@*/ char *stats_prefix_dump(int *length); diff --git a/t/bulk.t b/t/bulk.t index 717fc6c62..5712b0ea5 100644 --- a/t/bulk.t +++ b/t/bulk.t @@ -134,7 +134,8 @@ sub wrong_cmd_test{ } -sub non_bulk_cmd { + +sub slow_cmd { my $request_cnt = 10000; request_log("sop exist", $request_cnt, $start); @@ -155,22 +156,34 @@ sub non_bulk_cmd { } } +sub file_check { + + my $file_handle; + open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; + + my $line_cnt; + close($file_handle); +} + wrong_cmd_test(); -sleep(3); +sleep(1); #extremely small cases + + =pod request_log("bop insert", 1, $start); -do_bulk_coll_insert("bop", "bkey", 1); +do_bulk_coll_insert("bop", "bkey1", 1); + request_log("bop insert", 9, $start); do_bulk_coll_insert("bop", "bkey2", 9); =cut -request_log("sop insert", $bulk_size, $start); -do_bulk_coll_insert("sop", "skey", $bulk_size+1); +request_log("bop insert", $bulk_size, $start); +do_bulk_coll_insert("bop", "bkey3", $bulk_size+100); sleep(1); -#non_bulk_cmd(); +#slow_cmd(); release_memcached($engine, $server); From b633ab1da6a994bdbad604caa151fc6ae51d1d26 Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Wed, 7 Aug 2019 18:56:09 +0900 Subject: [PATCH 06/10] str to enum --- cmd_in_second.c | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index 4c341698d..e24112fe5 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -42,8 +42,8 @@ typedef struct cmd_in_second_timer { } timertype; struct cmd_in_second { - char cmd[20]; - char collection_name[10]; + int cmd; + char cmd_str[50]; struct cmd_in_second_buffer buffer; timertype timer; int32_t bulk_limit; @@ -71,15 +71,6 @@ static bool is_bulk_cmd() return last_time->tv_sec - front_time->tv_sec <= 1; } -static void get_whole_cmd(char* whole_cmd) -{ - if (strlen(this.collection_name)) { - snprintf(whole_cmd, 20, "%s %s", this.collection_name, this.cmd); - return; - } - snprintf(whole_cmd, 15, "%s", this.cmd); -} - static bool buffer_empty() { return this.buffer.front == this.buffer.rear; @@ -104,10 +95,7 @@ static void* buffer_flush_thread() return NULL; } - char whole_cmd[20] = ""; - get_whole_cmd(whole_cmd); - - const size_t whole_cmd_len = strlen(whole_cmd); + const size_t cmd_len = strlen(this.cmd_str); const int32_t whitespaces = 3; size_t expected_write_length = 0; @@ -138,10 +126,10 @@ static void* buffer_flush_thread() } char log[LOG_LENGTH] = ""; - snprintf(log, LOG_LENGTH, "%s%s %s %s\n", time_str, whole_cmd, front.key, front.client_ip); + snprintf(log, LOG_LENGTH, "%s%s %s %s\n", time_str, this.cmd_str, front.key, front.client_ip); strncat(log_str, log, LOG_LENGTH); - expected_write_length += whole_cmd_len + strlen(front.key) + strlen(front.client_ip) + whitespaces; + expected_write_length += cmd_len + strlen(front.key) + strlen(front.client_ip) + whitespaces; circular_log_counter = (circular_log_counter+1) % this.log_per_timer; } @@ -228,8 +216,7 @@ static bool is_cmd_to_log(const char* collection_name, const char* cmd) strcmp(this.cmd, cmd) == 0; } -bool cmd_in_second_write(const char* collection_name, const char* cmd, - const char* key, const char* client_ip) +bool cmd_in_second_write(const int cmd, const char* key, const char* client_ip) { pthread_mutex_lock(&this.lock); if (this.cur_state != ON_LOGGING || !is_cmd_to_log(collection_name, cmd)) { @@ -252,9 +239,9 @@ bool cmd_in_second_write(const char* collection_name, const char* cmd, return true; } +/* TODO mc_logger */ void cmd_in_second_init() { - assert("test"); this.cur_state = NOT_STARTED; pthread_mutex_init(&this.lock, NULL); @@ -270,8 +257,7 @@ void cmd_in_second_init() this.timer.ring = NULL; } -int32_t cmd_in_second_start(const char* collection_name, const char* cmd, - const int32_t bulk_limit) +int cmd_in_second_start(const int cmd, const int bulk_limit) { pthread_mutex_lock(&this.lock); @@ -281,6 +267,7 @@ int32_t cmd_in_second_start(const char* collection_name, const char* cmd, return CMD_IN_SECOND_STARTED_ALREADY; } + this.cmd = cmd; this.bulk_limit = bulk_limit; this.buffer.capacity = bulk_limit+1; @@ -302,9 +289,6 @@ int32_t cmd_in_second_start(const char* collection_name, const char* cmd, return CMD_IN_SECOND_NO_MEM; } - snprintf(this.collection_name, 5, "%s", collection_name); - snprintf(this.cmd, 15, "%s", cmd); - this.cur_state = ON_LOGGING; pthread_mutex_unlock(&this.lock); From 40b5b32f466d466aa9a83385282c1bd0430496c0 Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Fri, 9 Aug 2019 17:48:11 +0900 Subject: [PATCH 07/10] thread design --- cmd_in_second.c | 230 +++++++++++------ cmd_in_second.h | 10 +- include/memcached/types.h | 17 +- memcached.c | 509 +++++++++++++++++++++++--------------- t/bulk.t | 15 +- 5 files changed, 479 insertions(+), 302 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index e24112fe5..209f1792d 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -13,6 +13,7 @@ #define LOG_LENGTH 400 #define IP_LENGTH 16 #define KEY_LENGTH 256 +#define CMD_STR_LEN 30 typedef enum cmd_in_second_state { NOT_STARTED, @@ -41,14 +42,24 @@ typedef struct cmd_in_second_timer { int32_t circular_counter; } timertype; +typedef struct cmd_in_second_flush_thread { + pthread_t tid; + pthread_attr_t attr; + pthread_cond_t cond; + pthread_mutex_t lock; + bool sleep; +} flush_thread; + struct cmd_in_second { - int cmd; + int operation; char cmd_str[50]; struct cmd_in_second_buffer buffer; timertype timer; int32_t bulk_limit; int32_t log_per_timer; state cur_state; + flush_thread flusher; + EXTENSION_LOGGER_DESCRIPTOR *mc_logger; pthread_mutex_t lock; }; @@ -65,9 +76,6 @@ static bool is_bulk_cmd() const struct timeval* front_time = &this.timer.ring[this.timer.front]; const struct timeval* last_time = &this.timer.ring[this.timer.last_elem_idx]; - //printf("%d\n", this.timer.last_elem_idx); - //assert(0); - return last_time->tv_sec - front_time->tv_sec <= 1; } @@ -76,71 +84,124 @@ static bool buffer_empty() return this.buffer.front == this.buffer.rear; } -static void* buffer_flush_thread() -{ - const int32_t fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); +static bool buffer_full() { + return (this.buffer.rear+1) % this.buffer.capacity == this.buffer.front; +} - if (fd < 0) { - perror("Can't open cmd_in_second log file: cmd_in_second.log"); - return NULL; +static void put_flusher_to_sleep() { + struct timeval now; + struct timespec timeout; + + pthread_mutex_lock(&this.flusher.lock); + gettimeofday(&now, NULL); + + now.tv_usec += 50000; + + if (now.tv_usec >= 1000000) { + now.tv_sec += 1; + now.tv_usec -= 1000000; } - buffertype* const buffer = &this.buffer; - timertype* const timer = &this.timer; + timeout.tv_sec = now.tv_sec; + timeout.tv_nsec = now.tv_usec * 1000; + + flush_thread* const flusher = &this.flusher; - char* log_str = (char*)malloc(LOG_LENGTH * this.bulk_limit * sizeof(char)); + flusher->sleep = true; + pthread_cond_timedwait(&flusher->cond, &flusher->lock, &timeout); + flusher->sleep = false; - if (log_str == NULL) { - perror("Can't allocate memory"); - return NULL; + pthread_mutex_unlock(&this.flusher.lock); +} + +static void wake_flusher_up(){ + printf("in\n"); + pthread_mutex_lock(&this.flusher.lock); + if (this.flusher.sleep) { + printf("signaled\n"); + pthread_cond_signal(&this.flusher.cond); } + pthread_mutex_unlock(&this.flusher.lock); +} - const size_t cmd_len = strlen(this.cmd_str); - const int32_t whitespaces = 3; +static void* flush_buffer() +{ + const int32_t fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); + + while(1) { + if (fd < 0) { + perror("Can't open cmd_in_second log file: cmd_in_second.log"); + break; + } - size_t expected_write_length = 0; - int32_t circular_log_counter = 0; + if (this.cur_state != ON_FLUSHING) { + put_flusher_to_sleep(); + continue; + } - while (!buffer_empty()) { + buffertype* const buffer = &this.buffer; + timertype* const timer = &this.timer; - const logtype front = buffer->ring[buffer->front]; - buffer->front = (buffer->front+1) % buffer->capacity; + char* log_str = (char*)malloc(LOG_LENGTH * this.bulk_limit * sizeof(char)); + + if (log_str == NULL) { + break; + } - char time_str[50] = ""; + const size_t cmd_len = strlen(this.cmd_str); + const int32_t whitespaces = 3; - if (circular_log_counter == 0) { + size_t expected_write_length = 0; + int32_t circular_log_counter = 0; - const struct timeval* front_time = &timer->ring[timer->front]; - const struct tm* lt = localtime((time_t*)&front_time->tv_sec); + while (!buffer_empty()) { - timer->front = (timer->front+1) % timer->capacity; + const logtype front = buffer->ring[buffer->front]; + buffer->front = (buffer->front+1) % buffer->capacity; - if (lt == NULL) { - perror("localtime failed"); - continue; + char time_str[50] = ""; + + if (circular_log_counter == 0) { + + const struct timeval* front_time = &timer->ring[timer->front]; + const struct tm* lt = localtime((time_t*)&front_time->tv_sec); + + timer->front = (timer->front+1) % timer->capacity; + + if (lt == NULL) { + perror("localtime failed"); + continue; + } + + sprintf(time_str, "%04d-%02d-%02d %02d:%02d:%02d.%06d\n", lt ->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday, + lt->tm_hour, lt->tm_min, lt->tm_sec, (int32_t)front_time->tv_usec); + expected_write_length += 27; } - sprintf(time_str, "%04d-%02d-%02d %02d:%02d:%02d.%06d\n", lt ->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday, - lt->tm_hour, lt->tm_min, lt->tm_sec, (int32_t)front_time->tv_usec); - expected_write_length += 27; + char log[LOG_LENGTH] = ""; + snprintf(log, LOG_LENGTH, "%s%s %s %s\n", time_str, this.cmd_str, front.key, front.client_ip); + strncat(log_str, log, LOG_LENGTH); + + expected_write_length += cmd_len + strlen(front.key) + strlen(front.client_ip) + whitespaces; + circular_log_counter = (circular_log_counter+1) % this.log_per_timer; } - char log[LOG_LENGTH] = ""; - snprintf(log, LOG_LENGTH, "%s%s %s %s\n", time_str, this.cmd_str, front.key, front.client_ip); - strncat(log_str, log, LOG_LENGTH); + const int written_length = write(fd, log_str, expected_write_length); + printf("%s", log_str); + free(log_str); - expected_write_length += cmd_len + strlen(front.key) + strlen(front.client_ip) + whitespaces; - circular_log_counter = (circular_log_counter+1) % this.log_per_timer; - } + if (written_length != expected_write_length) { + perror("write length is difference to expectation."); + break; + } - if (write(fd, log_str, expected_write_length) != expected_write_length) { - perror("write length is difference to expectation."); + break; } - close(fd); - - free(log_str); - + if (fd >= 0) { + close(fd); + } + free(this.timer.ring); this.timer.ring = NULL; @@ -152,49 +213,32 @@ static void* buffer_flush_thread() return NULL; } -static int32_t buffer_flush() -{ - - pthread_t tid; - pthread_attr_t attr; - - int32_t ret = 0; - - if (pthread_attr_init(&attr) != 0 || - pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED) != 0 || - (ret = pthread_create(&tid, &attr, buffer_flush_thread, NULL)) != 0) - { - perror("Can't create buffer flush thread"); - return ret; - } - - return ret; -} - static void buffer_add(const logtype* log) { - + printf("add ready\n"); struct cmd_in_second_buffer* const buffer = &this.buffer; buffer->ring[buffer->rear] = *log; buffer->rear = (buffer->rear+1) % buffer->capacity; + printf("enqueued\n"); - const bool buffer_full = (buffer->rear+1) % buffer->capacity == buffer->front; - - if (buffer_full) { + if (buffer_full()) { + printf("buffer full\n"); if (is_bulk_cmd()) { + printf("bulk_cmd\n"); this.cur_state = ON_FLUSHING; - buffer_flush(); + wake_flusher_up(); + printf("flush woked up\n"); return; } buffer->front = (buffer->front+1) % buffer->capacity; + printf("buffer popped\n"); } } static void timer_add() { struct cmd_in_second_timer* const timer = &this.timer; - const bool timer_full = (timer->rear+1) % timer->capacity == timer->front; if (timer_full) { @@ -210,17 +254,19 @@ static void timer_add() timer->rear = (timer->rear+1) % timer->capacity; } -static bool is_cmd_to_log(const char* collection_name, const char* cmd) +static bool is_cmd_to_log(const int operation) { - return strcmp(this.collection_name, collection_name) == 0 && - strcmp(this.cmd, cmd) == 0; + return this.operation == operation; } -bool cmd_in_second_write(const int cmd, const char* key, const char* client_ip) +bool cmd_in_second_write(const int operation, const char* key, const char* client_ip) { pthread_mutex_lock(&this.lock); - if (this.cur_state != ON_LOGGING || !is_cmd_to_log(collection_name, cmd)) { + printf("locked %s %d \n", key, this.cur_state); + + if (this.cur_state != ON_LOGGING || !is_cmd_to_log(operation)) { pthread_mutex_unlock(&this.lock); + printf("already locked\n"); return false; } @@ -233,17 +279,19 @@ bool cmd_in_second_write(const int cmd, const char* key, const char* client_ip) } buffer_add(&log); + printf("buffer add finished\n"); this.timer.circular_counter = (this.timer.circular_counter+1) % this.log_per_timer; pthread_mutex_unlock(&this.lock); + printf("unlocked\n"); return true; } /* TODO mc_logger */ -void cmd_in_second_init() +void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *mc_logger) { + this.mc_logger = mc_logger; this.cur_state = NOT_STARTED; - pthread_mutex_init(&this.lock, NULL); this.buffer.front = 0; this.buffer.rear = 0; @@ -255,9 +303,16 @@ void cmd_in_second_init() this.timer.circular_counter = 0; this.timer.last_elem_idx = 0; this.timer.ring = NULL; + + pthread_mutex_init(&this.lock, NULL); + + flush_thread* const flusher = &this.flusher; + pthread_attr_init(&flusher->attr); + pthread_mutex_init(&flusher->lock, NULL); + pthread_cond_init(&flusher->cond, NULL); } -int cmd_in_second_start(const int cmd, const int bulk_limit) +int cmd_in_second_start(const int operation, const char cmd_str[], const int bulk_limit) { pthread_mutex_lock(&this.lock); @@ -267,8 +322,21 @@ int cmd_in_second_start(const int cmd, const int bulk_limit) return CMD_IN_SECOND_STARTED_ALREADY; } - this.cmd = cmd; + int thread_created = -1; + + flush_thread* const flusher = &this.flusher; + + if (pthread_attr_init(&flusher->attr) != 0 || + pthread_attr_setdetachstate(&flusher->attr, PTHREAD_CREATE_DETACHED) != 0 || + (thread_created = pthread_create(&flusher->tid, &flusher->attr, + flush_buffer, NULL)) != 0) + { + return CMD_IN_SECOND_THREAD_FAILED; + } + + this.operation = operation; this.bulk_limit = bulk_limit; + snprintf(this.cmd_str, CMD_STR_LEN, cmd_str); this.buffer.capacity = bulk_limit+1; this.buffer.ring = (logtype*)malloc(this.buffer.capacity * sizeof(logtype)); @@ -280,7 +348,6 @@ int cmd_in_second_start(const int cmd, const int bulk_limit) this.log_per_timer = bulk_limit / 10 + (bulk_limit % 10 != 0); this.timer.capacity = this.log_per_timer + 1; - this.timer.ring = (struct timeval*)malloc(this.timer.capacity * sizeof(struct timeval)); if (this.timer.ring == NULL) { @@ -291,6 +358,7 @@ int cmd_in_second_start(const int cmd, const int bulk_limit) this.cur_state = ON_LOGGING; + printf("log start\n"); pthread_mutex_unlock(&this.lock); return CMD_IN_SECOND_START; diff --git a/cmd_in_second.h b/cmd_in_second.h index 406b5b012..3e50d8eae 100644 --- a/cmd_in_second.h +++ b/cmd_in_second.h @@ -4,12 +4,14 @@ #include #include -#include +#include +#include #define CMD_IN_SECOND_START 0 #define CMD_IN_SECOND_STARTED_ALREADY 1 #define CMD_IN_SECOND_NO_MEM 2 +#define CMD_IN_SECOND_THREAD_FAILED 3 -void cmd_in_second_init(void); -int32_t cmd_in_second_start(const char* collection_name, const char* cmd, const int32_t bulk_limit); -bool cmd_in_second_write(const char* collection_name, const char* cmd, const char* key, const char* client_ip); +void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *mc_logger); +int32_t cmd_in_second_start(const int operation, const char cmd[], const int32_t bulk_limit); +bool cmd_in_second_write(const int operation, const char* key, const char* client_ip); diff --git a/include/memcached/types.h b/include/memcached/types.h index a0587bf34..481af7937 100644 --- a/include/memcached/types.h +++ b/include/memcached/types.h @@ -106,6 +106,19 @@ extern "C" { OPERATION_MGET /**< Retrieve with mget semantics */ } ENGINE_RETRIEVE_OPERATION; + typedef enum { + OPERATION_INCR = 31, + OPERATION_DECR, + } ENGINE_INCR_DECR_OPERATION; + + typedef enum { + OPERATION_DELETE = 51 + } ENGINE_DELETION_OPERATION; + + typedef enum { + OPERATION_GETATTR = 61, + OPERATION_SETATTR = 61, + } ENGINE_ITEM_ATTR_OPERATION; /* collection operation */ typedef enum { /* list operation */ @@ -142,7 +155,9 @@ extern "C" { // SUPPORT_BOP_MGET OPERATION_BOP_MGET, /**< B+tree operation with mget(multiple get) element semantics */ // SUPPORT_BOP_SMGET - OPERATION_BOP_SMGET /**< B+tree operation with smget(sort-merge get) element semantics */ + OPERATION_BOP_SMGET, /**< B+tree operation with smget(sort-merge get) element semantics */ + OPERATION_BOP_INCR, + OPERATION_BOP_DECR } ENGINE_COLL_OPERATION; /* item type */ diff --git a/memcached.c b/memcached.c index 26447836a..c0fe1d842 100644 --- a/memcached.c +++ b/memcached.c @@ -1629,11 +1629,10 @@ static void process_lop_insert_complete(conn *c) { } if (settings.detail_enabled) { stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "insert", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_INSERT, c->coll_key, c->client_ip); #endif - } - #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { if (! lqdetect_lop_insert(c->client_ip, c->coll_key, c->coll_index)) { @@ -1694,11 +1693,11 @@ static void process_sop_insert_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "insert", c->coll_key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_SOP_INSERT, c->coll_key, c->client_ip); +#endif switch (ret) { case ENGINE_SUCCESS: STATS_HITS(c, sop_insert, c->coll_key, c->coll_nkey); @@ -1751,11 +1750,11 @@ static void process_sop_delete_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "delete", c->coll_key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_SOP_DELETE, c->coll_key, c->client_ip); +#endif switch (ret) { case ENGINE_SUCCESS: STATS_ELEM_HITS(c, sop_delete, c->coll_key, c->coll_nkey); @@ -1802,10 +1801,10 @@ static void process_sop_exist_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "exist", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_EXIST, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -1876,10 +1875,10 @@ static void process_mop_insert_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_mop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "insert", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_MOP_INSERT, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -1929,10 +1928,10 @@ static void process_mop_update_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_mop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "update", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_MOP_UPDATE, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -2015,11 +2014,11 @@ static void process_mop_delete_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_mop_delete(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "delete", c->coll_key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_MOP_DELETE, c->coll_key, c->client_ip); +#endif #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { if (! lqdetect_mop_delete(c->client_ip, c->coll_key, del_count, @@ -2137,11 +2136,11 @@ static void process_mop_get_complete(conn *c) if (settings.detail_enabled) { stats_prefix_record_mop_get(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "get", c->coll_key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_MOP_GET, c->coll_key, c->client_ip); +#endif #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { if (! lqdetect_mop_get(c->client_ip, c->coll_key, elem_count, @@ -2314,11 +2313,11 @@ static void process_bop_insert_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "insert", c->coll_key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_BOP_INSERT, c->coll_key, c->client_ip); +#endif switch (ret) { case ENGINE_SUCCESS: STATS_HITS(c, bop_insert, c->coll_key, c->coll_nkey); @@ -2420,10 +2419,10 @@ static void process_bop_update_complete(conn *c) if (settings.detail_enabled) { stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "update", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_UPDATE, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -2516,10 +2515,10 @@ static void process_bop_mget_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_bop_get(key_tokens[k].value, key_tokens[k].length, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "get", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_MGET, c->coll_key, c->client_ip); #endif - } if (ret == ENGINE_SUCCESS) { do { @@ -3105,10 +3104,10 @@ static void process_mget_complete(conn *c) } if (settings.detail_enabled) { stats_prefix_record_get(key, nkey, NULL != it); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_MGET, key, c->client_ip); #endif - } if (it) { /* get_item_info() always returns true. */ (void)mc_engine.v1->get_item_info(mc_engine.v0, c, it, &c->hinfo); @@ -3697,17 +3696,19 @@ static void complete_incr_bin(conn *c) { if (settings.detail_enabled) { if (incr) { stats_prefix_record_incr(key, nkey); -#ifdef CMD_IN_SECOND - cmd_in_second_write("", "incr", key, c->client_ip); -#endif } else { stats_prefix_record_decr(key, nkey); -#ifdef CMD_IN_SECOND - cmd_in_second_write("", "decr", key, c->client_ip); -#endif } } +#ifdef CMD_IN_SECOND + if (incr) { + cmd_in_second_write(OPERATION_INCR, key, c->client_ip); + } else { + cmd_in_second_write(OPERATION_DECR, key, c->client_ip); + } +#endif + switch (ret) { case ENGINE_SUCCESS: rsp->message.body.value = htonll(rsp->message.body.value); @@ -3959,10 +3960,10 @@ static void process_bin_get(conn *c) { if (settings.detail_enabled && ret != ENGINE_EWOULDBLOCK) { stats_prefix_record_get(key, nkey, ret == ENGINE_SUCCESS); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "decr", key, c->client_ip); + cmd_in_second_write(OPERATION_GET, key, c->client_ip); #endif - } } static void append_bin_stats(const char *key, const uint16_t klen, @@ -4540,11 +4541,11 @@ static void process_bin_lop_create(conn *c) { if (settings.detail_enabled) { stats_prefix_record_lop_create(key, nkey); -#ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "create", key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_LOP_CREATE, key, c->client_ip); +#endif switch (ret) { case ENGINE_SUCCESS: STATS_OKS(c, lop_create, key, nkey); @@ -4611,10 +4612,10 @@ static void process_bin_lop_prepare_nread(conn *c) { if (settings.detail_enabled && ret != ENGINE_SUCCESS) { stats_prefix_record_lop_insert(key, nkey, false); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "insert", key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_INSERT, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -4681,10 +4682,11 @@ static void process_bin_lop_insert_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_lop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } + #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "insert", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_INSERT, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -4764,10 +4766,10 @@ static void process_bin_lop_delete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_lop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "delete", key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_DELETE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -4856,10 +4858,10 @@ static void process_bin_lop_get(conn *c) { if (settings.detail_enabled) { stats_prefix_record_lop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_GET, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -4975,10 +4977,10 @@ static void process_bin_sop_create(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_create(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "create", key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_CREATE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -5051,21 +5053,21 @@ static void process_bin_sop_prepare_nread(conn *c) { if (settings.detail_enabled && ret != ENGINE_SUCCESS) { if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) { stats_prefix_record_sop_insert(key, nkey, false); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "insert", key, c->client_ip); -#endif } else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) { stats_prefix_record_sop_delete(key, nkey, false); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "delete", key, c->client_ip); -#endif } else { stats_prefix_record_sop_exist(key, nkey, false); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "exist", key, c->client_ip); -#endif } } +#ifdef CMD_IN_SECOND + if (c->cmd == PROTOCOL_BINARY_CMD_SOP_INSERT) { + cmd_in_second_write(OPERATION_SOP_INSERT, key, c->client_ip); + } else if (c->cmd == PROTOCOL_BINARY_CMD_SOP_DELETE) { + cmd_in_second_write(OPERATION_SOP_DELETE, key, c->client_ip); + } else { + cmd_in_second_write(OPERATION_SOP_EXIST, key, c->client_ip); + } +#endif switch (ret) { case ENGINE_SUCCESS: @@ -5154,10 +5156,10 @@ static void process_bin_sop_insert_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "insert", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_INSERT, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -5217,11 +5219,10 @@ static void process_bin_sop_delete_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_delete(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "delete", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_DELETE, c->coll_key, c->client_ip); #endif - } - switch (ret) { case ENGINE_SUCCESS: STATS_ELEM_HITS(c, sop_delete, c->coll_key, c->coll_nkey); @@ -5269,10 +5270,10 @@ static void process_bin_sop_exist_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_exist(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "exist", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_EXIST, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -5368,10 +5369,10 @@ static void process_bin_sop_get(conn *c) { if (settings.detail_enabled) { stats_prefix_record_sop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_GET, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -5497,10 +5498,10 @@ static void process_bin_bop_create(conn *c) { } if (settings.detail_enabled) { stats_prefix_record_bop_create(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "create", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_CREATE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -5580,10 +5581,10 @@ static void process_bin_bop_prepare_nread(conn *c) { if (settings.detail_enabled && ret != ENGINE_SUCCESS) { stats_prefix_record_bop_insert(key, nkey, false); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "insert", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_INSERT, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -5662,10 +5663,10 @@ static void process_bin_bop_insert_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_bop_insert(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "insert", c->coll_key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_INSERT, c->coll_key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -5744,11 +5745,11 @@ static void process_bin_bop_update_complete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_bop_update(c->coll_key, c->coll_nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "insert", c->coll_key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_BOP_UPDATE, c->coll_key, c->client_ip); +#endif switch (ret) { case ENGINE_SUCCESS: STATS_ELEM_HITS(c, bop_update, c->coll_key, c->coll_nkey); @@ -5860,11 +5861,11 @@ static void process_bin_bop_update_prepare_nread(conn *c) { if (settings.detail_enabled && ret != ENGINE_SUCCESS) { stats_prefix_record_bop_update(key, nkey, false); -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "update", key, c->client_ip); -#endif } +#ifdef CMD_IN_SECOND + cmd_in_second_write(OPERATION_BOP_UPDATE, key, c->client_ip); +#endif switch (ret) { case ENGINE_SUCCESS: c->ritem = ((value_item *)elem)->ptr; @@ -5941,10 +5942,10 @@ static void process_bin_bop_delete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_bop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "delete", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_DELETE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -6049,10 +6050,10 @@ static void process_bin_bop_get(conn *c) { if (settings.detail_enabled) { stats_prefix_record_bop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_GET, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -6184,10 +6185,10 @@ static void process_bin_bop_count(conn *c) { if (settings.detail_enabled) { stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "count", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_COUNT, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -6830,10 +6831,10 @@ static void process_bin_getattr(conn *c) { if (settings.detail_enabled) { stats_prefix_record_getattr(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "getattr", key, c->client_ip); + cmd_in_second_write(OPERATION_GETATTR, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -6970,10 +6971,10 @@ static void process_bin_setattr(conn *c) { if (settings.detail_enabled) { stats_prefix_record_setattr(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "setattr", key, c->client_ip); + cmd_in_second_write(OPERATION_SETATTR, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -7458,10 +7459,10 @@ static void process_bin_update(conn *c) { if (settings.detail_enabled) { stats_prefix_record_set(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "set", key, c->client_ip); + cmd_in_second_write(OPERATION_SET, key, c->client_ip); #endif - } ENGINE_ERROR_CODE ret; ret = mc_engine.v1->allocate(mc_engine.v0, c, &it, key, nkey, vlen+2, @@ -7546,10 +7547,10 @@ static void process_bin_append_prepend(conn *c) { if (settings.detail_enabled) { stats_prefix_record_set(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "set", key, c->client_ip); + cmd_in_second_write(OPERATION_SET, key, c->client_ip); #endif - } ENGINE_ERROR_CODE ret; ret = mc_engine.v1->allocate(mc_engine.v0, c, &it, key, nkey, vlen+2, @@ -7720,10 +7721,10 @@ static void process_bin_delete(conn *c) { if (settings.detail_enabled) { stats_prefix_record_delete(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "delete", key, c->client_ip); + cmd_in_second_write(OPERATION_DELETE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -8643,10 +8644,10 @@ static inline void process_get_command(conn *c, token_t *tokens, size_t ntokens, } if (settings.detail_enabled) { stats_prefix_record_get(key, nkey, NULL != it); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_GET, key, c->client_ip); #endif - } if (it) { if (!mc_engine.v1->get_item_info(mc_engine.v0, c, it, &c->hinfo)) { @@ -8860,10 +8861,10 @@ static void process_update_command(conn *c, token_t *tokens, const size_t ntoken if (settings.detail_enabled) { stats_prefix_record_set(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "set", key, c->client_ip); + cmd_in_second_write(OPERATION_SET, key, c->client_ip); #endif - } ENGINE_ERROR_CODE ret; ret = mc_engine.v1->allocate(mc_engine.v0, c, &it, key, nkey, vlen, @@ -8952,16 +8953,19 @@ static void process_arithmetic_command(conn *c, token_t *tokens, const size_t nt if (settings.detail_enabled) { if (incr) { stats_prefix_record_incr(key, nkey); -#ifdef CMD_IN_SECOND - cmd_in_second_write("", "incr", key, c->client_ip); -#endif } else { stats_prefix_record_decr(key, nkey); -#ifdef CMD_IN_SECOND - cmd_in_second_write("", "decr", key, c->client_ip); -#endif } } +#ifdef CMD_IN_SECOND + if (incr) { + stats_prefix_record_incr(key, nkey); + cmd_in_second_write(OPERATION_INCR, key, c->client_ip); + } else { + stats_prefix_record_decr(key, nkey); + cmd_in_second_write(OPERATION_DECR, key, c->client_ip); + } +#endif ENGINE_ERROR_CODE ret; uint64_t cas; @@ -9053,10 +9057,10 @@ static void process_delete_command(conn *c, token_t *tokens, const size_t ntoken if (settings.detail_enabled) { stats_prefix_record_delete(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "delete", key, c->client_ip); + cmd_in_second_write(OPERATION_DELETE, key, c->client_ip); #endif - } ENGINE_ERROR_CODE ret; ret = mc_engine.v1->remove(mc_engine.v0, c, key, nkey, 0, 0); @@ -10090,10 +10094,10 @@ static void process_lop_get(conn *c, char *key, size_t nkey, if (settings.detail_enabled) { stats_prefix_record_lop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_GET, key, c->client_ip); #endif - } #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { @@ -10201,10 +10205,10 @@ static void process_lop_prepare_nread(conn *c, int cmd, size_t vlen, if (settings.detail_enabled && ret != ENGINE_SUCCESS) { stats_prefix_record_lop_insert(key, nkey, false); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "insert", key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_INSERT, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -10247,10 +10251,10 @@ static void process_lop_create(conn *c, char *key, size_t nkey, item_attr *attrp if (settings.detail_enabled) { stats_prefix_record_lop_create(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "create", key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_CREATE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -10289,10 +10293,10 @@ static void process_lop_delete(conn *c, char *key, size_t nkey, if (settings.detail_enabled) { stats_prefix_record_lop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("lop", "delete", key, c->client_ip); + cmd_in_second_write(OPERATION_LOP_DELETE, key, c->client_ip); #endif - } #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { @@ -10520,10 +10524,10 @@ static void process_sop_get(conn *c, char *key, size_t nkey, uint32_t count, if (settings.detail_enabled) { stats_prefix_record_sop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_GET, key, c->client_ip); #endif - } #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { @@ -10638,19 +10642,10 @@ static void process_sop_prepare_nread(conn *c, int cmd, size_t vlen, char *key, if (settings.detail_enabled && ret != ENGINE_SUCCESS) { if (cmd == (int)OPERATION_SOP_INSERT) { stats_prefix_record_sop_insert(key, nkey, false); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "insert", key, c->client_ip); -#endif } else if (cmd == (int)OPERATION_SOP_DELETE) { stats_prefix_record_sop_delete(key, nkey, false); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "delete", key, c->client_ip); -#endif } else { stats_prefix_record_sop_exist(key, nkey, false); -#ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "exist", key, c->client_ip); -#endif } } @@ -10705,10 +10700,10 @@ static void process_sop_create(conn *c, char *key, size_t nkey, item_attr *attrp if (settings.detail_enabled) { stats_prefix_record_sop_create(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("sop", "create", key, c->client_ip); + cmd_in_second_write(OPERATION_SOP_CREATE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -10929,10 +10924,10 @@ static void process_bop_get(conn *c, char *key, size_t nkey, if (settings.detail_enabled) { stats_prefix_record_bop_get(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_GET, key, c->client_ip); #endif - } #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { @@ -11048,10 +11043,10 @@ static void process_bop_count(conn *c, char *key, size_t nkey, if (settings.detail_enabled) { stats_prefix_record_bop_count(key, nkey, (ret==ENGINE_SUCCESS)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "get", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_COUNT, key, c->client_ip); #endif - } #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { @@ -11101,10 +11096,10 @@ static void process_bop_position(conn *c, char *key, size_t nkey, if (settings.detail_enabled) { stats_prefix_record_bop_position(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "position", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_POSITION, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -11163,10 +11158,10 @@ static void process_bop_pwg(conn *c, char *key, size_t nkey, const bkey_range *b if (settings.detail_enabled) { stats_prefix_record_bop_pwg(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "pwg", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_PWG, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -11279,10 +11274,10 @@ static void process_bop_gbp(conn *c, char *key, size_t nkey, ENGINE_BTREE_ORDER if (settings.detail_enabled) { stats_prefix_record_bop_gbp(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "gbp", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_GBP, key, c->client_ip); #endif - } #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { @@ -11392,10 +11387,10 @@ static void process_bop_update_prepare_nread(conn *c, int cmd, char *key, size_t if (settings.detail_enabled && ret != ENGINE_SUCCESS) { stats_prefix_record_bop_update(key, nkey, false); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "update", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_UPDATE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -11437,10 +11432,10 @@ static void process_bop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, if (settings.detail_enabled && ret != ENGINE_SUCCESS) { stats_prefix_record_bop_insert(key, nkey, false); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "insert", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_INSERT, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -11582,10 +11577,10 @@ static void process_bop_create(conn *c, char *key, size_t nkey, item_attr *attrp if (settings.detail_enabled) { stats_prefix_record_bop_create(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "create", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_CREATE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -11626,10 +11621,10 @@ static void process_bop_delete(conn *c, char *key, size_t nkey, if (settings.detail_enabled) { stats_prefix_record_bop_delete(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "delete", key, c->client_ip); + cmd_in_second_write(OPERATION_BOP_DELETE, key, c->client_ip); #endif - } #ifdef DETECT_LONG_QUERY if (lqdetect_in_use && ret == ENGINE_SUCCESS) { @@ -11683,22 +11678,30 @@ static void process_bop_arithmetic(conn *c, char *key, size_t nkey, bkey_range * c->ewouldblock = true; ret = ENGINE_SUCCESS; } - +#ifdef CMD_IN_SECOND + if (incr) { + cmd_in_second_write(OPERATION_BOP_INCR, key, c->client_ip); + if (settings.detail_enabled) { + stats_prefix_record_bop_incr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } + } else { + cmd_in_second_write(OPERATION_BOP_DECR, key, c->client_ip); + + if (settings.detail_enabled) { + stats_prefix_record_bop_decr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); + } + } +#else if (settings.detail_enabled) { if (incr) { stats_prefix_record_bop_incr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "incr", key, c->client_ip); -#endif } else { stats_prefix_record_bop_decr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); -#ifdef CMD_IN_SECOND - cmd_in_second_write("bop", "decr", key, c->client_ip); -#endif } } +#endif - switch (ret) { + switch (ret) { case ENGINE_SUCCESS: if (incr) { STATS_ELEM_HITS(c, bop_incr, key, nkey); @@ -11944,10 +11947,10 @@ static void process_mop_prepare_nread(conn *c, int cmd, char *key, size_t nkey, if (settings.detail_enabled && ret != ENGINE_SUCCESS) { stats_prefix_record_mop_insert(key, nkey, false); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "insert", key, c->client_ip); + cmd_in_second_write(OPERATION_MOP_INSERT, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -12037,10 +12040,10 @@ static void process_mop_create(conn *c, char *key, size_t nkey, item_attr *attrp if (settings.detail_enabled) { stats_prefix_record_mop_create(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("mop", "create", key, c->client_ip); + cmd_in_second_write(OPERATION_MOP_CREATE, key, c->client_ip); #endif - } switch (ret) { case ENGINE_SUCCESS: @@ -13057,10 +13060,10 @@ static void process_getattr_command(conn *c, token_t *tokens, const size_t ntoke if (settings.detail_enabled) { stats_prefix_record_getattr(key, nkey); + } #ifdef CMD_IN_SECOND - cmd_in_second_write("", "getattr", key, c->client_ip); + cmd_in_second_write(OPERATION_GETATTR, key, c->client_ip); #endif - } } switch (ret) { @@ -13217,7 +13220,7 @@ static void process_setattr_command(conn *c, token_t *tokens, const size_t ntoke if (settings.detail_enabled) { stats_prefix_record_setattr(key, nkey); #ifdef CMD_IN_SECOND - cmd_in_second_write("", "setattr", key, c->client_ip); + cmd_in_second_write(OPERATION_SETATTR, key, c->client_ip); #endif } } @@ -13248,57 +13251,159 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken assert(c != NULL); - bool is_collection_cmd = true; if(strlen(tokens[SUBCOMMAND_TOKEN].value) > 3){ out_string(c, "CLIENT_ERROR bad command line format"); return; } - if (strcmp("lop", tokens[SUBCOMMAND_TOKEN].value) == 0) { - if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value)) { + int operation_enum = 0; + char cmd[20] = ""; + bool is_collection_cmd = true; + + if (strcmp("lop", tokens[SUBCOMMAND_TOKEN].value) == 0) { + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "lop create"); + operation_enum = OPERATION_LOP_CREATE; + } else if (strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "lop insert"); + operation_enum = OPERATION_LOP_INSERT; + } else if (strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "lop delete"); + operation_enum = OPERATION_LOP_DELETE; + } else if (strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "lop get"); + operation_enum = OPERATION_LOP_GET; + } else { out_string(c, "CLIENT_ERROR bad command line format"); return; } } else if (strcmp("sop", tokens[SUBCOMMAND_TOKEN].value) == 0) { - if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("exist", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value) != 0) { - + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "sop create"); + operation_enum = OPERATION_SOP_CREATE; + } else if (strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "sop insert"); + operation_enum = OPERATION_SOP_INSERT; + } else if (strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "sop delete"); + operation_enum = OPERATION_SOP_DELETE; + } else if (strcmp("exist", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "sop exist"); + operation_enum = OPERATION_SOP_EXIST; + } else if (strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "sop get"); + operation_enum = OPERATION_SOP_GET; + } else { out_string(c, "CLIENT_ERROR bad command line format"); return; } } else if (strcmp("mop", tokens[SUBCOMMAND_TOKEN].value) == 0) { - if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("get", tokens[SUBCOMMAND_TOKEN].value) != 0) { - + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "mop create"); + operation_enum = OPERATION_MOP_CREATE; + } else if (strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "mop insert"); + operation_enum = OPERATION_MOP_INSERT; + } else if (strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "mop delete"); + operation_enum = OPERATION_MOP_DELETE; + } else if (strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "mop update"); + operation_enum = OPERATION_MOP_UPDATE; + } else if (strcmp("get", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "mop get"); + operation_enum = OPERATION_MOP_GET; + } else { out_string(c, "CLIENT_ERROR bad command line format"); return; } } else if (strcmp("bop", tokens[SUBCOMMAND_TOKEN].value) == 0) { - if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("upsert", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("count", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("incr", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("decr", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("mget", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("smget", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("position", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && - strcmp("gbp", tokens[SUBCOMMAND_TOKEN+1].value) != 0 && strcmp("pwg", tokens[SUBCOMMAND_TOKEN+1].value) != 0) { - - + if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop create"); + operation_enum = OPERATION_BOP_CREATE; + } else if (strcmp("insert", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop insert"); + operation_enum = OPERATION_BOP_INSERT; + } else if (strcmp("upsert", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop upsert"); + operation_enum = OPERATION_BOP_UPSERT; + } else if (strcmp("update", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop update"); + operation_enum = OPERATION_BOP_UPDATE; + } else if (strcmp("delete", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop delete"); + operation_enum = OPERATION_BOP_DELETE; + } else if (strcmp("count", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop count"); + operation_enum = OPERATION_BOP_COUNT; + } else if (strcmp("incr", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop incr"); + operation_enum = OPERATION_BOP_INCR; + } else if (strcmp("decr", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop decr"); + operation_enum = OPERATION_BOP_DECR; + } else if (strcmp("get", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop get"); + operation_enum = OPERATION_BOP_GET; + } else if (strcmp("mget", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop mget"); + operation_enum = OPERATION_BOP_MGET; + } else if (strcmp("smget", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop smget"); + operation_enum = OPERATION_BOP_SMGET; + } else if (strcmp("position", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop position"); + operation_enum = OPERATION_BOP_POSITION; + } else if (strcmp("gbp", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop gbp"); + operation_enum = OPERATION_BOP_GBP; + } else if (strcmp("pwg", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { + sprintf(cmd, "bop pwg"); + operation_enum = OPERATION_BOP_PWG; + } else { out_string(c, "CLIENT_ERROR bad command line format"); return; } } else { /* key value command */ - if (strcmp("set", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("add", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("replace", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("append", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("prepend", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("delete", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("get", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("gets", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("mget", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("cas", tokens[SUBCOMMAND_TOKEN].value) != 0 && - strcmp("incr", tokens[SUBCOMMAND_TOKEN].value) != 0 && strcmp("decr", tokens[SUBCOMMAND_TOKEN].value) != 0) { - + if (strcmp("set", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "set"); + operation_enum = OPERATION_SET; + } else if (strcmp("add", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "add"); + operation_enum = OPERATION_ADD; + } else if (strcmp("replace", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "replace"); + operation_enum = OPERATION_REPLACE; + } else if (strcmp("append", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "append"); + operation_enum = OPERATION_APPEND; + } else if (strcmp("prepend", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "prepend"); + operation_enum = OPERATION_PREPEND; + } else if (strcmp("delete", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "delete"); + operation_enum = OPERATION_DELETE; + } else if (strcmp("get", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "get"); + operation_enum = OPERATION_GET; + } else if (strcmp("gets", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "get"); + operation_enum = OPERATION_GETS; + } else if (strcmp("mget", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "mget"); + operation_enum = OPERATION_MGET; + } else if (strcmp("cas", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "cas"); + operation_enum = OPERATION_CAS; + } else if (strcmp("incr", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "incr"); + operation_enum = OPERATION_INCR; + } else if (strcmp("decr", tokens[SUBCOMMAND_TOKEN].value) == 0) { + sprintf(cmd, "decr"); + operation_enum = OPERATION_DECR; + } else { out_string(c, "CLIENT_ERROR bad command line format"); return; } @@ -13312,26 +13417,18 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken return; } - char collection_name[20] = ""; - char cmd[20] = ""; - int32_t cmd_idx = SUBCOMMAND_TOKEN; - - if (is_collection_cmd) { - sprintf(collection_name, "%s", tokens[cmd_idx++].value); - } + const int cmd_idx = SUBCOMMAND_TOKEN; + const int cnt_idx = cmd_idx + 1 + is_collection_cmd; - sprintf(cmd, "%s", tokens[cmd_idx].value); - int32_t cnt_to_log= 0; + int cnt_to_log= 0; - if (!safe_strtol(tokens[cmd_idx+1].value, &cnt_to_log) || cnt_to_log <= 0) { + if (!safe_strtol(tokens[cnt_idx].value, &cnt_to_log) || cnt_to_log <= 0) { out_string(c, "CLIENT_ERROR bad command line format"); return; } - STATS_LOCK(); - const int32_t start_code = cmd_in_second_start(collection_name, cmd, cnt_to_log); - STATS_UNLOCK(); + const int start_code = cmd_in_second_start(operation_enum, cmd, cnt_to_log); char response[100] = ""; @@ -13344,7 +13441,7 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken break; case CMD_IN_SECOND_START: settings.detail_enabled = true; - sprintf(response, "%s %s will be logged", collection_name, cmd); + sprintf(response, "%s will be logged", cmd); break; } out_string(c, response); @@ -16046,7 +16143,7 @@ int main (int argc, char **argv) { #endif #ifdef CMD_IN_SECOND - cmd_in_second_init(); + cmd_in_second_init(mc_logger); #endif #ifdef DETECT_LONG_QUERY diff --git a/t/bulk.t b/t/bulk.t index 5712b0ea5..be18e56b1 100644 --- a/t/bulk.t +++ b/t/bulk.t @@ -42,9 +42,6 @@ sub request_log{ sub do_bulk_coll_insert { - if (-e "cmd_in_second.log") { - unlink "cmd_in_second.log" or die "can't remove old file\n"; - } my ($collection, $key, $count) = @_; my $start_time = time; @@ -92,7 +89,7 @@ sub do_bulk_coll_insert { $rst = "CREATED_STORED"; } - if ($count >= 10) { + if ($count >= 10 and $count < $bulk_size) { if ($index % (int($count/10)) == 0) { request_log("mop insert", 1000, $on_logging); } @@ -106,6 +103,7 @@ sub do_bulk_coll_insert { my $end_time = time; cmp_ok($end_time - $start_time, "<=", 1000, "all commands are done in a second"); + sleep(1); my $file_handle; open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; @@ -169,18 +167,15 @@ sub file_check { wrong_cmd_test(); sleep(1); #extremely small cases - - -=pod request_log("bop insert", 1, $start); do_bulk_coll_insert("bop", "bkey1", 1); - request_log("bop insert", 9, $start); do_bulk_coll_insert("bop", "bkey2", 9); -=cut +=pod request_log("bop insert", $bulk_size, $start); -do_bulk_coll_insert("bop", "bkey3", $bulk_size+100); +do_bulk_coll_insert("bop", "bkey3", $bulk_size); +=cut sleep(1); From f22e658a3380190598e0296f1a59a6461016e01b Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Fri, 9 Aug 2019 18:19:47 +0900 Subject: [PATCH 08/10] logger add --- cmd_in_second.c | 39 ++++++++++++++------------------------- t/bulk.t | 2 -- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index 209f1792d..ad1de85f1 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -1,5 +1,4 @@ #include "cmd_in_second.h" -#include "include/memcached/extension.h" #include #include #include @@ -45,7 +44,7 @@ typedef struct cmd_in_second_timer { typedef struct cmd_in_second_flush_thread { pthread_t tid; pthread_attr_t attr; - pthread_cond_t cond; + pthread_cond_t cond; pthread_mutex_t lock; bool sleep; } flush_thread; @@ -59,11 +58,11 @@ struct cmd_in_second { int32_t log_per_timer; state cur_state; flush_thread flusher; - EXTENSION_LOGGER_DESCRIPTOR *mc_logger; pthread_mutex_t lock; }; static struct cmd_in_second this; +static EXTENSION_LOGGER_DESCRIPTOR *mc_logger; static bool is_bulk_cmd() { @@ -115,10 +114,8 @@ static void put_flusher_to_sleep() { } static void wake_flusher_up(){ - printf("in\n"); pthread_mutex_lock(&this.flusher.lock); if (this.flusher.sleep) { - printf("signaled\n"); pthread_cond_signal(&this.flusher.cond); } pthread_mutex_unlock(&this.flusher.lock); @@ -130,7 +127,8 @@ static void* flush_buffer() while(1) { if (fd < 0) { - perror("Can't open cmd_in_second log file: cmd_in_second.log"); + mc_logger->log(EXTENSION_LOG_WARNING, NULL, + "Can't open cmd_in_second.log"); break; } @@ -169,7 +167,8 @@ static void* flush_buffer() timer->front = (timer->front+1) % timer->capacity; if (lt == NULL) { - perror("localtime failed"); + mc_logger->log(EXTENSION_LOG_WARNING, NULL, + "localtime failed"); continue; } @@ -187,11 +186,12 @@ static void* flush_buffer() } const int written_length = write(fd, log_str, expected_write_length); - printf("%s", log_str); free(log_str); if (written_length != expected_write_length) { - perror("write length is difference to expectation."); + printf("log\n"); + mc_logger->log(EXTENSION_LOG_WARNING, NULL, + "write length is difference to expectation"); break; } @@ -201,7 +201,7 @@ static void* flush_buffer() if (fd >= 0) { close(fd); } - + free(this.timer.ring); this.timer.ring = NULL; @@ -215,24 +215,18 @@ static void* flush_buffer() static void buffer_add(const logtype* log) { - printf("add ready\n"); struct cmd_in_second_buffer* const buffer = &this.buffer; buffer->ring[buffer->rear] = *log; buffer->rear = (buffer->rear+1) % buffer->capacity; - printf("enqueued\n"); if (buffer_full()) { - printf("buffer full\n"); if (is_bulk_cmd()) { - printf("bulk_cmd\n"); this.cur_state = ON_FLUSHING; wake_flusher_up(); - printf("flush woked up\n"); return; } buffer->front = (buffer->front+1) % buffer->capacity; - printf("buffer popped\n"); } } @@ -262,11 +256,9 @@ static bool is_cmd_to_log(const int operation) bool cmd_in_second_write(const int operation, const char* key, const char* client_ip) { pthread_mutex_lock(&this.lock); - printf("locked %s %d \n", key, this.cur_state); if (this.cur_state != ON_LOGGING || !is_cmd_to_log(operation)) { pthread_mutex_unlock(&this.lock); - printf("already locked\n"); return false; } @@ -279,18 +271,16 @@ bool cmd_in_second_write(const int operation, const char* key, const char* clien } buffer_add(&log); - printf("buffer add finished\n"); this.timer.circular_counter = (this.timer.circular_counter+1) % this.log_per_timer; pthread_mutex_unlock(&this.lock); - printf("unlocked\n"); return true; } /* TODO mc_logger */ -void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *mc_logger) +void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *global_logger) { - this.mc_logger = mc_logger; + mc_logger = global_logger; this.cur_state = NOT_STARTED; this.buffer.front = 0; @@ -308,7 +298,7 @@ void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *mc_logger) flush_thread* const flusher = &this.flusher; pthread_attr_init(&flusher->attr); - pthread_mutex_init(&flusher->lock, NULL); + pthread_mutex_init(&flusher->lock, NULL); pthread_cond_init(&flusher->cond, NULL); } @@ -328,7 +318,7 @@ int cmd_in_second_start(const int operation, const char cmd_str[], const int bul if (pthread_attr_init(&flusher->attr) != 0 || pthread_attr_setdetachstate(&flusher->attr, PTHREAD_CREATE_DETACHED) != 0 || - (thread_created = pthread_create(&flusher->tid, &flusher->attr, + (thread_created = pthread_create(&flusher->tid, &flusher->attr, flush_buffer, NULL)) != 0) { return CMD_IN_SECOND_THREAD_FAILED; @@ -358,7 +348,6 @@ int cmd_in_second_start(const int operation, const char cmd_str[], const int bul this.cur_state = ON_LOGGING; - printf("log start\n"); pthread_mutex_unlock(&this.lock); return CMD_IN_SECOND_START; diff --git a/t/bulk.t b/t/bulk.t index be18e56b1..1e7dcb23a 100644 --- a/t/bulk.t +++ b/t/bulk.t @@ -172,10 +172,8 @@ do_bulk_coll_insert("bop", "bkey1", 1); request_log("bop insert", 9, $start); do_bulk_coll_insert("bop", "bkey2", 9); -=pod request_log("bop insert", $bulk_size, $start); do_bulk_coll_insert("bop", "bkey3", $bulk_size); -=cut sleep(1); From 0a97164db65d9b7298d9b1f771a08de2c199578e Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Fri, 9 Aug 2019 18:21:49 +0900 Subject: [PATCH 09/10] change int32_t to int --- cmd_in_second.c | 30 +++++++++++++++--------------- include/memcached/types.h | 2 +- memcached.c | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index ad1de85f1..0d9673a0f 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -27,18 +27,18 @@ typedef struct cmd_in_second_log { typedef struct cmd_in_second_buffer { logtype* ring; - int32_t front; - int32_t rear; - int32_t capacity; + int front; + int rear; + int capacity; } buffertype; typedef struct cmd_in_second_timer { struct timeval* ring; - int32_t front; - int32_t rear; - int32_t capacity; - int32_t last_elem_idx; - int32_t circular_counter; + int front; + int rear; + int capacity; + int last_elem_idx; + int circular_counter; } timertype; typedef struct cmd_in_second_flush_thread { @@ -54,8 +54,8 @@ struct cmd_in_second { char cmd_str[50]; struct cmd_in_second_buffer buffer; timertype timer; - int32_t bulk_limit; - int32_t log_per_timer; + int bulk_limit; + int log_per_timer; state cur_state; flush_thread flusher; pthread_mutex_t lock; @@ -123,7 +123,7 @@ static void wake_flusher_up(){ static void* flush_buffer() { - const int32_t fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); + const int fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); while(1) { if (fd < 0) { @@ -147,10 +147,10 @@ static void* flush_buffer() } const size_t cmd_len = strlen(this.cmd_str); - const int32_t whitespaces = 3; + const int whitespaces = 3; size_t expected_write_length = 0; - int32_t circular_log_counter = 0; + int circular_log_counter = 0; while (!buffer_empty()) { @@ -173,7 +173,7 @@ static void* flush_buffer() } sprintf(time_str, "%04d-%02d-%02d %02d:%02d:%02d.%06d\n", lt ->tm_year + 1900, lt->tm_mon + 1, lt->tm_mday, - lt->tm_hour, lt->tm_min, lt->tm_sec, (int32_t)front_time->tv_usec); + lt->tm_hour, lt->tm_min, lt->tm_sec, (int)front_time->tv_usec); expected_write_length += 27; } @@ -326,7 +326,7 @@ int cmd_in_second_start(const int operation, const char cmd_str[], const int bul this.operation = operation; this.bulk_limit = bulk_limit; - snprintf(this.cmd_str, CMD_STR_LEN, cmd_str); + snprintf(this.cmd_str, CMD_STR_LEN, "%s", cmd_str); this.buffer.capacity = bulk_limit+1; this.buffer.ring = (logtype*)malloc(this.buffer.capacity * sizeof(logtype)); diff --git a/include/memcached/types.h b/include/memcached/types.h index 481af7937..104cf2da1 100644 --- a/include/memcached/types.h +++ b/include/memcached/types.h @@ -38,7 +38,7 @@ struct iovec { #define SUPPORT_BOP_SMGET #define JHPARK_OLD_SMGET_INTERFACE #define MAX_EFLAG_COMPARE_COUNT 100 -#define CMD_IN_SECOND 1 +#define CMD_IN_SECOND #ifdef __cplusplus extern "C" { diff --git a/memcached.c b/memcached.c index c0fe1d842..763d7adc0 100644 --- a/memcached.c +++ b/memcached.c @@ -11686,7 +11686,7 @@ static void process_bop_arithmetic(conn *c, char *key, size_t nkey, bkey_range * } } else { cmd_in_second_write(OPERATION_BOP_DECR, key, c->client_ip); - + if (settings.detail_enabled) { stats_prefix_record_bop_decr(key, nkey, (ret==ENGINE_SUCCESS || ret==ENGINE_ELEM_ENOENT)); } From c9997e17bfd417c43c6ff1fc6a1ffc2104ee2699 Mon Sep 17 00:00:00 2001 From: computerphilosopher Date: Fri, 16 Aug 2019 11:31:21 +0900 Subject: [PATCH 10/10] fix coding style, add cmd_in_second_stop feature --- cmd_in_second.c | 317 +++++++++++++++++++++++++++--------------------- cmd_in_second.h | 16 ++- memcached.c | 35 ++++-- t/bulk.t | 35 ++++-- 4 files changed, 237 insertions(+), 166 deletions(-) diff --git a/cmd_in_second.c b/cmd_in_second.c index 0d9673a0f..167cbd424 100644 --- a/cmd_in_second.c +++ b/cmd_in_second.c @@ -1,4 +1,20 @@ -#include "cmd_in_second.h" +/* -*- Mode: C; tab-width: 4; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +/* + * arcus-memcached - Arcus memory cache server + * Copyright 2015 JaM2in Co., Ltd. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ #include #include #include @@ -9,28 +25,33 @@ #include #include +#include "cmd_in_second.h" + #define LOG_LENGTH 400 #define IP_LENGTH 16 #define KEY_LENGTH 256 #define CMD_STR_LEN 30 +static EXTENSION_LOGGER_DESCRIPTOR *mc_logger; + typedef enum cmd_in_second_state { NOT_STARTED, ON_LOGGING, - ON_FLUSHING + ON_FLUSHING, + STOPPED } state; typedef struct cmd_in_second_log { char key[KEY_LENGTH]; char client_ip[IP_LENGTH]; -} logtype; +} cmd_in_second_log; typedef struct cmd_in_second_buffer { - logtype* ring; + cmd_in_second_log* ring; int front; int rear; int capacity; -} buffertype; +} cmd_in_second_buffer; typedef struct cmd_in_second_timer { struct timeval* ring; @@ -39,7 +60,7 @@ typedef struct cmd_in_second_timer { int capacity; int last_elem_idx; int circular_counter; -} timertype; +} cmd_in_second_timer; typedef struct cmd_in_second_flush_thread { pthread_t tid; @@ -49,49 +70,38 @@ typedef struct cmd_in_second_flush_thread { bool sleep; } flush_thread; -struct cmd_in_second { +struct cmd_in_second_global { int operation; char cmd_str[50]; struct cmd_in_second_buffer buffer; - timertype timer; + cmd_in_second_timer timer; int bulk_limit; int log_per_timer; state cur_state; - flush_thread flusher; + flush_thread flush; pthread_mutex_t lock; }; - -static struct cmd_in_second this; -static EXTENSION_LOGGER_DESCRIPTOR *mc_logger; +static struct cmd_in_second_global cmd_in_second; static bool is_bulk_cmd() { - const bool timer_empty = this.timer.front == this.timer.rear; + bool timer_empty = cmd_in_second.timer.front == cmd_in_second.timer.rear; if (timer_empty) { return false; } - const struct timeval* front_time = &this.timer.ring[this.timer.front]; - const struct timeval* last_time = &this.timer.ring[this.timer.last_elem_idx]; + struct timeval* front_time = &cmd_in_second.timer.ring[cmd_in_second.timer.front]; + struct timeval* last_time = &cmd_in_second.timer.ring[cmd_in_second.timer.last_elem_idx]; return last_time->tv_sec - front_time->tv_sec <= 1; } -static bool buffer_empty() -{ - return this.buffer.front == this.buffer.rear; -} - -static bool buffer_full() { - return (this.buffer.rear+1) % this.buffer.capacity == this.buffer.front; -} - -static void put_flusher_to_sleep() { +static void do_flush_sleep() { struct timeval now; struct timespec timeout; - pthread_mutex_lock(&this.flusher.lock); + pthread_mutex_lock(&cmd_in_second.flush.lock); gettimeofday(&now, NULL); now.tv_usec += 50000; @@ -104,26 +114,28 @@ static void put_flusher_to_sleep() { timeout.tv_sec = now.tv_sec; timeout.tv_nsec = now.tv_usec * 1000; - flush_thread* const flusher = &this.flusher; - flusher->sleep = true; - pthread_cond_timedwait(&flusher->cond, &flusher->lock, &timeout); - flusher->sleep = false; + cmd_in_second.flush.sleep = true; + pthread_cond_timedwait(&cmd_in_second.flush.cond, &cmd_in_second.flush.lock, &timeout); + cmd_in_second.flush.sleep = false; - pthread_mutex_unlock(&this.flusher.lock); + pthread_mutex_unlock(&cmd_in_second.flush.lock); } -static void wake_flusher_up(){ - pthread_mutex_lock(&this.flusher.lock); - if (this.flusher.sleep) { - pthread_cond_signal(&this.flusher.cond); +static void do_flush_wakeup(){ + pthread_mutex_lock(&cmd_in_second.flush.lock); + if (cmd_in_second.flush.sleep) { + pthread_cond_signal(&cmd_in_second.flush.cond); } - pthread_mutex_unlock(&this.flusher.lock); + pthread_mutex_unlock(&cmd_in_second.flush.lock); } -static void* flush_buffer() +static void* do_flush_write() { - const int fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); + int fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); + + cmd_in_second_buffer* buffer = &cmd_in_second.buffer; + cmd_in_second_timer* timer = &cmd_in_second.timer; while(1) { if (fd < 0) { @@ -132,37 +144,42 @@ static void* flush_buffer() break; } - if (this.cur_state != ON_FLUSHING) { - put_flusher_to_sleep(); + if (cmd_in_second.cur_state == STOPPED) { + break; + } + + if (cmd_in_second.cur_state != ON_FLUSHING) { + do_flush_sleep(); continue; } - buffertype* const buffer = &this.buffer; - timertype* const timer = &this.timer; - char* log_str = (char*)malloc(LOG_LENGTH * this.bulk_limit * sizeof(char)); + char* log_str = (char*)malloc(LOG_LENGTH * cmd_in_second.bulk_limit * sizeof(char)); if (log_str == NULL) { break; } - const size_t cmd_len = strlen(this.cmd_str); + const size_t cmd_len = strlen(cmd_in_second.cmd_str); const int whitespaces = 3; size_t expected_write_length = 0; int circular_log_counter = 0; - while (!buffer_empty()) { + bool buffer_empty = buffer->front == buffer->rear; - const logtype front = buffer->ring[buffer->front]; + while (!buffer_empty) { + + cmd_in_second_log front = buffer->ring[buffer->front]; buffer->front = (buffer->front+1) % buffer->capacity; + buffer_empty = buffer->front == buffer->rear; char time_str[50] = ""; if (circular_log_counter == 0) { - const struct timeval* front_time = &timer->ring[timer->front]; - const struct tm* lt = localtime((time_t*)&front_time->tv_sec); + struct timeval* front_time = &timer->ring[timer->front]; + struct tm* lt = localtime((time_t*)&front_time->tv_sec); timer->front = (timer->front+1) % timer->capacity; @@ -178,18 +195,17 @@ static void* flush_buffer() } char log[LOG_LENGTH] = ""; - snprintf(log, LOG_LENGTH, "%s%s %s %s\n", time_str, this.cmd_str, front.key, front.client_ip); + snprintf(log, LOG_LENGTH, "%s%s %s %s\n", time_str, cmd_in_second.cmd_str, front.key, front.client_ip); strncat(log_str, log, LOG_LENGTH); expected_write_length += cmd_len + strlen(front.key) + strlen(front.client_ip) + whitespaces; - circular_log_counter = (circular_log_counter+1) % this.log_per_timer; + circular_log_counter = (circular_log_counter+1) % cmd_in_second.log_per_timer; } - const int written_length = write(fd, log_str, expected_write_length); + int written_length = write(fd, log_str, expected_write_length); free(log_str); if (written_length != expected_write_length) { - printf("log\n"); mc_logger->log(EXTENSION_LOG_WARNING, NULL, "write length is difference to expectation"); break; @@ -202,153 +218,176 @@ static void* flush_buffer() close(fd); } - free(this.timer.ring); - this.timer.ring = NULL; + free(buffer->ring); + buffer->ring = NULL; - free(this.buffer.ring); - this.buffer.ring = NULL; + free(timer->ring); + timer->ring = NULL; - this.cur_state = NOT_STARTED; + cmd_in_second.cur_state = NOT_STARTED; return NULL; } -static void buffer_add(const logtype* log) +static void do_buffer_add(const char* key, const char* client_ip) { - struct cmd_in_second_buffer* const buffer = &this.buffer; - buffer->ring[buffer->rear] = *log; + cmd_in_second_buffer* buffer = &cmd_in_second.buffer; + + snprintf(buffer->ring[buffer->rear].key, + KEY_LENGTH, "%s", key); + snprintf(buffer->ring[buffer->rear].client_ip, + IP_LENGTH, "%s", client_ip); + buffer->rear = (buffer->rear+1) % buffer->capacity; - if (buffer_full()) { + bool buffer_full = (buffer->rear+1) % buffer->capacity == + buffer->front; + + if (buffer_full) { if (is_bulk_cmd()) { - this.cur_state = ON_FLUSHING; - wake_flusher_up(); + cmd_in_second.cur_state = ON_FLUSHING; + do_flush_wakeup(); return; } buffer->front = (buffer->front+1) % buffer->capacity; } } -static void timer_add() +static void do_timer_add() { - struct cmd_in_second_timer* const timer = &this.timer; - const bool timer_full = (timer->rear+1) % timer->capacity == timer->front; - - if (timer_full) { - timer->front = (timer->front+1) % timer->capacity; - } + struct cmd_in_second_timer* timer = &cmd_in_second.timer; if (gettimeofday(&timer->ring[timer->rear], NULL) == -1) { - perror("gettimeofday failed"); + mc_logger->log(EXTENSION_LOG_WARNING, NULL, + "gettimeofday failed"); return; - }; + } - timer->last_elem_idx = timer->rear; timer->rear = (timer->rear+1) % timer->capacity; -} + timer->last_elem_idx = timer->rear; + bool timer_full = (timer->rear+1) % timer->capacity == timer->front; -static bool is_cmd_to_log(const int operation) -{ - return this.operation == operation; + if (timer_full) { + timer->front = (timer->front+1) % timer->capacity; + } } -bool cmd_in_second_write(const int operation, const char* key, const char* client_ip) +void cmd_in_second_write(int operation, const char* key, const char* client_ip) { - pthread_mutex_lock(&this.lock); + pthread_mutex_lock(&cmd_in_second.lock); - if (this.cur_state != ON_LOGGING || !is_cmd_to_log(operation)) { - pthread_mutex_unlock(&this.lock); - return false; + if (cmd_in_second.cur_state != ON_LOGGING || + cmd_in_second.operation != operation) { + pthread_mutex_unlock(&cmd_in_second.lock); + return; } - logtype log = {"", ""}; - snprintf(log.client_ip, IP_LENGTH, "%s", client_ip); - snprintf(log.key, KEY_LENGTH, "%s", key); - - if (this.timer.circular_counter == 0) { - timer_add(); + if (cmd_in_second.timer.circular_counter == 0) { + do_timer_add(); } - buffer_add(&log); - this.timer.circular_counter = (this.timer.circular_counter+1) % this.log_per_timer; + do_buffer_add(key, client_ip); + cmd_in_second.timer.circular_counter = (cmd_in_second.timer.circular_counter+1) % + cmd_in_second.log_per_timer; - pthread_mutex_unlock(&this.lock); - return true; + pthread_mutex_unlock(&cmd_in_second.lock); + return; } -/* TODO mc_logger */ -void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *global_logger) +void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *logger) { - mc_logger = global_logger; - this.cur_state = NOT_STARTED; - - this.buffer.front = 0; - this.buffer.rear = 0; - this.buffer.ring = NULL; - - this.timer.front = 0; - this.timer.rear = 0; - this.timer.capacity = 0; - this.timer.circular_counter = 0; - this.timer.last_elem_idx = 0; - this.timer.ring = NULL; - - pthread_mutex_init(&this.lock, NULL); - - flush_thread* const flusher = &this.flusher; - pthread_attr_init(&flusher->attr); - pthread_mutex_init(&flusher->lock, NULL); - pthread_cond_init(&flusher->cond, NULL); + mc_logger = logger; + cmd_in_second.cur_state = NOT_STARTED; + + pthread_mutex_init(&cmd_in_second.lock, NULL); + pthread_attr_init(&cmd_in_second.flush.attr); + pthread_mutex_init(&cmd_in_second.flush.lock, NULL); + pthread_cond_init(&cmd_in_second.flush.cond, NULL); + + cmd_in_second.buffer.ring = NULL; + cmd_in_second.timer.ring = NULL; } -int cmd_in_second_start(const int operation, const char cmd_str[], const int bulk_limit) +int cmd_in_second_start(int operation, const char cmd_str[], int bulk_limit) { - pthread_mutex_lock(&this.lock); + pthread_mutex_lock(&cmd_in_second.lock); - if (this.cur_state != NOT_STARTED) { - pthread_mutex_unlock(&this.lock); + if (cmd_in_second.cur_state != NOT_STARTED) { + pthread_mutex_unlock(&cmd_in_second.lock); return CMD_IN_SECOND_STARTED_ALREADY; } - int thread_created = -1; + int fd = open("cmd_in_second.log", O_CREAT | O_WRONLY | O_TRUNC, 0644); - flush_thread* const flusher = &this.flusher; + if (fd < 0) { + pthread_mutex_unlock(&cmd_in_second.lock); + return CMD_IN_SECOND_FILE_FAILED; + } + + close(fd); - if (pthread_attr_init(&flusher->attr) != 0 || - pthread_attr_setdetachstate(&flusher->attr, PTHREAD_CREATE_DETACHED) != 0 || - (thread_created = pthread_create(&flusher->tid, &flusher->attr, - flush_buffer, NULL)) != 0) + if (pthread_attr_init(&cmd_in_second.flush.attr) != 0 || + pthread_attr_setdetachstate(&cmd_in_second.flush.attr, PTHREAD_CREATE_DETACHED) != 0 || + (pthread_create(&cmd_in_second.flush.tid, &cmd_in_second.flush.attr, + do_flush_write, NULL)) != 0) { + pthread_mutex_unlock(&cmd_in_second.lock); return CMD_IN_SECOND_THREAD_FAILED; } - this.operation = operation; - this.bulk_limit = bulk_limit; - snprintf(this.cmd_str, CMD_STR_LEN, "%s", cmd_str); + cmd_in_second.operation = operation; + cmd_in_second.bulk_limit = bulk_limit; + snprintf(cmd_in_second.cmd_str, CMD_STR_LEN, "%s", cmd_str); - this.buffer.capacity = bulk_limit+1; - this.buffer.ring = (logtype*)malloc(this.buffer.capacity * sizeof(logtype)); + cmd_in_second.buffer.front = 0; + cmd_in_second.buffer.rear = 0; + cmd_in_second.buffer.capacity = bulk_limit+1; + cmd_in_second.buffer.ring = (cmd_in_second_log*)malloc(cmd_in_second.buffer.capacity * + sizeof(cmd_in_second_log)); - if (this.buffer.ring == NULL) { - pthread_mutex_unlock(&this.lock); + if (cmd_in_second.buffer.ring == NULL) { + pthread_mutex_unlock(&cmd_in_second.lock); return CMD_IN_SECOND_NO_MEM; } - this.log_per_timer = bulk_limit / 10 + (bulk_limit % 10 != 0); - this.timer.capacity = this.log_per_timer + 1; - this.timer.ring = (struct timeval*)malloc(this.timer.capacity * sizeof(struct timeval)); + cmd_in_second.log_per_timer = bulk_limit / 10 + (bulk_limit % 10 != 0); + + cmd_in_second.timer.front = 0; + cmd_in_second.timer.rear = 0; + cmd_in_second.timer.circular_counter = 0; + cmd_in_second.timer.last_elem_idx = 0; + cmd_in_second.timer.capacity = 12; + cmd_in_second.timer.ring = (struct timeval*) malloc (cmd_in_second.timer.capacity * + sizeof(struct timeval)); - if (this.timer.ring == NULL) { - free(this.buffer.ring); - pthread_mutex_unlock(&this.lock); + if (cmd_in_second.timer.ring == NULL) { + free(cmd_in_second.buffer.ring); + pthread_mutex_unlock(&cmd_in_second.lock); return CMD_IN_SECOND_NO_MEM; } - this.cur_state = ON_LOGGING; + cmd_in_second.cur_state = ON_LOGGING; - pthread_mutex_unlock(&this.lock); + pthread_mutex_unlock(&cmd_in_second.lock); return CMD_IN_SECOND_START; } + +void cmd_in_second_stop(bool* already_stop) { + + pthread_mutex_lock(&cmd_in_second.lock); + + if (cmd_in_second.cur_state != ON_LOGGING) { + *already_stop = true; + pthread_mutex_unlock(&cmd_in_second.lock); + return; + } + + *already_stop = false; + + cmd_in_second.cur_state = STOPPED; + + pthread_mutex_unlock(&cmd_in_second.lock); +} diff --git a/cmd_in_second.h b/cmd_in_second.h index 3e50d8eae..6490e84f2 100644 --- a/cmd_in_second.h +++ b/cmd_in_second.h @@ -7,11 +7,15 @@ #include #include -#define CMD_IN_SECOND_START 0 -#define CMD_IN_SECOND_STARTED_ALREADY 1 -#define CMD_IN_SECOND_NO_MEM 2 -#define CMD_IN_SECOND_THREAD_FAILED 3 +typedef enum { + CMD_IN_SECOND_START, + CMD_IN_SECOND_STARTED_ALREADY, + CMD_IN_SECOND_NO_MEM, + CMD_IN_SECOND_THREAD_FAILED, + CMD_IN_SECOND_FILE_FAILED, +} CMD_IN_SECOND_START_CODE; void cmd_in_second_init(EXTENSION_LOGGER_DESCRIPTOR *mc_logger); -int32_t cmd_in_second_start(const int operation, const char cmd[], const int32_t bulk_limit); -bool cmd_in_second_write(const int operation, const char* key, const char* client_ip); +int cmd_in_second_start(int operation, const char cmd[], int bulk_limit); +void cmd_in_second_write(int operation, const char* key, const char* client_ip); +void cmd_in_second_stop(bool* already_stop); diff --git a/memcached.c b/memcached.c index 763d7adc0..4bbbd680d 100644 --- a/memcached.c +++ b/memcached.c @@ -13251,16 +13251,29 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken assert(c != NULL); + if (ntokens == 3) { + if (strcmp("stop", tokens[SUBCOMMAND_TOKEN].value) != 0) { + out_string(c, "CLIENT_ERROR bad command line format"); + return; + } - if(strlen(tokens[SUBCOMMAND_TOKEN].value) > 3){ - out_string(c, "CLIENT_ERROR bad command line format"); + bool already_stop = false; + cmd_in_second_stop(&already_stop); + + if (already_stop) { + out_string(c, "cmd_in_second already stopped"); + return; + } + + out_string(c, "cmd_in_second stopped"); return; } + int operation_enum = 0; char cmd[20] = ""; - bool is_collection_cmd = true; + bool is_collection_cmd = true; if (strcmp("lop", tokens[SUBCOMMAND_TOKEN].value) == 0) { if (strcmp("create", tokens[SUBCOMMAND_TOKEN+1].value) == 0) { @@ -13412,14 +13425,12 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken if ((is_collection_cmd && ntokens != 5) || (!is_collection_cmd && ntokens != 4)) { - out_string(c, "CLIENT_ERROR bad command line format"); return; } - const int cmd_idx = SUBCOMMAND_TOKEN; - const int cnt_idx = cmd_idx + 1 + is_collection_cmd; - + int cmd_idx = SUBCOMMAND_TOKEN; + int cnt_idx = cmd_idx + 1 + is_collection_cmd; int cnt_to_log= 0; @@ -13428,7 +13439,7 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken return; } - const int start_code = cmd_in_second_start(operation_enum, cmd, cnt_to_log); + int start_code = cmd_in_second_start(operation_enum, cmd, cnt_to_log); char response[100] = ""; @@ -13439,6 +13450,12 @@ static void process_second_command(conn *c, token_t *tokens, const size_t ntoken case CMD_IN_SECOND_NO_MEM: sprintf(response, "SERVER_ERROR out of memory"); break; + case CMD_IN_SECOND_THREAD_FAILED: + sprintf(response, "cmd_in_second failed to create a flush thread"); + break; + case CMD_IN_SECOND_FILE_FAILED: + sprintf(response, "cmd_in_second failed to create a log file"); + break; case CMD_IN_SECOND_START: settings.detail_enabled = true; sprintf(response, "%s will be logged", cmd); @@ -13570,7 +13587,7 @@ static void process_command(conn *c, char *command, int cmdlen) process_config_command(c, tokens, ntokens); } #ifdef CMD_IN_SECOND - else if ((ntokens == 4 || ntokens == 5) && (strcmp(tokens[COMMAND_TOKEN].value, "cmd_in_second") == 0)) + else if ((ntokens >= 3 && ntokens <= 5) && (strcmp(tokens[COMMAND_TOKEN].value, "cmd_in_second") == 0)) { process_second_command(c, tokens, ntokens); } diff --git a/t/bulk.t b/t/bulk.t index 1e7dcb23a..8db53268e 100644 --- a/t/bulk.t +++ b/t/bulk.t @@ -19,7 +19,7 @@ my $bulk_size = 3461; #mem_cmd_is($sock, $cmd, $val, $rst); # -sub request_log{ +sub request_log { my ($whole_cmd, $cnt, $state) = @_; if ($cnt <= 0) { @@ -40,6 +40,12 @@ sub request_log{ } } +sub stop_log { + request_log("bop insert", 1000, $start); + mem_cmd_is($sock, "cmd_in_second stop", "", "cmd_in_second stopped"); + mem_cmd_is($sock, "cmd_in_second stop", "", "cmd_in_second already stopped"); +} + sub do_bulk_coll_insert { @@ -67,7 +73,6 @@ sub do_bulk_coll_insert { $whole_cmd = "$collection $cmd $key $field $vleng"; } - my $rst = "STORED"; if ($index == 0) { @@ -95,8 +100,7 @@ sub do_bulk_coll_insert { } } - mem_cmd_is($sock, $whole_cmd, $val, $rst) or die; - + mem_cmd_is($sock, $whole_cmd, $val, $rst); } @@ -105,9 +109,7 @@ sub do_bulk_coll_insert { cmp_ok($end_time - $start_time, "<=", 1000, "all commands are done in a second"); sleep(1); - my $file_handle; - open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; - close($file_handle); + file_check() } @@ -124,9 +126,10 @@ sub wrong_cmd_test{ mem_cmd_is($sock, "cmd_in_second bop insert", "", $bad_format); mem_cmd_is($sock, "cmd_in_second bop insert blahblah", "", $bad_format); mem_cmd_is($sock, "cmd_in_second set 1000 blahblah", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second bop", "", $bad_format); + mem_cmd_is($sock, "cmd_in_second stop blahblah", "", $bad_format); my $unknown = "ERROR unknown command"; - mem_cmd_is($sock, "cmd_in_second bop", "", $unknown); mem_cmd_is($sock, "cmd_in_second bop insert 1000 blahblah", "", $unknown); mem_cmd_is($sock, "cmd_in_second bop insert 1000 1000", "", $unknown); @@ -159,24 +162,32 @@ sub file_check { my $file_handle; open($file_handle, "cmd_in_second.log") or die "log file not exist\n"; + if (-s "cmd_in_second.log" == 0) { + die "empty log"; + } + my $line_cnt; close($file_handle); } wrong_cmd_test(); -sleep(1); +sleep(0.3); + +stop_log(); + +sleep(0.3); #extremely small cases request_log("bop insert", 1, $start); do_bulk_coll_insert("bop", "bkey1", 1); +sleep(0.3); + request_log("bop insert", 9, $start); do_bulk_coll_insert("bop", "bkey2", 9); +sleep(0.3); request_log("bop insert", $bulk_size, $start); do_bulk_coll_insert("bop", "bkey3", $bulk_size); -sleep(1); - #slow_cmd(); - release_memcached($engine, $server);