fix(cherry-pick): commits to v0.7.x from replication (#172)

* [TA3891] cherry-pick: Add missing metrics (#85)
* cherry-pick: Fixed typo in istgt/src/istgt_lu_ctl.c (#161)
* [TA3890] cherry-pick: push prod build

Signed-off-by: Utkarsh Mani Tripathi <[email protected]>

Author: utkarshmani1997

build_image.sh

@@ -1,6 +1,13 @@
 #!/bin/bash
 set -e
 
+pwd
+make clean
+sh autogen.sh
+./configure --enable-replication
+make clean
+make
+
 BUILD_DATE=$(date +'%Y%m%d%H%M%S')
 REPO_NAME="openebs/cstor-istgt"
 

src/istgt_iscsi.c

@@ -3467,7 +3467,9 @@ istgt_iscsi_op_scsi(CONN_Ptr conn, ISCSI_PDU_Ptr pdu)
 	lu_cmd.sense_alloc_len = 0; //conn->snsbufsize;
 	lu_cmd.sense_data_len = 0;
 	lu_cmd.connGone = 0;
-
+	#ifdef REPLICATION
+	clock_gettime(CLOCK_MONOTONIC_RAW, &lu_cmd.start_rw_time);
+	#endif
 	ISTGT_TRACELOG(ISTGT_TRACE_ISCSI,
 		"LU%d: CSN:%x ITT:%x (%lu/%u)[0x%x %lx+%x] PG=0x%4.4x, LUN=0x%lx "
 	    "ExpStatSN=%x StatSN=%x ExpCmdSN=%x MaxCmdSN=%x "
@@ -5462,7 +5464,7 @@ prof_log(ISTGT_LU_CMD_Ptr p, const char *caller)
 	struct timespec *_s = &(p->times[baseindx]);
 	struct timespec *_n = &(p->times[_inx]);
 	struct timespec _r;
-        ISTGT_LU_DISK *spec;
+	ISTGT_LU_DISK *spec;
 	unsigned long secs, nsecs;
 
 	if ((_n->tv_nsec - _s->tv_nsec) < 0) {
@@ -5480,7 +5482,7 @@ prof_log(ISTGT_LU_CMD_Ptr p, const char *caller)
 				len = (p->lblen * spec->blocklen)/ 1024;
 				for(; (x < len && ind < 9); x *= 2, ind++);
 			}
-			else 
+			else
 				ind = 0;
 			switch(p->cdb0) {
 				case SBC_WRITE_6:
@@ -5529,7 +5531,7 @@ prof_log(ISTGT_LU_CMD_Ptr p, const char *caller)
 							spec->IO_size[ind].cmp_n_write.tdiff[i].tv_sec = p->tdiff[i].tv_sec;
 							spec->IO_size[ind].cmp_n_write.tdiff[i].tv_nsec = p->tdiff[i].tv_nsec;
 						}
-					}	
+					}
 					break;
 				case SBC_UNMAP:
 					if((spec->IO_size[ind].unmp.total_time.tv_sec < _r.tv_sec) || (spec->IO_size[ind].unmp.total_time.tv_sec == _r.tv_sec && spec->IO_size[ind].unmp.total_time.tv_nsec < _r.tv_nsec)) {
@@ -5647,6 +5649,51 @@ prof_log(ISTGT_LU_CMD_Ptr p, const char *caller)
         }
 }
 
+#ifdef REPLICATION
+static void
+update_cummulative_rw_time(ISTGT_LU_TASK_Ptr lu_task)
+{
+	ISTGT_LU_DISK *spec = NULL;
+	struct timespec endtime, diff;
+	uint64_t ns = 0;
+
+	switch(lu_task->lu_cmd.cdb0) {
+		case SBC_WRITE_6:
+		case SBC_WRITE_10:
+		case SBC_WRITE_12:
+		case SBC_WRITE_16:
+		case SBC_WRITE_AND_VERIFY_10:
+		case SBC_WRITE_AND_VERIFY_12:
+		case SBC_WRITE_AND_VERIFY_16:
+				spec = (ISTGT_LU_DISK *)
+					lu_task->lu_cmd.lu->lun[0].spec;
+				clock_gettime(CLOCK_MONOTONIC_RAW, &endtime);
+				timesdiff(CLOCK_MONOTONIC_RAW,
+					lu_task->lu_cmd.start_rw_time,
+					endtime, diff);
+				ns = diff.tv_sec*1000000000;
+				ns += diff.tv_nsec;
+				__sync_fetch_and_add(&spec->totalwritetime, ns);
+				break;
+		case SBC_READ_6:
+		case SBC_READ_10:
+		case SBC_READ_12:
+		case SBC_READ_16:
+				spec = (ISTGT_LU_DISK *)
+					lu_task->lu_cmd.lu->lun[0].spec;
+				clock_gettime(CLOCK_MONOTONIC_RAW, &endtime);
+				timesdiff(CLOCK_MONOTONIC_RAW,
+					lu_task->lu_cmd.start_rw_time,
+					endtime, diff);
+				ns = diff.tv_sec*1000000000;
+				ns += diff.tv_nsec;
+				__sync_fetch_and_add(&spec->totalreadtime, ns);
+				break;
+        }
+	return;
+}
+#endif
+
 static void *
 sender(void *arg)
 {
@@ -5723,6 +5770,10 @@ sender(void *arg)
 			lu_task->lock = 1;
 			timediff(&lu_task->lu_cmd, 'r', __LINE__);
 			if (lu_task->type == ISTGT_LU_TASK_RESPONSE) {
+				#ifdef REPLICATION
+				update_cummulative_rw_time(lu_task);
+				#endif
+
 				/* send DATA-IN, SCSI status */
 				rc = istgt_iscsi_task_response(conn, lu_task);
 				if (rc < 0) {

src/istgt_iscsi.h

@@ -35,6 +35,7 @@
 #include "istgt_lu.h"
 #include "istgt_queue.h"
 #include <sys/uio.h>
+/* Basic header segment length */
 #define ISCSI_BHS_LEN 48
 #define ISCSI_ALIGNMENT 4
 /* support version - RFC3720(10.12.4) */

src/istgt_lu.c

@@ -3724,6 +3724,9 @@ istgt_lu_create_task(CONN_Ptr conn, ISTGT_LU_CMD_Ptr lu_cmd, int lun, ISTGT_LU_D
 
 	}
 
+	#ifdef REPLICATION
+	lu_task->lu_cmd.start_rw_time = lu_cmd->start_rw_time;
+	#endif
 	lu_task->condwait = 0;
 	lu_task->offset = 0;
 	lu_task->req_execute = 0;

src/istgt_lu.h

@@ -429,6 +429,7 @@ typedef struct istgt_lu_cmd_t {
 	uint8_t	   release_aborted;
 #ifdef REPLICATION
 	uint32_t   luworkerindx;
+	struct timespec start_rw_time;
 #endif
 } ISTGT_LU_CMD;
 typedef ISTGT_LU_CMD *ISTGT_LU_CMD_Ptr;
@@ -781,6 +782,10 @@ typedef struct istgt_lu_disk_t {
 	uint64_t reads;
 	uint64_t readbytes;
 	uint64_t writebytes;
+	uint64_t totalreadtime;
+	uint64_t totalwritetime;
+	uint64_t totalreadblockcount;
+	uint64_t totalwriteblockcount;
 #endif
 	/* modify lun */
 	int dofake;

src/istgt_lu_ctl.c

@@ -2980,10 +2980,17 @@ istgt_uctl_cmd_iostats(UCTL_Ptr uctl)
 	ISTGT_LU_Ptr lu;
 	int rc, length;
 	uint64_t usedlogicalblocks;
+	struct timespec now;
+	uint64_t time_diff;
 	/* instantiate json_object from json-c library. */
 	struct json_object *jobj;
-	/* these are utility variables that will be freed at the end of the function. */
-	char *writes, *reads, *totalreadbytes, *totalwritebytes, *size, *usedblocks, *sectorsize;
+	/*
+	 * these are utility variables that will be freed
+	 * at the end of the function.
+	 */
+	char *writes, *reads, *totalreadbytes, *totalwritebytes, *size,
+	*usedblocks, *sectorsize, *uptime, *totalreadtime, *totalwritetime,
+	*totalreadblockcount, *totalwriteblockcount;
 	ISTGT_LU_DISK *spec;
 	MTX_LOCK(&specq_mtx);
 	TAILQ_FOREACH(spec, &spec_q, spec_next) {
@@ -3011,33 +3018,89 @@ istgt_uctl_cmd_iostats(UCTL_Ptr uctl)
 		length = snprintf(NULL, 0, "%"PRIu64, spec->size);
 		size = malloc(length + 1);
 		snprintf(size, length + 1, "%"PRIu64, spec->size);
-		usedlogicalblocks = ( spec->stats.used / spec->blocklen ) ;
+
+		usedlogicalblocks = (spec->stats.used / spec->blocklen);
 
 		length = snprintf(NULL, 0, "%"PRIu64, usedlogicalblocks);
 		usedblocks = malloc(length + 1);
-		snprintf(usedblocks, length + 1, "%"PRIu64, usedlogicalblocks);
+		snprintf(usedblocks, length + 1, "%"PRIu64,
+			usedlogicalblocks);
 
 		length = snprintf(NULL, 0, "%"PRIu64, spec->blocklen);
 		sectorsize = malloc(length + 1);
 		snprintf(sectorsize, length + 1, "%"PRIu64, spec->blocklen);
 
+		length = snprintf(NULL, 0, "%"PRIu64, spec->totalreadtime);
+		totalreadtime = malloc(length + 1);
+		snprintf(totalreadtime, length + 1, "%"PRIu64,
+			spec->totalreadtime);
+
+		length = snprintf(NULL, 0, "%"PRIu64, spec->totalwritetime);
+		totalwritetime = malloc(length + 1);
+		snprintf(totalwritetime, length + 1, "%"PRIu64,
+			spec->totalwritetime);
+
+		length = snprintf(NULL, 0, "%"PRIu64,
+				spec->totalwriteblockcount);
+		totalwriteblockcount = malloc(length + 1);
+		snprintf(totalwriteblockcount, length + 1, "%"PRIu64,
+			spec->totalwriteblockcount);
+
+		length = snprintf(NULL, 0, "%"PRIu64,
+				spec->totalreadblockcount);
+		totalreadblockcount = malloc(length + 1);
+		snprintf(totalreadblockcount, length + 1, "%"PRIu64,
+			spec->totalreadblockcount);
+
+		clock_gettime(CLOCK_MONOTONIC_RAW, &now);
+		time_diff = (uint64_t)(now.tv_sec - istgt_start_time.tv_sec);
+		length = snprintf(NULL, 0, "%"PRIu64, time_diff);
+		uptime = malloc(length + 1);
+		snprintf(uptime, length + 1, "%"PRIu64, time_diff);
+
 		json_object *jreads = json_object_new_string(reads);	/* instantiate child object */
 		json_object *jwrites = json_object_new_string(writes);
-		json_object *jtotalreadbytes = json_object_new_string(totalreadbytes);
-		json_object *jtotalwritebytes = json_object_new_string(totalwritebytes);
+		json_object *jtotalreadbytes =
+				json_object_new_string(totalreadbytes);
+		json_object *jtotalwritebytes =
+				json_object_new_string(totalwritebytes);
 		json_object *jsize = json_object_new_string(size);
-		json_object *jusedlogicalblocks = json_object_new_string(usedblocks);
+		json_object *jusedlogicalblocks =
+				json_object_new_string(usedblocks);
 		json_object *jsectorsize = json_object_new_string(sectorsize);
-
-		json_object_object_add(jobj, "WriteIOPS", jwrites);	/* add values to object field */
+		json_object *juptime = json_object_new_string(uptime);
+		json_object *jtotalreadtime = json_object_new_string(
+						totalreadtime);
+		json_object *jtotalwritetime = json_object_new_string(
+						totalwritetime);
+		json_object *jtotalreadblockcount = json_object_new_string(
+							totalreadblockcount);
+		json_object *jtotalwriteblockcount = json_object_new_string(
+							totalwriteblockcount);
+
+		json_object_object_add(jobj, "WriteIOPS",
+			jwrites);	/* add values to object field */
 		json_object_object_add(jobj, "ReadIOPS", jreads);
-		json_object_object_add(jobj, "TotalWriteBytes", jtotalwritebytes);
-		json_object_object_add(jobj, "TotalReadBytes", jtotalreadbytes);
+		json_object_object_add(jobj, "TotalWriteBytes",
+			jtotalwritebytes);
+		json_object_object_add(jobj, "TotalReadBytes",
+			jtotalreadbytes);
 		json_object_object_add(jobj, "Size", jsize);
-		json_object_object_add(jobj, "UsedLogicalBlocks", jusedlogicalblocks);
+		json_object_object_add(jobj, "UsedLogicalBlocks",
+			jusedlogicalblocks);
 		json_object_object_add(jobj, "SectorSize", jsectorsize);
-
-		istgt_uctl_snprintf(uctl, "%s  %s\n", uctl->cmd, json_object_to_json_string(jobj));
+		json_object_object_add(jobj, "Uptime", juptime);
+		json_object_object_add(jobj, "TotalReadTime",
+			jtotalreadtime);
+		json_object_object_add(jobj, "TotalWriteTime",
+			jtotalwritetime);
+		json_object_object_add(jobj, "TotalReadBlockCount",
+			jtotalreadblockcount);
+		json_object_object_add(jobj, "TotalWriteBlockCount",
+			jtotalwriteblockcount);
+
+		istgt_uctl_snprintf(uctl, "%s  %s\n",
+			uctl->cmd, json_object_to_json_string(jobj));
 		rc = istgt_uctl_writeline(uctl);
 		if (rc != UCTL_CMD_OK){
 			// free the pointers
@@ -3048,6 +3111,11 @@ istgt_uctl_cmd_iostats(UCTL_Ptr uctl)
 			free(size);
 			free(usedblocks);
 			free(sectorsize);
+			free(uptime);
+			free(totalreadtime);
+			free(totalwritetime);
+			free(totalreadblockcount);
+			free(totalwriteblockcount);
 			/* freeing root json_object will free all the allocated memory
 			** associated with the json_object.
 			*/
@@ -3062,6 +3130,11 @@ istgt_uctl_cmd_iostats(UCTL_Ptr uctl)
 		free(size);
 		free(usedblocks);
 		free(sectorsize);
+		free(uptime);
+		free(totalreadtime);
+		free(totalwritetime);
+		free(totalreadblockcount);
+		free(totalwriteblockcount);
 		json_object_put(jobj);
 	}
 	MTX_UNLOCK(&specq_mtx);

src/istgt_lu_disk.c

@@ -5640,6 +5640,7 @@ istgt_lu_disk_lbread(ISTGT_LU_DISK *spec, CONN_Ptr conn __attribute__((__unused_
 	rc = pread(spec->fd, data, nbytes, offset);
 #endif
 	timediffw(lu_cmd, 'D');
+
 	exitblockingcall(endofmacro2)
 	if (markedForFree == 1 || markedForReturn == 1) {
 		ISTGT_TRACELOG(ISTGT_TRACE_NET, "c#%d connGone(%d)OrMarkedReturn(%d):%p:%d pendingIO:%d (read:%zd/%zd lba:%lu+%u)",

src/replication.c

@@ -32,6 +32,7 @@ cstor_conn_ops_t cstor_ops = {
 
 int replication_initialized = 0;
 size_t rcmd_mempool_count = RCMD_MEMPOOL_ENTRIES;
+struct timespec istgt_start_time;
 
 static int start_rebuild(void *buf, replica_t *replica, uint64_t data_len);
 static void handle_mgmt_conn_error(replica_t *r, int sfd, struct epoll_event *events,
@@ -44,6 +45,7 @@ static int handle_mgmt_event_fd(replica_t *replica);
 
 #define build_rcomm_cmd(rcomm_cmd, cmd, offset, nbytes) 						\
 	do {								\
+		uint64_t blockcnt = 0;                                  \
 		rcomm_cmd = malloc(sizeof (*rcomm_cmd));		\
 		memset(rcomm_cmd, 0, sizeof (*rcomm_cmd));		\
 		rcomm_cmd->copies_sent = 0;				\
@@ -72,7 +74,11 @@ static int handle_mgmt_event_fd(replica_t *replica);
 				rcomm_cmd->opcode = ZVOL_OPCODE_WRITE;	\
 				rcomm_cmd->iovcnt = cmd->iobufindx + 1;	\
 				__sync_add_and_fetch(&spec->writes, 1); \
-				__sync_add_and_fetch(&spec->writebytes, nbytes);\
+				__sync_add_and_fetch(&spec->writebytes, \
+							nbytes);        \
+				blockcnt = (nbytes/spec->blocklen);     \
+				__sync_add_and_fetch(&spec->totalwriteblockcount,\
+							blockcnt);      \
 				break;					\
 									\
 			case SBC_READ_6:				\
@@ -82,7 +88,11 @@ static int handle_mgmt_event_fd(replica_t *replica);
 				rcomm_cmd->opcode = ZVOL_OPCODE_READ;	\
 				rcomm_cmd->iovcnt = 0;			\
 				__sync_add_and_fetch(&spec->reads, 1);	\
-				__sync_add_and_fetch(&spec->readbytes, nbytes);\
+				__sync_add_and_fetch(&spec->readbytes,  \
+							nbytes);	\
+				blockcnt = (nbytes/spec->blocklen);     \
+				__sync_add_and_fetch(&spec->totalreadblockcount,\
+							blockcnt);      \
 				break;					\
 									\
 			case SBC_SYNCHRONIZE_CACHE_10:			\
@@ -2448,7 +2458,6 @@ replicate(ISTGT_LU_DISK *spec, ISTGT_LU_CMD_Ptr cmd, uint64_t offset, uint64_t n
 
 	ASSERT(spec->io_seq);
 	build_rcomm_cmd(rcomm_cmd, cmd, offset, nbytes);
-
 retry_read:
 	replica_choosen = false;
 	skip_count = 0;
@@ -2916,6 +2925,7 @@ initialize_replication()
 		REPLICA_ERRLOG("Failed to init specq_mtx err(%d)\n", rc);
 		return -1;
 	}
+	clock_gettime(CLOCK_MONOTONIC_RAW, &istgt_start_time);
 	return 0;
 }
 

src/replication.h

@@ -164,6 +164,8 @@ typedef struct known_replica_s {
 	uint64_t zvol_guid;
 } known_replica_t;
 
+extern struct timespec istgt_start_time;
+
 void *init_replication(void *);
 int make_socket_non_blocking(int);
 int send_mgmtack(int, zvol_op_code_t, void *, char *, int);