feat/6641435300-save-audit-in-self

docs/en/08-operation/16-security.md

@@ -99,6 +99,8 @@ Additionally, for an audit database:
 
 Audit databases created before version 3.4.0.0 are incompatible with audit features in version 3.4.0.0 and later. In older versions, the `is_audit` parameter was not enforced, so there were no mandatory requirements for `DURATION`, `WAL_LEVEL`, and `ENCRYPT_ALGORITHM`. To enable new audit features for an old audit database, it must be dropped and recreated. As a workaround to access data from a pre-3.4.0.0 audit database in a newer version (without new audit features), you can disable `auditUseToken` by setting it to 0.
 
+In versions 3.4.1.0 and later, audit information can be saved locally rather than sent to taoskeeper. To use this functionality, you need to set the parameter auditSaveInSelf to 1, and the number of vgroups in the audit database created must be limited to one.
+
 ### taosKeeper Configuration
 
 Configure the related parameters for audit logs in the taosKeeper configuration file `keeper.toml`, as shown in the table below

docs/en/14-reference/01-components/01-taosd.md

@@ -243,6 +243,7 @@ The effective value of charset is UTF-8.
 | auditHttps                 | After 3.4.0.0     | Supported, effective immediately   | Whether to use https to report audit data; Enterprise parameter; range 0 - 1, default value 0 (1: use https, 0: do not use).                                                                                                         |
 | auditUseToken              | After 3.4.0.0     | Supported, effective immediately   | Whether to use token to report audit data; Enterprise parameter; range 0 - 1, default value 1 (1: use token, 0: do not use).                                                                                                         |
 | auditCreateTable           |                   | Supported, effective immediately   | Whether to enable audit feature for creating subtables; Enterprise parameter                                                                                                                                                                                                   |
+| auditSaveInSelf            | After 3.4.1.0     | Supported, effective immediately   | Whether to save audit information locally instead of sending it to taoskeeper. Range: 0-1, default: 0 (1: enabled, 0: disabled).                                                                                                                                                                                                    |
 | encryptAlgorithm           |                   | Not supported                      | Data encryption algorithm; Enterprise parameter                                                                                                                                                                                                                                |
 | encryptScope               |                   | Not supported                      | Encryption scope; Enterprise parameter                                                                                                                                                                                                                                         |
 | encryptExtDir              | v3.4.0.0           | Not supported                      | User-defined encryption algorithms extensions path; Enterprise parameter                                                                                                                                                                                                                                         |

docs/en/14-reference/09-error-code.md

@@ -683,11 +683,12 @@ Below are the business error codes for each module.
 
 | Error Code | Description                                             | Possible Error Scenarios or Reasons                                                                                                                                  | Recommended Actions for Users                                                 |
 |------------|---------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------|
-| 0x80006103 | Audit database must be encrypted                       | Invalid param,eter                                                                                                 | Check and correct the SQL statement                           |
-| 0x80006104 | Audit database wal_level must be 2                       | Invalid param,eter                                                                                                 | Check and correct the SQL statement                           |
-| 0x80006105 | Audit database keep2 must be greater than 1825d                       | Invalid param,eter                                                                                                 | Check and correct the SQL statement                           |
-| 0x80006106 | Audit database already exist                       | Invalid param,eter                                                                                                 | Check and correct the SQL statement                           |
-| 0x80006107 | Audit database is not allowed to change                       | Invalid param,eter                                                                                                 | Check and correct the SQL statement                           |
+| 0x80006103 | Audit database must be encrypted                       | Invalid parameter                                                                                                 | Check and correct the SQL statement                           |
+| 0x80006104 | Audit database wal_level must be 2                       | Invalid parameter                                                                                                 | Check and correct the SQL statement                           |
+| 0x80006105 | Audit database keep2 must be greater than 1825d                       | Invalid parameter                                                                                                 | Check and correct the SQL statement                           |
+| 0x80006106 | Audit database already exist                       | Invalid parameter                                                                                                 | Check and correct the SQL statement                           |
+| 0x80006107 | Audit database is not allowed to change                       | Invalid parameter                                                                                                 | Check and correct the SQL statement                           |
+| 0x80006108 | Audit database is not allowed to keep multiple vgroups        | Invalid parameter                                                                                                 | Check and correct the SQL statement                           |
 
 #### virtual table
 

docs/zh/08-operation/16-security.md

@@ -92,6 +92,8 @@ database_option: {
 
 在 3.4.0.0 之前版本创建的审计库，与 3.4.0.0 及之后版本的审计库不兼容。3.4.0.0 之前版本的审计库无法开启 is_audit 参数，因此不会对 DURATION、WAL_LEVEL、ENCRYPT_ALGORITHM 做强制要求。对于 3.4.0.0 之前创建的审计库，如需使用新版本的审计能力，建议先 drop 该审计库后再重新创建。如果要在 3.4.0.0 之后的版本中继续使用由 3.4.0.0 之前版本创建的审计库，则需要将 auditUseToken 关闭（设置为 0）。
 
+在 3.4.1.0 之后的版本可以将审计信息保存在自身，而不发送给 taoskeeper，若要使用该功能，需要将参数 auditSaveInSelf 设置为 1，并且在使用该功能时，创建的审计库的 vgroups 的数量只能为 1。
+
 ### taosKeeper 配置
 
 在 taosKeeper 的配置文件 keeper.toml 中配置与审计日志有关的配置参数，如下表所示

docs/zh/14-reference/01-components/01-taosd.md

@@ -1264,6 +1264,17 @@ charset 的有效值是 UTF-8。
 - 动态修改：支持通过 SQL 修改，立即生效。
 - 支持版本：从 v3.1.0.0 版本开始引入
 
+#### auditSaveInSelf
+
+- 说明：审计数据保存在自身，而不发送给 taoskeeper
+- 类型：整数；0:关闭，1：开启。
+- 默认值：0
+- 最小值：0
+- 最大值：1
+- 参数类型：全局配置参数
+- 动态修改：支持通过 SQL 修改，立即生效。
+- 支持版本：从 v3.4.1.0 版本开始引入
+
 #### encryptAlgorithm
 
 - 说明：数据加密算法 **`企业版参数`**

docs/zh/14-reference/09-error-code.md

@@ -670,6 +670,7 @@ TSDB 错误码包括 taosc 客户端和服务端，所有语言的连接器无
 | 0x80006105 | Audit database keep2 must be greater than 1825d                           | 参数不正确                              | 检查并修正 SQL 语句               |
 | 0x80006106 | Audit database already exist                                              | 参数不正确                              | 检查并修正 SQL 语句               |
 | 0x80006107 | Audit database is not allowed to change                                   | 参数不正确                              | 检查并修正 SQL 语句               |
+| 0x80006108 | Audit database is not allowed to keep multiple vgroups                    | 参数不正确                              | 检查并修正 SQL 语句               |
 
 #### virtual table
 

include/common/tglobal.h

@@ -245,6 +245,7 @@ extern int32_t tsAuditLevel;
 extern int32_t tsAuditInterval;
 extern bool    tsAuditHttps;
 extern bool    tsAuditUseToken;
+extern bool    tsAuditSaveInSelf;
 
 // telem
 extern bool     tsEnableTelem;

include/common/tmsg.h

@@ -2813,6 +2813,8 @@ typedef struct {
   int64_t     timestamp;
   char        auditDB[TSDB_DB_FNAME_LEN];
   char        auditToken[TSDB_TOKEN_LEN];
+  SEpSet      auditEpSet;
+  int32_t     auditVgId;
 } SStatusReq;
 
 int32_t tSerializeSStatusReq(void* buf, int32_t bufLen, SStatusReq* pReq);
@@ -2912,6 +2914,8 @@ typedef struct {
   int64_t   timeWhiteVer;
   char      auditDB[TSDB_DB_FNAME_LEN];
   char      auditToken[TSDB_TOKEN_LEN];
+  SEpSet    auditEpSet;
+  int32_t   auditVgId;
 } SStatusRsp;
 
 int32_t tSerializeSStatusRsp(void* buf, int32_t bufLen, SStatusRsp* pRsp);
@@ -4069,6 +4073,13 @@ int32_t tSerializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetA
 int32_t tDeserializeSVArbSetAssignedLeaderReq(void* buf, int32_t bufLen, SVArbSetAssignedLeaderReq* pReq);
 void    tFreeSVArbSetAssignedLeaderReq(SVArbSetAssignedLeaderReq* pReq);
 
+typedef struct {
+  char* data;
+} SVAuditRecordReq;
+int32_t tSerializeSVAuditRecordReq(void* buf, int32_t bufLen, SVAuditRecordReq* pReq);
+int32_t tDeserializeSVAuditRecordReq(void* buf, int32_t bufLen, SVAuditRecordReq* pReq);
+void    tFreeSVAuditRecordReq(SVAuditRecordReq* pReq);
+
 typedef struct {
   char*   arbToken;
   char*   memberToken;

include/common/tmsgcb.h

@@ -43,6 +43,7 @@ typedef enum {
   STREAM_RUNNER_QUEUE,
   STREAM_READER_QUEUE,
   STREAM_TRIGGER_QUEUE,
+  AUDIT_QUEUE,
   QUEUE_MAX,
 } EQueueType;
 

include/common/tmsgdef.h

@@ -366,6 +366,7 @@
   TD_DEF_MSG_TYPE(TDMT_VND_QUERY_TRIM_PROGRESS, "vnode-query-trim-progress", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_SET_KEEP_VERSION, "vnode-set-keep-version", NULL, NULL)
   TD_DEF_MSG_TYPE(TDMT_VND_TRIM_WAL, "vnode-trim-wal", NULL, NULL)
+  TD_DEF_MSG_TYPE(TDMT_VND_AUDIT_RECORD, "vnode-audit-record", NULL, NULL)
   TD_CLOSE_MSG_SEG(TDMT_VND_MSG)
 
   TD_NEW_MSG_SEG(TDMT_SCH_MSG)  // 3<<8

include/libs/audit/audit.h

@@ -30,6 +30,8 @@ extern "C" {
 
 #define AUDIT_DETAIL_MAX 65472
 
+typedef SEpSet (*mndGetDnodeEpsetByIdFn)(void *pMnode, int32_t dnodeId);
+
 typedef struct {
   const char *server;
   uint16_t    port;
@@ -39,12 +41,12 @@ typedef struct {
 typedef struct {
   int64_t curTime;
   char    strClusterId[TSDB_CLUSTER_ID_LEN];
-  char    clientAddress[50];
+  char    clientAddress[AUDIT_CLIENT_ADD_LEN];
   char    user[TSDB_USER_LEN];
   char    operation[AUDIT_OPERATION_LEN];
   char    target1[TSDB_DB_NAME_LEN]; //put db name
   char    target2[TSDB_STREAM_NAME_LEN]; //put stb name, table name, topic name, user name, stream name, use max
-  char*   detail;
+  char   *detail;
   double  duration;
   int64_t affectedRows;
 } SAuditRecord;
@@ -53,11 +55,11 @@ int32_t auditInit(const SAuditCfg *pCfg);
 void    auditSetDnodeId(int32_t dnodeId);
 void    auditCleanup();
 
-void    auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, char *detail,
+void auditRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, char *detail,
+                 int32_t len, double duration, int64_t affectedRows);
+void auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, char *detail,
                     int32_t len, double duration, int64_t affectedRows);
-void    auditAddRecord(SRpcMsg *pReq, int64_t clusterId, char *operation, char *target1, char *target2, char *detail,
-                       int32_t len, double duration, int64_t affectedRows);
-void    auditSendRecordsInBatch();
+void auditSendRecordsInBatch();
 
 #ifdef __cplusplus
 }

include/util/taoserror.h

@@ -1255,6 +1255,7 @@ int32_t  taosGetErrSize();
 #define TSDB_CODE_AUDIT_MUST_KEEPFORCE           TAOS_DEF_ERROR_CODE(0, 0x6105)
 #define TSDB_CODE_AUDIT_DB_ALREADY_EXIST         TAOS_DEF_ERROR_CODE(0, 0x6106)
 #define TSDB_CODE_AUDIT_DB_NOT_ALLOW_CHANGE      TAOS_DEF_ERROR_CODE(0, 0x6107)
+#define TSDB_CODE_AUDIT_DB_NOT_MULTI_VGROUP      TAOS_DEF_ERROR_CODE(0, 0x6108)
 
 // VTABLE
 #define TSDB_CODE_VTABLE_SCAN_INTERNAL_ERROR     TAOS_DEF_ERROR_CODE(0, 0x6200)

include/util/tdef.h

@@ -815,6 +815,7 @@ enum { RAND_ERR_MEMORY = 1, RAND_ERR_FILE = 2, RAND_ERR_NETWORK = 4 };
 #define MONITOR_METRIC_NAME_LEN 100
 
 #define AUDIT_OPERATION_LEN 20
+#define AUDIT_CLIENT_ADD_LEN 256
 #define CONNECTOR_INFO_LEN  256
 
 typedef enum {
@@ -836,6 +837,8 @@ typedef enum {
   ANALY_ALGO_TYPE_END = 10,
 } EAnalyAlgoType;
 
+#define AUDIT_STABLE_NAME "operations"
+
 typedef enum {
   TSDB_VERSION_UNKNOWN = 0,
   TSDB_VERSION_OSS,

include/util/tjson.h

@@ -72,6 +72,12 @@ void    tjsonGetObjectValueBigInt(const SJson* pJson, int64_t* pVal);
 void    tjsonGetObjectValueDouble(const SJson* pJson, double* pVal);
 int32_t tjsonGetStringValue(const SJson* pJson, const char* pName, char* pVal);
 int32_t tjsonGetStringValue2(const SJson* pJson, const char* pName, char* pVal, int32_t maxLen);
+/**
+ * Returns a pointer to an internal string buffer owned by the underlying JSON object.
+ * The returned pointer must not be modified or freed by the caller and remains valid
+ * only as long as the associated SJson object (and its parent tree) is alive and unchanged.
+ */
+const char* tjsonGetStringPointer(const SJson* pJson, const char* pName);
 int32_t tjsonDupStringValue(const SJson* pJson, const char* pName, char** pVal);
 int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal);
 int32_t tjsonGetIntValue(const SJson* pJson, const char* pName, int32_t* pVal);

source/common/src/msg/tmsg.c

@@ -2105,6 +2105,14 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
     TAOS_CHECK_EXIT(tEncodeI64(&encoder, pload->syncTotalIndex));
   }
 
+  TAOS_CHECK_EXIT(tEncodeI8(&encoder, pReq->auditEpSet.numOfEps));
+  for (int32_t i = 0; i < pReq->auditEpSet.numOfEps; ++i) {
+    TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->auditEpSet.eps[i].fqdn));
+    TAOS_CHECK_EXIT(tEncodeI16(&encoder, pReq->auditEpSet.eps[i].port));
+    // do not need to encode InUse here, because inUse is not accurate at every time
+  }
+  TAOS_CHECK_EXIT(tEncodeI32(&encoder, pReq->auditVgId));
+
   tEndEncode(&encoder);
 
 _exit:
@@ -2281,6 +2289,20 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) {
     }
   }
 
+  if (!tDecodeIsEnd(&decoder)) {
+    TAOS_CHECK_EXIT(tDecodeI8(&decoder, &(pReq->auditEpSet.numOfEps)));
+    int32_t numEps = pReq->auditEpSet.numOfEps;
+    if (numEps < 0 || numEps > TSDB_MAX_REPLICA) {
+      code = TSDB_CODE_INVALID_MSG;
+      goto _exit;
+    }
+    for (int32_t i = 0; i < pReq->auditEpSet.numOfEps; ++i) {
+      TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pReq->auditEpSet.eps[i].fqdn));
+      TAOS_CHECK_EXIT(tDecodeI16(&decoder, &(pReq->auditEpSet.eps[i].port)));
+    }
+    TAOS_CHECK_EXIT(tDecodeI32(&decoder, &(pReq->auditVgId)));
+  }
+
   tEndDecode(&decoder);
 
 _exit:
@@ -2546,6 +2568,14 @@ int32_t tSerializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
   TAOS_CHECK_EXIT(tEncodeI64(&encoder, pRsp->timeWhiteVer));
   TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pRsp->auditDB));
   TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pRsp->auditToken));
+  TAOS_CHECK_EXIT(tEncodeI8(&encoder, pRsp->auditEpSet.numOfEps));
+  for (int32_t i = 0; i < pRsp->auditEpSet.numOfEps; ++i) {
+    TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pRsp->auditEpSet.eps[i].fqdn));
+    TAOS_CHECK_EXIT(tEncodeI16(&encoder, pRsp->auditEpSet.eps[i].port));
+    // do not need to encode InUse here, because inUse is not accurate at every time
+  }
+  TAOS_CHECK_EXIT(tEncodeI32(&encoder, pRsp->auditVgId));
+
   tEndEncode(&encoder);
 
 _exit:
@@ -2610,6 +2640,20 @@ int32_t tDeserializeSStatusRsp(void *buf, int32_t bufLen, SStatusRsp *pRsp) {
     TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pRsp->auditToken));
   }
 
+  if (!tDecodeIsEnd(&decoder)) {
+    TAOS_CHECK_EXIT(tDecodeI8(&decoder, &(pRsp->auditEpSet.numOfEps)));
+    int32_t numEps = pRsp->auditEpSet.numOfEps;
+    if (numEps < 0 || numEps > TSDB_MAX_REPLICA) {
+      code = TSDB_CODE_INVALID_MSG;
+      goto _exit;
+    }
+    for (int32_t i = 0; i < pRsp->auditEpSet.numOfEps; ++i) {
+      TAOS_CHECK_EXIT(tDecodeCStrTo(&decoder, pRsp->auditEpSet.eps[i].fqdn));
+      TAOS_CHECK_EXIT(tDecodeI16(&decoder, &(pRsp->auditEpSet.eps[i].port)));
+    }
+    TAOS_CHECK_EXIT(tDecodeI32(&decoder, &(pRsp->auditVgId)));
+  }
+
   tEndDecode(&decoder);
 _exit:
   tDecoderClear(&decoder);
@@ -12239,6 +12283,55 @@ void tFreeSVArbCheckSyncRsp(SVArbCheckSyncRsp *pRsp) {
   taosMemoryFreeClear(pRsp->member1Token);
 }
 
+int32_t tSerializeSVAuditRecordReq(void *buf, int32_t bufLen, SVAuditRecordReq *pReq) {
+  SEncoder encoder = {0};
+  int32_t  code = 0;
+  int32_t  lino = 0;
+  int32_t  tlen = 0;
+
+  tEncoderInit(&encoder, buf, bufLen);
+
+  TAOS_CHECK_EXIT(tStartEncode(&encoder));
+
+  TAOS_CHECK_EXIT(tEncodeCStr(&encoder, pReq->data));
+
+  tEndEncode(&encoder);
+
+_exit:
+  if (code) {
+    tlen = code;
+  } else {
+    tlen = encoder.pos;
+  }
+  tEncoderClear(&encoder);
+  return tlen;
+}
+
+int32_t tDeserializeSVAuditRecordReq(void *buf, int32_t bufLen, SVAuditRecordReq *pReq) {
+  SDecoder decoder = {0};
+  int32_t  code = 0;
+  int32_t  lino = 0;
+
+  tDecoderInit(&decoder, buf, bufLen);
+
+  TAOS_CHECK_EXIT(tStartDecode(&decoder));
+
+  TAOS_CHECK_EXIT(tDecodeCStrAlloc(&decoder, &(pReq->data)));
+
+  tEndDecode(&decoder);
+
+_exit:
+  tDecoderClear(&decoder);
+  return code;
+}
+
+void tFreeSVAuditRecordReq(SVAuditRecordReq *pReq) {
+  if (NULL == pReq) {
+    return;
+  }
+  taosMemoryFreeClear(pReq->data);
+}
+
 int32_t tSerializeSVArbSetAssignedLeaderReq(void *buf, int32_t bufLen, SVArbSetAssignedLeaderReq *pReq) {
   SEncoder encoder = {0};
   int32_t  code = 0;