enhance: add more params for json stats tools and add jemalloc-stats interface

models/metrics_info.go

@@ -29,6 +29,12 @@ type HardwareMetrics struct {
 	// how to metric disk & disk usage in distributed storage
 	Disk      uint64 `json:"disk"`
 	DiskUsage uint64 `json:"disk_usage"`
+
+	// jemalloc memory statistics
+	JemallocAvailable bool   `json:"jemalloc_available"`
+	JemallocAllocated uint64 `json:"jemalloc_allocated"`
+	JemallocResident  uint64 `json:"jemalloc_resident"`
+	JemallocCached    uint64 `json:"jemalloc_cached"`
 }
 
 const (

states/etcd/show/json_stats.go

@@ -29,6 +29,7 @@ type JSONStatsParam struct {
 	Detail              bool   `name:"detail" default:"false" desc:"print details including files & memory size"`
 	ShowSchema          bool   `name:"show-schema" default:"false" desc:"print schema"`
 	KeyPrefix           string `name:"prefix" default:"" desc:"filter json keys by prefix when parsing layout map"`
+	PrintShared         bool   `name:"print-shared" default:"true" desc:"print shared keys data layout"`
 }
 
 // JSONStatsCommand implements `show json-stats` using etcd meta (same source as show segment).
@@ -232,6 +233,33 @@ func readParquetMeta(ctx context.Context, client *minio.Client, bucket, key stri
 	}
 
 	kvMetaData := pqReader.MetaData().KeyValueMetadata()
+
+	// Print all metadata keys and values (except key_layout_type_map which is handled separately)
+	if kvMetaData != nil && kvMetaData.Len() > 0 {
+		fmt.Printf("      [parquet] metadata entries: %d\n", kvMetaData.Len())
+		for i := 0; i < kvMetaData.Len(); i++ {
+			key := kvMetaData.Keys()[i]
+			val := kvMetaData.Values()[i]
+			// Skip key_layout_type_map as it's processed separately below
+			if key == "key_layout_type_map" {
+				continue
+			}
+			// Special handling: row_group_metadata -> count groups by ';'
+			if key == "row_group_metadata" {
+				parts := strings.Split(val, ";")
+				cnt := 0
+				for _, p := range parts {
+					if strings.TrimSpace(p) != "" {
+						cnt++
+					}
+				}
+				fmt.Printf("        row_group_metadata groups: %d\n", cnt)
+				continue
+			}
+			fmt.Printf("        [%d] %s: %s\n", i, key, val)
+		}
+	}
+
 	v := kvMetaData.FindValue("key_layout_type_map")
 	if v != nil {
 		var mm map[string]string
@@ -240,23 +268,48 @@ func readParquetMeta(ctx context.Context, client *minio.Client, bucket, key stri
 			return
 		}
 
+		var nonSharedKeys []string
 		sharedCount := 0
-		nonCount := 0
-		// calculate and print layout summary at the end
+		// separate shared and non-shared keys
 		for k, val := range mm {
 			if p.KeyPrefix != "" && !strings.HasPrefix(k, p.KeyPrefix) {
 				continue
 			}
-			fmt.Printf("        layout: %s=%s\n", k, val)
-
 			if val == "SHARED" {
 				sharedCount++
 			} else {
-				nonCount++
+				nonSharedKeys = append(nonSharedKeys, k)
+			}
+		}
+
+		// print shared keys if enabled
+		if p.PrintShared {
+			fmt.Printf("        shared keys: ")
+			first := true
+			for k, val := range mm {
+				if p.KeyPrefix != "" && !strings.HasPrefix(k, p.KeyPrefix) {
+					continue
+				}
+				if val == "SHARED" {
+					if !first {
+						fmt.Printf(", ")
+					}
+					fmt.Printf("%s", k)
+					first = false
+				}
+			}
+			if sharedCount > 0 {
+				fmt.Println()
+			} else {
+				fmt.Println("(none)")
 			}
 		}
-		totalCount := sharedCount + nonCount
+		// print non-shared keys in one line
+		if len(nonSharedKeys) > 0 {
+			fmt.Printf("        non-shared keys: %s\n", strings.Join(nonSharedKeys, ", "))
+		}
+		totalCount := sharedCount + len(nonSharedKeys)
 		fmt.Printf("        layout summary: shared=%d items, non-shared=%d items, total=%d items\n",
-			sharedCount, nonCount, totalCount)
+			sharedCount, len(nonSharedKeys), totalCount)
 	}
 }

states/etcd/show/stats_task.go

@@ -3,12 +3,14 @@ package show
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/samber/lo"
 
 	"github.com/milvus-io/birdwatcher/framework"
 	"github.com/milvus-io/birdwatcher/models"
 	"github.com/milvus-io/birdwatcher/states/etcd/common"
+	"github.com/milvus-io/birdwatcher/utils"
 )
 
 type StatsTaskParam struct {
@@ -18,9 +20,23 @@ type StatsTaskParam struct {
 	State               string `name:"state" default:"" desc:"stats state"`
 	CollectionID        int64  `name:"collectionID" default:"0" desc:"collection id to filter with"`
 	SegmentID           int64  `name:"segmentID" default:"0" desc:"segmentID id to filter with"`
+	Since               string `name:"since" default:"" desc:"only show tasks created after this time, format: 2006-01-02 15:04:05 or duration like 1h, 30m"`
+	Failed              bool   `name:"failed" default:"false" desc:"only show tasks with fail reason"`
 }
 
 func (c *ComponentShow) StatsTaskCommand(ctx context.Context, p *StatsTaskParam) error {
+	var sinceTime time.Time
+	if p.Since != "" {
+		// try parse as duration first (e.g., "1h", "30m")
+		if duration, err := time.ParseDuration(p.Since); err == nil {
+			sinceTime = time.Now().Add(-duration)
+		} else if t, err := time.ParseInLocation("2006-01-02 15:04:05", p.Since, time.Local); err == nil {
+			sinceTime = t
+		} else {
+			return fmt.Errorf("invalid since format: %s, use format like '2006-01-02 15:04:05' or duration like '1h', '30m'", p.Since)
+		}
+	}
+
 	statsTasks, err := common.ListStatsTask(ctx, c.client, c.metaPath, func(task *models.StatsTask) bool {
 		if p.SubJobType != "" && p.SubJobType != task.GetProto().GetSubJobType().String() {
 			return false
@@ -37,6 +53,15 @@ func (c *ComponentShow) StatsTaskCommand(ctx context.Context, p *StatsTaskParam)
 		if p.SegmentID != 0 && task.GetProto().GetSegmentID() != p.SegmentID {
 			return false
 		}
+		if !sinceTime.IsZero() {
+			createTime, _ := utils.ParseTS(uint64(task.GetProto().GetTaskID()))
+			if createTime.Before(sinceTime) {
+				return false
+			}
+		}
+		if p.Failed && task.GetProto().GetFailReason() == "" {
+			return false
+		}
 		return true
 	})
 	if err != nil {
@@ -55,7 +80,16 @@ func (c *ComponentShow) StatsTaskCommand(ctx context.Context, p *StatsTaskParam)
 	for colID, tasks := range task2Col {
 		fmt.Printf("CollectionID: %d, Tasks: %d\n", colID, len(tasks))
 		for _, task := range tasks {
-			fmt.Printf("  info: %v\n", task.GetProto())
+			proto := task.GetProto()
+			createTime, _ := utils.ParseTS(uint64(proto.GetTaskID()))
+			fmt.Printf("  TaskID: %d, SegmentID: %d, Type: %s, State: %s, CanRecycle: %v, CreateTime: %s, FailReason: %s\n",
+				proto.GetTaskID(),
+				proto.GetSegmentID(),
+				proto.GetSubJobType().String(),
+				proto.GetState().String(),
+				proto.GetCanRecycle(),
+				createTime.Format("2006-01-02 15:04:05"),
+				proto.GetFailReason())
 		}
 	}
 	return nil