Skip to content

Add Capture & Replay dashboard with dynamic stage #1322

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 15 commits into from
Mar 19, 2025
Merged

Add Capture & Replay dashboard with dynamic stage #1322

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
3440220
Add capture and replay dashboard with dynamic stage
jugal-chauhan Mar 5, 2025
c57ae2b
Merge branch 'opensearch-project:main' into feature/capture-replay-da…
jugal-chauhan Mar 5, 2025
496868d
Add capture and replay dashboard with dynamic stage
jugal-chauhan Mar 5, 2025
56e6103
Merge branch 'feature/capture-replay-dashboard' of https://github.com…
jugal-chauhan Mar 6, 2025
8b87e8d
cleaning up work done in incorrect file
jugal-chauhan Mar 6, 2025
ad934eb
cleaning up work done in incorrect file
jugal-chauhan Mar 6, 2025
3a4bb56
Merge remote-tracking branch 'upstream/feature/capture-replay-dashboa…
jugal-chauhan Mar 6, 2025
7b50f93
applying changes in traffic-replayer-stack.ts
jugal-chauhan Mar 6, 2025
9900122
update widgets and metrics
jugal-chauhan Mar 7, 2025
bf53f8e
create new construct for cleaner code
jugal-chauhan Mar 10, 2025
4e12d98
RFS stack utilizes dashboard construct
jugal-chauhan Mar 10, 2025
00c1764
new widget update
jugal-chauhan Mar 14, 2025
1185664
Rename dashboard file and dashboard name
jugal-chauhan Mar 18, 2025
7fad58b
minor test error fix
jugal-chauhan Mar 19, 2025
7f8b731
Make statistic changes
jugal-chauhan Mar 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
369 changes: 369 additions & 0 deletions ...nsearch-service-migration/lib/components/migrationassistant-capture-replay-dashboard.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,369 @@
{
"variables": [
{
"type": "property",
"property": "region",
"inputType": "input",
"id": "REGION",
"label": "Region",
"defaultValue": "us-east-1",
"visible": false
},
{
"type": "property",
"property": "TargetDomainName",
"inputType": "select",
"id": "TC_DOMAIN_NAME",
"label": "Target Cluster Domain",
"search": "{AWS/ES,ClientId,DomainName} MetricName=ClusterStatus.green",
"populateFrom": "DomainName",
"defaultValue": "placeholder-target",
"visible": true
},
{
"type": "pattern",
"pattern": "MA_STAGE",
"inputType": "input",
"id": "MA_STAGE",
"label": "Migration Stage",
"defaultValue": "placeholder-stage",
"visible": false
},
{
"type": "pattern",
"pattern": "ACCOUNT_ID",
"inputType": "input",
"id": "ACCOUNT_ID",
"label": "Account ID",
"defaultValue": "ACCOUNT_ID",
"visible": false
}
],
"widgets": [
{
"height": 1,
"width": 24,
"y": 0,
"x": 0,
"type": "text",
"properties": {
"markdown": "# Traffic Capture",
"background": "transparent"
}
},
{
"height": 9,
"width": 12,
"y": 1,
"x": 0,
"type": "metric",
"properties": {
"metrics": [
[ { "expression": "(m1-m2)/(m2*100)", "label": "Data Loss %", "id": "e1", "region": "REGION" } ],
[ "OpenSearchMigrations", "bytesWrittenToTarget", "OTelLib", "replayer", { "region": "REGION", "id": "m1", "label": "Replayed on T" } ],
[ ".", "bytesRead", ".", "captureProxy", { "label": "Captured from S", "region": "REGION", "id": "m2" } ]
],
"view": "timeSeries",
"stacked": false,
"region": "REGION",
"stat": "Sum",
"period": 60,
"title": "Captured vs Replayed Bytes with Data Loss %",
"yAxis": {
"left": {
"label": "Bytes"
}
}
}
},
{
"height": 9,
"width": 12,
"y": 1,
"x": 12,
"type": "metric",
"properties": {
"metrics": [
[ "OpenSearchMigrations", "activeConnection", "OTelLib", "captureProxy" ],
[ "OpenSearchMigrations", "activeTargetConnections", "OTelLib", "replayer" ],
[ "OpenSearchMigrations", "connectionsOpened", "OTelLib", "replayer" ],
[ "OpenSearchMigrations", "connectionsClosedCount", "OTelLib", "replayer" ],
[ "OpenSearchMigrations", "gatheringRequestCount", "OTelLib", "captureProxy" ],
[ "OpenSearchMigrations", "gatheringResponseCount", "OTelLib", "captureProxy" ]
],
"sparkline": true,
"view": "singleValue",
"region": "REGION",
"stat": "Average",
"period": 60,
"title": "Active Capture Proxy & Replayer Connections",
"liveData": true
}
},
{
"height": 1,
"width": 24,
"y": 10,
"x": 0,
"type": "text",
"properties": {
"markdown": "# Error Analysis",
"background": "transparent"
}
},
{
"height": 1,
"width": 24,
"y": 29,
"x": 0,
"type": "text",
"properties": {
"markdown": "# Kafka Integration",
"background": "transparent"
}
},
{
"height": 10,
"width": 12,
"y": 30,
"x": 0,
"type": "metric",
"properties": {
"metrics": [
[ "OpenSearchMigrations", "kafkaCommitCount", "OTelLib", "replayer" ],
[ ".", "kafkaCommitDuration", ".", ".", { "stat": "Average" } ],
[ "AWS/Kafka", "MessagesInPerSec", "Cluster Name", "migration-msk-cluster-MA_STAGE", "Broker ID", "1" ],
[ "...", "2" ]
],
"view": "timeSeries",
"stacked": false,
"title": "Kafka Message Commit Rate",
"region": "REGION",
"stat": "Sum",
"period": 60,
"setPeriodToTimeRange": true
}
},
{
"height": 8,
"width": 12,
"y": 50,
"x": 12,
"type": "metric",
"properties": {
"metrics": [
[ { "expression": "m1/1000000", "label": "Source/Target Request Lag (sec)", "region": "REGION", "stat": "Sum", "period": 60, "id": "m3" } ],
[ "OpenSearchMigrations", "lagBetweenSourceAndTargetRequests", "OTelLib", "replayer", { "id": "m1", "visible": false, "region": "REGION" } ],
[ { "expression": "m2/1000000", "label": "Schedule Lag (sec)", "region": "REGION", "stat": "Sum", "period": 60, "id": "m4" } ],
[ "OpenSearchMigrations", "scheduleLag", "OTelLib", "replayer", { "id": "m2", "visible": false, "region": "REGION" } ]
],
"view": "timeSeries",
"title": "Replayer Performance Analysis",
"region": "REGION",
"yAxis": {
"left": {
"label": "Lag (seconds)"
}
},
"stat": "Sum",
"period": 60
}
},
{
"height": 1,
"width": 24,
"y": 40,
"x": 0,
"type": "text",
"properties": {
"markdown": "# Replayer Performance"
}
},
{
"height": 9,
"width": 12,
"y": 20,
"x": 0,
"type": "metric",
"properties": {
"metrics": [
[ { "expression": "(m1/m2)*10", "label": "Replay Speed Multiplier", "id": "e1" } ],
[ "OpenSearchMigrations", "kafkaBytesRead", "OTelLib", "replayer", { "id": "m1", "visible": false } ],
[ ".", "bytesWrittenToTarget", ".", ".", { "id": "m2", "visible": false } ]
],
"view": "gauge",
"title": "Current Replay Speed Multiplier",
"region": "REGION",
"stat": "Sum",
"period": 60,
"yAxis": {
"left": {
"min": 1,
"max": 10
}
}
}
},
{
"height": 9,
"width": 12,
"y": 41,
"x": 12,
"type": "metric",
"properties": {
"metrics": [
[ "AWS/ECS", "CPUUtilization", "ServiceName", "migration-MA_STAGE-traffic-replayer-default", "ClusterName", "migration-MA_STAGE-ecs-cluster", { "stat": "Minimum", "label": "CPU Min" } ],
[ ".", ".", ".", ".", ".", ".", { "stat": "Maximum", "label": "CPU Max" } ],
[ ".", ".", ".", ".", ".", ".", { "stat": "Average", "label": "CPU Avg" } ]
],
"view": "timeSeries",
"stacked": false,
"title": "Replayer CPU Utilization",
"region": "REGION",
"stat": "Average",
"period": 60,
"yAxis": {
"left": {
"label": "CPU Utilization %"
}
}
}
},
{
"height": 9,
"width": 12,
"y": 20,
"x": 12,
"type": "metric",
"properties": {
"metrics": [
[ { "expression": "(m1/(m1+m2))*100", "label": "Mismatch Percentage", "id": "e1" } ],
[ "OpenSearchMigrations", "tupleComparison", "OTelLib", "replayer", "statusCodesMatch", "false", { "label": "Mismatched Status Code", "id": "m1" } ],
[ "...", "true", { "label": "Matched Status Codes", "id": "m2" } ]
],
"view": "timeSeries",
"stacked": false,
"title": "Status Code Mismatch Analyzer",
"region": "REGION",
"stat": "Sum",
"period": 60
}
},
{
"height": 9,
"width": 12,
"y": 11,
"x": 12,
"type": "metric",
"properties": {
"metrics": [
[ "OpenSearchMigrations", "tupleComparison", "OTelLib", "replayer", "statusCodesMatch", "false", { "label": "All Mismatches" } ],
[ "...", "method", "GET", "OTelLib", "replayer", { "label": "GET Mismatches" } ],
[ "...", "PUT", ".", ".", { "label": "PUT Mismatches" } ]
],
"view": "timeSeries",
"stacked": true,
"region": "REGION",
"stat": "Sum",
"period": 60,
"title": "HTTP Method-Specific Error Analysis"
}
},
{
"height": 8,
"width": 12,
"y": 50,
"x": 0,
"type": "metric",
"properties": {
"metrics": [
[ "OpenSearchMigrations", "transformedJsonSucceeded", "OTelLib", "replayer", { "label": "JSON Transformations Succeeded" } ],
[ ".", "transformedTextSucceeded", ".", ".", { "label": "Text Transformations Succeeded" } ]
],
"view": "singleValue",
"stacked": false,
"title": "Transformation Success Counts",
"region": "REGION",
"stat": "Sum",
"period": 60,
"sparkline": true,
"singleValueFullPrecision": true
}
},
{
"height": 9,
"width": 12,
"y": 41,
"x": 0,
"type": "metric",
"properties": {
"metrics": [
[ "AWS/ECS", "MemoryUtilization", "ServiceName", "migration-MA_STAGE-traffic-replayer-default", "ClusterName", "migration-MA_STAGE-ecs-cluster", { "stat": "Minimum", "label": "Memory Min" } ],
[ ".", ".", ".", ".", ".", ".", { "stat": "Maximum", "label": "Memory Max" } ],
[ ".", ".", ".", ".", ".", ".", { "stat": "Average", "label": "Memory Avg" } ]
],
"view": "timeSeries",
"stacked": false,
"title": "Replayer Memory Utilization",
"region": "${REGION}",
"period": 60,
"yAxis": {
"left": {
"label": "Memory %"
}
}
}
},
{
"height": 9,
"width": 12,
"y": 11,
"x": 0,
"type": "metric",
"properties": {
"metrics": [
[ "OpenSearchMigrations", "blockingRequest", "OTelLib", "captureProxy", { "label": "Blocked Requests", "color": "#d13212" } ],
[ ".", "removed", ".", ".", { "label": "Removed Connections", "color": "#ff7f0e" } ],
[ ".", "unregistered", ".", ".", { "label": "Unregistered Connections", "color": "#2ca02c" } ],
[ ".", "gatheringRequestCount", ".", ".", { "label": "Total Requests", "yAxis": "right" } ]
],
"view": "timeSeries",
"stacked": false,
"region": "REGION",
"stat": "Sum",
"period": 60,
"title": "CaptureProxy Connection Issues",
"yAxis": {
"left": {
"label": "Error Count"
},
"right": {
"label": "Request Count"
}
}
}
},
{
"height": 10,
"width": 12,
"y": 30,
"x": 12,
"type": "metric",
"properties": {
"metrics": [
[ { "expression": "(m2/m3)", "label": "Commit Success Ratio", "id": "e1" } ],
[ "OpenSearchMigrations", "kafkaCommitCount", "OTelLib", "captureProxy", { "id": "m1" } ],
[ "...", "replayer", { "id": "m2" } ],
[ ".", "kafkaRecordsRead", ".", ".", { "visible": false, "id": "m3" } ],
[ ".", "numPartitionsAssigned", ".", ".", { "label": "Partition Reassignments", "id": "m4" } ]
],
"sparkline": true,
"view": "singleValue",
"region": "REGION",
"period": 60,
"stat": "Sum",
"title": "Kafka Error Detection Monitor"
}
}
]
}
Loading
Loading