Skip to content

Commit 10035d6

Browse files
committed
Address review comments
- Change command structure from subcommand to flag-based - Make command idempotent - allow re-running without errors - Add progression check before starting dry-run - Auto-generate report directory with timestamp - Add version checking - fail gracefully if dry-run not supported - Remove misleading language about safety from documentation - Add accurate warnings about split-brain and full resync - Remove automation and pre-migration use case examples - Simplify documentation to focus on what command does - Update all kubectl examples to include --context flag - Move implementation details from docs to code comments - Update dependencies to get DryRun support - Fix all lint issues Signed-off-by: Aman Agrawal <aman_31dec@yahoo.in>
1 parent e5dbb8d commit 10035d6

7 files changed

Lines changed: 316 additions & 245 deletions

File tree

cmd/commands/failover.go

Lines changed: 22 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -10,43 +10,48 @@ import (
1010
"github.com/ramendr/ramenctl/pkg/failover"
1111
)
1212

13+
var (
14+
dryRun bool
15+
abortDryRun bool
16+
)
17+
1318
var FailoverCmd = &cobra.Command{
1419
Use: "failover",
15-
Short: "Manage application failover operations",
16-
}
17-
18-
var FailoverDryRunCmd = &cobra.Command{
19-
Use: "dry-run",
20-
Short: "Test failover without affecting the primary application (DRY-RUN mode)",
20+
Short: "Test failover without affecting the primary application",
2121
Long: `Test failover to the secondary cluster without affecting the primary application.
2222
23-
This is a DRY-RUN operation that:
23+
This performs a dry-run failover that:
2424
- Starts the application on the secondary cluster
2525
- Keeps the primary application running
26-
- Allows you to verify DR readiness without risk
26+
- Allows you to verify DR readiness
2727
2828
The application reaches "TestingFailover" progression when the dry-run succeeds.
2929
3030
Use --abort to revert the dry-run test and return to the original state.`,
3131
Run: func(c *cobra.Command, args []string) {
3232
if abortDryRun {
33-
if err := failover.AbortDryRun(configFile, drpcName, drpcNamespace); err != nil {
33+
if err := failover.AbortDryRun(
34+
configFile, drpcName, drpcNamespace); err != nil {
3435
console.Fatal(err)
3536
}
3637
} else {
37-
if err := failover.TestDryRun(configFile, outputDir, drpcName, drpcNamespace); err != nil {
38+
if err := failover.TestDryRun(
39+
configFile, outputDir, drpcName, drpcNamespace); err != nil {
3840
console.Fatal(err)
3941
}
4042
}
4143
},
4244
}
4345

44-
var abortDryRun bool
45-
4646
func init() {
47-
addDRPCFlags(FailoverDryRunCmd)
48-
FailoverDryRunCmd.Flags().BoolVar(&abortDryRun, "abort", false, "abort the dry-run failover test and revert to original state")
49-
FailoverDryRunCmd.Flags().StringVarP(&outputDir, "output", "o", "", "output directory for test report (optional, only used without --abort)")
50-
51-
FailoverCmd.AddCommand(FailoverDryRunCmd)
47+
addDRPCFlags(FailoverCmd)
48+
FailoverCmd.Flags().BoolVar(&dryRun, "dry-run", false, "perform dry-run failover test")
49+
FailoverCmd.Flags().BoolVar(&abortDryRun, "abort", false,
50+
"abort the dry-run failover test and revert to original state")
51+
FailoverCmd.Flags().StringVarP(&outputDir, "output", "o", "",
52+
"output directory for test report (default: ./failover-dry-run-<name>-<timestamp>)")
53+
54+
if err := FailoverCmd.MarkFlagRequired("dry-run"); err != nil {
55+
panic(err)
56+
}
5257
}

docs/failover.md

Lines changed: 80 additions & 121 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,12 @@ Manage application failover operations
1313
Usage:
1414
ramenctl failover [command]
1515

16-
Available Commands:
17-
dry-run Test failover without affecting the primary application (DRY-RUN mode)
16+
Flags:
17+
--abort abort the dry-run failover test and revert to original state
18+
--dry-run perform dry-run failover test (required)
19+
-n, --name string name of the DRPlacementControl resource
20+
--namespace string namespace of the DRPlacementControl resource
21+
-o, --output string output directory for test report (default: ./failover-dry-run-<name>-<timestamp>)
1822

1923
Flags:
2024
-h, --help help for failover
@@ -29,26 +33,44 @@ Use "ramenctl failover [command] --help" for more information about a command.
2933
> The failover command requires a configuration file. See [init](docs/init.md) to
3034
> learn how to create one.
3135
32-
> [!IMPORTANT]
33-
> This command requires Ramen PR [#2416](https://github.com/RamenDR/ramen/pull/2416)
34-
> to be merged. The `DryRun` field is not yet available in the current Ramen API version.
35-
36-
## failover dry-run
36+
## failover --dry-run
3737

38-
The failover dry-run command tests failover to the secondary cluster without
39-
affecting the primary application. This allows you to verify DR readiness without
40-
risk to production workloads.
38+
The failover dry-run command tests failover to the secondary cluster while
39+
keeping the primary application running. This allows you to verify DR readiness,
40+
but has significant implications for data synchronization.
4141

4242
### What is a dry-run failover?
4343

44-
A dry-run failover is a non-destructive test that:
44+
A dry-run failover is a test that:
4545
- Starts the application on the secondary cluster
46-
- Keeps the primary application running
46+
- Keeps the primary application running (creating a temporary split-brain)
4747
- Validates that failover would work in a real disaster
48-
- Can be safely aborted and reverted
48+
- Can be aborted, but requires full data resynchronization afterward
49+
50+
> [!WARNING]
51+
> **Data Sync Implications**: Aborting a dry-run creates a split-brain scenario
52+
> where both clusters had the application running. After abort, a full data sync
53+
> is required from the primary to secondary cluster to ensure consistency. This
54+
> sync can take significant time depending on data volume.
55+
56+
> [!WARNING]
57+
> **Not Risk-Free**: While the primary application continues running during the
58+
> test, the abort operation requires full resynchronization of all data, which
59+
> can impact performance and take considerable time.
60+
61+
### Preconditions
62+
63+
Before running a dry-run failover, the following must be true:
64+
65+
1. **No active action**: The DRPC must not have an ongoing action (empty `spec.action`)
66+
2. **Progression completed**: The DRPC `status.progression` must be `Completed` (not stuck in cleanup or other operation)
67+
3. **Ramen version**: Must have Ramen with dry-run support (v0.17.0+)
68+
69+
The command validates all preconditions and fails with a clear error if not met.
70+
If already in dry-run mode, the command continues the existing dry-run instead of failing.
4971

50-
This is achieved by setting `dryRun: true` in the DRPC spec along with
51-
`action: Failover`.
72+
> [!NOTE]
73+
> Dry-run is only supported with `action: Failover`. There is no dry-run mode for relocate operations.
5274
5375
### Looking up applications
5476

@@ -63,40 +85,32 @@ argocd appset-deploy-rbd 6m16s dr1
6385

6486
### Starting a dry-run failover test
6587

66-
To test failover for the application `appset-deploy-rbd` in namespace `argocd`,
67-
run the following command:
88+
To test failover for the application `appset-deploy-rbd` in namespace `argocd`:
6889

6990
```console
70-
$ ramenctl failover dry-run --name appset-deploy-rbd --namespace argocd -o dry-run-test
71-
⭐ Using config "config.yaml"
72-
🔎 Starting DRY-RUN failover test
73-
74-
🧪 DRY-RUN MODE: Testing failover to cluster "dr2" without affecting primary
75-
✅ DRY-RUN failover triggered on cluster "dr2"
91+
$ ramenctl failover --dry-run --name appset-deploy-rbd --namespace argocd
92+
validating config
93+
failover dry run
94+
```
7695

77-
🔎 Waiting for DRY-RUN to complete (this may take several minutes)
78-
✅ DRY-RUN: Application "appset-deploy-rbd" is available on cluster "dr2"
79-
✅ DRY-RUN: Primary application remains on original cluster
96+
This starts the application on the secondary cluster while keeping the primary
97+
running, allowing you to verify that failover would work in a real disaster.
98+
The command waits for the test to complete and generates a test report.
8099

81-
✅ DRY-RUN failover test passed
100+
The report is automatically saved to `./failover-dry-run-<name>-<timestamp>/`.
101+
You can specify a custom location with `-o`:
82102

83-
💡 To abort this dry-run: ramenctl failover dry-run --abort --name appset-deploy-rbd --namespace argocd
103+
```console
104+
$ ramenctl failover --dry-run --name appset-deploy-rbd --namespace argocd -o my-report
84105
```
85106

86-
The command will:
87-
1. Validate preconditions (not already in dry-run, no active action)
88-
2. Determine the secondary cluster from the DR peer relationship
89-
3. Update the DRPC to trigger dry-run failover
90-
4. Wait for the application to become available on the secondary cluster
91-
5. Generate a test report if `-o` option is provided
92-
93107
### Checking the test report
94108

95-
If you specified an output directory with `-o`, you can examine the test results:
109+
Examine the test results in the auto-generated output directory:
96110

97111
```console
98-
$ tree dry-run-test
99-
dry-run-test
112+
$ tree failover-dry-run-appset-deploy-rbd-20260415-143022
113+
failover-dry-run-appset-deploy-rbd-20260415-143022
100114
├── failover-dry-run.log
101115
└── failover-dry-run.yaml
102116
```
@@ -108,96 +122,39 @@ The YAML report contains the test execution details and timing information.
108122
To abort the dry-run test and return the application to its original state:
109123

110124
```console
111-
$ ramenctl failover dry-run --abort --name appset-deploy-rbd --namespace argocd
112-
🔎 Aborting DRY-RUN failover test
113-
114-
⚠️ Aborting dry-run failover for "appset-deploy-rbd"
115-
✅ DRY-RUN aborted
116-
117-
🔎 Waiting for application to return to original state
118-
✅ Application "appset-deploy-rbd" restored to original state
119-
120-
✅ DRY-RUN abort completed
125+
$ ramenctl failover --dry-run --abort --name appset-deploy-rbd --namespace argocd
126+
validating config
127+
abort dry run
121128
```
122129

123-
The abort command will:
124-
1. Verify the DRPC is in dry-run mode
125-
2. Read the `last-action` label to determine the original state
126-
3. Restore the DRPC spec to its pre-dry-run configuration
127-
4. Wait for the application to return to the original phase
128-
129-
### How abort restores state
130-
131-
The abort logic uses Ramen's `last-action` label to intelligently restore the
132-
DRPC to its state before the dry-run:
133-
134-
| Original State | last-action label | Restored DRPC Spec |
135-
|----------------|-------------------|-------------------|
136-
| Deployed | `""` (empty) | `action=""`, `failoverCluster=""`, `dryRun=false` |
137-
| FailedOver | `"Failover"` | `action="Failover"`, `failoverCluster=preferredCluster`, `dryRun=false` |
138-
| Relocated | `"Relocate"` | `action="Relocate"`, `failoverCluster=""`, `dryRun=false` |
139-
140-
**Important**: The `last-action` label is NOT updated during dry-run (per Ramen
141-
PR #2416), which allows safe state restoration.
130+
The abort operation stops the test and returns the application to the state it was
131+
in before the dry-run started. After abort completes, a full data resynchronization
132+
occurs from the primary to secondary cluster.
142133

143134
## Use cases
144135

145136
### Testing DR readiness
146137

147-
Before a real disaster, test that failover will work:
138+
Users can use this feature to test failover in advance and verify that they are prepared for a real disaster.
148139

149140
```console
150-
# Test failover
151-
$ ramenctl failover dry-run --name my-app --namespace argocd -o test-$(date +%Y%m%d)
141+
# Test failover (report auto-generated in ./failover-dry-run-my-app-<timestamp>/)
142+
$ ramenctl failover --dry-run --name my-app --namespace argocd
152143

153144
# Review results
154-
$ cat test-20260406/failover-dry-run.yaml
145+
$ ls failover-dry-run-my-app-*/
146+
$ cat failover-dry-run-my-app-*/failover-dry-run.yaml
155147

156148
# Clean up
157-
$ ramenctl failover dry-run --abort --name my-app --namespace argocd
158-
```
159-
160-
### Periodic DR drills
161-
162-
Schedule regular dry-run tests to ensure DR readiness:
163-
164-
```bash
165-
#!/bin/bash
166-
# Monthly DR drill script
167-
168-
APPS=("app1" "app2" "app3")
169-
NAMESPACE="argocd"
170-
REPORT_DIR="dr-drill-$(date +%Y%m)"
171-
172-
for app in "${APPS[@]}"; do
173-
echo "Testing $app..."
174-
ramenctl failover dry-run --name "$app" --namespace "$NAMESPACE" -o "$REPORT_DIR/$app"
175-
sleep 60 # Wait between tests
176-
ramenctl failover dry-run --abort --name "$app" --namespace "$NAMESPACE"
177-
done
178-
```
179-
180-
### Pre-migration validation
181-
182-
Before performing an actual failover or relocation, verify it will work:
183-
184-
```console
185-
# Test first
186-
$ ramenctl failover dry-run --name critical-app --namespace prod
187-
188-
# If test passes, perform actual failover
189-
$ kubectl patch drpc critical-app -n prod --type merge -p '{"spec":{"action":"Failover","failoverCluster":"dr2"}}'
149+
$ ramenctl failover --dry-run --abort --name my-app --namespace argocd
190150
```
191151

192152
## Troubleshooting
193153

194-
### Error: "DRPC is already in dry-run mode"
154+
### Error: "dry-run failover is not supported"
195155

196-
You're trying to start a dry-run when one is already active. Abort first:
197-
198-
```console
199-
$ ramenctl failover dry-run --abort --name my-app --namespace argocd
200-
```
156+
Your Ramen installation does not support dry-run failover. This feature requires
157+
Ramen v0.17.0 or later. Upgrade Ramen to use this command.
201158

202159
### Error: "DRPC has active action"
203160

@@ -210,29 +167,31 @@ The command waits up to 10 minutes for completion. If stuck:
210167

211168
1. Check DRPC status:
212169
```console
213-
$ kubectl get drpc my-app -n argocd -o yaml
170+
$ kubectl get drpc my-app -n argocd --context hub -o yaml
214171
```
215172

216173
2. Check Ramen operator logs:
217174
```console
218-
$ kubectl logs -n ramen-system deployment/ramen-hub-operator
175+
$ kubectl logs -n ramen-system deployment/ramen-hub-operator --context hub
219176
```
220177

221178
3. Cancel the operation with Ctrl+C and abort:
222179
```console
223-
$ ramenctl failover dry-run --abort --name my-app --namespace argocd
180+
$ ramenctl failover --dry-run --abort --name my-app --namespace argocd
224181
```
225182

226-
## Safety features
183+
## Risks and Implications
184+
185+
**Split-Brain During Dry-Run**: Both clusters run the application as PRIMARY and write data
186+
independently. This creates a true split-brain scenario where data diverges between sites.
227187

228-
The dry-run failover command includes several safety features:
188+
**Full Resynchronization Required on Abort**: When aborting the dry-run:
189+
- All data written on the secondary during the test is discarded
190+
- The entire dataset must be resynchronized from primary to secondary
191+
- This operation consumes significant time, network bandwidth, and may impact performance
192+
- There is no shortcut - the full resync is unavoidable
229193

230-
1. **Precondition validation**: Prevents starting if already in dry-run or has active action
231-
2. **Non-destructive**: Primary application continues running during test
232-
3. **State preservation**: Uses `last-action` label for safe abort
233-
4. **Timeout protection**: 10-minute timeout prevents indefinite hangs
234-
5. **Error detection**: Monitors DRPC conditions for failures
235-
6. **Cancellation support**: Handles Ctrl+C gracefully
194+
Use this command only when you understand and accept the resync cost for your data volume.
236195

237196
## Comparison with actual failover
238197

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@ require (
1515
github.com/aymanbagabas/go-udiff v0.4.1
1616
github.com/go-logr/zapr v1.3.0
1717
github.com/nirs/kubectl-gather v0.12.0
18-
github.com/ramendr/ramen/api v0.0.0-20260302102746-0080ff0b2f30
19-
github.com/ramendr/ramen/e2e v0.0.0-20260303090636-b77204c8e780
18+
github.com/ramendr/ramen/api v0.0.0-20260414113435-d7767d533779
19+
github.com/ramendr/ramen/e2e v0.0.0-20260414113435-d7767d533779
2020
github.com/spf13/cobra v1.10.2
2121
github.com/spf13/viper v1.19.0
2222
github.com/yosssi/gohtml v0.0.0-20201013000340-ee4748c638f4

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -237,8 +237,12 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg
237237
github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
238238
github.com/ramendr/ramen/api v0.0.0-20260302102746-0080ff0b2f30 h1:q6ci5ht39oiDf5Kg+u05Q/0klUEDTVT27kKc2/9LeLI=
239239
github.com/ramendr/ramen/api v0.0.0-20260302102746-0080ff0b2f30/go.mod h1:G3q4wmHUdOVefjCKSacg3McDodLwtfFe84wSrGxQ69w=
240+
github.com/ramendr/ramen/api v0.0.0-20260414113435-d7767d533779 h1:9ac+kP2cVSsyktZpeQ6zsilXRGiYixBhyaKa3D5iooM=
241+
github.com/ramendr/ramen/api v0.0.0-20260414113435-d7767d533779/go.mod h1:F6Iuq5ywxI4TJL2VvoJNQvn2cMWOHOdysbaoeT9RMig=
240242
github.com/ramendr/ramen/e2e v0.0.0-20260303090636-b77204c8e780 h1:6TJ2D6N8sfFYWXXTyzkLQg7nBFu0yNYZdGmnRkhcOOU=
241243
github.com/ramendr/ramen/e2e v0.0.0-20260303090636-b77204c8e780/go.mod h1:BhAPwG+WRu/nJ3Qr9BM+pEMNu6N4QYGftCczD8QWsHk=
244+
github.com/ramendr/ramen/e2e v0.0.0-20260414113435-d7767d533779 h1:Us9JXR+WodP196RN418hkXJ8zw0MYWqf8gmDmRFyoEU=
245+
github.com/ramendr/ramen/e2e v0.0.0-20260414113435-d7767d533779/go.mod h1:NwS/S9KVuEstUG9e1KkKgjalgT93zBGGmcopI0T8j/o=
242246
github.com/ramendr/recipe v0.0.0-20250507125257-0295a01da567 h1:QRcHe6GTJAgLK7zgF6ivwZ6B0SFe1tONbA7VMQMWjMM=
243247
github.com/ramendr/recipe v0.0.0-20250507125257-0295a01da567/go.mod h1:dGXrk743fq6VG8u6lflEce7ITM7d/9xSBeAbI2RXl9s=
244248
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=

0 commit comments

Comments
 (0)