Skip to content

fix: serialize state in error cases #40

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: na/catalyst-types
Choose a base branch
from
Open

fix: serialize state in error cases #40

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
48e0da8
serialize state in error cases
nadim-az Apr 6, 2025
34f6326
basic lifecycle tests
nadim-az Apr 6, 2025
28f7841
wip
nadim-az Apr 7, 2025
2fb968f
remove tests
nadim-az Apr 16, 2025
65723ab
add back nodes to PackagedState
nadim-az Apr 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 139 additions & 10 deletions activities/loadtest/loadtest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,8 @@ func generateLoadTestSpec(ctx context.Context, logger *zap.Logger, chain *chain.
node := validators[len(validators)-1]
err := node.RecoverKey(ctx, "faucet", faucetWallet.Mnemonic())
if err != nil {
logger.Fatal("failed to recover faucet wallet key", zap.Error(err))
logger.Error("failed to recover faucet wallet key", zap.Error(err))
return nil, fmt.Errorf("failed to recover faucet wallet key: %w", err)
}
time.Sleep(1 * time.Second)

Expand Down Expand Up @@ -167,12 +168,34 @@ func (a *Activity) RunLoadTest(ctx context.Context, chainState []byte,

chain, err := chain.RestoreChain(ctx, logger, p, chainState, node.RestoreNode, testnet.CosmosWalletConfig)
if err != nil {
return PackagedState{}, err
newProviderState, serializeErr := p.SerializeProvider(ctx)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we extract this out into a function? where we serialize as much as possible and just return it

if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after chain restore error: %v, original error: %w", serializeErr, err)
}

return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to restore chain: %w", err)
}

configBytes, err := generateLoadTestSpec(ctx, logger, chain, chain.GetConfig().ChainId, loadTestSpec)
if err != nil {
return PackagedState{}, err
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after config generation error: %v, original error: %w", serializeErr, err)
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after config generation error: %v, original error: %w", chainErr, err)
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
}, fmt.Errorf("failed to generate load test config: %w", err)
}

task, err := p.CreateTask(ctx, provider.TaskDefinition{
Expand All @@ -196,16 +219,61 @@ func (a *Activity) RunLoadTest(ctx context.Context, chainState []byte,
})

if err != nil {
return PackagedState{}, err
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after task creation error: %v, original error: %w", serializeErr, err)
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after task creation error: %v, original error: %w", chainErr, err)
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
}, fmt.Errorf("failed to create task: %w", err)
}

if err := task.WriteFile(ctx, "loadtest.yml", configBytes); err != nil {
return PackagedState{}, fmt.Errorf("failed to write config file to task: %w", err)
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after write file error: %v, original error: %w", serializeErr, err)
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after write file error: %v, original error: %w", chainErr, err)
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
}, fmt.Errorf("failed to write config file to task: %w", err)
}

logger.Info("starting load test")
if err := task.Start(ctx); err != nil {
return PackagedState{}, err
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after task start error: %v, original error: %w", serializeErr, err)
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after task start error: %v, original error: %w", chainErr, err)
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
}, fmt.Errorf("failed to start task: %w", err)
}

ticker := time.NewTicker(10 * time.Second)
Expand All @@ -214,7 +282,22 @@ func (a *Activity) RunLoadTest(ctx context.Context, chainState []byte,
for {
select {
case <-ctx.Done():
return PackagedState{}, ctx.Err()
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after context done: %v, original error: %w", serializeErr, ctx.Err())
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after context done: %v, original error: %w", chainErr, ctx.Err())
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
}, ctx.Err()
case <-ticker.C:
status, err := task.GetStatus(ctx)
if err != nil {
Expand All @@ -227,17 +310,63 @@ func (a *Activity) RunLoadTest(ctx context.Context, chainState []byte,

resultBytes, err := task.ReadFile(ctx, "load_test.json")
if err != nil {
return PackagedState{}, fmt.Errorf("failed to read result file: %w", err)
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after read file error: %v, original error: %w", serializeErr, err)
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after read file error: %v, original error: %w", chainErr, err)
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
}, fmt.Errorf("failed to read result file: %w", err)
}

var result types.LoadTestResult
if err := json.Unmarshal(resultBytes, &result); err != nil {
return PackagedState{}, fmt.Errorf("failed to parse result file: %w", err)
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after unmarshal error: %v, original error: %w", serializeErr, err)
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after unmarshal error: %v, original error: %w", chainErr, err)
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
}, fmt.Errorf("failed to parse result file: %w", err)
}
logger.Info("load test result", zap.Any("result", result))

if err := task.Destroy(ctx); err != nil {
return PackagedState{}, fmt.Errorf("failed to destroy task: %w", err)
newProviderState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after task destroy error: %v, original error: %w", serializeErr, err)
}

newChainState, chainErr := chain.Serialize(ctx, p)
if chainErr != nil {
return PackagedState{
ProviderState: newProviderState,
}, fmt.Errorf("failed to serialize chain after task destroy error: %v, original error: %w", chainErr, err)
}

return PackagedState{
ProviderState: newProviderState,
ChainState: newChainState,
Result: result,
}, fmt.Errorf("failed to destroy task: %w", err)
}

newProviderState, err := p.SerializeProvider(ctx)
Expand Down
126 changes: 116 additions & 10 deletions activities/observability/observability.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,38 @@ func (a *Activity) LaunchObservabilityStack(ctx context.Context, opts Options) (
prometheusTask, err := monitoring.SetupPrometheusTask(ctx, logger, p, prometheusOptions)

if err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after prometheus setup error: %v, original error: %w", serializeErr, err)
}

return PackagedState{
ProviderState: providerState,
}, fmt.Errorf("failed to setup prometheus task: %w", err)
}

if err := prometheusTask.Start(ctx); err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after prometheus start error: %v, original error: %w", serializeErr, err)
}

return PackagedState{
ProviderState: providerState,
}, fmt.Errorf("failed to start prometheus task: %w", err)
}

prometheusIp, err := prometheusTask.GetIP(ctx)

if err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after prometheus IP error: %v, original error: %w", serializeErr, err)
}

return PackagedState{
ProviderState: providerState,
}, fmt.Errorf("failed to get prometheus IP: %w", err)
}

grafanaOptions := monitoring.GrafanaOptions{
Expand All @@ -102,41 +123,126 @@ func (a *Activity) LaunchObservabilityStack(ctx context.Context, opts Options) (
grafanaTask, err := monitoring.SetupGrafanaTask(ctx, logger, p, grafanaOptions)

if err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after grafana setup error: %v, original error: %w", serializeErr, err)
}

prometheusState, promErr := p.SerializeTask(ctx, prometheusTask)
if promErr != nil {
return PackagedState{
ProviderState: providerState,
}, fmt.Errorf("failed to serialize prometheus task after grafana setup error: %v, original error: %w", promErr, err)
}

return PackagedState{
ProviderState: providerState,
PrometheusState: prometheusState,
}, fmt.Errorf("failed to setup grafana task: %w", err)
}

if err := grafanaTask.Start(ctx); err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after grafana start error: %v, original error: %w", serializeErr, err)
}

prometheusState, promErr := p.SerializeTask(ctx, prometheusTask)
if promErr != nil {
return PackagedState{
ProviderState: providerState,
}, fmt.Errorf("failed to serialize prometheus task after grafana start error: %v, original error: %w", promErr, err)
}

return PackagedState{
ProviderState: providerState,
PrometheusState: prometheusState,
}, fmt.Errorf("failed to start grafana task: %w", err)
}

grafanaIp, err := grafanaTask.GetIP(ctx)

if err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after grafana IP error: %v, original error: %w", serializeErr, err)
}

prometheusState, promErr := p.SerializeTask(ctx, prometheusTask)
if promErr != nil {
return PackagedState{
ProviderState: providerState,
}, fmt.Errorf("failed to serialize prometheus task after grafana IP error: %v, original error: %w", promErr, err)
}

return PackagedState{
ProviderState: providerState,
PrometheusState: prometheusState,
}, fmt.Errorf("failed to get grafana IP: %w", err)
}

externalGrafanaIp, err := grafanaTask.GetExternalAddress(ctx, "3000")

if err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after external grafana IP error: %v, original error: %w", serializeErr, err)
}

prometheusState, promErr := p.SerializeTask(ctx, prometheusTask)
if promErr != nil {
return PackagedState{
ProviderState: providerState,
}, fmt.Errorf("failed to serialize prometheus task after external grafana IP error: %v, original error: %w", promErr, err)
}

return PackagedState{
ProviderState: providerState,
PrometheusState: prometheusState,
GrafanaURL: fmt.Sprintf("http://%s:3000", grafanaIp),
}, fmt.Errorf("failed to get external grafana IP: %w", err)
}

prometheusState, err := p.SerializeTask(ctx, prometheusTask)

if err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after prometheus state error: %v, original error: %w", serializeErr, err)
}

return PackagedState{
ProviderState: providerState,
GrafanaURL: fmt.Sprintf("http://%s:3000", grafanaIp),
ExternalGrafanaURL: fmt.Sprintf("http://%s", externalGrafanaIp),
}, fmt.Errorf("failed to serialize prometheus task: %w", err)
}

grafanaState, err := p.SerializeTask(ctx, grafanaTask)

if err != nil {
return PackagedState{}, err
providerState, serializeErr := p.SerializeProvider(ctx)
if serializeErr != nil {
return PackagedState{}, fmt.Errorf("failed to serialize provider after grafana state error: %v, original error: %w", serializeErr, err)
}

return PackagedState{
ProviderState: providerState,
PrometheusState: prometheusState,
GrafanaURL: fmt.Sprintf("http://%s:3000", grafanaIp),
ExternalGrafanaURL: fmt.Sprintf("http://%s", externalGrafanaIp),
}, fmt.Errorf("failed to serialize grafana task: %w", err)
}

providerState, err := p.SerializeProvider(ctx)

if err != nil {
return PackagedState{}, err
return PackagedState{
PrometheusState: prometheusState,
GrafanaState: grafanaState,
GrafanaURL: fmt.Sprintf("http://%s:3000", grafanaIp),
ExternalGrafanaURL: fmt.Sprintf("http://%s", externalGrafanaIp),
}, fmt.Errorf("failed to serialize provider: %w", err)
}

return PackagedState{
Expand Down
Loading