Skip to content

feat: add otel span attributes #729

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat: add otel span attributes #729

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
86ab69f
feat: add span attributes for pod-delete
suhyenim Dec 20, 2024
ec90972
Merge branch 'master' into add-span-attributes
suhyenim Jan 19, 2025
307a084
feat: add span attributes
suhyenim Mar 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions bin/experiment/experiment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"flag"
"go.opentelemetry.io/otel/attribute"
"os"

// Uncomment to load all auth plugins
Expand Down Expand Up @@ -102,6 +103,7 @@ func main() {

// parse the experiment name
experimentName := flag.String("name", "pod-delete", "name of the chaos experiment")
span.SetAttributes(attribute.String("experiment.name", *experimentName))

//Getting kubeConfig and Generate ClientSets
if err := clients.GenerateClientSetFromKubeConfig(); err != nil {
Expand Down
2 changes: 2 additions & 0 deletions bin/helper/helper.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"errors"
"flag"
"go.opentelemetry.io/otel/attribute"
"os"

// Uncomment to load all auth plugins
Expand Down Expand Up @@ -59,6 +60,7 @@ func main() {

// parse the helper name
helperName := flag.String("name", "", "name of the helper pod")
span.SetAttributes(attribute.String("helper.name", *helperName))

//Getting kubeConfig and Generate ClientSets
if err := clients.GenerateClientSetFromKubeConfig(); err != nil {
Expand Down
26 changes: 24 additions & 2 deletions chaoslib/litmus/aws-ssm-chaos/lib/ssm-chaos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ package lib

import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"os"
"strings"
"time"
Expand All @@ -21,7 +23,17 @@ import (

// InjectChaosInSerialMode will inject the aws ssm chaos in serial mode that is one after other
func InjectChaosInSerialMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceIDList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails, inject chan os.Signal) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAWSSSMFaultInSerialMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAWSSSMFaultInSerialMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
attribute.String("aws.ec2.id", experimentsDetails.EC2InstanceID),
attribute.String("aws.ec2.tag", experimentsDetails.EC2InstanceTag),
attribute.String("aws.ssm.document", experimentsDetails.DocumentName),
attribute.String("aws.region", experimentsDetails.Region),
),
)
defer span.End()

select {
Expand Down Expand Up @@ -91,7 +103,17 @@ func InjectChaosInSerialMode(ctx context.Context, experimentsDetails *experiment

// InjectChaosInParallelMode will inject the aws ssm chaos in parallel mode that is all at once
func InjectChaosInParallelMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceIDList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails, inject chan os.Signal) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAWSSSMFaultInParallelMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAWSSSMFaultInParallelMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
attribute.String("aws.ec2.id", experimentsDetails.EC2InstanceID),
attribute.String("aws.ec2.tag", experimentsDetails.EC2InstanceTag),
attribute.String("aws.ssm.document", experimentsDetails.DocumentName),
attribute.String("aws.region", experimentsDetails.Region),
),
)
defer span.End()

select {
Expand Down
6 changes: 5 additions & 1 deletion chaoslib/litmus/aws-ssm-chaos/lib/ssm/aws-ssm-chaos-by-id.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package ssm
import (
"context"
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"os"
"os/signal"
"strings"
Expand All @@ -28,7 +30,9 @@ var (

// PrepareAWSSSMChaosByID contains the prepration and injection steps for the experiment
func PrepareAWSSSMChaosByID(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAWSSSMFaultByID")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAWSSSMFaultByID",
trace.WithAttributes(attribute.Int("experiment.ramptime", experimentsDetails.RampTime)),
)
defer span.End()

// inject channel is used to transmit signal notifications.
Expand Down
6 changes: 5 additions & 1 deletion chaoslib/litmus/aws-ssm-chaos/lib/ssm/aws-ssm-chaos-by-tag.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package ssm
import (
"context"
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"os"
"os/signal"
"strings"
Expand All @@ -23,7 +25,9 @@ import (

// PrepareAWSSSMChaosByTag contains the prepration and injection steps for the experiment
func PrepareAWSSSMChaosByTag(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAWSSSMFaultByTag")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAWSSSMFaultByTag",
trace.WithAttributes(attribute.Int("experiment.ramptime", experimentsDetails.RampTime)),
)
defer span.End()

// inject channel is used to transmit signal notifications.
Expand Down
26 changes: 23 additions & 3 deletions chaoslib/litmus/azure-disk-loss/lib/azure-disk-loss.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package lib
import (
"context"
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"os"
"os/signal"
"strings"
Expand Down Expand Up @@ -33,7 +35,9 @@ var (

// PrepareChaos contains the prepration and injection steps for the experiment
func PrepareChaos(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAzureDiskLossFault")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAzureDiskLossFault",
trace.WithAttributes(attribute.Int("experiment.ramptime", experimentsDetails.RampTime)),
)
defer span.End()

// inject channel is used to transmit signal notifications.
Expand Down Expand Up @@ -106,7 +110,15 @@ func PrepareChaos(ctx context.Context, experimentsDetails *experimentTypes.Exper

// injectChaosInParallelMode will inject the Azure disk loss chaos in parallel mode that is all at once
func injectChaosInParallelMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNamesWithDiskNames map[string][]string, attachedDisksWithInstance map[string]*[]compute.DataDisk, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureDiskLossFaultInParallelMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureDiskLossFaultInParallelMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
attribute.String("azure.disk.name", experimentsDetails.VirtualDiskNames),
attribute.String("azure.resource.group", experimentsDetails.ResourceGroup),
),
)
defer span.End()

//ChaosStartTimeStamp contains the start timestamp, when the chaos injection begin
Expand Down Expand Up @@ -186,7 +198,15 @@ func injectChaosInParallelMode(ctx context.Context, experimentsDetails *experime

// injectChaosInSerialMode will inject the Azure disk loss chaos in serial mode that is one after other
func injectChaosInSerialMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNamesWithDiskNames map[string][]string, attachedDisksWithInstance map[string]*[]compute.DataDisk, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureDiskLossFaultInSerialMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureDiskLossFaultInSerialMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
attribute.String("azure.disk.name", experimentsDetails.VirtualDiskNames),
attribute.String("azure.resource.group", experimentsDetails.ResourceGroup),
),
)
defer span.End()

//ChaosStartTimeStamp contains the start timestamp, when the chaos injection begin
Expand Down
26 changes: 23 additions & 3 deletions chaoslib/litmus/azure-instance-stop/lib/azure-instance-stop.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package lib
import (
"context"
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"os"
"os/signal"
"strings"
Expand Down Expand Up @@ -31,7 +33,9 @@ var (

// PrepareAzureStop will initialize instanceNameList and start chaos injection based on sequence method selected
func PrepareAzureStop(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAzureInstanceStopFault")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareAzureInstanceStopFault",
trace.WithAttributes(attribute.Int("experiment.ramptime", experimentsDetails.RampTime)),
)
defer span.End()

// inject channel is used to transmit signal notifications
Expand Down Expand Up @@ -81,7 +85,15 @@ func PrepareAzureStop(ctx context.Context, experimentsDetails *experimentTypes.E

// injectChaosInSerialMode will inject the Azure instance termination in serial mode that is one after the other
func injectChaosInSerialMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceStopFaultInSerialMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceStopFaultInSerialMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
attribute.String("azure.vm.name", experimentsDetails.AzureInstanceNames),
attribute.String("azure.resource.group", experimentsDetails.ResourceGroup),
),
)
defer span.End()

select {
Expand Down Expand Up @@ -162,7 +174,15 @@ func injectChaosInSerialMode(ctx context.Context, experimentsDetails *experiment

// injectChaosInParallelMode will inject the Azure instance termination in parallel mode that is all at once
func injectChaosInParallelMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, instanceNameList []string, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceStopFaultInParallelMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectAzureInstanceStopFaultInParallelMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
attribute.String("azure.vm.name", experimentsDetails.AzureInstanceNames),
attribute.String("azure.resource.group", experimentsDetails.ResourceGroup),
),
)
defer span.End()

select {
Expand Down
6 changes: 6 additions & 0 deletions chaoslib/litmus/container-kill/helper/container-kill.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"github.com/litmuschaos/litmus-go/pkg/telemetry"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"os/exec"
"strconv"
"time"
Expand Down Expand Up @@ -49,6 +50,11 @@ func Helper(ctx context.Context, clients clients.ClientSets) {
// Initialise Chaos Result Parameters
types.SetResultAttributes(&resultDetails, chaosDetails)

span.SetAttributes(
attribute.String("container.runtime", experimentsDetails.ContainerRuntime),
attribute.String("kill.signal", experimentsDetails.Signal),
)

if err := killContainer(&experimentsDetails, clients, &eventsDetails, &chaosDetails, &resultDetails); err != nil {
// update failstep inside chaosresult
if resultErr := result.UpdateFailedStepFromHelper(&resultDetails, &chaosDetails, clients, err); resultErr != nil {
Expand Down
29 changes: 25 additions & 4 deletions chaoslib/litmus/container-kill/lib/container-kill.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package lib
import (
"context"
"fmt"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"os"
"strconv"
"strings"
Expand All @@ -27,7 +29,9 @@ import (

// PrepareContainerKill contains the preparation steps before chaos injection
func PrepareContainerKill(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, clients clients.ClientSets, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails, chaosDetails *types.ChaosDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareContainerKillFault")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "PrepareContainerKillFault",
trace.WithAttributes(attribute.Int("experiment.ramptime", experimentsDetails.RampTime)),
)
defer span.End()

var err error
Expand Down Expand Up @@ -93,7 +97,13 @@ func PrepareContainerKill(ctx context.Context, experimentsDetails *experimentTyp

// injectChaosInSerialMode kill the container of all target application serially (one by one)
func injectChaosInSerialMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, targetPodList apiv1.PodList, clients clients.ClientSets, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectContainerKillFaultInSerialMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectContainerKillFaultInSerialMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
),
)
defer span.End()
// run the probes during chaos
if len(resultDetails.ProbeDetails) != 0 {
Expand Down Expand Up @@ -145,7 +155,13 @@ func injectChaosInSerialMode(ctx context.Context, experimentsDetails *experiment

// injectChaosInParallelMode kill the container of all target application in parallel mode (all at once)
func injectChaosInParallelMode(ctx context.Context, experimentsDetails *experimentTypes.ExperimentDetails, targetPodList apiv1.PodList, clients clients.ClientSets, chaosDetails *types.ChaosDetails, resultDetails *types.ResultDetails, eventsDetails *types.EventDetails) error {
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectContainerKillFaultInParallelMode")
ctx, span := otel.Tracer(telemetry.TracerName).Start(ctx, "InjectContainerKillFaultInParallelMode",
trace.WithAttributes(
attribute.Int("chaos.duration", experimentsDetails.ChaosDuration),
attribute.Int("chaos.interval", experimentsDetails.ChaosInterval),
attribute.String("chaos.namespace", experimentsDetails.ChaosNamespace),
),
)
defer span.End()
// run the probes during chaos
if len(resultDetails.ProbeDetails) != 0 {
Expand Down Expand Up @@ -262,10 +278,15 @@ func createHelperPod(ctx context.Context, experimentsDetails *experimentTypes.Ex
helperPod.Spec.Volumes = append(helperPod.Spec.Volumes, common.GetSidecarVolumes(chaosDetails)...)
}

_, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).Create(context.Background(), helperPod, v1.CreateOptions{})
createdHelperPod, err := clients.KubeClient.CoreV1().Pods(experimentsDetails.ChaosNamespace).Create(context.Background(), helperPod, v1.CreateOptions{})
if err != nil {
return cerrors.Error{ErrorCode: cerrors.ErrorTypeGeneric, Reason: fmt.Sprintf("unable to create helper pod: %s", err.Error())}
}
span.SetAttributes(
attribute.String("helper.pod.name", createdHelperPod.Name),
attribute.String("helper.image.name", createdHelperPod.Spec.Containers[0].Image),
)

return nil
}

Expand Down
3 changes: 3 additions & 0 deletions chaoslib/litmus/disk-fill/helper/disk-fill.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"github.com/litmuschaos/litmus-go/pkg/telemetry"
"github.com/palantir/stacktrace"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"os"
"os/exec"
"os/signal"
Expand Down Expand Up @@ -64,6 +65,8 @@ func Helper(ctx context.Context, clients clients.ClientSets) {
// Set the chaos result uid
result.SetResultUID(&resultDetails, clients, &chaosDetails)

span.SetAttributes(attribute.String("container.runtime", experimentsDetails.ContainerRuntime))

if err := diskFill(&experimentsDetails, clients, &eventsDetails, &chaosDetails, &resultDetails); err != nil {
// update failstep inside chaosresult
if resultErr := result.UpdateFailedStepFromHelper(&resultDetails, &chaosDetails, clients, err); resultErr != nil {
Expand Down
Loading
Loading