@@ -88,6 +88,9 @@ const (
8888 setALLOWPolicy = "ALLOW"
8989 bgpMED = 25
9090 aclStatement3 = "30"
91+ gnmiRetryCount = 3
92+ gnmiDeleteRetryCount = 3
93+ gnmiSleepDuration = 30 * time .Second
9194)
9295
9396var (
@@ -202,7 +205,7 @@ func configureRoutePolicy(t *testing.T, dut *ondatra.DUTDevice, name string, pr
202205 t .Fatal (err )
203206 }
204207 st .GetOrCreateActions ().PolicyResult = pr
205- gnmi .Replace (t , dut , gnmi .OC ().RoutingPolicy ().Config (), rp )
208+ gnmi .Update (t , dut , gnmi .OC ().RoutingPolicy ().Config (), rp )
206209}
207210
208211func configInterfaceDUT (t * testing.T , i * oc.Interface , me * attrs.Attributes , subIntfIndex uint32 , vlan uint16 , dut * ondatra.DUTDevice ) {
@@ -280,8 +283,13 @@ func configureDUT(t *testing.T, dut *ondatra.DUTDevice) {
280283 fptest .SetPortSpeed (t , dut .Port (t , "port2" ))
281284 }
282285 if deviations .ExplicitInterfaceInDefaultVRF (dut ) {
283- fptest .AssignToNetworkInstance (t , dut , i1 .GetName (), deviations .DefaultNetworkInstance (dut ), 0 )
284286 fptest .AssignToNetworkInstance (t , dut , i2 .GetName (), deviations .DefaultNetworkInstance (dut ), 0 )
287+ if deviations .RequireRoutedSubinterface0 (dut ) {
288+ fptest .AssignToNetworkInstance (t , dut , i1 .GetName (), deviations .DefaultNetworkInstance (dut ), 0 )
289+ }
290+ for _ , subIntf := range []uint32 {10 , 20 , 30 , 40 , 50 , 60 } {
291+ fptest .AssignToNetworkInstance (t , dut , i1 .GetName (), deviations .DefaultNetworkInstance (dut ), subIntf )
292+ }
285293 }
286294}
287295
@@ -638,7 +646,7 @@ func removeNewPeers(t *testing.T, dut *ondatra.DUTDevice, nbrs []*bgpNeighbor) {
638646 t .Helper ()
639647 dutConfPath := gnmi .OC ().NetworkInstance (deviations .DefaultNetworkInstance (dut )).Protocol (oc .PolicyTypes_INSTALL_PROTOCOL_TYPE_BGP , "BGP" ).Bgp ()
640648 for _ , nbr := range nbrs {
641- gnmi . Delete (t , dut , dutConfPath .Neighbor (nbr .neighborip ).Config ())
649+ deleteWithRetry (t , dut , dutConfPath .Neighbor (nbr .neighborip ).Config ())
642650 }
643651 fptest .LogQuery (t , "DUT BGP Config" , dutConfPath .Config (), gnmi .Get (t , dut , dutConfPath .Config ()))
644652}
@@ -658,20 +666,20 @@ func setBgpPolicy(t *testing.T, dut *ondatra.DUTDevice, d *oc.Root) {
658666 actions5 .GetOrCreateBgpActions ().SetMedAction = oc .BgpPolicy_BgpSetMedAction_SET
659667 }
660668 actions5 .GetOrCreateBgpActions ().SetLocalPref = ygot .Uint32 (100 )
661- gnmi . Update (t , dut , gnmi .OC ().RoutingPolicy ().Config (), rp )
669+ updateWithRetry (t , dut , gnmi .OC ().RoutingPolicy ().Config (), rp )
662670
663671 dutConfPath := gnmi .OC ().NetworkInstance (deviations .DefaultNetworkInstance (dut )).Protocol (oc .PolicyTypes_INSTALL_PROTOCOL_TYPE_BGP , "BGP" ).Bgp ()
664672
665673 if deviations .RoutePolicyUnderAFIUnsupported (dut ) {
666- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
667- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
668- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
669- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
674+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
675+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
676+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
677+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
670678 } else {
671- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
672- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
673- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
674- gnmi . Update (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
679+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
680+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv4GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
681+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
682+ updateWithRetry (t , dut , dutConfPath .PeerGroup (peerv6GrpName ).AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" , setMEDPolicy })
675683 }
676684}
677685
@@ -699,7 +707,7 @@ func configureDUTNewPeers(t *testing.T, dut *ondatra.DUTDevice, nbrs []*bgpNeigh
699707 af6 := nv4 .GetOrCreateAfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST )
700708 af6 .Enabled = ygot .Bool (false )
701709 }
702- gnmi . Update (t , dut , dutConfPath .Config (), niProto )
710+ updateWithRetry (t , dut , dutConfPath .Config (), niProto )
703711 fptest .LogQuery (t , "DUT BGP Config" , dutConfPath .Config (), gnmi .Get (t , dut , dutConfPath .Config ()))
704712}
705713
@@ -751,26 +759,56 @@ func verifyGracefulRestart(t *testing.T, dut *ondatra.DUTDevice) {
751759 t .Errorf ("Expected Graceful restart timer: got %v, want %v" , grTimerVal , grRestartTime )
752760 }
753761
754- if llgrTimer := gnmi .Get (t , dut , nbrPath .GracefulRestart ().StaleRoutesTime ().State ()); llgrTimer != grStaleRouteTime {
755- t .Errorf ("LLGR timer is incorrect, want %v, got %v" , grStaleRouteTime , llgrTimer )
756- }
757- if grState := gnmi .Get (t , dut , nbrPath .GracefulRestart ().Enabled ().State ()); grState != true {
758- t .Errorf ("Graceful restart enabled state is incorrect, want true, got %v" , grState )
759- }
760- if peerRestartTime := gnmi .Get (t , dut , nbrPath .GracefulRestart ().PeerRestartTime ().State ()); peerRestartTime != 0 {
761- t .Errorf ("Peer restart time is incorrect, want 0, got %v" , peerRestartTime )
762+ if ! deviations .BgpLlgrOcUndefined (dut ) {
763+ if llgrTimer := gnmi .Get (t , dut , nbrPath .GracefulRestart ().StaleRoutesTime ().State ()); llgrTimer != grStaleRouteTime {
764+ t .Errorf ("LLGR timer is incorrect, want %v, got %v" , grStaleRouteTime , llgrTimer )
765+ }
762766 }
763- if peerRestartState := gnmi .Get (t , dut , nbrPath .GracefulRestart ().PeerRestarting ().State ()); peerRestartState != true {
764- t .Errorf ("Peer restart state is incorrect, want true , got %v" , peerRestartState )
767+ grState , present := gnmi .Lookup (t , dut , nbrPath .GracefulRestart ().Enabled ().State ()).Val ()
768+ if ! present && deviations .MissingValueForDefaults (dut ) {
769+ grState = true
770+ } else if ! present {
771+ t .Errorf ("Graceful restart enabled state is not present" )
765772 }
766- if localRestartState := gnmi . Get ( t , dut , nbrPath . GracefulRestart (). LocalRestarting (). State ()); localRestartState != false {
767- t .Errorf ("Local restart state is incorrect, want false , got %v" , localRestartState )
773+ if grState != true {
774+ t .Errorf ("Graceful restart enabled state is incorrect, want true , got %v" , grState )
768775 }
769- if grMode := gnmi .Get (t , dut , nbrPath .GracefulRestart ().Mode ().State ()); grMode != oc .GracefulRestart_Mode_HELPER_ONLY {
770- t .Errorf ("Graceful restart mode is incorrect, want oc.GracefulRestart_Mode_HELPER_ONLY, got %v" , grMode )
776+ if ! deviations .BgpLlgrOcUndefined (dut ) {
777+ peerRestartState , present := gnmi .Lookup (t , dut , nbrPath .GracefulRestart ().PeerRestarting ().State ()).Val ()
778+ if ! present && deviations .MissingValueForDefaults (dut ) {
779+ peerRestartState = true
780+ } else if ! present {
781+ t .Errorf ("Graceful restart peer-restarting state is not present" )
782+ }
783+ if peerRestartState != true {
784+ peerRestartTime , present := gnmi .Lookup (t , dut , nbrPath .GracefulRestart ().PeerRestartTime ().State ()).Val ()
785+ if ! present && deviations .MissingValueForDefaults (dut ) {
786+ peerRestartTime = 0
787+ } else if ! present {
788+ t .Errorf ("Peer restart time is not present" )
789+ }
790+ if peerRestartTime != 0 {
791+ t .Errorf ("Peer restart time is incorrect, want 0, got %v" , peerRestartTime )
792+ }
793+ t .Errorf ("Peer restart state is incorrect, want true, got %v" , peerRestartState )
794+ }
795+ if localRestartState := gnmi .Get (t , dut , nbrPath .GracefulRestart ().LocalRestarting ().State ()); localRestartState != false {
796+ t .Errorf ("Local restart state is incorrect, want false, got %v" , localRestartState )
797+ }
798+ if grMode := gnmi .Get (t , dut , nbrPath .GracefulRestart ().Mode ().State ()); grMode != oc .GracefulRestart_Mode_HELPER_ONLY && grMode != oc .GracefulRestart_Mode_BILATERAL {
799+ t .Errorf ("Graceful restart mode is incorrect, want %v or %v, got %v" , oc .GracefulRestart_Mode_HELPER_ONLY , oc .GracefulRestart_Mode_BILATERAL , grMode )
800+ }
771801 }
772- if nbrAfiSafiGrState := gnmi .Get (t , dut , nbrPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).GracefulRestart ().Enabled ().State ()); nbrAfiSafiGrState != true {
773- t .Errorf ("Neighbor AFI-SAFI graceful restart state is incorrect, want true, got %v" , nbrAfiSafiGrState )
802+ if ! deviations .BgpGracefulRestartUnderAfiSafiUnsupported (dut ) {
803+ nbrAfiSafiGrState , present := gnmi .Lookup (t , dut , nbrPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).GracefulRestart ().Enabled ().State ()).Val ()
804+ if ! present && deviations .MissingValueForDefaults (dut ) {
805+ nbrAfiSafiGrState = true
806+ } else if ! present {
807+ t .Errorf ("Neighbor AFI-SAFI Graceful restart enabled state is not present" )
808+ }
809+ if nbrAfiSafiGrState != true {
810+ t .Errorf ("Neighbor AFI-SAFI Graceful restart status: got %v, want Enabled" , nbrAfiSafiGrState )
811+ }
774812 }
775813}
776814
@@ -791,6 +829,58 @@ func buildCliConfigRequest(config string) *gpb.SetRequest {
791829 return gpbSetRequest
792830}
793831
832+ func replaceWithRetry [T any ](t * testing.T , dut * ondatra.DUTDevice , q ygnmi.ConfigQuery [T ], val T ) {
833+ t .Helper ()
834+ gnmiOperationWithRetry (t , "Replace" , gnmiRetryCount , func () error {
835+ gnmiClient := dut .RawAPIs ().GNMI (t )
836+ c , err := ygnmi .NewClient (gnmiClient , ygnmi .WithTarget (dut .Name ()))
837+ if err != nil {
838+ return fmt .Errorf ("failed to create ygnmi client: %w" , err )
839+ }
840+ _ , err = ygnmi .Replace (context .Background (), c , q , val )
841+ return err
842+ })
843+ }
844+
845+ func updateWithRetry [T any ](t * testing.T , dut * ondatra.DUTDevice , q ygnmi.ConfigQuery [T ], val T ) {
846+ t .Helper ()
847+ gnmiOperationWithRetry (t , "Update" , gnmiRetryCount , func () error {
848+ gnmiClient := dut .RawAPIs ().GNMI (t )
849+ c , err := ygnmi .NewClient (gnmiClient , ygnmi .WithTarget (dut .Name ()))
850+ if err != nil {
851+ return fmt .Errorf ("failed to create ygnmi client: %w" , err )
852+ }
853+ _ , err = ygnmi .Update (context .Background (), c , q , val )
854+ return err
855+ })
856+ }
857+
858+ func deleteWithRetry [T any ](t * testing.T , dut * ondatra.DUTDevice , q ygnmi.ConfigQuery [T ]) {
859+ t .Helper ()
860+ gnmiOperationWithRetry (t , "Delete" , gnmiDeleteRetryCount , func () error {
861+ gnmiClient := dut .RawAPIs ().GNMI (t )
862+ c , err := ygnmi .NewClient (gnmiClient , ygnmi .WithTarget (dut .Name ()))
863+ if err != nil {
864+ return fmt .Errorf ("failed to create ygnmi client: %w" , err )
865+ }
866+ _ , err = ygnmi .Delete (context .Background (), c , q )
867+ return err
868+ })
869+ }
870+
871+ func gnmiOperationWithRetry (t * testing.T , opName string , retryCount int , op func () error ) {
872+ t .Helper ()
873+ for i := 0 ; i < retryCount ; i ++ {
874+ err := op ()
875+ if err == nil {
876+ return
877+ }
878+ t .Logf ("%s failed, retrying... Attempt %d/%d. Error: %v" , opName , i + 1 , retryCount , err )
879+ time .Sleep (gnmiSleepDuration )
880+ }
881+ t .Fatalf ("%s failed after %d attempts" , opName , retryCount )
882+ }
883+
794884func TestTrafficWithGracefulRestartLLGR (t * testing.T ) {
795885 dut := ondatra .DUT (t , "dut" )
796886 ate := ondatra .ATE (t , "ate" )
@@ -887,33 +977,45 @@ func TestTrafficWithGracefulRestartLLGR(t *testing.T) {
887977 })
888978
889979 t .Run ("Restart routing" , func (t * testing.T ) {
980+ if deviations .RoutingRestartViaGnoiUnsupported (dut ) {
981+ t .Skip ("Skipping routing restart via gNOI due to deviation" )
982+ }
890983 gnoi .KillProcess (t , dut , gnoi .ROUTING , gnoi .SigTerm , true , true )
891984 })
892985
893986 var bgpIxPeer []* ixnet.BGP
894987 t .Run ("configure 5 more new BGP peers" , func (t * testing.T ) {
988+ if deviations .BgpConfigDuringGracefulRestartUnsupported (dut ) {
989+ t .Skip ("Skipping BGP Peer configuration during graceful restart due to deviation" )
990+ }
895991 configureDUTNewPeers (t , dut , dutNbrs )
896992 bgpIxPeer = configureATENewPeers (t , topo , ateIntfList )
897993 })
898994
899995 t .Run ("Remove newly added 5 BGP peers" , func (t * testing.T ) {
996+ if deviations .BgpConfigDuringGracefulRestartUnsupported (dut ) {
997+ t .Skip ("Skipping BGP Peer removal during graceful restart due to deviation" )
998+ }
900999 removeNewPeers (t , dut , dutNbrs )
9011000 removeATENewPeers (t , topo , bgpIxPeer )
9021001 })
9031002
9041003 t .Run ("Remove policy configured" , func (t * testing.T ) {
1004+ if deviations .BgpConfigDuringGracefulRestartUnsupported (dut ) {
1005+ t .Skip ("Skipping BGP Policy removal during graceful restart due to deviation" )
1006+ }
9051007 dutBgpV4PeerGroupPath := dutConfPath .Bgp ().PeerGroup (peerv4GrpName )
9061008 dutBgpV6PeerGroupPath := dutConfPath .Bgp ().PeerGroup (peerv6GrpName )
9071009 if deviations .RoutePolicyUnderAFIUnsupported (dut ) {
908- gnmi . Replace (t , dut , dutBgpV4PeerGroupPath .ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
909- gnmi . Replace (t , dut , dutBgpV4PeerGroupPath .ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
910- gnmi . Replace (t , dut , dutBgpV6PeerGroupPath .ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
911- gnmi . Replace (t , dut , dutBgpV6PeerGroupPath .ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
1010+ replaceWithRetry (t , dut , dutBgpV4PeerGroupPath .ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
1011+ replaceWithRetry (t , dut , dutBgpV4PeerGroupPath .ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
1012+ replaceWithRetry (t , dut , dutBgpV6PeerGroupPath .ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
1013+ replaceWithRetry (t , dut , dutBgpV6PeerGroupPath .ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
9121014 } else {
913- gnmi . Replace (t , dut , dutBgpV4PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
914- gnmi . Replace (t , dut , dutBgpV4PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
915- gnmi . Replace (t , dut , dutBgpV6PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
916- gnmi . Replace (t , dut , dutBgpV6PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
1015+ replaceWithRetry (t , dut , dutBgpV4PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
1016+ replaceWithRetry (t , dut , dutBgpV4PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV4_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
1017+ replaceWithRetry (t , dut , dutBgpV6PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ImportPolicy ().Config (), []string {"ALLOW" })
1018+ replaceWithRetry (t , dut , dutBgpV6PeerGroupPath .AfiSafi (oc .BgpTypes_AFI_SAFI_TYPE_IPV6_UNICAST ).ApplyPolicy ().ExportPolicy ().Config (), []string {"ALLOW" })
9171019 }
9181020 })
9191021 })
0 commit comments