Skip to content

Commit 5df593b

Browse files
l0wl3vell0wl3vel
authored
Fix route reconciliation by enabling fpm (#315)
1 parent c327986 commit 5df593b

6 files changed

Lines changed: 231 additions & 31 deletions

File tree

‎Makefile‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -79,20 +79,12 @@ up: env gen-certs verify-deployment-image control-plane-bake partition-bake
7979
@chmod 600 files/ssh/id_ed25519
8080
docker compose $(COMPOSE_ARGS) up --pull=always --abort-on-container-failure --remove-orphans --force-recreate control-plane partition
8181
@$(MAKE) --no-print-directory start-machines
82-
# for some reason an allocated machine will not be able to phone home
83-
# without restarting the metal-core
84-
# TODO: should be investigated and fixed if possible
85-
# check that underlay gets working
82+
83+
# on old dell_sonic flavor metal-core needs a restart for nodes to register
84+
ifneq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),)
8685
sleep 10
8786
ssh -F files/ssh/config leaf01 'systemctl restart metal-core'
8887
ssh -F files/ssh/config leaf02 'systemctl restart metal-core'
89-
90-
# for community SONiC versions > 202311 a bgp restart is needed in the virtual environment
91-
# TODO: should be investigated and fixed if possible
92-
ifeq ($(filter $(MINI_LAB_FLAVOR),dell_sonic capms_dell_sonic),)
93-
sleep 15
94-
ssh -F files/ssh/config leaf01 'systemctl restart bgp'
95-
ssh -F files/ssh/config leaf02 'systemctl restart bgp'
9688
endif
9789

9890
.PHONY: restart

‎deploy_partition.yaml‎

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,8 +132,33 @@
132132
when: monitoring_enabled
133133
tags: sonic-exporter
134134

135-
- name: Deploy metal-core
136-
hosts: leaves
135+
# So, route propagation was broken in community sonic for a while now. Turns out
136+
# during a change in upstream sonic some time in the end of 2024 sonic switched
137+
# the module used for FRR->fpmsyncd communication, which does not provide a
138+
# default connection endpoint. We just needed to add `fpm address 127.0.0.1`
139+
# to the frr.conf template and could remove all route-propagation related
140+
# workarounds
141+
- name: Deploy metal-core (SONiC with dplane_fpm_sonic)
142+
hosts: leaves:!dell_sonic
143+
any_errors_fatal: true
144+
become: true
145+
pre_tasks:
146+
- name: copy custom sonic_frr.tpl with fpm configured
147+
ansible.builtin.copy:
148+
dest: /sonic_frr.tpl
149+
src: /mini-lab/files/sonic_frr_with_fpm.tpl
150+
roles:
151+
- name: ansible-common
152+
tags: always
153+
- name: metal-roles/partition/roles/metal-core
154+
tags: metal-core
155+
vars:
156+
metal_core_frr_tpl_file: /sonic_frr.tpl
157+
metal_core_additional_volume_mounts:
158+
- /sonic_frr.tpl:/sonic_frr.tpl
159+
160+
- name: Deploy metal-core (Dell SONiC without dplane_fpm_sonic)
161+
hosts: dell_sonic
137162
any_errors_fatal: true
138163
become: true
139164
pre_tasks:

‎files/sonic_frr_with_fpm.tpl‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
{{- $ASN := .ASN -}}{{- $RouterId := .Loopback -}}! The frr version is not rendered since it seems to be optional.
2+
frr defaults datacenter
3+
# This is the only line changed from upstream: https://github.com/metal-stack/metal-core/blob/master/cmd/internal/switcher/templates/tpl/sonic_frr.tpl
4+
# Follow-up issue: https://github.com/metal-stack/metal-core/issues/199
5+
fpm address 127.0.0.1
6+
hostname {{ .Name }}
7+
password zebra
8+
enable password zebra
9+
!
10+
agentx
11+
log syslog {{ .LogLevel }}
12+
log facility local4
13+
debug bgp updates
14+
debug bgp nht
15+
debug bgp update-groups
16+
debug bgp zebra
17+
debug zebra events
18+
debug zebra nexthop detail
19+
debug zebra rib detailed
20+
debug zebra nht detailed
21+
{{- range $vrf, $t := .Ports.Vrfs }}
22+
!
23+
vrf Vrf{{ $t.VNI }}
24+
vni {{ $t.VNI }}
25+
exit-vrf
26+
{{- end }}
27+
{{- range .Ports.Underlay }}
28+
!
29+
interface {{ . }}
30+
ipv6 nd ra-interval 6
31+
no ipv6 nd suppress-ra
32+
{{- end }}
33+
{{- range .Ports.Firewalls }}
34+
!
35+
interface {{ .Port }}
36+
ipv6 nd ra-interval 6
37+
no ipv6 nd suppress-ra
38+
{{- end }}
39+
{{- range $vrf, $t := .Ports.Vrfs }}
40+
{{- range $t.Neighbors }}
41+
!
42+
interface {{ . }} vrf {{ $vrf }}
43+
ipv6 nd ra-interval 6
44+
no ipv6 nd suppress-ra
45+
{{- end }}
46+
{{- end }}
47+
!
48+
router bgp {{ $ASN }}
49+
bgp router-id {{ $RouterId }}
50+
bgp bestpath as-path multipath-relax
51+
neighbor FABRIC peer-group
52+
neighbor FABRIC remote-as external
53+
neighbor FABRIC timers 2 8
54+
{{- range .Ports.Underlay }}
55+
neighbor {{ . }} interface peer-group FABRIC
56+
{{- end }}
57+
neighbor FIREWALL peer-group
58+
neighbor FIREWALL remote-as external
59+
neighbor FIREWALL timers 2 8
60+
{{- range .Ports.Firewalls }}
61+
neighbor {{ .Port }} interface peer-group FIREWALL
62+
{{- end }}
63+
!
64+
address-family ipv4 unicast
65+
redistribute connected route-map DENY_MGMT
66+
redistribute kernel route-map DENY_MGMT
67+
neighbor FIREWALL allowas-in 2
68+
{{- range $k, $f := .Ports.Firewalls }}
69+
neighbor {{ $f.Port }} route-map fw-{{ $k }}-in in
70+
{{- end }}
71+
exit-address-family
72+
!
73+
address-family ipv6 unicast
74+
redistribute connected route-map DENY_MGMT
75+
redistribute kernel route-map DENY_MGMT
76+
neighbor FIREWALL allowas-in 2
77+
# see https://docs.frrouting.org/en/latest/bgp.html#clicmd-neighbor-A.B.C.D-activate
78+
# why activate is required
79+
neighbor FIREWALL activate
80+
{{- range $k, $f := .Ports.Firewalls }}
81+
neighbor {{ $f.Port }} route-map fw-{{ $k }}-in in
82+
{{- end }}
83+
exit-address-family
84+
!
85+
address-family l2vpn evpn
86+
advertise-all-vni
87+
neighbor FABRIC activate
88+
neighbor FABRIC allowas-in 2
89+
neighbor FIREWALL activate
90+
neighbor FIREWALL allowas-in 2
91+
{{- range $k, $f := .Ports.Firewalls }}
92+
neighbor {{ $f.Port }} route-map fw-{{ $k }}-vni out
93+
{{- end }}
94+
exit-address-family
95+
!
96+
route-map DENY_MGMT deny 10
97+
match interface eth0
98+
route-map DENY_MGMT permit 20
99+
!
100+
{{- range $k, $f := .Ports.Firewalls }}
101+
# route-maps for firewall@{{ $k }}
102+
{{- range $f.IPPrefixLists }}
103+
ip prefix-list {{ .Name }} {{ .Spec }}
104+
{{- end}}
105+
{{- range $f.RouteMaps }}
106+
route-map {{ .Name }} {{ .Policy }} {{ .Order }}
107+
{{- range .Entries }}
108+
{{ . }}
109+
{{- end }}
110+
{{- end }}
111+
!
112+
{{- end }}
113+
{{- if .Ports.Eth0.Gateway }}
114+
ip route 0.0.0.0/0 {{ .Ports.Eth0.Gateway }} nexthop-vrf mgmt
115+
{{- end }}
116+
!
117+
{{- range $vrf, $t := .Ports.Vrfs }}
118+
router bgp {{ $ASN }} vrf {{ $vrf }}
119+
bgp router-id {{ $RouterId }}
120+
bgp bestpath as-path multipath-relax
121+
neighbor MACHINE peer-group
122+
neighbor MACHINE remote-as external
123+
neighbor MACHINE timers 2 8
124+
{{- range $t.Neighbors }}
125+
neighbor {{ . }} interface peer-group MACHINE
126+
{{- end }}
127+
!
128+
{{- if $t.Has4 }}
129+
address-family ipv4 unicast
130+
redistribute connected
131+
neighbor MACHINE maximum-prefix 24000
132+
{{- if gt (len $t.IPPrefixLists) 0 }}
133+
neighbor MACHINE route-map {{ $vrf }}-in in
134+
{{- end }}
135+
exit-address-family
136+
!
137+
{{- end }}
138+
{{- if $t.Has6 }}
139+
address-family ipv6 unicast
140+
redistribute connected
141+
neighbor MACHINE maximum-prefix 24000
142+
# see https://docs.frrouting.org/en/latest/bgp.html#clicmd-neighbor-A.B.C.D-activate
143+
# why activate is required
144+
neighbor MACHINE activate
145+
{{- if gt (len $t.IPPrefixLists) 0 }}
146+
neighbor MACHINE route-map {{ $vrf }}-in6 in
147+
{{- end }}
148+
exit-address-family
149+
!
150+
{{- end }}
151+
address-family l2vpn evpn
152+
{{- if $t.Has4 }}
153+
advertise ipv4 unicast
154+
{{- end }}
155+
{{- if $t.Has6 }}
156+
advertise ipv6 unicast
157+
{{- end }}
158+
exit-address-family
159+
!
160+
{{- if gt (len $t.IPPrefixLists) 0 }}
161+
# route-maps for {{ $vrf }}
162+
{{- range $t.IPPrefixLists }}
163+
ip prefix-list {{ .Name }} {{ .Spec }}
164+
{{- end}}
165+
{{- range $t.RouteMaps }}
166+
route-map {{ .Name }} {{ .Policy }} {{ .Order }}
167+
{{- range .Entries }}
168+
{{ . }}
169+
{{- end }}
170+
{{- end }}
171+
!{{- end }}{{- end }}
172+
{{- if .SetSrcLoopback }}
173+
route-map RM_SET_SRC permit 10
174+
set src {{ .Loopback }}
175+
exit
176+
!
177+
ip protocol bgp route-map RM_SET_SRC
178+
!
179+
{{- end }}
180+
line vty
181+
!

‎images/sonic/Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,6 @@ ENTRYPOINT ["/launch.py"]
2121

2222
COPY mirror_tap_to_eth.sh mirror_tap_to_front_panel.sh port_config.ini launch.py /
2323

24-
HEALTHCHECK --start-period=10s --interval=5s --retries=20 CMD test -f /healthy
24+
# Readiness now waits for the dataplane (front-panel LLDP), which converges later than the
25+
# mgmt plane, so allow more time before the container is considered unhealthy.
26+
HEALTHCHECK --start-period=30s --interval=5s --retries=40 CMD test -f /healthy

‎images/sonic/launch.py‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -102,8 +102,12 @@ def initial_configuration(g: GuestFS, hwsku: str) -> None:
102102

103103
# Workaround: Speed up lldp startup by remove hardcoded wait of 90 seconds
104104
g.ln_s(linkname=systemd_system + 'aaastatsd.timer', target='/dev/null') # Radius
105-
g.ln_s(linkname=systemd_system + 'featured.timer', target='/dev/null') # Feature handling not necessary
106-
g.ln_s(linkname=systemd_system + 'hostcfgd.timer', target='/dev/null') # After boot Host configuration
105+
# NOTE: featured.timer and hostcfgd.timer are intentionally NOT masked. They run
106+
# SONiC's feature-manager and host-config reconciliation and provide the startup
107+
# ordering that EVPN/FRR bring-up relies on. Masking them raced the remote L3VNI
108+
# VTEP RMAC programming on the leaves, which then required a manual
109+
# `systemctl restart bgp` to recover. lldp/pmon are still hand-wired below so they
110+
# start immediately instead of waiting for featured's delayed timer.
107111
g.ln_s(linkname=systemd_system + 'rasdaemon.timer', target='/dev/null') # After boot Host configuration
108112
g.ln_s(linkname=systemd_system + 'tacacs-config.timer', target='/dev/null') # After boot Host configuration
109113
# Started by featured
@@ -203,11 +207,17 @@ def main():
203207
logger.info('Start QEMU')
204208
vm.start()
205209

206-
# SONiC will start sending LLDP packets after PortConfigDone is set in APPL database
207-
logger.info('Wait until eth0 has an IPv4 address')
208-
sniff(iface='eth0', filter='ether proto 0x88cc', stop_filter=has_an_IPv4_address('eth0'), store=0)
209-
210-
logger.info('eth0 has a configured IPv4 address')
210+
# Readiness: wait until SONiC forwards its own LLDP out a *front-panel* port, not just
211+
# mgmt eth0. LLDP egress on a front-panel port requires PortConfigDone AND the port to
212+
# be programmed/oper-up in the (v)ASIC dataplane, so this signals dataplane readiness
213+
# instead of mgmt-plane-only readiness (which let the lab proceed, and metal-core
214+
# configure the fabric, while the switch dataplane/EVPN was still converging).
215+
# We match on SONiC's own mgmt address, so neighbor LLDP on the port is ignored.
216+
dataplane_ifaces = get_ethernet_interfaces()
217+
logger.info(f'Wait until a front-panel port forwards LLDP (dataplane up): {dataplane_ifaces}')
218+
sniff(iface=dataplane_ifaces, filter='ether proto 0x88cc', stop_filter=has_an_IPv4_address('eth0'), store=0)
219+
220+
logger.info('dataplane is forwarding LLDP')
211221
with open('/healthy', 'w'):
212222
pass
213223

‎tasks/check_queued.yaml‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -21,13 +21,3 @@
2121
register: queued_check
2222
delay: 5
2323
changed_when: false
24-
25-
- name: restart bgp
26-
ansible.builtin.systemd:
27-
name: bgp
28-
state: restarted
29-
when: queued_check.stdout == "true"
30-
31-
- name: recheck for queued routes
32-
include_tasks: tasks/check_queued.yaml
33-
when: queued_check.stdout == "true"

0 commit comments

Comments
 (0)