Redundant IPsec site to site VPNs with BGP ECMP

General questions
1

Hi Team,

I am trying to setup a test environment between two appliances where I have installed vyos 1.4 rolling update and VPN failover is not happening. I have used iBGP between two routers but unable to send traffic from secondary tunnel if primary fails

ECMP-BGP
ECMP-BGP1280×720 34.3 KB

Here is my config
R1

set interfaces ethernet eth0 address '100.1.1.10/24'
set interfaces ethernet eth0 hw-id '00:03:2d:4d:dd:73'
set interfaces ethernet eth1 address '200.1.1.10/24'
set interfaces ethernet eth1 hw-id '00:03:2d:4d:dd:72'
set interfaces ethernet eth2 address '10.10.1.10/24'
set interfaces ethernet eth2 hw-id '00:03:2d:4d:dd:71'
set interfaces ethernet eth3 hw-id '00:03:2d:4d:dd:70'
set interfaces ethernet eth4 hw-id '00:03:2d:4d:dd:6f'
set interfaces ethernet eth5 hw-id '00:03:2d:4d:dd:6e'
set interfaces ethernet eth6 hw-id '00:03:2d:4d:dd:6d'
set interfaces ethernet eth7 hw-id '00:03:2d:4d:dd:6c'
set interfaces loopback lo
set interfaces vti vti5 address '169.254.254.254/32'
set interfaces vti vti10 address '169.170.170.171/32'
set protocols bgp address-family ipv4-unicast network 10.10.1.0/24
set protocols bgp local-as '65001'
set protocols bgp neighbor 169.170.170.170 remote-as '65001'
set protocols bgp neighbor 169.170.170.170 update-source '169.170.170.171'
set protocols bgp neighbor 169.254.254.253 remote-as '65001'
set protocols bgp neighbor 169.254.254.253 update-source '169.254.254.254'
set protocols bgp parameters graceful-restart stalepath-time '5'
set protocols bgp timers holdtime '5'
set protocols bgp timers keepalive '2'
set protocols static route 169.170.170.170/32 interface vti10
set protocols static route 169.254.254.253/32 interface vti5
set vpn ipsec esp-group ESPG compression 'disable'
set vpn ipsec esp-group ESPG mode 'tunnel'
set vpn ipsec esp-group ESPG pfs 'enable'
set vpn ipsec esp-group ESPG proposal 1 encryption 'aes256'
set vpn ipsec esp-group ESPG proposal 1 hash 'sha256'
set vpn ipsec ike-group IKEG dead-peer-detection action 'restart'
set vpn ipsec ike-group IKEG dead-peer-detection interval '5'
set vpn ipsec ike-group IKEG dead-peer-detection timeout '5'
set vpn ipsec ike-group IKEG key-exchange 'ikev1'
set vpn ipsec ike-group IKEG lifetime '28800'
set vpn ipsec ike-group IKEG proposal 1 dh-group '2'
set vpn ipsec ike-group IKEG proposal 1 encryption 'aes256'
set vpn ipsec ike-group IKEG proposal 1 hash 'sha256'
set vpn ipsec ipsec-interfaces interface 'eth0'
set vpn ipsec ipsec-interfaces interface 'eth1'
set vpn ipsec site-to-site peer 100.1.1.20 authentication id '100.1.1.10'
set vpn ipsec site-to-site peer 100.1.1.20 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 100.1.1.20 authentication pre-shared-secret 'admin@123'
set vpn ipsec site-to-site peer 100.1.1.20 authentication remote-id '100.1.1.20'
set vpn ipsec site-to-site peer 100.1.1.20 ike-group 'IKEG'
set vpn ipsec site-to-site peer 100.1.1.20 local-address '100.1.1.10'
set vpn ipsec site-to-site peer 100.1.1.20 vti bind 'vti5'
set vpn ipsec site-to-site peer 100.1.1.20 vti esp-group 'ESPG'
set vpn ipsec site-to-site peer 200.1.1.20 authentication id '200.1.1.10'
set vpn ipsec site-to-site peer 200.1.1.20 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 200.1.1.20 authentication pre-shared-secret 'admin@123'
set vpn ipsec site-to-site peer 200.1.1.20 authentication remote-id '200.1.1.20'
set vpn ipsec site-to-site peer 200.1.1.20 ike-group 'IKEG'
set vpn ipsec site-to-site peer 200.1.1.20 local-address '200.1.1.10'
set vpn ipsec site-to-site peer 200.1.1.20 vti bind 'vti10'
set vpn ipsec site-to-site peer 200.1.1.20 vti esp-group 'ESPG'

And R2

set interfaces ethernet eth0 address '100.1.1.20/24'
set interfaces ethernet eth0 hw-id '00:03:2d:4d:dd:7b'
set interfaces ethernet eth1 address '200.1.1.20/24'
set interfaces ethernet eth1 hw-id '00:03:2d:4d:dd:7a'
set interfaces ethernet eth2 address '192.168.10.1/24'
set interfaces ethernet eth2 hw-id '00:03:2d:4d:dd:79'
set interfaces ethernet eth3 hw-id '00:03:2d:4d:dd:78'
set interfaces ethernet eth4 hw-id '00:03:2d:4d:dd:77'
set interfaces ethernet eth5 hw-id '00:03:2d:4d:dd:76'
set interfaces ethernet eth6 hw-id '00:03:2d:4d:dd:75'
set interfaces ethernet eth7 hw-id '00:03:2d:4d:dd:74'
set interfaces loopback lo
set interfaces vti vti5 address '169.254.254.253/32'
set interfaces vti vti10 address '169.170.170.170/32'
set protocols bgp address-family ipv4-unicast network 192.168.10.0/24
set protocols bgp local-as '65001'
set protocols bgp neighbor 169.170.170.171 remote-as '65001'
set protocols bgp neighbor 169.170.170.171 update-source '169.170.170.1'
set protocols bgp neighbor 169.254.254.254 remote-as '65001'
set protocols bgp neighbor 169.254.254.254 update-source '169.254.254.253'
set protocols bgp parameters graceful-restart stalepath-time '5'
set protocols bgp timers holdtime '5'
set protocols bgp timers keepalive '2'
set protocols static route 169.170.170.171/32 interface vti10
set protocols static route 169.254.254.254/32 interface vti5
set vpn ipsec esp-group ESPG compression 'disable'
set vpn ipsec esp-group ESPG mode 'tunnel'
set vpn ipsec esp-group ESPG pfs 'enable'
set vpn ipsec esp-group ESPG proposal 1 encryption 'aes256'
set vpn ipsec esp-group ESPG proposal 1 hash 'sha256'
set vpn ipsec ike-group IKEG dead-peer-detection action 'restart'
set vpn ipsec ike-group IKEG dead-peer-detection interval '5'
set vpn ipsec ike-group IKEG dead-peer-detection timeout '5'
set vpn ipsec ike-group IKEG key-exchange 'ikev1'
set vpn ipsec ike-group IKEG lifetime '28800'
set vpn ipsec ike-group IKEG proposal 1 dh-group '2'
set vpn ipsec ike-group IKEG proposal 1 encryption 'aes256'
set vpn ipsec ike-group IKEG proposal 1 hash 'sha256'
set vpn ipsec ipsec-interfaces interface 'eth0'
set vpn ipsec ipsec-interfaces interface 'eth1'
set vpn ipsec site-to-site peer 100.1.1.10 authentication id '100.1.1.20'
set vpn ipsec site-to-site peer 100.1.1.10 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 100.1.1.10 authentication pre-shared-secret 'admin@123'
set vpn ipsec site-to-site peer 100.1.1.10 authentication remote-id '100.1.1.10'
set vpn ipsec site-to-site peer 100.1.1.10 ike-group 'IKEG'
set vpn ipsec site-to-site peer 100.1.1.10 local-address '100.1.1.20'
set vpn ipsec site-to-site peer 100.1.1.10 vti bind 'vti5'
set vpn ipsec site-to-site peer 100.1.1.10 vti esp-group 'ESPG'
set vpn ipsec site-to-site peer 200.1.1.10 authentication id '200.1.1.20'
set vpn ipsec site-to-site peer 200.1.1.10 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 200.1.1.10 authentication pre-shared-secret 'admin@123'
set vpn ipsec site-to-site peer 200.1.1.10 authentication remote-id '200.1.1.10'
set vpn ipsec site-to-site peer 200.1.1.10 ike-group 'IKEG'
set vpn ipsec site-to-site peer 200.1.1.10 local-address '200.1.1.20'
set vpn ipsec site-to-site peer 200.1.1.10 vti bind 'vti10'
set vpn ipsec site-to-site peer 200.1.1.10 vti esp-group 'ESPG'

Now if take the eth0 cable out i.e. 100.1.1.10 connection does not restore from other tunnel while I see when my both the tunnels are up when the links are up
And I feel an issue with BGP configuration since show ip bgp route from R1 or R2 shows 10.10.1.0/24 and 192.168.10.0/24 still being learned from both the links from respective routers I believe neighbors 169.254.254.254 should disappear?

2

Dang - I again simulated with 1.2.7 and it worked properly; not sure if 1.4 since its in development stage has broke something?

3

Hi @blason

I hope you are well! when you use bgp , it has a mechanism that only select the best route to install (it’s to loop-prevention) . So if you want to enable ecmp on BGP , you should enable that it allow equal path in your mechanism of path selection :

vyos-cli:

 set protocols bgp <asn> parameters bestpath as-path multipath-relax
This command specifies that BGP decision process should consider paths of equal AS_PATH length candidates for multipath computation. Without the knob, the entire AS_PATH must match for multipath computation.

PD: I recommend if you could use a version as 1,3.X on vyos , because the 1.4.x is more for experimental feature .

4

Hi,

Is 1.3 experimental? I believe that one is not considered as stable, right?

And when I enable BGP I guess it does pick same cost correct?

5

Sorry !maybe my words were a little confusing , 1.3.x is more stable than 1.4.x , better to test those kinds of features.

with respect to bgp,yes it should has the same path cost.

6

So in by default config I believe BGP has a same cost right? I am sorry I am not a pro in BGP.

7

It’ depends on how another router announce the prefixes (as path/ prefix lentgh / IGP metric) ,but if you have the same configuration in both router ,it should be equal.

also , you can setting this command :
set protocols bgp <asn> maximum-paths <ebgp|ibgp> <number>

if you attributes are differents

1 Like