Ipsec restarts all interfaces simutaneously

Hello,

1.3-rolling-202009160118

I met the problem that ipsec restarts ALL vti nterfaces if one ipsec tunnel goes down for some reason.
All three tunnels connected to one cisco router.
And I wonder - this a bug or a feature ?
this is parts of my configuration and /etc/ipsec.conf file.

thank you in advance.

interfaces {
ethernet eth0 {
hw-id 00:22:64:04:2b:3a
vif 63 {
address 192.168.230.29/24
description “BEELINE L2”
}
vif 80 {
address 192.168.221.29/29
description “MTS L3”
}
vif 496 {
address 77.36.229.162/24
description “ENFORTA INET via RADIO”
firewall {
local {
name FW_FROM_INET
}
}
vrf INET-VRF
}
vif 999 {
address 172.25.0.7/24
description LAN
ip {
ospf {
authentication {
md5 {
key-id 1 {
md5-key 59113449
}
}
}
cost 7
dead-interval 40
hello-interval 10
priority 1
retransmit-interval 5
transmit-delay 1
}
}
policy {
route PR_DSMARKER
}
}
}
ethernet eth1 {
address 172.25.1.1/31
description “PTP LINK TO PRIMARY ROUTER”
hw-id 00:22:64:04:3b:84
ip {
ospf {
authentication {
md5 {
key-id 1 {
md5-key 59113449
}
}
}
bfd
cost 5
dead-interval 40
hello-interval 10
network point-to-point
priority 1
retransmit-interval 5
transmit-delay 1
}
}
}
loopback lo {
address 10.40.73.50/32
description “+LOOPBACK OSPF RID”
}
vti vti01 {
address 10.40.81.175/31
description “IPSEC TO CROC VIA BEELINE”
ip {
ospf {
authentication {
md5 {
key-id 1 {
md5-key 59113449
}
}
}
bfd
cost 40
dead-interval 40
hello-interval 10
mtu-ignore
network point-to-point
priority 1
retransmit-interval 5
transmit-delay 1
}
}
mtu 1436
traffic-policy {
out HTB7-POLICY
}
}
vti vti02 {
address 10.40.81.177/31
description “IPSEC TO CROC VIA MTS”
ip {
ospf {
authentication {
md5 {
key-id 1 {
md5-key 59113449
}
}
}
cost 40
dead-interval 40
hello-interval 10
mtu-ignore
network point-to-point
priority 1
retransmit-interval 5
transmit-delay 1
}
}
mtu 1436
traffic-policy {
out HTB7-POLICY
}
}
vti vti03 {
address 10.40.81.179/31
description “IPSEC TO CROC VIA ENFORTA”
ip {
ospf {
authentication {
md5 {
key-id 1 {
md5-key 59113449
}
}
}
bfd
cost 150
dead-interval 40
hello-interval 10
mtu-ignore
priority 1
retransmit-interval 5
transmit-delay 1
}
}
mtu 1436
traffic-policy {
out HTB7-POLICY
}
}
}
protocols {
bfd {
peer 10.40.81.174 {
}
peer 10.40.81.176 {
}
peer 10.40.81.178 {
}
}
ospf {
area 0.12.12.0 {
authentication md5
network 10.40.81.174/31
network 10.40.81.176/31
network 172.25.0.0/24
network 10.40.81.178/31
network 172.25.1.0/31
network 10.40.73.50/32
}
log-adjacency-changes {
detail
}
parameters {
abr-type cisco
router-id 10.40.73.50
}
passive-interface default
passive-interface-exclude eth0.999
passive-interface-exclude vti01
passive-interface-exclude vti02
passive-interface-exclude vti03
passive-interface-exclude eth1
}
static {
route 0.0.0.0/0 {
next-hop 172.25.0.2 {
distance 200
}
}
route 82.204.148.110/32 {
next-hop 77.36.229.1 {
next-hop-vrf INET-VRF
}
}
route 192.168.221.16/29 {
next-hop 192.168.221.25 {
}
}
route 194.222.83.144/29 {
next-hop 77.36.229.1 {
next-hop-vrf INET-VRF
}
}
route 194.222.83.215/32 {
next-hop 77.36.229.1 {
next-hop-vrf INET-VRF
}
}
}
vrf INET-VRF {
static {
route 0.0.0.0/0 {
next-hop 77.36.229.1 {
}
}
}
}
}

vpn {
ipsec {
esp-group ESP01 {
compression disable
lifetime 3600
mode tunnel
pfs disable
proposal 1 {
encryption aes256
hash sha1
}
}
ike-group IKE00 {
close-action restart
dead-peer-detection {
action restart
interval 15
timeout 120
}
ikev2-reauth no
key-exchange ikev2
lifetime 86400
proposal 1 {
dh-group 14
encryption aes256
hash sha256
}
}
ike-group IKE01 {
close-action none
dead-peer-detection {
action restart
interval 10
timeout 120
}
ikev2-reauth no
key-exchange ikev2
lifetime 86400
proposal 1 {
dh-group 14
encryption aes256
hash sha256
}
}
logging {
log-level 0
log-modes ike
log-modes knl
log-modes cfg
}
nat-traversal disable
site-to-site {
peer 192.168.221.22 {
authentication {
id ntop2-m-gw.zenit.ru
mode pre-shared-secret
pre-shared-secret xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
remote-id ccr38.zenit.ru
}
connection-type initiate
description MTS-TO-CCR38
force-encapsulation disable
ike-group IKE00
ikev2-reauth inherit
local-address 192.168.221.29
vti {
bind vti02
esp-group ESP01
}
}
peer 192.168.230.12 {
authentication {
id ntop2-b-gw.zenit.ru
mode pre-shared-secret
pre-shared-secret xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
remote-id ccr38.zenit.ru
}
connection-type initiate
description BEELINE-TO-CCR38
force-encapsulation disable
ike-group IKE01
ikev2-reauth inherit
local-address 192.168.230.29
vti {
bind vti01
esp-group ESP01
}
}
peer 194.222.83.147 {
authentication {
id ntop2-e-gw.zenit.ru
mode pre-shared-secret
pre-shared-secret xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
remote-id ccr38.zenit.ru
}
connection-type initiate
description ENFORTA-INET-TO-CCR38
ike-group IKE01
ikev2-reauth inherit
local-address 77.36.229.162
vti {
bind vti03
esp-group ESP01
}
}
}
}
}
vrf {
bind-to-all
name INET-VRF {
table 200
}
}

generated by /opt/vyatta/sbin/vpn-config.pl

config setup
nat_traversal=no

conn %default
keyexchange=ikev1

conn peer-192.168.221.22-tunnel-vti
left=192.168.221.29
leftid=“ntop2-m-gw.anywhere.ru
right=192.168.221.22
rightid=“ccr38.anywhere.ru
leftsubnet=0.0.0.0/0
rightsubnet=0.0.0.0/0
leftsubnet=0.0.0.0/0
ike=aes256-sha256-modp2048!
keyexchange=ikev2
reauth=no
ikelifetime=86400s
dpddelay=15s
dpdtimeout=120s
dpdaction=restart
closeaction=restart
forceencaps=no
esp=aes256-sha1!
keylife=3600s
rekeymargin=540s
type=tunnel
compress=no
authby=secret
mark=9437185
leftupdown=“/usr/lib/ipsec/vti-up-down vti02”
auto=start
keyingtries=%forever
#conn peer-192.168.221.22-tunnel-vti

conn peer-192.168.230.12-tunnel-vti
left=192.168.230.29
leftid=“ntop2-b-gw.anywhere.ru
right=192.168.230.12
rightid=“ccr38.anywhere.ru
leftsubnet=0.0.0.0/0
rightsubnet=0.0.0.0/0
leftsubnet=0.0.0.0/0
ike=aes256-sha256-modp2048!
keyexchange=ikev2
reauth=no
ikelifetime=86400s
dpddelay=10s
dpdtimeout=120s
dpdaction=restart
closeaction=none
forceencaps=no
esp=aes256-sha1!
keylife=3600s
rekeymargin=540s
type=tunnel
compress=no
authby=secret
mark=9437186
leftupdown=“/usr/lib/ipsec/vti-up-down vti01”
auto=start
keyingtries=%forever
#conn peer-192.168.230.12-tunnel-vti

conn peer-194.222.83.147-tunnel-vti
left=77.36.229.162
leftid=“ntop2-e-gw.anywhere.ru
right=194.222.83.147
rightid=“ccr38.anywhere.ru
leftsubnet=0.0.0.0/0
rightsubnet=0.0.0.0/0
leftsubnet=0.0.0.0/0
ike=aes256-sha256-modp2048!
keyexchange=ikev2
reauth=no
ikelifetime=86400s
dpddelay=10s
dpdtimeout=120s
dpdaction=restart
closeaction=none
esp=aes256-sha1!
keylife=3600s
rekeymargin=540s
type=tunnel
compress=no
authby=secret
mark=9437187
leftupdown=“/usr/lib/ipsec/vti-up-down vti03”
auto=start
keyingtries=%forever
#conn peer-194.222.83.147-tunnel-vti

Hello @vzotov,

I think you need to mask all your private data includes peer’s public IP addresses.
For a better understanding of what happened, I propose to research logs

show log | match charon

ps:// As a best practice <command> | strip-private can help.

Dmitry,
thank you for reply.
this is not ipsec. I looked into log file and discovered that vti interface link up restarts whole ip stack.
furher investingation showed that this is frr. Destroying vrf eliminated the problem. It is a pitty because I need vrf.

here is sample from log file during interface state change. Probably you can find this bug in frr development forum.
Oct 20 14:43:21 vyos-g2 zebra[1021]: Extended Error: Label >= configured maximum in platform_labels
Oct 20 14:43:21 vyos-g2 zebra[1021]: [EC 4043309093] netlink-dp (NS 0) error: Invalid argument, type=RTM_NEWROUTE(24), seq=642079, pid=2772610761
Oct 20 14:43:21 vyos-g2 zebra[1021]: [EC 4043309108] LSP Install Failure: in-label 1467
Oct 20 14:45:21 vyos-g2 zebra[1021]: Extended Error: Label >= configured maximum in platform_labels
Oct 20 14:45:21 vyos-g2 zebra[1021]: [EC 4043309093] netlink-dp (NS 0) error: Invalid argument, type=RTM_NEWROUTE(24), seq=642082, pid=2772610761
Oct 20 14:45:21 vyos-g2 zebra[1021]: [EC 4043309108] LSP Install Failure: in-label 1467
Oct 20 14:46:06 vyos-g2 zebra[1021]: Extended Error: Label >= configured maximum in platform_labels
Oct 20 14:46:06 vyos-g2 zebra[1021]: [EC 4043309093] netlink-dp (NS 0) error: Invalid argument, type=RTM_NEWROUTE(24), seq=642085, pid=2772610761
Oct 20 14:46:06 vyos-g2 zebra[1021]: [EC 4043309108] LSP Install Failure: in-label 860
Oct 20 14:47:21 vyos-g2 zebra[1021]: Extended Error: Label >= configured maximum in platform_labels
Oct 20 14:47:21 vyos-g2 zebra[1021]: [EC 4043309093] netlink-dp (NS 0) error: Invalid argument, type=RTM_NEWROUTE(24), seq=642088, pid=2772610761
Oct 20 14:47:21 vyos-g2 zebra[1021]: [EC 4043309108] LSP Install Failure: in-label 1467
Oct 20 14:48:50 vyos-g2 zebra[1021]: Extended Error: Label >= configured maximum in platform_labels
Oct 20 14:48:50 vyos-g2 zebra[1021]: [EC 4043309093] netlink-dp (NS 0) error: Invalid argument, type=RTM_NEWROUTE(24), seq=642091, pid=2772610761
Oct 20 14:48:50 vyos-g2 zebra[1021]: [EC 4043309108] LSP Install Failure: in-label 860
Oct 20 14:49:21 vyos-g2 zebra[1021]: Extended Error: Label >= configured maximum in platform_labels
Oct 20 14:49:21 vyos-g2 zebra[1021]: [EC 4043309093] netlink-dp (NS 0) error: Invalid argument, type=RTM_NEWROUTE(24), seq=642094, pid=2772610761

Hello @vzotov,

Will be helpful to describe this bug with minimal reproducing steps on the development portal https://phabricator.vyos.net/
Thanks in advance