Wireguard can't reconnect unless vyos router is rebooted

[ACiD_GRiM]

I have a vyos router used for an sdwan to bond 3 connections together. When the client device reboots or its wireguard tunnels are reset, they often cant reconnect even though i see packets in tcpdump on the sdwan vyos side. However no traffic is returned to the client.

Ive found that a simple reboot of the vyos router allows all 3 interfaces to connect instantly. I have another router that acts as a hub in a wireguard hub/spoke WAN for several sites running the same version as this sdwan router that doesnt experience this issue.

The configuration for the sdwan is very simple, just some basic firewall rules, ospf over wireguard and the three wireguard interfaces as well as cake qos for each egress queue. The client to the sdwan router connects to the hub router wireguard through the bonded wireguard tunnels with a smaller mtu, there are no issues reconnecting as long as the sdwan tunnels are up.

Im using 1.4 nightly from 1/30/2023, ive seen some mentions in searching that this is a behavior since 2021, but im hoping theres some insight into how to improve this issue

[n.fort]

Can you share wireguard configuration? If possible, ensure that ports (local and peer ports) are configured.

[ACiD_GRiM]

here is the full configuation, I’m currently re-configuring the qos, so that is the only thing that I’m aware of being in consistent. the client IPs are all dynamic, so im not sure setting the ports will matter…
I also am trying with WAN_v4-to-LOCAL rule 100 with state new enable turned off, thinking that it was related to the connection states. I recently had this issue, and rebooting fixed again, so its consistent with every reboot of the client.

set firewall ipv6-name LOCAL_v6-to-SDWAN default-action 'accept'
set firewall ipv6-name LOCAL_v6-to-WAN default-action 'accept'
set firewall ipv6-name SDWAN_v6-to-LOCAL default-action 'accept'
set firewall ipv6-name SDWAN_v6-to-WAN default-action 'accept'
set firewall ipv6-name WAN_v6-to-LOCAL default-action 'drop'
set firewall ipv6-name WAN_v6-to-LOCAL enable-default-log
set firewall ipv6-name WAN_v6-to-LOCAL rule 9998 action 'accept'
set firewall ipv6-name WAN_v6-to-LOCAL rule 9998 icmpv6 type-name 'nd-neighbor-advert'
set firewall ipv6-name WAN_v6-to-LOCAL rule 9998 protocol 'ipv6-icmp'
set firewall ipv6-name WAN_v6-to-LOCAL rule 9999 action 'accept'
set firewall ipv6-name WAN_v6-to-LOCAL rule 9999 icmpv6 type '134'
set firewall ipv6-name WAN_v6-to-LOCAL rule 9999 protocol 'ipv6-icmp'
set firewall ipv6-name WAN_v6-to-SDWAN default-action 'drop'
set firewall log-martians 'disable'
set firewall name LOCAL_v4-to-SDWAN default-action 'accept'
set firewall name LOCAL_v4-to-WAN default-action 'accept'
set firewall name SDWAN_v4-to-LOCAL default-action 'accept'
set firewall name SDWAN_v4-to-WAN default-action 'accept'
set firewall name WAN_v4-to-LOCAL default-action 'drop'
set firewall name WAN_v4-to-LOCAL enable-default-log
set firewall name WAN_v4-to-LOCAL rule 1 action 'accept'
set firewall name WAN_v4-to-LOCAL rule 1 protocol 'icmp'
set firewall name WAN_v4-to-LOCAL rule 100 action 'accept'
set firewall name WAN_v4-to-LOCAL rule 100 destination port '53,9468,9469'
set firewall name WAN_v4-to-LOCAL rule 100 protocol 'udp'
set firewall name WAN_v4-to-LOCAL rule 1000 action 'accept'
set firewall name WAN_v4-to-LOCAL rule 1000 destination port '13698'
set firewall name WAN_v4-to-LOCAL rule 1000 protocol 'tcp'
set firewall name WAN_v4-to-LOCAL rule 1000 state new 'enable'
set firewall name WAN_v4-to-LOCAL rule 1001 action 'accept'
set firewall name WAN_v4-to-LOCAL rule 1001 destination port '161'
set firewall name WAN_v4-to-LOCAL rule 1001 protocol 'udp'
set firewall name WAN_v4-to-LOCAL rule 1001 source address 'xx'
set firewall name WAN_v4-to-LOCAL rule 1001 state new 'enable'
set firewall name WAN_v4-to-SDWAN default-action 'drop'
set firewall name WAN_v4-to-SDWAN rule 9 action 'accept'
set firewall name WAN_v4-to-SDWAN rule 9 destination port '24680,24681'
set firewall name WAN_v4-to-SDWAN rule 9 protocol 'udp'
set firewall name WAN_v4-to-SDWAN rule 9 state new 'enable'
set firewall name WAN_v4-to-SDWAN rule 10 action 'accept'
set firewall name WAN_v4-to-SDWAN rule 10 destination port '80,443'
set firewall name WAN_v4-to-SDWAN rule 10 protocol 'tcp'
set firewall name WAN_v4-to-SDWAN rule 10 state new 'enable'
set firewall state-policy established action 'accept'
set firewall state-policy invalid action 'drop'
set firewall state-policy invalid log enable
set firewall state-policy related action 'accept'
set firewall zone LOCAL from SDWAN firewall ipv6-name 'SDWAN_v6-to-LOCAL'
set firewall zone LOCAL from SDWAN firewall name 'SDWAN_v4-to-LOCAL'
set firewall zone LOCAL from WAN firewall ipv6-name 'LOCAL_v6-to-WAN'
set firewall zone LOCAL from WAN firewall name 'WAN_v4-to-LOCAL'
set firewall zone LOCAL local-zone
set firewall zone SDWAN from LOCAL firewall ipv6-name 'LOCAL_v6-to-SDWAN'
set firewall zone SDWAN from LOCAL firewall name 'LOCAL_v4-to-SDWAN'
set firewall zone SDWAN from WAN firewall ipv6-name 'WAN_v6-to-SDWAN'
set firewall zone SDWAN from WAN firewall name 'WAN_v4-to-SDWAN'
set firewall zone SDWAN interface 'dum0'
set firewall zone SDWAN interface 'wg0'
set firewall zone SDWAN interface 'wg1'
set firewall zone SDWAN interface 'wg2'
set firewall zone WAN from LOCAL firewall ipv6-name 'LOCAL_v6-to-WAN'
set firewall zone WAN from LOCAL firewall name 'LOCAL_v4-to-WAN'
set firewall zone WAN from SDWAN firewall ipv6-name 'SDWAN_v6-to-WAN'
set firewall zone WAN from SDWAN firewall name 'SDWAN_v4-to-WAN'
set firewall zone WAN interface 'eth0'
set interfaces dummy dum0 address '10.23.22.1/32'
set interfaces ethernet eth0 address 
set interfaces ethernet eth0 address
set interfaces ethernet eth0 ip adjust-mss 'clamp-mss-to-pmtu'
set interfaces ethernet eth0 ipv6 address autoconf
set interfaces ethernet eth0 offload gro
set interfaces ethernet eth0 offload gso
set interfaces ethernet eth0 offload rps
set interfaces loopback lo
set interfaces wireguard wg0 address '10.33.23.1/30'
set interfaces wireguard wg0 ip adjust-mss 'clamp-mss-to-pmtu'
set interfaces wireguard wg0 peer npancwangw01-wan allowed-ips '0.0.0.0/0'
set interfaces wireguard wg0 peer npancwangw01-wan allowed-ips '::/0'
set interfaces wireguard wg0 peer npancwangw01-wan persistent-keepalive '5'
set interfaces wireguard wg0 peer npancwangw01-wan preshared-key.
set interfaces wireguard wg0 peer npancwangw01-wan public-key.
set interfaces wireguard wg0 port '9468'
set interfaces wireguard wg0 private-key.
set interfaces wireguard wg1 address '10.33.23.5/30'
set interfaces wireguard wg1 ip adjust-mss 'clamp-mss-to-pmtu'
set interfaces wireguard wg1 peer npancwangw-wwan allowed-ips '0.0.0.0/0'
set interfaces wireguard wg1 peer npancwangw-wwan allowed-ips '::/0'
set interfaces wireguard wg1 peer npancwangw-wwan persistent-keepalive '5'
set interfaces wireguard wg1 peer npancwangw-wwan preshared-key.
set interfaces wireguard wg1 peer npancwangw-wwan public-key.
set interfaces wireguard wg1 port '53'
set interfaces wireguard wg1 private-key.
set interfaces wireguard wg2 address '10.33.23.9/30'
set interfaces wireguard wg2 ip adjust-mss 'clamp-mss-to-pmtu'
set interfaces wireguard wg2 peer npancwangw-ofdn allowed-ips '0.0.0.0/0'
set interfaces wireguard wg2 peer npancwangw-ofdn allowed-ips '::/0'
set interfaces wireguard wg2 peer npancwangw-ofdn persistent-keepalive '5'
set interfaces wireguard wg2 peer npancwangw-ofdn preshared-key.
set interfaces wireguard wg2 peer npancwangw-ofdn public-key.
set interfaces wireguard wg2 port '9469'
set interfaces wireguard wg2 private-key.
set nat destination rule 9 destination port '24680,24681'
set nat destination rule 9 inbound-interface 'eth0'
set nat destination rule 9 protocol 'udp'
set nat destination rule 9 translation address '10.23.32.1'
set nat destination rule 10 destination port '80,443'
set nat destination rule 10 inbound-interface 'eth0'
set nat destination rule 10 protocol 'tcp'
set nat destination rule 10 translation address '10.23.32.1'
set nat source rule 10 outbound-interface 'eth0'
set nat source rule 10 source address '10.0.0.0/8'
set nat source rule 10 translation address 'masquerade'
set nat66 source rule 10 outbound-interface 'eth0'
set nat66 source rule 10 translation address 'masquerade'
set protocols ospf interface dum0 area '0.0.0.5'
set protocols ospf interface wg0 area '0.0.0.0'
set protocols ospf interface wg0 authentication md5 key-id 1 md5-key.
set protocols ospf interface wg0 cost '1'
set protocols ospf interface wg0 dead-interval '4'
set protocols ospf interface wg0 hello-interval '1'
set protocols ospf interface wg1 area '0.0.0.0'
set protocols ospf interface wg1 authentication md5 key-id 1 md5-key.
set protocols ospf interface wg1 cost '5'
set protocols ospf interface wg1 hello-interval '1'
set protocols ospf interface wg1 hello-multiplier '2'
set protocols ospf interface wg1 passive disable
set protocols ospf interface wg2 area '0.0.0.0'
set protocols ospf interface wg2 authentication md5 key-id 1 md5-key.
set protocols ospf interface wg2 cost '10'
set protocols ospf interface wg2 dead-interval '600'
set protocols ospf interface wg2 hello-interval '1'
set protocols ospf interface wg2 passive disable
set protocols ospf passive-interface 'default'
set protocols ospfv3 interface dum0 area '0.0.0.5'
set protocols ospfv3 interface eth0 passive
set protocols ospfv3 interface wg0 area '0.0.0.0'
set protocols ospfv3 interface wg0 dead-interval '2'
set protocols ospfv3 interface wg0 hello-interval '1'
set protocols ospfv3 interface wg1 area '0.0.0.0'
set protocols ospfv3 interface wg1 cost '10'
set protocols ospfv3 interface wg1 dead-interval '2'
set protocols ospfv3 interface wg1 hello-interval '1'
set protocols ospfv3 interface wg2 area '0.0.0.0'
set protocols ospfv3 interface wg2 cost '100'
set protocols ospfv3 interface wg2 dead-interval '600'
set protocols ospfv3 interface wg2 hello-interval '1'
set protocols static route 0.0.0.0/0 next-hop 
set protocols static route6 ::/0 next-hop 
set qos interface eth0 egress 'eth0_symm'
set qos interface wg2 egress 'wg2_egress'
set qos policy cake eth0_symm bandwidth '1gbit'
set qos policy cake vbond0_egress bandwidth '5mbit'
set qos policy cake vbond0_egress flow-isolation flow
set qos policy cake vbond0_ingress bandwidth '5mbit'
set qos policy cake vbond0_ingress flow-isolation flow
set qos policy cake wg2_egress bandwidth '5mbit'
set qos policy cake wg2_egress flow-isolation flow
set qos policy cake wg2_ingress bandwidth '5mbit'
set qos policy cake wg2_ingress flow-isolation flow
set service ntp server pool.ntp.org
set service snmp v3 engineid ''
set service snmp v3 group monitor mode 'ro'
set service snmp v3 group monitor seclevel 'priv'
set service snmp v3 group monitor view 'default'
set service snmp v3 user zabbix auth encrypted-password ''
set service snmp v3 user zabbix auth type 'sha'
set service snmp v3 user zabbix group 'monitor'
set service snmp v3 user zabbix mode 'ro'
set service snmp v3 user zabbix privacy encrypted-password ''
set service snmp v3 user zabbix privacy type 'aes'
set service snmp v3 view default oid 1
set service ssh listen-address '0.0.0.0'
set service ssh listen-address '::'
set service ssh port '13698'
set system config-management commit-revisions '100'
set system conntrack modules ftp
set system conntrack modules h323
set system conntrack modules nfs
set system conntrack modules pptp
set system conntrack modules sip
set system conntrack modules sqlnet
set system conntrack modules tftp
set system console device ttyS0 speed '115200'
set system domain-name '0xcbf.net'
set system host-name 'nvrnsdwangw01'
set system ip multipath layer4-hashing
set system login banner pre-login '* * * * * * * * * ■■■■ OFF WERE FULL * * * * * * * * * * *\nTHIS SYSTEM IS RESTRICTED TO AUTHORIZED USERS FOR AUTHORIZED\nUSE ONLY. UNAUTHORIZED ACCESS IS STRICTLY PROHIBITED AND MAY\nBE PUNISHABLE UNDER THE COMPUTER FRAUD AND ABUSE ACT OF 1986\nOR OTHER APPLICABLE LAWS. IF NOT AUTHORIZED TO ACCESS THIS\nSYSTEM, DISCONNECT NOW.\n'
set system login user admin authentication encrypted-password 'xx'
set system login user admin authentication otp key ''
set system name-server '1.1.1.1'
set system name-server '1.0.0.1'
set system option performance 'latency'
set system option reboot-on-panic
set system option root-partition-auto-resize
set system syslog global facility all level 'info'
set system syslog global facility protocols level 'notice'
set system time-zone 'America/Los_Angeles'

[ACiD_GRiM]

Any thoughts on this?

Ive updated the vyos router recently to a nightly build and it still needs to be rebooted to allow the client to connect over any of the wireguard tunnels.

My theory right now remains that theres some conntrack state that is maintaining the UDP connection from the original dynamic IPs so when a new connection from the same source port but from a new IP, the connection isnt established.
If this is true, im not sure how to decrease the state hold open time.

[pepe]

Try to remove persistent-keepalive option from peers configuration.

[ACiD_GRiM]

This seems to improve it, but it also takes 5 or more minutes for it to begin to reconnect, but immediate after rebooting the vyos router

[pepe]

Did you remove persistent-keepalive option on client side too ?

Do you use state new enable when allowing inbound wireguard connection ?
It can be conntrack timeout, default for UDP is:

net.netfilter.nf_conntrack_udp_timeout = 30
net.netfilter.nf_conntrack_udp_timeout_stream = 180