I am trying to setup a VXLAN to work over IPSec across the public internet. I am using two virtual Vyos Routers running VyOS 1.3.0-rc6 and followed this guide https://www.datai.net/article/vyos-encrypted-vxlan-tunneling/
Quick and dirty network diagram
Device A → vyos-rtr-1 ->UDM Pro Edge Router (NAT) → INTERNET <-vyos-rtr-2 ← Device B
So device A is uses vyos-rtr-1 which has a private IP that gets a NAT from the UDM Pro. All the tunnels are up and look good. I can ping between the GRE tunnel interfaces, I can see MAC addresses across the GRE Tunnels from either side. The problem I seem to be having is that traffic from Device A is getting dropped I think on the br0 interface of the vyos-rtr-1 interface.
So for example Device B sends out a broadcast, Device A receives the broadcast and sends a response. I can see the response hit the eth1 interface, br0 interface on the vyos-rtr-1 using “monitor traffic interface br0”. However when I check “monitoring traffic interface tun0” I do not see the response anymore. Like wise if I do the same packet captures on the vyos-rtr-2 I do not see the Device A to Device B response. I am not seeing drops on the tunnel interfaces. This is also broader than just this communication, for example I have a DHCP server on the A side, DHCP Server A will respond with offers to servers on B side, however those servers will never receive the DHCP-offer and thus never assign themselves the IP address.
So basically it looks like I have one way traffic. From B to A works, but A to B doesnt work. The frames im trying to send are pretty small, only 60 bytes.
vyos-rtr-1:~$ show bridge br0 macs
port no mac addr is local? ageing timer
1 00:0c:29:5c:5e:75 yes 0.00
1 00:0c:29:5c:5e:75 yes 0.00
2 26:8b:78:16:44:a1 yes 0.00
2 26:8b:78:16:44:a1 yes 0.00
1 3c:7c:3f:ee:22:55 no 29.73
2 52:54:00:21:4b:f7 no 41.94
2 52:54:00:28:b8:6e no 52.07
2 52:54:00:3a:0d:5e no 3.33
1 68:d7:9a:21:af:bc no 0.42
1 78:45:58:6d:e7:be no 48.01
2 ce:f8:9f:b5:46:e4 no 43.90
vyos-rtr-2:~$ show bridge br0 macs
port no mac addr is local? ageing timer
2 3c:7c:3f:ee:22:55 no 57.87
1 52:54:00:21:4b:f7 yes 0.00
1 52:54:00:21:4b:f7 yes 0.00
1 52:54:00:28:b8:6e no 7.00
1 52:54:00:3a:0d:5e no 1.29
2 68:d7:9a:21:af:bc no 0.75
2 78:45:58:6d:e7:be no 76.15
2 ce:0e:c6:6c:fd:31 yes 0.00
2 ce:0e:c6:6c:fd:31 yes 0.00
1 ce:f8:9f:b5:46:e4 no 72.05
1 fe:54:00:21:4b:f7 no 25.05
xxxx@vyos-rtr-1:~$ show conf com
set interfaces bridge br0 member interface eth1
set interfaces bridge br0 member interface vxlan4000
set interfaces dummy dum0 address ‘10.10.0.1/32’
set interfaces ethernet eth0 address ‘10.10.9.10/24’
set interfaces ethernet eth0 hw-id ‘00:0c:29:5c:5e:6b’
set interfaces ethernet eth1 hw-id ‘00:0c:29:5c:5e:75’
set interfaces loopback lo
set interfaces tunnel tun0 address ‘10.0.0.1/30’
set interfaces tunnel tun0 encapsulation ‘gre’
set interfaces tunnel tun0 mtu ‘1500’
set interfaces tunnel tun0 multicast ‘disable’
set interfaces tunnel tun0 remote ‘10.10.0.2’
set interfaces tunnel tun0 source-address ‘10.10.0.1’
set interfaces vxlan vxlan4000 group ‘239.0.0.241’
set interfaces vxlan vxlan4000 mtu ‘1300’
set interfaces vxlan vxlan4000 port ‘4789’
set interfaces vxlan vxlan4000 source-interface ‘tun0’
set interfaces vxlan vxlan4000 vni ‘4000’
set vpn ipsec esp-group aes256-sha256 compression ‘disable’
set vpn ipsec esp-group aes256-sha256 lifetime ‘28800’
set vpn ipsec esp-group aes256-sha256 mode ‘tunnel’
set vpn ipsec esp-group aes256-sha256 pfs ‘dh-group19’
set vpn ipsec esp-group aes256-sha256 proposal 1 encryption ‘aes256’
set vpn ipsec esp-group aes256-sha256 proposal 1 hash ‘sha256’
set vpn ipsec ike-group aes256-sha256 dead-peer-detection action ‘restart’
set vpn ipsec ike-group aes256-sha256 dead-peer-detection interval ‘30’
set vpn ipsec ike-group aes256-sha256 dead-peer-detection timeout ‘120’
set vpn ipsec ike-group aes256-sha256 ikev2-reauth ‘no’
set vpn ipsec ike-group aes256-sha256 key-exchange ‘ikev2’
set vpn ipsec ike-group aes256-sha256 lifetime ‘3600’
set vpn ipsec ike-group aes256-sha256 proposal 1 dh-group ‘19’
set vpn ipsec ike-group aes256-sha256 proposal 1 encryption ‘aes256’
set vpn ipsec ike-group aes256-sha256 proposal 1 hash ‘sha256’
set vpn ipsec ipsec-interfaces interface ‘eth0’
set vpn ipsec site-to-site peer xxx.xxx.152.7 authentication mode ‘pre-shared-secret’
set vpn ipsec site-to-site peer xxx.xxx.152.7 authentication pre-shared-secret ‘xxxxxxxxxxxx’
set vpn ipsec site-to-site peer xxx.xxx.152.7 authentication remote-id ‘xxx.xxx.152.7’
set vpn ipsec site-to-site peer xxx.xxx.152.7 connection-type ‘initiate’
set vpn ipsec site-to-site peer xxx.xxx.152.7 ike-group ‘aes256-sha256’
set vpn ipsec site-to-site peer xxx.xxx.152.7 ikev2-reauth ‘inherit’
set vpn ipsec site-to-site peer xxx.xxx.152.7 local-address ‘10.10.9.10’
set vpn ipsec site-to-site peer xxx.xxx.152.7 tunnel 1 allow-nat-networks ‘disable’
set vpn ipsec site-to-site peer xxx.xxx.152.7 tunnel 1 allow-public-networks ‘disable’
set vpn ipsec site-to-site peer xxx.xxx.152.7 tunnel 1 esp-group ‘aes256-sha256’
set vpn ipsec site-to-site peer xxx.xxx.152.7 tunnel 1 local prefix ‘10.10.0.1/32’
set vpn ipsec site-to-site peer xxx.xxx.152.7 tunnel 1 protocol ‘gre’
set vpn ipsec site-to-site peer xxx.xxx.152.7 tunnel 1 remote prefix ‘10.10.0.2/32’
xxxx@vyos-rtr-2:~$ show conf com
set interfaces bridge br0 member interface eth1
set interfaces bridge br0 member interface vxlan4000
set interfaces dummy dum0 address ‘10.10.0.2/32’
set interfaces ethernet eth0 address ‘xxx.xxx.152.7/28’
set interfaces ethernet eth0 hw-id ‘52:54:00:bc:e7:e8’
set interfaces ethernet eth1 hw-id ‘52:54:00:21:4b:f7’
set interfaces loopback lo
set interfaces tunnel tun0 address ‘10.0.0.2/30’
set interfaces tunnel tun0 encapsulation ‘gre’
set interfaces tunnel tun0 mtu ‘1500’
set interfaces tunnel tun0 multicast ‘disable’
set interfaces tunnel tun0 remote ‘10.10.0.1’
set interfaces tunnel tun0 source-address ‘10.10.0.2’
set interfaces vxlan vxlan4000 group ‘239.0.0.241’
set interfaces vxlan vxlan4000 mtu ‘1300’
set interfaces vxlan vxlan4000 port ‘4789’
set interfaces vxlan vxlan4000 source-interface ‘tun0’
set interfaces vxlan vxlan4000 vni ‘4000’
set vpn ipsec esp-group aes256-sha256 compression ‘disable’
set vpn ipsec esp-group aes256-sha256 lifetime ‘28800’
set vpn ipsec esp-group aes256-sha256 mode ‘tunnel’
set vpn ipsec esp-group aes256-sha256 pfs ‘dh-group19’
set vpn ipsec esp-group aes256-sha256 proposal 1 encryption ‘aes256’
set vpn ipsec esp-group aes256-sha256 proposal 1 hash ‘sha256’
set vpn ipsec ike-group aes256-sha256 dead-peer-detection action ‘restart’
set vpn ipsec ike-group aes256-sha256 dead-peer-detection interval ‘30’
set vpn ipsec ike-group aes256-sha256 dead-peer-detection timeout ‘120’
set vpn ipsec ike-group aes256-sha256 ikev2-reauth ‘no’
set vpn ipsec ike-group aes256-sha256 key-exchange ‘ikev2’
set vpn ipsec ike-group aes256-sha256 lifetime ‘3600’
set vpn ipsec ike-group aes256-sha256 proposal 1 dh-group ‘19’
set vpn ipsec ike-group aes256-sha256 proposal 1 encryption ‘aes256’
set vpn ipsec ike-group aes256-sha256 proposal 1 hash ‘sha256’
set vpn ipsec ipsec-interfaces interface ‘eth0’
set vpn ipsec site-to-site peer xxx.xxx.74.143 authentication mode ‘pre-shared-secret’
set vpn ipsec site-to-site peer xxx.xxx.74.143 authentication pre-shared-secret ‘xxxxxxxx’
set vpn ipsec site-to-site peer xxx.xxx.74.143 authentication remote-id ‘10.10.9.10’
set vpn ipsec site-to-site peer xxx.xxx.74.143 connection-type ‘initiate’
set vpn ipsec site-to-site peer xxx.xxx.74.143 ike-group ‘aes256-sha256’
set vpn ipsec site-to-site peer xxx.xxx.74.143 ikev2-reauth ‘inherit’
set vpn ipsec site-to-site peer xxx.xxx.74.143 local-address ‘xxx.xxx.152.7’
set vpn ipsec site-to-site peer xxx.xxx.74.143 tunnel 1 allow-nat-networks ‘disable’
set vpn ipsec site-to-site peer xxx.xxx.74.143 tunnel 1 allow-public-networks ‘disable’
set vpn ipsec site-to-site peer xxx.xxx.74.143 tunnel 1 esp-group ‘aes256-sha256’
set vpn ipsec site-to-site peer xxx.xxx.74.143 tunnel 1 local prefix ‘10.10.0.2/32’
set vpn ipsec site-to-site peer xxx.xxx.74.143 tunnel 1 protocol ‘gre’
set vpn ipsec site-to-site peer xxx.xxx.74.143 tunnel 1 remote prefix ‘10.10.0.1/32’