Hi there,
I’m trying to setup an ipsec tunnel to Cloudflare the tunnel seems to be up but I’m facing a few issues:
- Health checks are failing
- I can’t ping the other side of the tunnel
- Routing any traffic to the vti loses connection for the specifc subnet or IP that I send to it via PBR
- Lots of packet drops on vti0
Here’s my config:
container {
name pihole {
cap-add net-admin
environment WEBPASSWORD {
value xxx
}
host-name xxxxxx
image pihole/pihole:2023.05.2
memory 1024
network pihole {
address xxx.xxx.0.2
}
port tcp {
destination 53
protocol tcp
source 5053
}
port udp {
destination 53
protocol udp
source 5053
}
port webui {
destination 80
protocol tcp
source 8080
}
restart on-failure
volume dnsmasq {
destination /etc/dnsmasq.d
source /home/erfi/pihole/etc-dnsmasq.d
}
volume pihole {
destination /etc/pihole
source /home/erfi/pihole/etc-pihole
}
}
name unbound {
host-name xxxxxx
image klutchell/unbound:v1.17.1
network pihole {
address xxx.xxx.0.3
}
port tcp {
destination 53
protocol tcp
source 5054
}
port udp {
destination 53
protocol udp
source 5054
}
volume unbound {
destination /etc/unbound/unbound.conf
mode rw
source /home/erfi/unbound/unbound.conf
}
}
network pihole {
description pihole
prefix xxx.xxx.0.0/16
}
}
firewall {
all-ping enable
broadcast-ping disable
config-trap disable
group {
ipv6-network-group cf-ipv6 {
network xxxx:xxxx::/32
network xxxx:xxxx::/32
network xxxx:xxxx::/32
network xxxx:xxxx::/32
network xxxx:xxxx::/32
network xxxx:xxxx::/29
network xxxx:xxxx::/32
}
network-group cf-ipv4 {
network xxx.xxx.48.0/20
network xxx.xxx.244.0/22
network xxx.xxx.200.0/22
network xxx.xxx.4.0/22
network xxx.xxx.64.0/18
network xxx.xxx.192.0/18
network xxx.xxx.240.0/20
network xxx.xxx.96.0/20
network xxx.xxx.240.0/22
network xxx.xxx.128.0/17
network xxx.xxx.0.0/15
network xxx.xxx.0.0/13
network xxx.xxx.0.0/14
network xxx.xxx.0.0/13
network xxx.xxx.72.0/22
}
}
interface eth1.100 {
in {
name VLAN-100
}
}
interface pppoe0 {
in {
name EXTERNAL-IN
}
local {
name EXTERNAL-LOCAL
}
}
ipv6-receive-redirects disable
ipv6-src-route disable
ip-src-route disable
log-martians enable
name EXTERNAL-IN {
default-action drop
enable-default-log
rule 10 {
action accept
state {
established enable
related enable
}
}
}
name EXTERNAL-LOCAL {
default-action drop
enable-default-log
rule 10 {
action accept
log enable
state {
established enable
related enable
}
}
rule 20 {
action accept
icmp {
type-name echo-request
}
log enable
protocol icmp
state {
new enable
}
}
rule 30 {
action drop
description ssh
destination {
port 22
}
log enable
protocol tcp
recent {
count 20
time minute
}
state {
new enable
}
}
rule 31 {
action accept
destination {
port 22
}
log enable
protocol tcp
state {
new enable
}
}
rule 40 {
action accept
description magic-wan
log enable
protocol gre
source {
group {
network-group cf-ipv4
}
}
}
rule 41 {
action accept
description magic-wan-ipsec
log enable
protocol esp
source {
group {
network-group cf-ipv4
}
}
}
rule 50 {
action accept
icmp {
type-name echo-reply
}
log enable
protocol icmp
}
}
name VLAN-100 {
default-action accept
enable-default-log
rule 10 {
action accept
log enable
state {
established enable
related enable
}
}
rule 50 {
action drop
description "Restrict Access to INTERNAL1 network"
destination {
address xxx.xxx.69.0/24
}
}
}
receive-redirects disable
send-redirects enable
source-validation disable
syn-cookies enable
}
interfaces {
ethernet eth0 {
description EXTERNAL
duplex auto
hw-id xx:xx:xx:xx:xx:71
offload {
gro
gso
rps
sg
tso
}
speed auto
vif 6 {
}
}
ethernet eth1 {
address xxx.xxx.69.1/24
description INTERNAL1
duplex auto
hw-id xx:xx:xx:xx:xx:72
ip {
arp-cache-timeout 30
}
offload {
gro
gso
rps
sg
tso
}
speed auto
vif 100 {
address xxx.xxx.70.1/24
description beryl
}
}
ethernet eth2 {
hw-id xx:xx:xx:xx:xx:73
}
ethernet eth3 {
hw-id xx:xx:xx:xx:xx:74
}
ethernet eth4 {
hw-id xx:xx:xx:xx:xx:75
}
ethernet eth5 {
hw-id xx:xx:xx:xx:xx:76
}
loopback lo {
}
pppoe pppoe0 {
authentication {
password xxxxxx
username xxxxxx
}
description kpn
ip {
adjust-mss clamp-mss-to-pmtu
}
no-peer-dns
source-interface eth0.6
}
tunnel tun0 {
address xxx.xxx.99.20/31
description magic-wan
encapsulation gre
ip {
adjust-mss clamp-mss-to-pmtu
}
remote xxx.xxx.66.5
source-address xxx.xxx.81.42
}
vti vti0 {
address xxx.xxx.100.20/31
address xxx.xxx.240.252/31
description magic-wan-ipsec
ip {
adjust-mss clamp-mss-to-pmtu
}
}
}
nat {
source {
rule 100 {
description pppoe
log
outbound-interface pppoe0
source {
address xxx.xxx.0.0/16
}
translation {
address masquerade
}
}
}
}
policy {
route magic-wan-gre {
enable-default-log
interface eth1.100
rule 100 {
description magic-wan-gre
destination {
port 80,443
}
log enable
protocol tcp_udp
set {
table 100
}
source {
address xxx.xxx.70.10
}
}
}
route magic-wan-ipsec {
enable-default-log
rule 200 {
description magic-wan-ipsec
destination {
port 80,443
}
log enable
protocol tcp_udp
set {
table 200
}
source {
address xxx.xxx.70.10
}
}
}
route magic-wan-ipsec-health-checks {
enable-default-log
rule 300 {
description magic-wan-ipsec-health-checks
log enable
set {
table 200
}
source {
address xxx.xxx.240.253
}
}
}
}
protocols {
static {
table 100 {
route xxx.xxx.0.0/0 {
interface tun0 {
}
}
}
table 200 {
route xxx.xxx.0.0/0 {
interface vti0 {
}
}
}
}
}
service {
dhcp-server {
shared-network-name xxxxxx {
subnet xxx.xxx.69.0/24 {
default-router xxx.xxx.69.1
domain-name xxxxxx
lease 300
name-server xxx.xxx.69.1
name-server xxx.xxx.1.1
name-server xxx.xxx.8.8
range 0 {
start xxx.xxx.69.2
stop xxx.xxx.69.254
}
static-mapping xxxxxx {
ip-address xxx.xxx.69.3
mac-address xx:xx:xx:xx:xx:b6
}
static-mapping xxxxxx {
ip-address xxx.xxx.69.4
mac-address xx:xx:xx:xx:xx:9e
}
}
}
shared-network-name xxxxxx {
subnet xxx.xxx.70.0/24 {
default-router xxx.xxx.70.1
domain-name xxxxxx
lease 300
name-server xxx.xxx.70.1
name-server xxx.xxx.1.1
name-server xxx.xxx.8.8
range 0 {
start xxx.xxx.70.2
stop xxx.xxx.70.254
}
static-mapping xxxxxx {
ip-address xxx.xxx.70.2
mac-address xx:xx:xx:xx:xx:b1
}
}
}
}
dns {
forwarding {
allow-from xxx.xxx.0.0/16
cache-size 0
listen-address xxx.xxx.69.1
listen-address xxx.xxx.70.1
name-server xxx.xxx.0.2 {
port 53
}
}
}
ntp {
allow-client xxxxxx
address xxx.xxx.0.0/0
address ::/0
}
server xxxxx.tld {
}
server xxxxx.tld {
}
server xxxxx.tld {
}
server xxxxx.tld {
}
server xxxxx.tld {
}
server xxxxx.tld {
}
server xxxxx.tld {
}
}
ssh {
disable-password-authentication
loglevel verbose
port 22
}
}
system {
config-management {
commit-revisions 100
}
conntrack {
modules {
ftp
h323
pptp
sip
sqlnet
tftp
}
}
console {
device ttyS0 {
speed 115200
}
}
host-name xxxxxx
login {
user xxxxxx {
authentication {
encrypted-password xxxxxx
public-keys xxxx@xxx.xxx {
key xxxxxx
type ecdsa-sha2-nistp256
}
}
}
}
name-server xxx.xxx.69.1
name-server xxx.xxx.70.1
name-server xxx.xxx.1.1
name-server xxx.xxx.8.8
sysctl {
parameter net.ipv4.conf.all.accept_local {
value 1
}
parameter net.ipv4.conf.all.accept_redirects {
value 0
}
parameter net.ipv4.conf.all.rp_filter {
value 0
}
parameter net.ipv4.conf.all.send_redirects {
value 0
}
parameter net.ipv4.ip_forward {
value 1
}
}
syslog {
global {
facility all {
level all
}
facility local7 {
level all
}
}
}
time-zone Europe/Amsterdam
}
vpn {
ipsec {
authentication {
psk cf-ipsec {
id xxx
id xxx.xxx.66.5
secret xxx
}
}
esp-group vyos-nl-esp {
lifetime 14400
mode tunnel
pfs enable
proposal 1 {
encryption aes256gcm128
hash sha512
}
}
ike-group vyos-nl-ike {
close-action restart
dead-peer-detection {
action restart
interval 30
timeout 120
}
disable-mobike
key-exchange ikev2
lifetime 14400
proposal 1 {
dh-group 14
encryption aes256gcm128
hash sha512
}
}
interface pppoe0
log {
level 2
subsystem any
}
options {
disable-route-autoinstall
interface vti0
}
site-to-site {
peer magic-wan-ipsec {
authentication {
local-id xxx
mode pre-shared-secret
remote-id xxx.xxx.66.5
}
connection-type initiate
ike-group vyos-nl-ike
ikev2-reauth no
local-address xxx.xxx.81.42
remote-address xxx.xxx.66.5
vti {
bind vti0
esp-group vyos-nl-esp
}
}
}
}
}
magic-wan-ipsec: #5, ESTABLISHED, IKEv2, ee0984a3249a6df7_i* 4fcda780592b9275_r
local 'xxx' @ 195.240.xx.xx[500]
remote '162.159.xx.xx' @ 162.159.xx.xx[500]
AES_GCM_16-256/PRF_HMAC_SHA2_512/MODP_2048
established 9800s ago, rekeying in 4057s
magic-wan-ipsec-vti: #6, reqid 1, INSTALLED, TUNNEL, ESP:AES_GCM_16-256/MODP_2048
installed 534s ago, rekeying in 11421s, expires in 13867s
in cec5fb76 (-|0x00000001), 76188 bytes, 907 packets, 1s ago
out 2f2bf71b (-|0x00000001), 0 bytes, 0 packets
local 0.0.0.0/0
remote 0.0.0.0/0
magic-wan-ipsec: IKEv2, no reauthentication, rekeying every 14400s, dpd delay 30s
local: 195.240.xx.xx
remote: 162.159.xx.xx
local pre-shared key authentication:
id: xxx
remote pre-shared key authentication:
id: 162.159.xx.xx
magic-wan-ipsec-vti: TUNNEL, rekeying every 13090s, dpd action is start
local: 0.0.0.0/0 ::/0
remote: 0.0.0.0/0 ::/0
src 195.240.xx.xx dst 162.159.xx.xx
proto esp spi 0x2f2bf71b reqid 1 mode tunnel
replay-window 0 flag af-unspec
aead rfc4106(gcm(aes)) 0x5057d68b01ba0ccd97f690a91b189a0a0bd3e2d69df97f76bb72d009ceb356b216b8119c 128
anti-replay context: seq 0x0, oseq 0x0, bitmap 0x00000000
if_id 0x1
src 162.159.xx.xxdst 195.240.xx.xx
proto esp spi 0xcec5fb76 reqid 1 mode tunnel
replay-window 32 flag af-unspec
aead rfc4106(gcm(aes)) 0x140702e81d057e745c069864cbc41a041ae700600787c2ce3716d0fed494bd3e2c10883b 128
anti-replay context: seq 0x424, oseq 0x0, bitmap 0xffffffff
if_id 0x1
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
RX: bytes packets errors dropped missed mcast
9667800 130346 0 0 0 0
TX: bytes packets errors dropped carrier collsns
9667800 130346 0 0 0 0
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:71 brd ff:ff:ff:ff:ff:ff
alias EXTERNAL
RX: bytes packets errors dropped missed mcast
10822862039 42967673 0 0 0 2075
TX: bytes packets errors dropped carrier collsns
2896380958 20857480 0 0 0 0
3: eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:72 brd ff:ff:ff:ff:ff:ff
alias INTERNAL1
RX: bytes packets errors dropped missed mcast
927862999 3097526 0 0 0 91538
TX: bytes packets errors dropped carrier collsns
5302664344 5776260 0 0 0 0
4: eth2: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:73 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
5: eth3: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:74 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
6: eth4: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:75 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
7: eth5: <NO-CARRIER,BROADCAST,MULTICAST,UP> mtu 1500 qdisc mq state DOWN mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:76 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
8: eth0.6@eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:71 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
10049333265 42965598 0 0 0 0
TX: bytes packets errors dropped carrier collsns
2812723592 20854709 0 0 0 0
9: eth1.100@eth1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
link/ether 34:1a:4c:03:a6:72 brd ff:ff:ff:ff:ff:ff
alias beryl
RX: bytes packets errors dropped missed mcast
391915451 981277 0 0 0 8685
TX: bytes packets errors dropped carrier collsns
2793667711 2872176 0 0 0 0
10: gre0@NONE: <NOARP> mtu 1476 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/gre 0.0.0.0 brd 0.0.0.0
RX: bytes packets errors dropped missed mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
11: gretap0@NONE: <BROADCAST,MULTICAST> mtu 1462 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
12: erspan0@NONE: <BROADCAST,MULTICAST> mtu 1450 qdisc noop state DOWN mode DEFAULT group default qlen 1000
link/ether 00:00:00:00:00:00 brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
0 0 0 0 0 0
TX: bytes packets errors dropped carrier collsns
0 0 0 0 0 0
13: tun0@NONE: <POINTOPOINT,NOARP,UP,LOWER_UP> mtu 1476 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/gre 195.240.81.42 peer 162.159.66.5
alias magic-wan
RX: bytes packets errors dropped missed mcast
1568273467 17793589 0 0 0 0
TX: bytes packets errors dropped carrier collsns
5995334 36870 1 2 0 0
14: vti0@NONE: <NOARP,UP,LOWER_UP> mtu 1500 qdisc noqueue state UNKNOWN mode DEFAULT group default qlen 1000
link/none
alias magic-wan-ipsec
RX: bytes packets errors dropped missed mcast
11882859 141438 17652521 17652521 0 0
TX: bytes packets errors dropped carrier collsns
1335922 15388 0 0 0 0
15: pppoe0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1492 qdisc pfifo_fast state UNKNOWN mode DEFAULT group default qlen 3
link/ppp
alias kpn
RX: bytes packets errors dropped missed mcast
9705337212 42958375 0 0 0 0
TX: bytes packets errors dropped carrier collsns
2353861564 20847478 0 0 0 0
16: pod-pihole: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP mode DEFAULT group default qlen 1000
link/ether 2a:5c:28:20:1b:3a brd ff:ff:ff:ff:ff:ff
RX: bytes packets errors dropped missed mcast
39586041 101362 0 0 0 76
TX: bytes packets errors dropped carrier collsns
31342324 126141 0 0 0 0
17: veth0@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master pod-pihole state UP mode DEFAULT group default qlen 1000
link/ether d6:15:82:e7:58:c9 brd ff:ff:ff:ff:ff:ff link-netns netns-971685e3-f44c-6c26-17ec-b7d8ee824974
RX: bytes packets errors dropped missed mcast
36776399 55005 0 0 0 0
TX: bytes packets errors dropped carrier collsns
20434772 80347 0 0 0 0
18: veth1@if5: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master pod-pihole state UP mode DEFAULT group default qlen 1000
link/ether 2a:5c:28:20:1b:3a brd ff:ff:ff:ff:ff:ff link-netns netns-233dd149-050f-405e-de92-4e3e95cb5eef
RX: bytes packets errors dropped missed mcast
6727869 66118 0 0 0 0
TX: bytes packets errors dropped carrier collsns
13412533 65636 0 0 0 0
0: from all lookup local
100: from all fwmark 0x7fffff9b lookup 100
200: from all fwmark 0x7fffff37 lookup 200
220: from all lookup 220
32766: from all lookup main
32767: from all lookup default
default nhid 30 dev pppoe0 proto static metric 20
10.68.xx.xx/24 dev eth1 proto kernel scope link src 10.68.xx.xx
10.68.xx.xx/24 dev eth1.100 proto kernel scope link src 10.68.xx.xx
10.68.xx.xx/31 dev tun0 proto kernel scope link src 10.68.xx.xx
10.68.xx.xx/31 dev vti0 proto kernel scope link src 10.68.xx.xx
172.20.xx.xx/16 dev pod-pihole proto kernel scope link src 172.20.xx.xx
172.64.xx.xx/31 dev vti0 proto kernel scope link src 172.64.xx.xx
195.190.xx.xxdev pppoe0 proto kernel scope link src 195.240.xx.xx
sudo ip r show table 220
Error: ipv4: FIB table does not exist.
Dump terminated
sudo journalctl /usr/lib/ipsec/charon > /tmp/charon.log
erfi@vyos:~$ cat /tmp/charon.log
-- No entries --
Let me know if you need more info