Traffic through gre/ipsec tunnel stops on enabling wan loadbalancing

mafiosa · December 10, 2020, 8:39pm

site 1 and site 1 are connected to VyOS box. VyOS box is connected to a lan router (mikrotik with ospf) and it has three lans. I cannot access the routes learnt by ospf over the tunnels. They appear on the routing table but on doing traceroute I can see the traffic leaving through WAN interface. Load balancing is enabled.
Here is my vyos config.
=~=~=~=~=~=~=~=~=~=~=~= PuTTY log 2020.12.11 02:05:57 =~=~=~=~=~=~=~=~=~=~=~=
show config

interfaces {
bridge br700 {
aging 300
description osixp
hello-time 2
max-age 20
priority 32768
stp false
}
ethernet eth0 {
address 10.28.115.18/30
duplex auto
hw-id 6c:3b:6b:69:69:69
smp-affinity auto
speed auto
}
ethernet eth1 {
address 172.18.108.142/24
duplex auto
hw-id 6c:3b:6b:96:96:96
smp-affinity auto
speed auto
}
ethernet eth2 {
duplex auto
smp-affinity auto
speed auto
vif 400 {
address 192.168.168.1/30
}
vif 700 {
bridge-group {
bridge br700
}
description osixp
}
}
loopback lo {
address 192.168.254.1/32
}
tunnel tun4 {
address 192.168.168.5/30
encapsulation gre
ip {
ospf {
dead-interval 40
hello-interval 10
network broadcast
priority 1
retransmit-interval 5
transmit-delay 1
}
}
local-ip 10.28.115.18
mtu 1422
multicast disable
remote-ip 10.14.96.109
}
tunnel tun20 {
address 192.168.168.21/30
encapsulation gre
ip {
:
ospf {
:
dead-interval 40
:
hello-interval 10
:
network broadcast
:
priority 1
:
retransmit-interval 5
:
transmit-delay 1
:
}
:
}
:
local-ip 10.28.115.18
:
mtu 1422
:
multicast disable
:
remote-ip 10.14.94.232
:
}
:
vxlan vxlan3 {
:
bridge-group {
:
bridge br700
:
}
:
link eth2.400
:
remote 192.168.72.8
:
remote-port 4789
:
vni 3
:
}
:
}
:
load-balancing {
:
wan {
:
interface-health eth0 {
:
failure-count 5
:
nexthop 10.28.115.17
:
success-count 5
:
test 10 {
:
resp-time 5
:
target 4.2.2.2
:
ttl-limit 1
:
type ping
:
}
:
}
:
interface-health eth1 {
:
failure-count 5
:
nexthop 172.18.108.1
:
success-count 5
:
test 10 {
:
resp-time 5
:
target 4.2.2.1
:
ttl-limit 1
:
type ping
:
}
:
}
:
rule 1 {
:
inbound-interface eth2.400
:
interface eth0 {
:
weight 2
:
}
:
interface eth1 {
:
weight 1
:
}
:
protocol all
:
}
:
sticky-connections {
:
inbound
:
}
:
}
:
}
:
nat {
:
destination {
:
rule 10 {
:
description vxlan-fw
:
destination {
:
port 4789
:
}
:
inbound-interface eth0
:
protocol udp
:
translation {
:
address 192.168.72.8
:
}
:
}
:
}
:
source {
:
rule 100 {
:
outbound-interface eth0
:
source {
:
address 0.0.0.0/0
:
}
:
translation {
:
address masquerade
:
}
:
}
:
rule 200 {
:
outbound-interface eth1
:
source {
:
address 0.0.0.0/0
:
}
:
translation {
:
address masquerade
:
}
:
}
:
}
:
}
:
protocols {
:
ospf {
:
area 0.0.0.0 {
:
network 192.168.168.0/30
:
network 192.168.254.1/32
:
}
:
area 0.0.0.1 {
:
network 192.168.168.20/30
:
}
:
area 0.0.0.2 {
:
network 192.168.168.4/30
:
}
:
parameters {
:
abr-type cisco
:
router-id 192.168.254.1
:
}
:
}
:
static {
:
route 0.0.0.0/0 {
:
next-hop 10.28.115.17 {
:
}
:
next-hop 172.18.108.1 {
:
}
:
}
:
route 4.2.2.1/32 {
:
next-hop 172.18.108.1 {
:
}
:
}
:
route 4.2.2.2/32 {
:
next-hop 10.28.115.17 {
:
}
:
}
:
route 10.14.94.128/25 {
:
next-hop 10.28.115.17 {
:
}
:
}
:
route 10.14.96.0/25 {
:
next-hop 10.28.115.17 {
:
}
:
}
:
}
:
}
:
service {
:
ssh {
:
}
:
}
:
system {
:
config-management {
:
commit-revisions 100
:
}
:
console {
:
device ttyS0 {
:
speed 9600
:
}
:
}
:
host-name vyos
:
login {
:
user vyos {
:
authentication {
:
encrypted-password ****************
:
plaintext-password ****************
:
}
:
level admin
:
}
:
}
:
name-server 208.67.222.222
:
name-server 208.67.220.220
:
ntp {
:
server 0.pool.ntp.org {
:
}
:
server 1.pool.ntp.org {
:
}
:
server 2.pool.ntp.org {
:
}
:
}
:
syslog {
:
global {
:
facility all {
:
level info
:
}
:
facility protocols {
:
level debug
:
}
:
}
:
}
:
time-zone UTC
:
}
:
vpn {
:
ipsec {
:
esp-group pops {
:
compression disable
:
lifetime 86400
:
mode transport
:
pfs dh-group14
:
proposal 1 {
:
encryption aes256
:
hash sha256
:
}
:
}
:
ike-group pops {
:
close-action none
:
ikev2-reauth no
:
key-exchange ikev1
:
lifetime 28800
:
proposal 1 {
:
dh-group 14
:
encryption aes256
:
hash sha256
:
}
:
}
:
ipsec-interfaces {
:
interface eth0
:
}
:
site-to-site {
:
peer 10.14.94.232 {
:
authentication {
:
mode pre-shared-secret
:
pre-shared-secret ****************
:
}
:
connection-type initiate
:
default-esp-group pops
:
ike-group pops
:
ikev2-reauth inherit
:
local-address 10.28.115.18
:
tunnel 20 {
:
allow-nat-networks disable
:
allow-public-networks disable
:
protocol gre
:
}
:
}
:
peer 10.14.96.109 {
:
authentication {
:
mode pre-shared-secret
:
pre-shared-secret ****************
:
}
:
connection-type initiate
:
default-esp-group pops
:
ike-group pops
:
ikev2-reauth inherit
:
local-address 10.28.115.18
:
tunnel 4 {
:
allow-nat-networks disable
:
allow-public-networks disable
:
protocol gre
:
}
:
}
:
}
:
}
:
}
:

mafiosa · December 16, 2020, 8:11am

Can anyone help me? Really need some help regarding this.

Viacheslav · December 16, 2020, 4:30pm

@mafiosa Can you attach a simple network diagram? Which version do you use?

Dmitry · December 16, 2020, 4:54pm

Hello @mafiosa, it also will be helpful to decorate code/config for better understanding.

mafiosa · December 17, 2020, 2:29pm

Here you go!
`interfaces {
bridge br700 {
aging 300
description osixp
hello-time 2
max-age 20
priority 32768
stp false
}
ethernet eth0 {
address 10.28.115.18/30
duplex auto
hw-id 6c:3b:6b:69:69:69
smp-affinity auto
speed auto
}
ethernet eth1 {
address 172.18.108.142/24
duplex auto
hw-id 6c:3b:6b:96:96:96
smp-affinity auto
speed auto
}
ethernet eth2 {
duplex auto
smp-affinity auto
speed auto
vif 400 {
address 192.168.168.1/30
}
vif 700 {
bridge-group {
bridge br700
}
description osixp
}
}
loopback lo {
address 192.168.254.1/32
}
tunnel tun4 {
address 192.168.168.5/30
encapsulation gre
ip {
ospf {
dead-interval 40
hello-interval 10
network broadcast
priority 1
retransmit-interval 5
transmit-delay 1
}
}
local-ip 10.28.115.18
mtu 1422
multicast disable
remote-ip 10.14.96.109
}
tunnel tun20 {
address 192.168.168.21/30
encapsulation gre
ip {
:
ospf {
:
dead-interval 40
:
hello-interval 10
:
network broadcast
:
priority 1
:
retransmit-interval 5
:
transmit-delay 1
:
}
:
}
:
local-ip 10.28.115.18
:
mtu 1422
:
multicast disable
:
remote-ip 10.14.94.232
:
}
:
vxlan vxlan3 {
:
bridge-group {
:
bridge br700
:
}
:
link eth2.400
:
remote 192.168.72.8
:
remote-port 4789
:
vni 3
:
}
:
}
:
load-balancing {
:
wan {
:
interface-health eth0 {
:
failure-count 5
:
nexthop 10.28.115.17
:
success-count 5
:
test 10 {
:
resp-time 5
:
target 4.2.2.2
:
ttl-limit 1
:
type ping
:
}
:
}
:
interface-health eth1 {
:
failure-count 5
:
nexthop 172.18.108.1
:
success-count 5
:
test 10 {
:
resp-time 5
:
target 4.2.2.1
:
ttl-limit 1
:
type ping
:
}
:
}
:
rule 1 {
:
inbound-interface eth2.400
:
interface eth0 {
:
weight 2
:
}
:
interface eth1 {
:
weight 1
:
}
:
protocol all
:
}
:
sticky-connections {
:
inbound
:
}
:
}
:
}
:
nat {
:
destination {
:
rule 10 {
:
description vxlan-fw
:
destination {
:
port 4789
:
}
:
inbound-interface eth0
:
protocol udp
:
translation {
:
address 192.168.72.8
:
}
:
}
:
}
:
source {
:
rule 100 {
:
outbound-interface eth0
:
source {
:
address 0.0.0.0/0
:
}
:
translation {
:
address masquerade
:
}
:
}
:
rule 200 {
:
outbound-interface eth1
:
source {
:
address 0.0.0.0/0
:
}
:
translation {
:
address masquerade
:
}
:
}
:
}
:
}
:
protocols {
:
ospf {
:
area 0.0.0.0 {
:
network 192.168.168.0/30
:
network 192.168.254.1/32
:
}
:
area 0.0.0.1 {
:
network 192.168.168.20/30
:
}
:
area 0.0.0.2 {
:
network 192.168.168.4/30
:
}
:
parameters {
:
abr-type cisco
:
router-id 192.168.254.1
:
}
:
}
:
static {
:
route 0.0.0.0/0 {
:
next-hop 10.28.115.17 {
:
}
:
next-hop 172.18.108.1 {
:
}
:
}
:
route 4.2.2.1/32 {
:
next-hop 172.18.108.1 {
:
}
:
}
:
route 4.2.2.2/32 {
:
next-hop 10.28.115.17 {
:
}
:
}
:
route 10.14.94.128/25 {
:
next-hop 10.28.115.17 {
:
}
:
}
:
route 10.14.96.0/25 {
:
next-hop 10.28.115.17 {
:
}
:
}
:
}
:
}
:
service {
:
ssh {
:
}
:
}
:
system {
:
config-management {
:
commit-revisions 100
:
}
:
console {
:
device ttyS0 {
:
speed 9600
:
}
:
}
:
host-name vyos
:
login {
:
user vyos {
:
authentication {
:
encrypted-password ****************
:
plaintext-password ****************
:
}
:
level admin
:
}
:
}
:
name-server 208.67.222.222
:
name-server 208.67.220.220
:
ntp {
:
server 0.pool.ntp.org {
:
}
:
server 1.pool.ntp.org {
:
}
:
server 2.pool.ntp.org {
:
}
:
}
:
syslog {
:
global {
:
facility all {
:
level info
:
}
:
facility protocols {
:
level debug
:
}
:
}
:
}
:
time-zone UTC
:
}
:
vpn {
:
ipsec {
:
esp-group pops {
:
compression disable
:
lifetime 86400
:
mode transport
:
pfs dh-group14
:
proposal 1 {
:
encryption aes256
:
hash sha256
:
}
:
}
:
ike-group pops {
:
close-action none
:
ikev2-reauth no
:
key-exchange ikev1
:
lifetime 28800
:
proposal 1 {
:
dh-group 14
:
encryption aes256
:
hash sha256
:
}
:
}
:
ipsec-interfaces {
:
interface eth0
:
}
:
site-to-site {
:
peer 10.14.94.232 {
:
authentication {
:
mode pre-shared-secret
:
pre-shared-secret ****************
:
}
:
connection-type initiate
:
default-esp-group pops
:
ike-group pops
:
ikev2-reauth inherit
:
local-address 10.28.115.18
:
tunnel 20 {
:
allow-nat-networks disable
:
allow-public-networks disable
:
protocol gre
:
}
:
}
:
peer 10.14.96.109 {
:
authentication {
:
mode pre-shared-secret
:
pre-shared-secret ****************
:
}
:
connection-type initiate
:
default-esp-group pops
:
ike-group pops
:
ikev2-reauth inherit
:
local-address 10.28.115.18
:
tunnel 4 {
:
allow-nat-networks disable
:
allow-public-networks disable
:
protocol gre
:
}
:
}
:
}
:
}
:
}
:

`

bcstechdept · December 17, 2020, 7:21pm

I manually formatted the config-

interfaces {
    bridge br700 {
        aging 300
        description osixp
        hello-time 2
        max-age 20
        priority 32768
        stp false
    }
    ethernet eth0 {
        address 10.28.115.18/30
        duplex auto
        hw-id 6c:3b:6b:69:69:69
        smp-affinity auto
        speed auto
    }
    ethernet eth1 {
        address 172.18.108.142/24
        duplex auto
        hw-id 6c:3b:6b:96:96:96
        smp-affinity auto
        speed auto
    }
    ethernet eth2 {
        duplex auto
        smp-affinity auto
        speed auto
        vif 400 {
            address 192.168.168.1/30
        }
        vif 700 {
            bridge-group {
            bridge br700
            }
        description osixp
        }
    }
    loopback lo {
        address 192.168.254.1/32
    }
    tunnel tun4 {
        address 192.168.168.5/30
        encapsulation gre
        ip {
            ospf {
                dead-interval 40
                hello-interval 10
                network broadcast
                priority 1
                retransmit-interval 5
                transmit-delay 1
            }
        }
        local-ip 10.28.115.18
        mtu 1422
        multicast disable
        remote-ip 10.14.96.109
    }
    tunnel tun20 {
        address 192.168.168.21/30
        encapsulation gre
        ip {
            ospf {
                dead-interval 40
                hello-interval 10
                network broadcast
                priority 1
                retransmit-interval 5
                transmit-delay 1
            }
        }
        local-ip 10.28.115.18
        mtu 1422
        multicast disable
        remote-ip 10.14.94.232
    }
    vxlan vxlan3 {
    bridge-group {
        bridge br700
    }
    link eth2.400
    remote 192.168.72.8
    remote-port 4789
    vni 3
    }
}
load-balancing {
    wan {
        interface-health eth0 {
            failure-count 5
            nexthop 10.28.115.17
            success-count 5
            test 10 {
                resp-time 5
                target 4.2.2.2
                ttl-limit 1
                type ping
            }
        }
        interface-health eth1 {
            failure-count 5
            nexthop 172.18.108.1
            success-count 5
            test 10 {
                resp-time 5
                target 4.2.2.1
                ttl-limit 1
                type ping
            }
        }
        rule 1 {
            inbound-interface eth2.400
            interface eth0 {
                weight 2
            }
            interface eth1 {
                weight 1
            }
            protocol all
        }
        sticky-connections {
            inbound
        }
    }
}
nat {
    destination {
        rule 10 {
            description vxlan-fw
            destination {
                port 4789
            }
            inbound-interface eth0
            protocol udp
            translation {
                address 192.168.72.8
            }
        }
    }
    source {
        rule 100 {
            outbound-interface eth0
            source {
                address 0.0.0.0/0
            }
            translation {
                address masquerade
            }
        }
        rule 200 {
            outbound-interface eth1
            source {
                address 0.0.0.0/0
            }
            translation {
                address masquerade
            }
        }
    }
}
protocols {
    ospf {
        area 0.0.0.0 {
            network 192.168.168.0/30
            network 192.168.254.1/32
        }
        area 0.0.0.1 {
            network 192.168.168.20/30
        }
        area 0.0.0.2 {
            network 192.168.168.4/30
        }
        parameters {
            abr-type cisco
            router-id 192.168.254.1
        }
    }
    static {
        route 0.0.0.0/0 {
            next-hop 10.28.115.17 {
            }
            next-hop 172.18.108.1 {
            }
        }
        route 4.2.2.1/32 {
            next-hop 172.18.108.1 {
            }
        }
        route 4.2.2.2/32 {
            next-hop 10.28.115.17 {
            }
        }
        route 10.14.94.128/25 {
            next-hop 10.28.115.17 {
            }
        }
        route 10.14.96.0/25 {
            next-hop 10.28.115.17 {
            }
        }
    }
}
service {
    ssh {
    }
}
system {
    config-management {
        commit-revisions 100
    }
    console {
        device ttyS0 {
            speed 9600
        }
    }
    host-name vyos
    login {
        user vyos {
            authentication {
                encrypted-password ****************
                plaintext-password ****************
            }
            level admin
        }
    }
    name-server 208.67.222.222
    name-server 208.67.220.220
    ntp {
        server 0.pool.ntp.org {
        }
        server 1.pool.ntp.org {
        }
        server 2.pool.ntp.org {
        }
    }
    syslog {
        global {
            facility all {
                level info
            }
            facility protocols {
                level debug
            }
        }
    }
    time-zone UTC
}
vpn {
    ipsec {
        esp-group pops {
            compression disable
            lifetime 86400
            mode transport
            pfs dh-group14
            proposal 1 {
                encryption aes256
                hash sha256
            }
        }
        ike-group pops {
            close-action none
            ikev2-reauth no
            key-exchange ikev1
            lifetime 28800
            proposal 1 {
                dh-group 14
                encryption aes256
                hash sha256
            }
        }
        ipsec-interfaces {
            interface eth0
        }
        site-to-site {
            peer 10.14.94.232 {
                authentication {
                    mode pre-shared-secret
                    pre-shared-secret ****************
                }
                connection-type initiate
                default-esp-group pops
                ike-group pops
                ikev2-reauth inherit
                local-address 10.28.115.18
                tunnel 20 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    protocol gre
                }
            }
            peer 10.14.96.109 {
                authentication {
                    mode pre-shared-secret
                    pre-shared-secret ****************
                }
                connection-type initiate
                default-esp-group pops
                ike-group pops
                ikev2-reauth inherit
                local-address 10.28.115.18
                tunnel 4 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    protocol gre
                }
            }
        }
    }
}

Do you have a topology drawing?

mafiosa · December 28, 2020, 9:38pm

here you go!

I want the routes learnt from the tunnels via ospf to be excluded from loadbalancing.

mafiosa · December 28, 2020, 9:44pm

I am using vyos1.2.6s1

Dmitry · January 5, 2021, 4:44pm

Can you check default route?

show ip route 0.0.0.0

What about IPSec session status?

mafiosa · January 29, 2021, 1:06pm

I have two default routes working fine. Also ipsec is up