Inbound traffic using Load Balancing not working

General questions
, ,
1

I’d like someone to review my configuration, I think I’m over looking something rather than finding a bug.

I am working on a lab to confirm the use of VyOS as a replacement for our Watchguard firewalls. So I’m working to do a feature by feature review and confirmation. I have my baseline NAT/Firewall/DHCP/Routing configuration documented and working, I have added IPSEC VPN using VTI and that is working. So I added a second WAN connection and Failover Load-balancing, and that mostly works. It appears to be forcing my outbound traffic onto the primary interface, health checks are working, I saw some failure counts earlier in the week. However, once I added load-balancing, my inbound traffic stopped working. I could no longer ping the firewall from outside sources and the VTI VPN to my data center failed to connect. See my configuration below; what am I missing?

NOTE: eth3 is my primary WAN (even though I have it listed as WAN2), eth2 is the secondary WAN.
NOTE: If I remove load-balancing and disable eth2, everything goes back to working.
Question: Also, how do I set the Hardware info in show version?

protech@caswell-0231:~$ show version

Version:          VyOS 1.3-rolling-202010270217
Release Train:    equuleus

Built by:         autobuild@vyos.net
Built on:         Tue 27 Oct 2020 02:17 UTC
Build UUID:       28b7c9ca-a8a3-4bdf-a6ae-2aa34f10277a
Build Commit ID:  b0a54fc929f870

Architecture:     x86_64
Boot via:         installed image
System type:      bare metal

Hardware vendor:  To be filled by O.E.M.
Hardware model:   To be filled by O.E.M.
Hardware S/N:     Unknown
Hardware UUID:    Unknown

Copyright:        VyOS maintainers and contributors

protech@caswell-0231:~$ show configuration commands
# Cleaned up and some sections stripped
# Firewall
set firewall all-ping 'enable'
set firewall broadcast-ping 'disable'
set firewall config-trap 'disable'
set firewall group address-group PROTECH-COU address '40.40.40.2'
set firewall group address-group PROTECH-COU address '30.30.30.18'
set firewall group address-group PROTECH-COU description 'Professional Technologies Columbia Office'
set firewall group port-group PROTECH-SUPPORT description 'Ports used for remote access of routerby ProTech staff'
set firewall group port-group PROTECH-SUPPORT port '22'
set firewall ipv6-receive-redirects 'disable'
set firewall ipv6-src-route 'disable'
set firewall ip-src-route 'disable'
set firewall log-martians 'enable'
set firewall name WAN-LAN default-action 'drop'
set firewall name WAN-LAN rule 10 action 'accept'
set firewall name WAN-LAN rule 10 state established 'enable'
set firewall name WAN-LAN rule 10 state related 'enable'
set firewall name WAN-LAN rule 20 action 'accept'
set firewall name WAN-LAN rule 20 description 'SSH to NUC'
set firewall name WAN-LAN rule 20 destination address '10.10.10.11'
set firewall name WAN-LAN rule 20 destination port '22'
set firewall name WAN-LAN rule 20 protocol 'tcp'
set firewall name WAN-LAN rule 20 state new 'enable'
set firewall name WAN-LOCAL default-action 'drop'
set firewall name WAN-LOCAL rule 10 action 'accept'
set firewall name WAN-LOCAL rule 10 state established 'enable'
set firewall name WAN-LOCAL rule 10 state related 'enable'
set firewall name WAN-LOCAL rule 20 action 'accept'
set firewall name WAN-LOCAL rule 20 icmp type-name 'echo-request'
set firewall name WAN-LOCAL rule 20 protocol 'icmp'
set firewall name WAN-LOCAL rule 30 action 'accept'
set firewall name WAN-LOCAL rule 30 description 'Accept remote access from ProTech staff'
set firewall name WAN-LOCAL rule 30 destination group port-group 'PROTECH-SUPPORT'
set firewall name WAN-LOCAL rule 30 log 'enable'
set firewall name WAN-LOCAL rule 30 source group address-group 'PROTECH-COU'
set firewall name WAN-LOCAL rule 40 action 'accept'
set firewall name WAN-LOCAL rule 40 description 'Accept ESP'
set firewall name WAN-LOCAL rule 40 log 'enable'
set firewall name WAN-LOCAL rule 40 protocol 'esp'
set firewall name WAN-LOCAL rule 41 action 'accept'
set firewall name WAN-LOCAL rule 41 description 'Accept IKE'
set firewall name WAN-LOCAL rule 41 destination port '500'
set firewall name WAN-LOCAL rule 41 log 'enable'
set firewall name WAN-LOCAL rule 41 protocol 'udp'
set firewall name WAN-LOCAL rule 42 action 'accept'
set firewall name WAN-LOCAL rule 42 description 'Accept AH'
set firewall name WAN-LOCAL rule 42 log 'enable'
set firewall name WAN-LOCAL rule 42 protocol 'ah'
set firewall name WAN-LOCAL rule 43 action 'accept'
set firewall name WAN-LOCAL rule 43 description 'Accept NAT-T'
set firewall name WAN-LOCAL rule 43 destination port '4500'
set firewall name WAN-LOCAL rule 43 log 'enable'
set firewall name WAN-LOCAL rule 43 protocol 'udp'
set firewall receive-redirects 'disable'
set firewall send-redirects 'enable'
set firewall source-validation 'disable'
set firewall syn-cookies 'enable'
set firewall twa-hazards-protection 'disable'

# Interfaces
set interfaces ethernet eth0 address '10.10.10.1/24'
set interfaces ethernet eth0 description 'LAN1'
set interfaces ethernet eth0 hw-id '08:35:71:fe:1b:3c'
set interfaces ethernet eth0 vif 100 address '10.10.100.1/24'
set interfaces ethernet eth0 vif 100 description 'vlan100 - Test Subnet'
set interfaces ethernet eth1 description 'LAN2'
set interfaces ethernet eth1 hw-id '08:35:71:fe:1b:3d'
set interfaces ethernet eth2 address '30.30.30.30/28'
set interfaces ethernet eth2 description 'WAN1_CTL'
set interfaces ethernet eth2 firewall in name 'WAN-LAN'
set interfaces ethernet eth2 firewall local name 'WAN-LOCAL'
set interfaces ethernet eth2 hw-id '08:35:71:fe:1b:3e'
set interfaces ethernet eth3 address '40.40.40.14/28'
set interfaces ethernet eth3 description 'WAN2_MDC'
set interfaces ethernet eth3 firewall in name 'WAN-LAN'
set interfaces ethernet eth3 firewall local name 'WAN-LOCAL'
set interfaces ethernet eth3 hw-id '08:35:71:fe:1b:3f'
set interfaces loopback lo
set interfaces vti vti0 address '169.254.56.0/31'
set interfaces vti vti0 description 'Test VTI'

# WAN Load-Balancing
set load-balancing wan interface-health eth2 failure-count '5'
set load-balancing wan interface-health eth2 nexthop '30.30.30.17'
set load-balancing wan interface-health eth2 success-count '1'
set load-balancing wan interface-health eth2 test 10 resp-time '5'
set load-balancing wan interface-health eth2 test 10 target '8.8.8.8'
set load-balancing wan interface-health eth2 test 10 ttl-limit '1'
set load-balancing wan interface-health eth2 test 10 type 'ping'
set load-balancing wan interface-health eth2 test 20 resp-time '5'
set load-balancing wan interface-health eth2 test 20 target '1.1.1.1'
set load-balancing wan interface-health eth2 test 20 ttl-limit '1'
set load-balancing wan interface-health eth2 test 20 type 'ping'
set load-balancing wan interface-health eth3 failure-count '5'
set load-balancing wan interface-health eth3 nexthop '40.40.40.1'
set load-balancing wan interface-health eth3 success-count '1'
set load-balancing wan interface-health eth3 test 10 resp-time '5'
set load-balancing wan interface-health eth3 test 10 target '8.8.8.8'
set load-balancing wan interface-health eth3 test 10 ttl-limit '1'
set load-balancing wan interface-health eth3 test 10 type 'ping'
set load-balancing wan interface-health eth3 test 20 resp-time '5'
set load-balancing wan interface-health eth3 test 20 target '1.1.1.1'
set load-balancing wan interface-health eth3 test 20 ttl-limit '1'
set load-balancing wan interface-health eth3 test 20 type 'ping'
set load-balancing wan rule 1 failover
set load-balancing wan rule 1 inbound-interface 'eth0'
set load-balancing wan rule 1 interface eth2 weight '1'
set load-balancing wan rule 1 interface eth3 weight '10'
set load-balancing wan rule 1 protocol 'all'
set load-balancing wan sticky-connections inbound

# NAT Rules
set nat destination rule 10 description 'SSH to NUC via WAN1'
set nat destination rule 10 destination address '30.30.30.30'
set nat destination rule 10 destination port '8022'
set nat destination rule 10 inbound-interface 'eth2'
set nat destination rule 10 protocol 'tcp'
set nat destination rule 10 translation address '10.10.10.11'
set nat destination rule 10 translation port '22'
set nat destination rule 11 description 'SSH to NUC via WAN2'
set nat destination rule 11 destination address '40.40.40.14'
set nat destination rule 11 destination port '8022'
set nat destination rule 11 inbound-interface 'eth3'
set nat destination rule 11 protocol 'tcp'
set nat destination rule 11 translation address '10.10.10.11'
set nat destination rule 11 translation port '22'
set nat source rule 10 destination address '192.168.10.0/24'
set nat source rule 10 exclude
set nat source rule 10 outbound-interface 'eth2'
set nat source rule 10 source address '10.10.10.0/24'
set nat source rule 11 destination address '192.168.10.0/24'
set nat source rule 11 exclude
set nat source rule 11 outbound-interface 'eth3'
set nat source rule 11 source address '10.10.10.0/24'
set nat source rule 100 outbound-interface 'eth2'
set nat source rule 100 source address '10.10.10.0/24'
set nat source rule 100 translation address 'masquerade'
set nat source rule 101 outbound-interface 'eth3'
set nat source rule 101 source address '10.10.10.0/24'
set nat source rule 101 translation address 'masquerade'

# Static Routes
set protocols static route 0.0.0.0/0 next-hop 40.40.40.1
set protocols static route 0.0.0.0/0 next-hop 30.30.30.17
set protocols static route 192.168.10.0/24 next-hop 169.254.56.1 distance '1'

# VPN IPSEC
set vpn ipsec esp-group VTI0_Phase2 compression 'disable'
set vpn ipsec esp-group VTI0_Phase2 lifetime '3600'
set vpn ipsec esp-group VTI0_Phase2 mode 'tunnel'
set vpn ipsec esp-group VTI0_Phase2 pfs 'disable'
set vpn ipsec esp-group VTI0_Phase2 proposal 1 encryption 'aes256'
set vpn ipsec esp-group VTI0_Phase2 proposal 1 hash 'sha1'
set vpn ipsec ike-group VTI0_Phase1 close-action 'none'
set vpn ipsec ike-group VTI0_Phase1 ikev2-reauth 'no'
set vpn ipsec ike-group VTI0_Phase1 key-exchange 'ikev1'
set vpn ipsec ike-group VTI0_Phase1 lifetime '86400'
set vpn ipsec ike-group VTI0_Phase1 proposal 1 dh-group '2'
set vpn ipsec ike-group VTI0_Phase1 proposal 1 encryption 'aes256'
set vpn ipsec ike-group VTI0_Phase1 proposal 1 hash 'sha1'
set vpn ipsec ipsec-interfaces interface 'eth3'
set vpn ipsec site-to-site peer 65.65.65.65 authentication id '40.40.40.14'
set vpn ipsec site-to-site peer 65.65.65.65 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 65.65.65.65 authentication pre-shared-secret 'superSECRET'
set vpn ipsec site-to-site peer 65.65.65.65 authentication remote-id '65.65.65.65'
set vpn ipsec site-to-site peer 65.65.65.65 connection-type 'initiate'
set vpn ipsec site-to-site peer 65.65.65.65 ike-group 'VTI0_Phase1'
set vpn ipsec site-to-site peer 65.65.65.65 ikev2-reauth 'inherit'
set vpn ipsec site-to-site peer 65.65.65.65 local-address '40.40.40.14'
set vpn ipsec site-to-site peer 65.65.65.65 vti bind 'vti0'
set vpn ipsec site-to-site peer 65.65.65.65 vti esp-group 'VTI0_Phase2'
1 Like
3

Posted to Reddit, got 1 response, finally got things working after fiddling with it for a while.

  • Added distance metric to 0.0.0.0/0 routes to keep the router responding and routing local traffic.
  • Change the load-balancing interface-health to use different targets
  • Added static routes for the test targets so they go out the proper interface

Below is the current working configuration.

set firewall group address-group PROTECH-COU address '40.40.40.2'
set firewall group address-group PROTECH-COU address '30.30.30.18'
set firewall group address-group PROTECH-COU address '47.233.69.124'
set firewall group network-group RFC1918 description 'List of internal subnets'
set firewall group network-group RFC1918 network '10.0.0.0/8'
set firewall group network-group RFC1918 network '172.16.0.0/12'
set firewall group network-group RFC1918 network '192.168.0.0/16'
set firewall group port-group PROTECH-SUPPORT description 'Ports used for remote access'
set firewall group port-group PROTECH-SUPPORT port '22'
set firewall name WAN-LAN default-action 'drop'
set firewall name WAN-LAN rule 10 action 'accept'
set firewall name WAN-LAN rule 10 state established 'enable'
set firewall name WAN-LAN rule 10 state related 'enable'
set firewall name WAN-LAN rule 11 action 'drop'
set firewall name WAN-LAN rule 11 state invalid 'enable'
set firewall name WAN-LAN rule 20 action 'accept'
set firewall name WAN-LAN rule 20 description 'SSH to NUC'
set firewall name WAN-LAN rule 20 destination address '10.10.10.11'
set firewall name WAN-LAN rule 20 destination port '22'
set firewall name WAN-LAN rule 20 protocol 'tcp'
set firewall name WAN-LAN rule 20 state new 'enable'
set firewall name WAN-LOCAL default-action 'drop'
set firewall name WAN-LOCAL rule 10 action 'accept'
set firewall name WAN-LOCAL rule 10 state established 'enable'
set firewall name WAN-LOCAL rule 10 state related 'enable'
set firewall name WAN-LOCAL rule 11 action 'drop'
set firewall name WAN-LOCAL rule 11 state invalid 'enable'
set firewall name WAN-LOCAL rule 20 action 'accept'
set firewall name WAN-LOCAL rule 20 description 'Allow ICMP Ping'
set firewall name WAN-LOCAL rule 20 icmp type-name 'any'
set firewall name WAN-LOCAL rule 20 protocol 'icmp'
set firewall name WAN-LOCAL rule 20 state new 'enable'
set firewall name WAN-LOCAL rule 30 action 'drop'
set firewall name WAN-LOCAL rule 30 description 'Block SSH brute force - 4 tries in 60 sec'
set firewall name WAN-LOCAL rule 30 destination port '22'
set firewall name WAN-LOCAL rule 30 protocol 'tcp'
set firewall name WAN-LOCAL rule 30 recent count '4'
set firewall name WAN-LOCAL rule 30 recent time '60'
set firewall name WAN-LOCAL rule 30 state new 'enable'
set firewall name WAN-LOCAL rule 31 action 'accept'
set firewall name WAN-LOCAL rule 31 description 'Accept remote access PROTECH-COU'
set firewall name WAN-LOCAL rule 31 destination group port-group 'PROTECH-SUPPORT'
set firewall name WAN-LOCAL rule 31 log 'enable'
set firewall name WAN-LOCAL rule 31 source group address-group 'PROTECH-COU'
set firewall name WAN-LOCAL rule 40 action 'accept'
set firewall name WAN-LOCAL rule 40 description 'Accept ESP'
set firewall name WAN-LOCAL rule 40 log 'enable'
set firewall name WAN-LOCAL rule 40 protocol 'esp'
set firewall name WAN-LOCAL rule 41 action 'accept'
set firewall name WAN-LOCAL rule 41 description 'Accept IKE'
set firewall name WAN-LOCAL rule 41 destination port '500'
set firewall name WAN-LOCAL rule 41 log 'enable'
set firewall name WAN-LOCAL rule 41 protocol 'udp'
set firewall name WAN-LOCAL rule 42 action 'accept'
set firewall name WAN-LOCAL rule 42 description 'Accept AH'
set firewall name WAN-LOCAL rule 42 log 'enable'
set firewall name WAN-LOCAL rule 42 protocol 'ah'
set firewall name WAN-LOCAL rule 43 action 'accept'
set firewall name WAN-LOCAL rule 43 description 'Accept NAT-T'
set firewall name WAN-LOCAL rule 43 destination port '4500'
set firewall name WAN-LOCAL rule 43 log 'enable'
set firewall name WAN-LOCAL rule 43 protocol 'udp'
set interfaces ethernet eth0 address '10.10.10.1/24'
set interfaces ethernet eth0 description 'LAN1'
set interfaces ethernet eth0 vif 100 address '10.10.100.1/24'
set interfaces ethernet eth0 vif 100 description 'vlan100 - Test Subnet'
set interfaces ethernet eth1 description 'LAN2'
set interfaces ethernet eth2 address '30.30.30.30/28'
set interfaces ethernet eth2 description 'WAN1_CTL'
set interfaces ethernet eth2 firewall in name 'WAN-LAN'
set interfaces ethernet eth2 firewall local name 'WAN-LOCAL'
set interfaces ethernet eth3 address '40.40.40.14/28'
set interfaces ethernet eth3 description 'WAN2_MDC'
set interfaces ethernet eth3 firewall in name 'WAN-LAN'
set interfaces ethernet eth3 firewall local name 'WAN-LOCAL'
set interfaces vti vti0 address '169.254.56.0/31'
set interfaces vti vti0 description 'Test VTI'
set load-balancing wan interface-health eth2 failure-count '5'
set load-balancing wan interface-health eth2 nexthop '30.30.30.17'
set load-balancing wan interface-health eth2 test 1 target '8.8.4.4'
set load-balancing wan interface-health eth2 test 1 type 'ping'
set load-balancing wan interface-health eth2 test 2 target '1.1.1.1'
set load-balancing wan interface-health eth2 test 2 type 'ping'
set load-balancing wan interface-health eth3 failure-count '5'
set load-balancing wan interface-health eth3 nexthop '40.40.40.1'
set load-balancing wan interface-health eth3 test 1 target '8.8.8.8'
set load-balancing wan interface-health eth3 test 1 type 'ping'
set load-balancing wan interface-health eth3 test 2 target '1.0.0.1'
set load-balancing wan interface-health eth3 test 2 type 'ping'
set load-balancing wan rule 1 failover
set load-balancing wan rule 1 inbound-interface 'eth0'
set load-balancing wan rule 1 interface eth2 weight '10'
set load-balancing wan rule 1 interface eth3 weight '2'
set load-balancing wan sticky-connections inbound
set nat destination rule 10 description 'SSH to NUC via WAN1'
set nat destination rule 10 destination address '30.30.30.30'
set nat destination rule 10 destination port '8022'
set nat destination rule 10 inbound-interface 'eth2'
set nat destination rule 10 protocol 'tcp'
set nat destination rule 10 translation address '10.10.10.11'
set nat destination rule 10 translation port '22'
set nat destination rule 11 description 'SSH to NUC via WAN2'
set nat destination rule 11 destination address '40.40.40.14'
set nat destination rule 11 destination port '8022'
set nat destination rule 11 inbound-interface 'eth3'
set nat destination rule 11 protocol 'tcp'
set nat destination rule 11 translation address '10.10.10.11'
set nat destination rule 11 translation port '22'
set nat source rule 10 destination address '192.168.10.0/24'
set nat source rule 10 exclude
set nat source rule 10 outbound-interface 'eth2'
set nat source rule 10 source address '10.10.10.0/24'
set nat source rule 11 destination address '192.168.10.0/24'
set nat source rule 11 exclude
set nat source rule 11 outbound-interface 'eth3'
set nat source rule 11 source address '10.10.10.0/24'
set nat source rule 100 outbound-interface 'eth2'
set nat source rule 100 source address '10.10.10.0/24'
set nat source rule 100 translation address 'masquerade'
set nat source rule 101 outbound-interface 'eth3'
set nat source rule 101 source address '10.10.10.0/24'
set nat source rule 101 translation address 'masquerade'
set protocols static route 0.0.0.0/0 next-hop 40.40.40.1 distance '5'
set protocols static route 0.0.0.0/0 next-hop 30.30.30.17 distance '10'
set protocols static route 1.0.0.1/32 next-hop 40.40.40.1
set protocols static route 1.1.1.1/32 next-hop 30.30.30.17
set protocols static route 8.8.4.4/32 next-hop 30.30.30.17
set protocols static route 8.8.8.8/32 next-hop 40.40.40.1
set protocols static route 192.168.10.0/24 next-hop 169.254.56.1 distance '1'
set vpn ipsec esp-group VTI0_Phase2 compression 'disable'
set vpn ipsec esp-group VTI0_Phase2 lifetime '3600'
set vpn ipsec esp-group VTI0_Phase2 mode 'tunnel'
set vpn ipsec esp-group VTI0_Phase2 pfs 'disable'
set vpn ipsec esp-group VTI0_Phase2 proposal 1 encryption 'aes256'
set vpn ipsec esp-group VTI0_Phase2 proposal 1 hash 'sha1'
set vpn ipsec ike-group VTI0_Phase1 close-action 'none'
set vpn ipsec ike-group VTI0_Phase1 ikev2-reauth 'no'
set vpn ipsec ike-group VTI0_Phase1 key-exchange 'ikev1'
set vpn ipsec ike-group VTI0_Phase1 lifetime '86400'
set vpn ipsec ike-group VTI0_Phase1 proposal 1 dh-group '2'
set vpn ipsec ike-group VTI0_Phase1 proposal 1 encryption 'aes256'
set vpn ipsec ike-group VTI0_Phase1 proposal 1 hash 'sha1'
set vpn ipsec ipsec-interfaces interface 'eth3'
set vpn ipsec site-to-site peer 99.99.99.99 authentication id '40.40.40.14'
set vpn ipsec site-to-site peer 99.99.99.99 authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer 99.99.99.99 authentication pre-shared-secret 'vyos2WATCHGUARD'
set vpn ipsec site-to-site peer 99.99.99.99 authentication remote-id '99.99.99.99'
set vpn ipsec site-to-site peer 99.99.99.99 connection-type 'initiate'
set vpn ipsec site-to-site peer 99.99.99.99 description 'Tunnel to Orion Data Center'
set vpn ipsec site-to-site peer 99.99.99.99 ike-group 'VTI0_Phase1'
set vpn ipsec site-to-site peer 99.99.99.99 ikev2-reauth 'inherit'
set vpn ipsec site-to-site peer 99.99.99.99 local-address '40.40.40.14'
set vpn ipsec site-to-site peer 99.99.99.99 vti bind 'vti0'
set vpn ipsec site-to-site peer 99.99.99.99 vti esp-group 'VTI0_Phase2'
1 Like
4

This topic was automatically closed 2 days after the last reply. New replies are no longer allowed.