I have a 3 WAN setup with policy routing. I have port 80 forwarded on all public interfaces, eth0 eth1 and eth2. However, incoming connections only work on eth0. Anyone have a working configuration where this works?
Show policy, interface and nat rules
Which version?
me@gw01# run show version
Version: VyOS 1.4-rolling-202204162001
Release train: sagitta
Built by: autobuild@vyos.net
Built on: Sat 16 Apr 2022 20:01 UTC
Build UUID: 200f84d9-9805-4087-9fd9-e63473b1110c
Build commit ID: 50c10220997f6b
Architecture: x86_64
Boot via: installed image
System type: bare metal
Hardware vendor: Supermicro
Hardware model: Super Server
Hardware S/N: 0123456789
Hardware UUID: 00000000-0000-0000-0000-3cecef01a24c
Copyright: VyOS maintainers and contributors
me@gw01# show policy
prefix-list LOC-Network-v4 {
rule 10 {
action permit
le 32
prefix 10.0.0.0/8
}
}
prefix-list BlockIPConflicts {
description "Prevent Conflicting Routes"
rule 10 {
action permit
description "Internal IP Space"
le 32
prefix 172.18.60.0/24
}
rule 20 {
action permit
description "Internal IP Space"
le 32
prefix 10.0.0.0/16
}
}
prefix-list DN42-Network-v4 {
rule 10 {
action permit
le 32
prefix 172.20.0.0/14
}
}
prefix-list6 LOC-Network-v6 {
rule 10 {
action permit
le 128
prefix fc00::/56
}
}
prefix-list6 BlockIPConflicts-v6 {
description "Prevent Conflicting Routes"
rule 10 {
action permit
description "Internal IP Space"
le 128
prefix fc00::/64
}
}
prefix-list6 DN42-Network-v6 {
rule 10 {
action permit
le 128
prefix fd00::/8
}
}
route LAN-Policy {
rule 10 {
description "IRC Through Secure VPN Tunnel"
destination {
group {
address-group Unsafe_IRC_Servers
}
}
set {
table 114
}
}
rule 20 {
description "IRC Ports Through Secure VPN Tunnel"
destination {
group {
port-group Unsafe_IRC_Ports
}
}
protocol tcp
set {
table 114
}
}
rule 30 {
description "Local Net Hosts Through VPN Tunnel"
set {
table 114
}
source {
group {
network-group InternalNetVPNOnly
}
}
}
rule 40 {
description "Youtube & Google Through NW"
destination {
group {
network-group YouTube-Google-IPv4
}
}
set {
table 102
}
}
}
route-map LOC-Peering-Export {
rule 10 {
action permit
description "Allow LOC-Network"
match {
ip {
address {
prefix-list LOC-Network-v4
}
}
}
}
rule 20 {
action permit
description "Allow LOC-Network"
match {
ipv6 {
address {
prefix-list LOC-Network-v6
}
}
}
}
rule 30 {
action permit
description "Allow DN42-Network"
match {
ip {
address {
prefix-list DN42-Network-v4
}
}
}
}
rule 40 {
action permit
description "Allow DN42-Network"
match {
ipv6 {
address {
prefix-list DN42-Network-v6
}
}
}
}
rule 50 {
action deny
match {
rpki invalid
}
}
rule 100 {
action deny
}
}
route-map LOC-Peering-Import {
rule 10 {
action deny
description "Prevent IP Conflicts"
match {
ip {
address {
prefix-list BlockIPConflicts
}
}
}
}
rule 20 {
action deny
description "Prevent IP Conflicts"
match {
ipv6 {
address {
prefix-list BlockIPConflicts-v6
}
}
}
}
rule 30 {
action permit
description "Allow LOC-Network"
match {
ip {
address {
prefix-list LOC-Network-v4
}
}
}
}
rule 40 {
action permit
description "Allow LOC-Network"
match {
ipv6 {
address {
prefix-list LOC-Network-v6
}
}
}
}
rule 50 {
action deny
match {
rpki invalid
}
}
rule 100 {
action deny
}
}
route6 LAN-Policy {
rule 10 {
description "IRC Through Secure VPN Tunnel"
destination {
group {
address-group Unsafe_IRC_Servers
}
}
set {
table 114
}
}
rule 20 {
description "IRC Ports Through Secure VPN Tunnel"
destination {
group {
port-group Unsafe_IRC_Ports
}
}
protocol tcp
set {
table 114
}
}
}
me@gw01# show interfaces
ethernet eth0 {
address dhcp
description VZ
firewall {
in {
name OUTSIDE-IN
}
local {
name OUTSIDE-LOCAL
}
out {
ipv6-name OUTSIDE-6-OUT
name OUTSIDE-OUT
}
}
traffic-policy {
out VZFiOSOut
}
}
ethernet eth1 {
address dhcp
address dhcpv6
description SpectrumTWC
dhcpv6-options {
pd 0 {
interface eth3 {
sla-id 0
}
length 56
}
}
firewall {
in {
ipv6-name OUTSIDE-6-IN
name OUTSIDE-IN
}
local {
ipv6-name OUTSIDE-6-LOCAL
name OUTSIDE-LOCAL
}
out {
ipv6-name OUTSIDE-6-OUT
name OUTSIDE-OUT
}
}
traffic-policy {
out SpectrumOut
}
}
ethernet eth2 {
address dhcp
description NaturalWireless
firewall {
in {
name OUTSIDE-IN
}
local {
name OUTSIDE-LOCAL
}
out {
ipv6-name OUTSIDE-6-OUT
name OUTSIDE-OUT
}
}
traffic-policy {
out NWOut
}
}
ethernet eth3 {
address 172.18.50.1/24
description NetworkTest
ipv6 {
address {
autoconf
}
}
}
ethernet eth4 {
address 10.0.0.1/16
address fc00::1/64
description LAN
firewall {
in {
name LAN-IN
}
out {
}
}
ipv6 {
address {
autoconf
}
}
policy {
route LAN-Policy
route6 LAN-Policy
}
}
ethernet eth5 {
disable
hw-id 40:a6:b7:20:4d:f9
}
loopback lo {
}
wireguard wg100 {
address 192.168.10.1/24
address fc00:0:0:1::1/64
description "WireGuard VPN RoadWarrior"
peer MyiPad {
allowed-ips 192.168.10.2/32
allowed-ips fc00:0:0:1::2/128
persistent-keepalive 15
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
peer MyiPhone {
allowed-ips 192.168.10.3/32
allowed-ips fc00:0:0:1::3/128
persistent-keepalive 15
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
port 51820
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg101 {
address 10.66.98.57/32
address fc00:bbbb:bbbb:bb01::3:6238/128
description "WireGuard Mullvad NYC"
firewall {
in {
ipv6-name PUB-WIREGUARD-6-IN
name PUB-WIREGUARD-IN
}
local {
ipv6-name PUB-WIREGUARD-6-LOCAL
name PUB-WIREGUARD-LOCAL
}
}
peer mullvad {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::0/0
persistent-keepalive 10
port 51820
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg102 {
address 10.66.32.65/32
address fc00:bbbb:bbbb:bb01::3:2040/128
description "WireGuard Mullvad us123"
firewall {
in {
ipv6-name PUB-WIREGUARD-6-IN
name PUB-WIREGUARD-IN
}
local {
ipv6-name PUB-WIREGUARD-6-LOCAL
name PUB-WIREGUARD-LOCAL
}
}
peer mullvad {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::0/0
persistent-keepalive 10
port 51820
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg103 {
address 10.64.215.200/32
address fc00:bbbb:bbbb:bb01::1:d7c7/128
description "WireGuard Mullvad us119"
firewall {
in {
ipv6-name PUB-WIREGUARD-6-IN
name PUB-WIREGUARD-IN
}
local {
ipv6-name PUB-WIREGUARD-6-LOCAL
name PUB-WIREGUARD-LOCAL
}
}
peer mullvad {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::0/0
persistent-keepalive 10
port 51820
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg104 {
address 10.64.126.218/32
address fc00:bbbb:bbbb:bb01::1:7ed9/128
description "Mullvad Multihop"
firewall {
in {
ipv6-name PUB-WIREGUARD-6-IN
name PUB-WIREGUARD-IN
}
local {
ipv6-name PUB-WIREGUARD-6-LOCAL
name PUB-WIREGUARD-LOCAL
}
}
peer mullvad {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::0/0
port 3049
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg110 {
address 172.18.60.2/32
description aws.example.com
mtu 1420
peer gw01.aws.example.com {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::/0
persistent-keepalive 10
port 51820
preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
port 51822
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg111 {
address 172.18.60.8/32
description colorado.example.com
mtu 1420
peer gw01.aws.example.com {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::/0
persistent-keepalive 10
port 51821
preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
port 51824
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg112 {
address 172.18.60.4/32
description ia.example.com
mtu 1420
peer location1 {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::/0
persistent-keepalive 10
port 51822
preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
port 51823
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg113 {
address 172.18.60.12/32
description zaius.example.com
mtu 1420
peer location1 {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::/0
persistent-keepalive 10
port 51822
preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
port 51825
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
wireguard wg120 {
address 172.18.60.21/31
address fc00:0000:0000:20::2/64
description dn42-us-nj01.exabyte.network
peer dn42-us-nj {
address x.x.x.x
allowed-ips 0.0.0.0/0
allowed-ips ::0/0
persistent-keepalive 10
port 51820
preshared-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
public-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
private-key xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
}
me@gw01# show nat
destination {
rule 10 {
description "Port Forward: PLEX to 10.0.10.1"
destination {
port 32408
}
inbound-interface eth0
protocol tcp
translation {
address 10.0.10.1
port 32400
}
}
rule 20 {
description "Port Forward: PLEX to 10.0.10.1"
destination {
port 32408
}
inbound-interface eth1
protocol tcp
translation {
address 10.0.10.1
port 32400
}
}
rule 30 {
description "Port Forward: PLEX to 10.0.10.1"
destination {
port 32408
}
inbound-interface eth2
protocol tcp
translation {
address 10.0.10.1
port 32400
}
}
rule 40 {
description "Port Forward: HTTP to 10.0.10.10 for NGINX"
destination {
port 80
}
inbound-interface eth0
protocol tcp
translation {
address 10.0.10.10
port 80
}
}
rule 41 {
description "Port Forward: HTTP to 10.0.10.10 for NGINX"
destination {
port 80
}
inbound-interface eth1
protocol tcp
translation {
address 10.0.10.10
port 80
}
}
rule 42 {
description "Port Forward: HTTP to 10.0.10.10 for NGINX"
destination {
port 80
}
inbound-interface eth2
protocol tcp
translation {
address 10.0.10.10
port 80
}
}
rule 50 {
description "Port Forward: HTTPS to 10.0.10.10 for NGINX"
destination {
port 443
}
inbound-interface eth0
protocol tcp
translation {
address 10.0.10.10
port 443
}
}
rule 51 {
description "Port Forward: HTTPS to 10.0.10.10 for NGINX"
destination {
port 443
}
inbound-interface eth1
protocol tcp
translation {
address 10.0.10.10
port 443
}
}
rule 52 {
description "Port Forward: HTTPS to 10.0.10.10 for NGINX"
destination {
port 443
}
inbound-interface eth2
protocol tcp
translation {
address 10.0.10.10
port 443
}
}
}
source {
rule 100 {
outbound-interface eth0
source {
address 10.0.0.0/16
}
translation {
address masquerade
}
}
rule 101 {
outbound-interface eth0
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 102 {
outbound-interface eth0
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
rule 110 {
outbound-interface eth1
source {
address 10.0.0.0/16
}
translation {
address masquerade
}
}
rule 111 {
outbound-interface eth1
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 112 {
outbound-interface eth1
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
rule 120 {
outbound-interface eth2
source {
address 10.0.0.0/16
}
translation {
address masquerade
}
}
rule 121 {
outbound-interface eth2
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 122 {
outbound-interface eth2
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
rule 130 {
outbound-interface wg101
source {
address 10.0.0.0/16
}
translation {
address masquerade
}
}
rule 131 {
outbound-interface wg101
source {
address 172.18.60.0/24
}
translation {
address masquerade
}
}
rule 132 {
outbound-interface wg101
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 133 {
outbound-interface wg101
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
rule 140 {
outbound-interface wg102
source {
address 10.0.0.0/16
}
translation {
address masquerade
}
}
rule 141 {
outbound-interface wg102
source {
address 172.18.60.0/24
}
translation {
address masquerade
}
}
rule 142 {
outbound-interface wg102
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 143 {
outbound-interface wg102
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
rule 150 {
outbound-interface wg103
source {
address 10.0.0.0/16
}
translation {
address masquerade
}
}
rule 151 {
outbound-interface wg103
source {
address 172.18.60.0/24
}
translation {
address masquerade
}
}
rule 152 {
outbound-interface wg103
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 153 {
outbound-interface wg103
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
rule 160 {
outbound-interface wg104
source {
address 10.0.0.0/16
}
translation {
address masquerade
}
}
rule 161 {
outbound-interface wg104
source {
address 172.18.60.0/24
}
translation {
address masquerade
}
}
rule 162 {
outbound-interface wg104
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 163 {
outbound-interface wg104
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
rule 170 {
outbound-interface wg120
source {
address 10.0.0.0/8
}
translation {
address masquerade
}
}
rule 171 {
outbound-interface wg120
source {
address 172.18.60.0/24
}
translation {
address masquerade
}
}
rule 172 {
outbound-interface wg120
source {
address 172.18.50.0/24
}
translation {
address masquerade
}
}
rule 173 {
outbound-interface wg120
source {
address 192.168.10.0/24
}
translation {
address masquerade
}
}
}
Interesting, thanks.
This partially works when I specify the IP of the outgoing interface. I can now ping and ssh directly into eth1 and eth2 which previously did not work. Port forwarding (DNAT) still does not work.
set policy local-route rule 100 source 1.2.3.4
set policy local-route rule 100 set table 100
set policy local-route rule 101 source 2.3.4.5
set policy local-route rule 101 set table 101
set policy local-route rule 102 source 3.4.5.6
set policy local-route rule 102 set table 102
However, since these are dhcp assigned interfaces that’s not sustainable. I tried something like this but it disables all outgoing traffic from the network:
set policy local-route rule 100 inbound-interface eth0
set policy local-route rule 100 set table 100
set policy local-route rule 101 inbound-interface eth1
set policy local-route rule 101 set table 101
set policy local-route rule 102 inbound-interface eth2
set policy local-route rule 102 set table 102
table 100 {
route 0.0.0.0/0 {
dhcp-interface eth0
}
route6 ::/0 {
interface eth0 {
}
}
}
table 101 {
route 0.0.0.0/0 {
dhcp-interface eth1
}
route6 ::/0 {
interface eth1 {
}
}
}
table 102 {
route 0.0.0.0/0 {
dhcp-interface eth2
}
route6 ::/0 {
interface eth2 {
}
}
}
Perhaps this is possible in a dynamic ip friendly way by marking packets? Help greatly appreciated. Thanks
Set destination address not source. Or you can do it like in “normal” linux and create another rules with destination addresses only (Routing for multiple uplinks/providers)
set policy local-route rule 100 source 1.2.3.4
set policy local-route rule 100 set table 100
set policy local-route rule 101 source 2.3.4.5
set policy local-route rule 101 set table 101
set policy local-route rule 102 source 3.4.5.6
set policy local-route rule 102 set table 102
set policy local-route rule 200 destination 1.2.3.4
set policy local-route rule 200 set table 100
set policy local-route rule 201 destination 2.3.4.5
set policy local-route rule 201 set table 101
set policy local-route rule 202 destination 3.4.5.6
set policy local-route rule 202 set table 102
Thanks. I tried this configuration and it enables local connections to the router (ssh, icmp) but DNAT still only works on eth0.
Additionally, is there a way to do this without hardcoding the IPs on eth0, eth1, and eth2? These are dhcp-assigned.
I have configuration with 2 providers on 1.3 and DNAT working on both links. But in 1.3 configuration is simpler and you can only set destination address. Try to remove rules with source addresses, maybe it will help.
Looks like it can’t be done right now.
I just tested and 1.3 and this works. Looks like a bug. I’ll submit a report.
Not sure what VyOS can do.
But in generic, the way is to mark the connection (state=new) coming in on WAN.
(WAN1 → connmark =1 , WAN2 → connmark=2…)
All subsequent packets , including return traffic, will have this same connection mark.
Then use policy routing rules, to select proper router table for return traffic.
This is what I am using so far to test incoming connections to port 80 on eth0, eth1 eth2. eth0 is the default route, and incoming connections work fine. Incoming connections on eth1 and eth2 don’t work.
set policy route mark-in-eth0 rule 10 destination port '80'
set policy route mark-in-eth0 rule 10 protocol 'tcp'
set policy route mark-in-eth0 rule 10 set mark '100'
set policy route mark-in-eth0 rule 10 set table '100'
set policy route mark-in-eth1 rule 10 destination port '80'
set policy route mark-in-eth1 rule 10 protocol 'tcp'
set policy route mark-in-eth1 rule 10 set mark '101'
set policy route mark-in-eth1 rule 10 set table '101'
set policy route mark-in-eth2 rule 10 destination port '80'
set policy route mark-in-eth2 rule 10 protocol 'tcp'
set policy route mark-in-eth2 rule 10 set mark '102'
set policy route mark-in-eth2 rule 10 set table '102'
set policy local-route rule 110 fwmark '100'
set policy local-route rule 110 set table '100'
set policy local-route rule 111 fwmark '101'
set policy local-route rule 111 set table '101'
set policy local-route rule 112 fwmark '102'
set policy local-route rule 112 set table '102'
set interfaces ethernet eth0 policy route mark-in-eth0
set interfaces ethernet eth1 policy route mark-in-eth1
set interfaces ethernet eth2 policy route mark-in-eth2
Just tried on 1.2.7 , VyOS lacks connection marking.
Seems like something like below isn’t implemented:
Note a packet mark is applied to skbuf object of the packet, and is lost when packet is sent out
A connection mark is a marking in conntrack table, which remains in place
Setting table on in-eth0 ,1 and 2 is useless, as this packet should be routed out on your LAN interface. If tables tables 100 101 102 don’t have entries for connected networks, you only break extra things
Route table selection should be done on LAN-IN
Also setting packet mark on in-eth0 ,1 and 2 is useless. It only affects packets towards your internal server, not return traffic
Ugly workaround:
Make your webserver listen on 3 ports, map each wan interface to a different port. Then for return traffic , do PBR on source port
WAN1 nat 80->81
WAN2 nat 80->82
WAN3 nat 80->83
The use PBR rules on LAN-IN:
source=webserver sourceport=81 , use table 101
source=webserver sourceport=82 , use table 102
source=webserver sourceport=83 , use table 103
Thanks - That workaround indeed is messy but will work. Is there really no better way to do this on VyOS Rolling 1.4+?
One other option allows balancing between several backends, but it seems not your case as you don’t have one address, and use several external addresses
https://docs.vyos.io/en/latest/configuration/highavailability/index.html#virtual-server
So to confirm, there is currently no way to get this to work with packet marking on the LAN interface?
For example, if I mark the packets on the way in:
set policy route mark-in-eth0 rule 10 destination port '80'
set policy route mark-in-eth0 rule 10 protocol 'tcp'
set policy route mark-in-eth0 rule 10 set mark '100'
set policy route mark-in-eth0 rule 10 set table '100'
set policy route mark-in-eth1 rule 10 destination port '80'
set policy route mark-in-eth1 rule 10 protocol 'tcp'
set policy route mark-in-eth1 rule 10 set mark '101'
set policy route mark-in-eth1 rule 10 set table '101'
set policy route mark-in-eth2 rule 10 destination port '80'
set policy route mark-in-eth2 rule 10 protocol 'tcp'
set policy route mark-in-eth2 rule 10 set mark '102'
set policy route mark-in-eth2 rule 10 set table '102'
Is there a way to keep track of the marks, and mark them on the return LAN interface?
This is being nat’d at each interface eth0 eth1 and eth2.
Note that the below enables direct connections to ssh on the router at eth0, eth1, and eth2, but traffic passed through has no effect
set policy local-route rule 100 source 1.2.3.4
set policy local-route rule 100 set table 100
set policy local-route rule 101 source 2.3.4.5
set policy local-route rule 101 set table 101
set policy local-route rule 102 source 3.4.5.6
set policy local-route rule 102 set table 102
Try to dump trafiic
Fo example, Do you see request from eth1 and responces from eth0?
eth0 = WAN1
eth1 = WAN2
eth2 = WAN3
eth4 = LAN
s.s.s.s = external internet machine
x.x.x.x = public IP address of eth0
y.y.y.y = public IP address of eth1
z.z.z.z = public IP address of eth2
10.0.10.1 = internal web server
Configuration A - local routes based on source and destination WAN ips
(With this configuration, SSH connections to WAN1, WAN2, WAN3 work.)
me@gw01# run show configuration commands | grep local-route
set policy local-route rule 100 set table '100'
set policy local-route rule 100 source 'x.x.x.x'
set policy local-route rule 101 set table '101'
set policy local-route rule 101 source 'y.y.y.y'
set policy local-route rule 102 set table '102'
set policy local-route rule 102 source 'z.z.z.z'
set policy local-route rule 200 destination 'x.x.x.x'
set policy local-route rule 200 set table '100'
set policy local-route rule 201 destination 'y.y.y.y'
set policy local-route rule 201 set table '101'
set policy local-route rule 202 destination 'z.z.z.z'
set policy local-route rule 202 set table '102'
When trying to connect to port 80 on WAN2 (eth1) from an external internet machine, here is what I see:
the packet arrives properly on eth1:
me@gw01# sudo tcpdump -i eth1
10:12:24.206410 IP s.s.s.s.33620 > y.y.y.y.http: Flags [S], seq 3740166210, win 62727, options [mss 1460,sackOK,TS val 448025207 ecr 0,nop,wscale 7], length 0
10:12:25.209157 IP s.s.s.s.33620 > y.y.y.y.http: Flags [S], seq 3740166210, win 62727, options [mss 1460,sackOK,TS val 448026209 ecr 0,nop,wscale 7], length 0
10:12:27.225196 IP s.s.s.s.33620 > y.y.y.y.http: Flags [S], seq 3740166210, win 62727, options [mss 1460,sackOK,TS val 448028225 ecr 0,nop,wscale 7], length 0
the packet leaves eth4 heading for internal IP 10.0.10.1 on port 80
sudo tcpdump dst port 80 -n
10:25:18.276384 IP y.y.y.y.33646 > 10.0.10.1.80: Flags [S], seq 1239361556, win 62727, options [mss 1460,sackOK,TS val 448799274 ecr 0,nop,wscale 7], length 0
10:25:19.292100 IP y.y.y.y.33646 > 10.0.10.1.80: Flags [S], seq 1239361556, win 62727, options [mss 1460,sackOK,TS val 448800289 ecr 0,nop,wscale 7], length 0
The reply from 10.0.10.1 on port 80 returns back to the router with destination y.y.y.y
me@gw01# sudo tcpdump -n -i eth4 dst y.y.y.y
10:27:45.269290 IP 10.0.10.1.80 > y.y.y.y.33656: Flags [S.], seq 3878569655, ack 3772537332, win 43440, options [mss 1460,sackOK,TS val 2128561991 ecr 448944196,nop,wscale 9], length 0
10:27:47.317339 IP 10.0.10.1.80 > y.y.y.y.33656: Flags [S.], seq 3878569655, ack 3772537332, win 43440, options [mss 1460,sackOK,TS val 2128564039 ecr 448944196,nop,wscale 9], length 0
The reply incorrectly leaves from eth0 instead of eth1
me@gw01# sudo tcpdump -n -i eth0 tcp | grep y.y.y.y
10:37:18.686940 IP y.y.y.y.80 > s.s.s.s.33700: Flags [S.], seq 2258282197, ack 1996502720, win 43440, options [mss 1460,sackOK,TS val 2129135407 ecr 449519682,nop,wscale 9], length 0
10:37:19.709661 IP y.y.y.y.80 > s.s.s.s.33700: Flags [S.], seq 2258282197, ack 1996502720, win 43440, options [mss 1460,sackOK,TS val 2129136430 ecr 449519682,nop,wscale 9], length 0
Configuration B - Marking incoming packets on eth0/eth1/eth2
(With this configuration, incoming SSH connections to eth1 and eth2 time out. eth0 works.)
set policy route mark-in-eth0 rule 10 destination port '80'
set policy route mark-in-eth0 rule 10 protocol 'tcp'
set policy route mark-in-eth0 rule 10 set mark '100'
set policy route mark-in-eth0 rule 10 set table '100'
set policy route mark-in-eth1 rule 10 destination port '80'
set policy route mark-in-eth1 rule 10 protocol 'tcp'
set policy route mark-in-eth1 rule 10 set mark '101'
set policy route mark-in-eth1 rule 10 set table '101'
set policy route mark-in-eth2 rule 10 destination port '80'
set policy route mark-in-eth2 rule 10 protocol 'tcp'
set policy route mark-in-eth2 rule 10 set mark '102'
set policy route mark-in-eth2 rule 10 set table '102'
set policy local-route rule 110 fwmark '100'
set policy local-route rule 110 set table '100'
set policy local-route rule 111 fwmark '101'
set policy local-route rule 111 set table '101'
set policy local-route rule 112 fwmark '102'
set policy local-route rule 112 set table '102'
set interfaces ethernet eth0 policy route mark-in-eth0
set interfaces ethernet eth1 policy route mark-in-eth1
set interfaces ethernet eth2 policy route mark-in-eth2
Packet arrives on eth1 from s.s.s.s and attempts to leave eth1 for 10.0.10.1
me@gw01# sudo tcpdump -n -I eth1 tcp
10:45:16.945774 IP s.s.s.s.33708 > y.y.y.y.80: Flags [S], seq 1380001604, win 62727, options [mss 1460,sackOK,TS val 449997941 ecr 0,nop,wscale 7], length 0
10:45:16.945859 IP y.y.y.y.33708 > 10.0.10.1.80: Flags [S], seq 1380001604, win 62727, options [mss 1460,sackOK,TS val 449997941 ecr 0,nop,wscale 7], length 0
Packet never detected on internal network
me@gw01# sudo tcpdump dst port 80 -n
[edit]
Configuration C - combination of A + B
(Local SSH connections to eth0, eth1, eth2 all work.)
Connection initiates from internet (s.s.s.s) and received by eth1 (y.y.y.y)
me@gw01# sudo tcpdump -n -i eth1 tcp
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth1, link-type EN10MB (Ethernet), snapshot length 262144 bytes
10:54:12.665137 IP s.s.s.s.33752 > y.y.y.y.80: Flags [S], seq 2021677267, win 62727, options [mss 1460,sackOK,TS val 450533660 ecr 0,nop,wscale 7], length 0
10:54:13.694085 IP s.s.s.s.33752 > y.y.y.y.80: Flags [S], seq 2021677267, win 62727, options [mss 1460,sackOK,TS val 450534689 ecr 0,nop,wscale 7], length 0
Leaves LAN (eth4) for web server (10.0.10.1)
me@gw01# sudo tcpdump -n -i eth4 dst 10.0.10.1
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth4, link-type EN10MB (Ethernet), snapshot length 262144 bytes
10:56:02.655744 IP y.y.y.y.33754 > 10.0.10.1.80: Flags [S], seq 3807013775, win 62727, options [mss 1460,sackOK,TS val 450643651 ecr 0,nop,wscale 7], length 0
10:56:03.678490 IP y.y.y.y.33754 > 10.0.10.1.80: Flags [S], seq 3807013775, win 62727, options [mss 1460,sackOK,TS val 450644673 ecr 0,nop,wscale 7], length 0
Reply goes back, destined for eth1
me@gw01# sudo tcpdump -n -i eth4 src 10.0.10.1
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth4, link-type EN10MB (Ethernet), snapshot length 262144 bytes
10:57:16.062697 IP 10.0.10.1.80 > y.y.y.y.33758: Flags [S.], seq 1699893293, ack 3248160270, win 43440, options [mss 1460,sackOK,TS val 2130332782 ecr 450716042,nop,wscale 9], length 0
10:57:17.111305 IP 10.0.10.1.80 > y.y.y.y.33758: Flags [S.], seq 1699893293, ack 3248160270, win 43440, options [mss 1460,sackOK,TS val 2130333831 ecr 450716042,nop,wscale 9], length 0
Packet incorrectly leaves eth0 instead of eth1
me@gw01# sudo tcpdump -n -i eth0
tcpdump: verbose output suppressed, use -v[v]... for full protocol decode
listening on eth0, link-type EN10MB (Ethernet), snapshot length 262144 bytes
10:58:54.848122 IP y.y.y.y.80 > s.s.s.s.33762: Flags [S.], seq 771191046, ack 169155848, win 43440, options [mss 1460,sackOK,TS val 2130431568 ecr 450815842,nop,wscale 9], length 0
10:58:55.863465 IP y.y.y.y.80 > s.s.s.s.33762: Flags [S.], seq 771191046, ack 169155848, win 43440, options [mss 1460,sackOK,TS val 2130432583 ecr 450815842,nop,wscale 9], length 0
It looks like Configuration A gets me the closest: packet arrives on eth1, goes to internal web server, returns from web server, but then incorrectly leaves eth0 with a destination containing the IP of eth1. I am surprised the below rules don’t force the reply to leave eth1
set policy local-route rule 200 destination 'x.x.x.x'
set policy local-route rule 200 set table '100'
set policy local-route rule 201 destination 'y.y.y.y'
set policy local-route rule 201 set table '101'
set policy local-route rule 202 destination 'z.z.z.z'
set policy local-route rule 202 set table '102'
Isn’t this something that should be solvable with wan load balance and sticky inbound connections?
I tested the below configuration but it also does not work.
set load-balancing wan rule 1000 description "Default rule"
set load-balancing wan rule 1000 inbound-interface eth4
set load-balancing wan rule 1000 interface eth0 weight 1
set load-balancing wan rule 1000 interface eth1 weight 2
set load-balancing wan rule 1000 interface eth2 weight 3
set load-balancing wan rule 1000 failover
set load-balancing wan sticky-connections inbound
I would like to help, but I’m kind of lost with so many entries with multiple configurations.
So far, what I get is:
- 3 WAN interfaces
- SSH connections from remote hosts to vyos router should be possible through those 3 WAN connections.
- DNAT to port 80 (internal ip x.x.x.x) should be possible through those 3 WANs
What do you want to achieve with policy based routing? Force certain networks to reach internet through specific WAN connection?
With a more detailed info of what is needed (without commands), maybe I get a clear picture of the requirements and possible solutions.
Maybe I’m the only one who got confused with so many entries. Sorry for that!
1 Like
Hi - Thanks so much in advance for your help, and I apologize for the multiple confusing posts. Yes you are exactly right with your assumptions:
- 3 WAN interfaces
- SSH connections from remote hosts to vyos router should be possible through those 3 WAN connections.
- DNAT to port 80 (internal ip x.x.x.x) should be possible through those 3 WANs
- Policy routing to direct traffic from LAN out WAN1, WAN2, WAN3 depending based on destination IP, destination port, source IP, etc…