VYOS Dual WAN Issues

josh@ROUTER:~$ sh conf
firewall {
    all-ping enable
    broadcast-ping disable
    config-trap disable
    ipv6-receive-redirects disable
    ipv6-src-route disable
    ip-src-route disable
    log-martians enable
    name OUTSIDE-IN {
        default-action drop
        rule 10 {
            action accept
            state {
                established enable
                related enable
            }
        }
        rule 20 {
            action accept
            description OpenVPN
            destination {
                address 192.168.10.13
                port 1194
            }
            protocol udp
            state {
                established enable
                new enable
                related enable
            }
        }
        rule 30 {
            action accept
            description Plex
            destination {
                address 192.168.10.16
                port 32400
            }
            protocol tcp_udp
            state {
                established enable
                new enable
                related enable
            }
        }
        rule 40 {
            action accept
            description "Ethan Plex"
            destination {
                address 192.168.10.4
                port 32400
            }
            protocol tcp_udp
            state {
                established enable
                new enable
                related enable
            }
        }
    }
    name OUTSIDE-LOCAL {
        default-action drop
        rule 10 {
            action accept
            state {
                established enable
                related enable
            }
        }
        rule 20 {
            action accept
            icmp {
                type-name echo-request
            }
            protocol icmp
            state {
                new enable
            }
        }
        rule 30 {
            action drop
            destination {
                port 22
            }
            log enable
            protocol tcp
            state {
                new enable
            }
        }
    }
    receive-redirects disable
    send-redirects enable
    source-validation disable
    syn-cookies enable
    twa-hazards-protection disable
}
interfaces {
    ethernet eth0 {
        address dhcp
        description ValleyWAN
        duplex auto
        firewall {
            in {
                name OUTSIDE-IN
            }
            local {
                name OUTSIDE-LOCAL
            }
        }
        hw-id d4:ae:52:a6:26:75
        smp-affinity auto
        speed auto
    }
    ethernet eth1 {
        duplex auto
        hw-id d4:ae:52:a6:26:76
        smp-affinity auto
        speed auto
        vif 7 {
            address 192.168.7.1/24
            description TEST
        }
        vif 10 {
            address 192.168.10.1/24
            description Servers
        }
        vif 20 {
            address 192.168.20.1/24
            description Desktops
        }
        vif 30 {
            address 192.168.30.1/24
            description WiFi
        }
        vif 40 {
            address 192.168.40.1/24
            description "Smart Home"
        }
        vif 50 {
            address 192.168.50.1/24
            description Management
        }
        vif 60 {
            address 192.168.60.1/24
            description Surveillance
        }
        vif 70 {
            address 192.168.70.1/24
            description "Guest WiFi"
        }
        vif 80 {
            address 192.168.80.1/24
            description HTB
        }
        vif 110 {
            address 192.168.110.1/24
            description Smoqueed
        }
    }
    ethernet eth2 {
        address 192.168.90.1/30
        description "Lab Environment"
        duplex auto
        hw-id d4:ae:52:a6:26:77
        smp-affinity auto
        speed auto
    }
    ethernet eth3 {
        address dhcp
        description RiseWAN
        duplex auto
        firewall {
            in {
                name OUTSIDE-IN
            }
            local {
                name OUTSIDE-LOCAL
            }
        }
        hw-id d4:ae:52:a6:26:78
        smp-affinity auto
        speed auto
    }
    loopback lo {
        description Loopback
    }
}
nat {
    destination {
        rule 20 {
            description OpenVPN
            destination {
                address 158.140.34.8
                port 1194
            }
            inbound-interface eth0
            protocol udp
            translation {
                address 192.168.10.13
                port 1194
            }
        }
        rule 30 {
            description Plex
            destination {
                address 158.140.34.8
                port 32400
            }
            inbound-interface eth0
            protocol tcp_udp
            translation {
                address 192.168.10.16
                port 32400
            }
        }
        rule 40 {
            description "Ethan Plex"
            destination {
                address 158.140.34.8
                port 32500
            }
            inbound-interface eth0
            protocol tcp_udp
            translation {
                address 192.168.10.4
                port 32400
            }
        }
    }
    source {
        rule 7 {
            description TEST
            outbound-interface eth3
            source {
                address 192.168.7.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 10 {
            outbound-interface eth0
            source {
                address 192.168.10.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 20 {
            outbound-interface eth3
            source {
                address 192.168.20.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 30 {
            outbound-interface eth3
            source {
                address 192.168.30.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 40 {
            outbound-interface eth3
            source {
                address 192.168.40.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 50 {
            outbound-interface eth3
            source {
                address 192.168.50.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 60 {
            outbound-interface eth3
            source {
                address 192.168.60.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 70 {
            outbound-interface eth3
            source {
                address 192.168.70.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 80 {
            outbound-interface eth3
            source {
                address 192.168.80.0/24
            }
            translation {
                address masquerade
            }
        }
        rule 110 {
            outbound-interface eth3
            source {
                address 192.168.110.0/24
            }
            translation {
                address masquerade
            }
        }
    }
}
protocols {
    static {
        route 10.1.1.0/24 {
            next-hop 192.168.90.2 {
            }
        }
    }
}
service {
    dhcp-server {
        disabled false
        hostfile-update disable
        shared-network-name Desktops {
            authoritative disable
            subnet 192.168.20.0/24 {
                default-router 192.168.20.1
                dns-server 192.168.10.7
                dns-server 192.168.10.14
                domain-name sharrer.us
                lease 86400
                start 192.168.20.11 {
                    stop 192.168.20.254
                }
            }
        }
        shared-network-name Guest_WiFi {
            authoritative disable
            subnet 192.168.70.0/24 {
                default-router 192.168.70.1
                dns-server 192.168.10.14
                domain-name sharrer.us
                lease 86400
                start 192.168.70.10 {
                    stop 192.168.70.254
                }
            }
        }
        shared-network-name HTB {
            authoritative disable
            subnet 192.168.80.0/24 {
                default-router 192.168.80.1
                dns-server 192.168.10.14
                domain-name sharrer.us
                lease 86400
                start 192.168.80.10 {
                    stop 192.168.80.20
                }
            }
        }
        shared-network-name Smart_Home {
            authoritative disable
            subnet 192.168.40.0/24 {
                default-router 192.168.40.1
                dns-server 192.168.10.14
                domain-name sharrer.us
                lease 86400
                start 192.168.40.10 {
                    stop 192.168.40.254
                }
            }
        }
        shared-network-name Surveillance {
            authoritative disable
            subnet 192.168.60.0/24 {
                default-router 192.168.60.1
                dns-server 192.168.10.14
                domain-name sharrer.us
                lease 86400
                start 192.168.60.10 {
                    stop 192.168.60.254
                }
            }
        }
        shared-network-name WiFi {
            authoritative disable
            subnet 192.168.30.0/24 {
                default-router 192.168.30.1
                dns-server 192.168.10.14
                domain-name sharrer.us
                lease 86400
                start 192.168.30.5 {
                    stop 192.168.30.254
                }
            }
        }
    }
    dns {
        forwarding {
            cache-size 0
            listen-on eth1.10
            listen-on eth1.20
            listen-on eth1.30
            listen-on eth1.40
            listen-on eth1.50
            listen-on eth1.60
            listen-on eth1.70
            listen-on eth1.80
            listen-on eth1.110
            listen-on eth2
            name-server 192.168.10.14
        }
    }
    lldp {
        interface eth1 {
        }
        interface eth1.10 {
        }
        interface eth1.20 {
        }
        interface eth2 {
        }
    }
    mdns {
        repeater {
            interface eth1.20
            interface eth1.30
            interface eth1.40
        }
    }
    ssh {
        port 22
    }
}
system {
    config-management {
        commit-revisions 100
    }
    console {
        device ttyS0 {
            speed 9600
        }
    }
    domain-name sharrer.us
    host-name ROUTER
    ip {
        arp {
            table-size 8192
        }
    }
    ipv6 {
        disable
    }
    login {
        user josh {
            authentication {
                encrypted-password ****************
                plaintext-password ****************
            }
            level admin
        }
    }
    ntp {
        server 0.pool.ntp.org {
        }
        server 1.pool.ntp.org {
        }
        server 2.pool.ntp.org {
        }
    }
    syslog {
        global {
            facility all {
                level notice
            }
            facility protocols {
                level debug
            }
        }
    }
    time-zone US/Pacific
}

Above is the config Im using. The issue Im experiencing is that when I have interface 0 and 3 enabled at the same time I experience network issues such as packet loss, and connectivity issues. If I enable int eth0 and disable eth3, set all the source nat outbound-interface to the eth0, everything works. If I enable int eth3 and disable eth0, set all the source nat outbound-interface to the eth3, everything works. But It will not work with both enabled. Any suggestions?

1 Like

I don’t think that your issue is firewall related, I rather think you lose the tcp state via routing. Syn goes out eth0, ayn/ack comes back eth0, ack being sent now via eth3 (different src IP). Have a tcpdump on both interfaces, try to initiate a connection and see what and where it goes out.

1 Like

hagbard is right. If you have more than one active default routes and want to use NAT, you must configure load balancing or routing policy to be ensure that all traffic for same connection will be forwarded through one external interface.
Otherwise, problems is inevitable. :slight_smile:

1 Like

This topic was automatically closed 2 days after the last reply. New replies are no longer allowed.