I am facing a very weird issue. Need some pointers on how to troubleshoot
Picked up latest rolling and tried it on the 1.3.2 LTS. Both hit the exact same issue so I guess something is not right in my config.
I have two WAN connections – WAN1is PPPoE (gives public IPv4 and IPv6), WAN2 is a static IP (public IPv4 only). WAN1 is primary and WAN2 will become failover (once this issue is resolved).
I have NextDNS as well as ControlD subscription and individual devices are configured for either DoT or DoH
When WAN1 is active, All the devices complains about DNS unavailability and when I look at the conntrack table, I see the secure DNS connections are in Close state. If I disconnect the WAN1 and once WAN2 takes over due to change in routing table, all secure DNS connections become fine.
The whole config is below. Please suggest what can be changed.
firewall {
all-ping enable
broadcast-ping disable
config-trap disable
group {
address-group google_dns {
address 8.8.8.8
address 8.8.4.4
address 114.114.114.114
}
ipv6-address-group google-ipv6-dns {
address 2001:4860:4860::8888
address 2001:4860:4860::8844
}
network-group RFC1918 {
description "RFC1918 Address"
network 172.16.0.0/12
network 192.168.0.0/16
network 10.0.0.0/8
}
network-group apac {
network xxx.xxx.182.0/24
network xxx.xxx.184.0/21
}
port-group dnsports {
port 53
port 853
port 443
}
}
ipv6-name v6-local-north {
default-action drop
enable-default-log
rule 100 {
action accept
}
}
ipv6-name v6-local-south {
default-action drop
enable-default-log
rule 100 {
action accept
}
}
ipv6-name v6-north-local {
default-action drop
enable-default-log
rule 100 {
action accept
description "Accept Established/Related"
protocol all
state {
established enable
related enable
}
}
rule 200 {
action accept
description "Accept ICMPv6"
protocol icmpv6
}
rule 300 {
action accept
description "Accept DHCPv6"
destination {
port 546
}
protocol udp
source {
port 547
}
}
rule 400 {
action drop
description "Drop Invalid state"
protocol all
state {
invalid enable
}
}
}
ipv6-name v6-north-south {
default-action drop
enable-default-log
rule 100 {
action accept
description "Accept Established/Related"
protocol all
state {
established enable
related enable
}
}
rule 200 {
action accept
description "Accept ICMP"
protocol icmpv6
}
rule 300 {
action drop
description "Drop Invalid state"
protocol all
state {
invalid enable
}
}
}
ipv6-name v6-south-local {
default-action drop
enable-default-log
rule 100 {
action accept
}
}
ipv6-name v6-south-north {
default-action drop
enable-default-log
rule 100 {
action drop
destination {
group {
address-group google-ipv6-dns
port-group dnsports
}
}
protocol tcp
}
rule 200 {
action drop
destination {
group {
address-group google-ipv6-dns
port-group dnsports
}
}
protocol udp
}
rule 300 {
action accept
}
}
ip-src-route disable
log-martians enable
name local-north {
default-action drop
description "This Router to WAN IPv4"
enable-default-log
rule 100 {
action accept
}
}
name local-south {
default-action drop
description "This Firewall to LAN IPv4"
enable-default-log
rule 100 {
action accept
description "Firewall to LAN IPv4!"
}
}
name north-local {
default-action drop
description "WAN to This Device IPv4"
enable-default-log
rule 100 {
action accept
state {
established enable
related enable
}
}
rule 200 {
action drop
state {
invalid enable
}
}
}
name north-south {
default-action drop
description "North to South IPv4"
enable-default-log
rule 100 {
action accept
state {
established enable
related enable
}
}
rule 200 {
action accept
destination {
address 192.168.50.128/25
}
source {
group {
network-group apac
}
}
}
rule 300 {
action drop
state {
invalid enable
}
}
}
name south-local {
default-action drop
description "LAN to This Router IPv4"
enable-default-log
rule 100 {
action accept
description "Allow Access to Firewall IPv4"
}
}
name south-north {
default-action drop
description "LAN to WAN IPv4"
enable-default-log
rule 100 {
action drop
description "No Access to Google DNS"
destination {
group {
address-group google_dns
port-group dnsports
}
}
protocol tcp
}
rule 200 {
action drop
description "No Access to Google DNS"
destination {
group {
address-group google_dns
port-group dnsports
}
}
protocol udp
}
rule 300 {
action accept
description "Allow Everything Else!"
}
}
send-redirects enable
syn-cookies enable
twa-hazards-protection disable
zone local {
from north {
firewall {
ipv6-name v6-north-local
name north-local
}
}
from south {
firewall {
ipv6-name v6-south-local
name south-local
}
}
local-zone
}
zone north {
from local {
firewall {
ipv6-name v6-local-north
name local-north
}
}
from south {
firewall {
ipv6-name v6-south-north
name south-north
}
}
interface eth1
interface pppoe0
}
zone south {
from local {
firewall {
ipv6-name v6-local-south
name local-south
}
}
from north {
firewall {
ipv6-name v6-north-south
name north-south
}
}
interface eth2
}
}
interfaces {
ethernet eth0 {
description EMPTY
duplex auto
hw-id 40:62:31:12:7e:84
speed auto
}
ethernet eth1 {
address xxx.xxx.42.189/24
description "Static Interface"
duplex auto
hw-id 40:62:31:12:7e:85
speed auto
}
ethernet eth2 {
address 192.168.50.129/25
description VLAN314
duplex auto
hw-id 40:62:31:12:7e:86
speed auto
}
ethernet eth3 {
description "PPPoE Interface"
duplex auto
hw-id 40:62:31:12:7e:87
offload {
gro
gso
tso
}
speed auto
}
loopback lo {
}
pppoe pppoe0 {
authentication {
password ****************
username ****************
}
default-route-distance 1
dhcpv6-options {
pd 0 {
interface eth2 {
address 1
sla-id 0
}
length 64
}
rapid-commit
}
ipv6 {
address {
autoconf
}
}
source-interface eth3
}
}
nat {
source {
rule 5010 {
outbound-interface pppoe0
source {
address 192.168.50.128/25
}
translation {
address masquerade
}
}
rule 5020 {
outbound-interface eth1
source {
address 192.168.50.128/25
}
translation {
address masquerade
}
}
}
}
protocols {
static {
route 0.0.0.0/0 {
next-hop xxx.xxx.42.1 {
distance 220
}
}
}
}
service {
dhcp-server {
shared-network-name VLAN314 {
subnet 192.168.50.128/25 {
default-router 192.168.50.129
lease 7200
name-server 9.9.9.9
name-server 1.1.1.1
range 0 {
start 192.168.50.140
stop 192.168.50.240
}
static-mapping deco-lobby {
ip-address 192.168.50.247
mac-address 54:af:97:4b:0d:78
}
static-mapping deco-office {
ip-address 192.168.50.246
mac-address 54:af:97:4b:16:00
}
static-mapping printer {
ip-address 192.168.50.250
mac-address 14:cb:19:13:c4:83
}
}
}
}
dns {
forwarding {
allow-from 192.168.50.128/25
cache-size 0
listen-address 192.168.50.129
name-server 2606:4700:4700::1111
name-server 2606:4700:4700::1001
name-server 9.9.9.9
name-server 1.1.1.1
}
}
ntp {
allow-client {
address 0.0.0.0/0
address ::/0
}
server time1.vyos.net {
}
server time2.vyos.net {
}
server time3.vyos.net {
}
server time.apple.com {
}
server time.google.com {
}
}
router-advert {
interface eth2 {
default-lifetime 1000
default-preference medium
hop-limit 64
interval {
max 600
min 200
}
link-mtu 1492
prefix ::/64 {
preferred-lifetime 300
valid-lifetime 900
}
reachable-time 900000
retrans-timer 0
route ::/0 {
route-preference medium
}
}
}
ssh {
listen-address 192.168.50.129
port 22
}
}
system {
config-management {
commit-revisions 50
}
conntrack {
modules {
ftp
h323
nfs
pptp
sip
sqlnet
tftp
}
}
console {
device ttyS0 {
speed 115200
}
}
host-name vyos-pghome
login {
user vyos {
authentication {
encrypted-password ****************
plaintext-password ****************
}
}
}
name-server 9.9.9.9
name-server 1.0.0.3
syslog {
global {
facility all {
level info
}
facility protocols {
level debug
}
}
}
}
vyos@vyos-pghome:~$ show conntrack table ipv4 | match CLOSE
96990961 192.168.50.163:56864 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56864 tcp CLOSE 0
1755131400 192.168.50.163:56868 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56868 tcp CLOSE 0
109362680 192.168.50.163:56871 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56871 tcp CLOSE 4
1376176983 192.168.50.163:56886 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56886 tcp CLOSE 8
2081159336 192.168.50.163:56884 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56884 tcp CLOSE 8
2540851035 192.168.50.163:56878 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56878 tcp CLOSE 8
3741069042 192.168.50.163:56874 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56874 tcp CLOSE 4
3685412744 192.168.50.163:56885 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56885 tcp CLOSE 8
3203357945 192.168.50.163:56882 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56882 tcp CLOSE 8
4139253757 192.168.50.163:56863 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56863 tcp CLOSE 0
2892239778 192.168.50.163:56876 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56876 tcp CLOSE 8
1484574281 192.168.50.163:56869 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56869 tcp CLOSE 0
4088457300 192.168.50.163:56866 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56866 tcp CLOSE 0
2968011344 192.168.50.163:56887 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56887 tcp CLOSE 8
959950871 192.168.50.163:56879 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56879 tcp CLOSE 8
810116120 192.168.50.163:56883 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56883 tcp CLOSE 8
1294691336 192.168.50.163:56888 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56888 tcp CLOSE 8
4001782502 192.168.50.163:56873 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56873 tcp CLOSE 4
2381457619 192.168.50.163:56881 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56881 tcp CLOSE 8
3681586203 192.168.50.163:56865 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56865 tcp CLOSE 0
2367763093 192.168.50.163:56889 76.76.10.22:443 76.76.10.22:443 xxx.xxx196.35:56889 tcp CLOSE 8
1597342096 192.168.50.163:56875 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56875 tcp CLOSE 8
2749612757 192.168.50.163:56867 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56867 tcp CLOSE 0
3169830273 192.168.50.163:56877 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56877 tcp CLOSE 8
1740576987 192.168.50.163:56872 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56872 tcp CLOSE 4
2284616856 192.168.50.163:56880 76.76.2.22:443 76.76.2.22:443 xxx.xxx196.35:56880 tcp CLOSE 8
vyos@vyos-pghome:~$ show version
Version: VyOS 1.4-rolling-202303170317
Release train: current
Built by: autobuild@vyos.net
Built on: Fri 17 Mar 2023 03:17 UTC
Build UUID: 45391302-1240-4cc7-95a8-da8ee6390765
Build commit ID: e887f582cfd7de
Architecture: x86_64
Boot via: installed image
System type: guest
Hardware vendor: Default string
Hardware model: Default string
Hardware S/N: Default string
Hardware UUID: 03000200-0400-0500-0006-000700080009
Copyright: VyOS maintainers and contributors