This is an odd one, I am losing my main internet uplink interface every ~2.5 hours and it looks like it is caused by the VPN.
root@router2:~# show ip bgp sum
IPv4 Unicast Summary:
BGP router identifier 204.89.189.3, local AS number 398334 vrf-id 0
BGP table version 11482347
RIB entries 1537949, using 270 MiB of memory
Peers 6, using 123 KiB of memory
Neighbor V AS MsgRcvd MsgSent TblVer InQ OutQ Up/Down State/PfxRcd
144.121.130.193 4 46887 714047 673 0 0 0 00:16:29 Active
169.254.0.5 4 65534 1919 1933 0 0 0 00:18:39 Active
204.89.189.2 4 398334 447619 489764 0 0 0 02:54:23 5
204.89.189.4 4 398334 1217127 464061 0 0 0 02:53:46 840778
204.89.189.56 4 398334 889 1695315 0 0 0 02:53:30 32
If I ping the other side of my main uplink 144.121.130.193 I get no reply. However, if I do a “reset vpn ipsec-peer 35.245.110.63” then both interfaces come back up, I can ping over them and BGP comes back.
Any ideas? This is driving me crazy. I am running 1.3-rolling-202011201558
My config:
high-availability {
vrrp {
group management {
interface bond0.3
virtual-address 10.0.2.1/24
vrid 3
}
group private {
interface bond0.2
priority 220
virtual-address 10.0.0.1/23
vrid 2
}
group public {
interface bond0
virtual-address 204.89.189.1/24
virtual-address 204.117.64.1/24
vrid 1
}
}
}
interfaces {
bonding bond0 {
address 204.89.189.3/24
address 204.117.64.3/24
description "Public Internet"
hash-policy layer2
member {
interface eth0
interface eth1
}
mode 802.3ad
mtu 9000
policy {
route PBR
}
vif 2 {
address 10.0.0.3/23
description "Vocinity Private"
mtu 9000
}
vif 3 {
address 10.0.2.3/24
description "Vocinity Management"
mtu 9000
}
}
ethernet eth0 {
hw-id 00:02:c9:07:76:96
mtu 9000
}
ethernet eth1 {
hw-id 00:02:c9:0d:03:3c
mtu 9000
}
ethernet eth2 {
address 144.121.130.194/30
description "Crown Castle 10Gig"
hw-id 00:02:c9:07:76:97
}
ethernet eth3 {
hw-id 00:02:c9:0d:03:3d
}
loopback lo {
}
vti vti10 {
address 169.254.0.6/30
mtu 1436
}
}
nat {
source {
rule 10 {
outbound-interface eth2
protocol all
source {
address 10.0.0.0/22
}
translation {
address 204.89.189.3
}
}
rule 11 {
outbound-interface eth2
protocol all
source {
address 172.16.0.0/16
}
translation {
address 204.89.189.3
}
}
rule 20 {
outbound-interface bond0
protocol all
source {
address 10.0.0.0/22
}
translation {
address 204.89.189.3
}
}
rule 21 {
outbound-interface bond0
protocol all
source {
address 172.16.0.0/16
}
translation {
address 204.89.189.3
}
}
}
}
policy {
prefix-list bogons {
rule 10 {
action permit
description "this network [RFC1122]"
le 32
prefix 0.0.0.0/8
}
rule 11 {
action permit
description "private space [RFC1918]"
le 32
prefix 10.0.0.0/8
}
rule 12 {
action permit
description "CGN shared [RFC6598]"
le 32
prefix 100.64.0.0/10
}
rule 13 {
action permit
description "localhost [RFC1122]"
le 32
prefix 127.0.0.0/8
}
rule 14 {
action permit
description "link local [RFC3927]"
le 32
prefix 169.254.0.0/16
}
rule 15 {
action permit
description "private space [RFC1918]"
le 32
prefix 172.16.0.0/12
}
rule 16 {
action permit
description "TEST-NET-1 [RFC5737]"
le 32
prefix 192.0.2.0/24
}
rule 17 {
action permit
description "6to4 anycast relay [RFC7526]"
le 32
prefix 192.88.99.0/24
}
rule 18 {
action permit
description "private space [RFC1918]"
le 32
prefix 192.168.0.0/16
}
rule 19 {
action permit
description "benchmarking [RFC2544]"
le 32
prefix 198.18.0.0/15
}
rule 20 {
action permit
description "TEST-NET-2 [RFC5737]"
le 32
prefix 198.51.100.0/24
}
rule 21 {
action permit
description "TEST-NET-3 [RFC5737]"
le 32
prefix 203.0.113.0/24
}
rule 22 {
action permit
description multicast
le 32
prefix 224.0.0.0/4
}
rule 23 {
action permit
description "reserved for future use"
le 32
prefix 240.0.0.0/4
}
}
prefix-list default {
rule 10 {
action permit
prefix 0.0.0.0/0
}
}
prefix-list internal {
rule 10 {
action permit
prefix 10.0.0.0/8
}
}
prefix-list local {
rule 10 {
action permit
prefix 204.89.189.0/24
}
rule 11 {
action permit
prefix 204.117.64.0/24
}
}
route PBR {
rule 20 {
description "AS12189 irp probe traffic to table 10"
set {
table 10
}
source {
address 204.89.189.28/32
}
}
rule 30 {
description "AS46887 irp probe traffic to table 11"
set {
table 11
}
source {
address 204.89.189.29/32
}
}
rule 40 {
description "AS29838 irp probe traffic to table 12"
set {
table 12
}
source {
address 204.89.189.27/32
}
}
}
route-map external-in {
rule 10 {
action deny
match {
ip {
address {
prefix-list bogons
}
}
}
}
rule 11 {
action deny
match {
ip {
address {
prefix-list default
}
}
}
}
rule 20 {
action permit
set {
as-path-prepend "46887 46887"
}
}
}
route-map internal-out {
rule 10 {
action permit
match {
ip {
address {
prefix-list internal
}
}
}
}
}
route-map local-out {
rule 10 {
action permit
match {
ip {
address {
prefix-list local
}
}
}
set {
as-path-prepend 398334
}
}
}
}
protocols {
bgp 398334 {
address-family {
ipv4-unicast {
network 10.0.0.0/8 {
}
network 204.89.189.0/24 {
}
network 204.117.64.0/24 {
}
redistribute {
connected {
}
}
}
}
neighbor 144.121.130.193 {
address-family {
ipv4-unicast {
route-map {
export local-out
import external-in
}
soft-reconfiguration {
inbound
}
}
}
password ****************
remote-as 46887
}
neighbor 169.254.0.5 {
address-family {
ipv4-unicast {
route-map {
export internal-out
}
soft-reconfiguration {
inbound
}
}
}
remote-as 65534
}
neighbor 204.89.189.2 {
address-family {
ipv4-unicast {
nexthop-self {
}
soft-reconfiguration {
inbound
}
}
}
remote-as 398334
}
neighbor 204.89.189.4 {
address-family {
ipv4-unicast {
nexthop-self {
}
soft-reconfiguration {
inbound
}
}
}
remote-as 398334
}
neighbor 204.89.189.56 {
address-family {
ipv4-unicast {
route-reflector-client
soft-reconfiguration {
inbound
}
}
}
remote-as 398334
}
parameters {
router-id 204.89.189.3
}
}
static {
route 10.0.3.0/24 {
next-hop 10.0.0.9 {
distance 1
}
}
route 10.40.0.0/14 {
next-hop 10.0.0.251 {
distance 1
}
}
route 10.88.64.0/22 {
next-hop 10.0.0.9 {
distance 1
}
}
route 172.16.0.0/16 {
next-hop 10.0.0.53 {
distance 1
}
}
route 204.89.189.0/24 {
blackhole {
distance 254
}
}
route 204.117.64.0/24 {
blackhole {
distance 254
}
}
table 10 {
route 0.0.0.0/0 {
next-hop 172.24.255.14 {
}
}
}
table 11 {
route 0.0.0.0/0 {
next-hop 144.121.130.193 {
}
}
}
table 12 {
route 0.0.0.0/0 {
next-hop 69.9.42.177 {
}
}
}
}
}
service {
snmp {
community vocinity {
client 10.0.0.32
client 10.0.0.56
}
}
ssh {
listen-address 10.0.0.3
listen-address 10.0.2.3
listen-address 204.89.189.3
}
}
system {
config-management {
commit-revisions 100
}
console {
device ttyS0 {
speed 115200
}
}
domain-name vocinity.com
flow-accounting {
buffer-size 1024
interface eth2
interface bond0
netflow {
sampling-rate 1
server 10.0.0.36 {
}
server 10.0.0.56 {
port 2055
}
timeout {
expiry-interval 10
}
version 9
}
syslog-facility daemon
}
host-name router2
login {
user vyos {
authentication {
encrypted-password ****************
plaintext-password ****************
}
}
}
name-server 10.0.0.5
name-server 8.8.8.8
ntp {
server 0.pool.ntp.org {
}
server 1.pool.ntp.org {
}
server 2.pool.ntp.org {
}
}
syslog {
global {
facility all {
level info
}
facility protocols {
level debug
}
}
}
}
vpn {
ipsec {
esp-group gcp-esp {
compression disable
lifetime 10800
mode tunnel
pfs enable
proposal 1 {
encryption aes256
hash sha1
}
}
ike-group gcp-ike {
close-action none
ikev2-reauth no
key-exchange ikev2
lifetime 36000
proposal 1 {
dh-group 14
encryption aes256
hash sha1
}
}
ipsec-interfaces {
interface bond0
}
site-to-site {
peer 35.245.110.63 {
authentication {
mode pre-shared-secret
pre-shared-secret ****************
}
connection-type initiate
ike-group gcp-ike
ikev2-reauth inherit
local-address 204.89.189.3
vti {
bind vti10
esp-group gcp-esp
}
}
}
}
}