L2TP/IPsec VPN with PPPoE and Firewall - VyOS 1.4

felipesc · December 6, 2022, 9:41pm

Hello, I’m attempting to set up a L2TP/IPsec VPN, but my firewall rules are interfering with internet access. The issue is that I didn’t find a way to permit it and didn’t find the rule blocking it either. What is the best way to do it?

VyOS

set firewall name LAN-LOCAL default-action 'accept'
set firewall name LAN-WAN default-action 'accept'
set firewall name LOCAL-LAN default-action 'accept'
set firewall name LOCAL-WAN default-action 'accept'
set firewall name WAN-LAN default-action 'drop'
set firewall name WAN-LAN rule 5 action 'accept'
set firewall name WAN-LAN rule 5 description 'Allow EST/Related Traffic'
set firewall name WAN-LAN rule 5 state established 'enable'
set firewall name WAN-LAN rule 5 state related 'enable'
set firewall name WAN-LAN rule 20 action 'accept'
set firewall name WAN-LAN rule 20 protocol 'icmp'
set firewall name WAN-LAN rule 20 state new 'enable'
set firewall name WAN-LOCAL default-action 'drop'
set firewall name WAN-LOCAL rule 5 action 'accept'
set firewall name WAN-LOCAL rule 5 description 'Allow EST/Related Traffic'
set firewall name WAN-LOCAL rule 5 state established 'enable'
set firewall name WAN-LOCAL rule 5 state related 'enable'
set firewall name WAN-LOCAL rule 20 action 'accept'
set firewall name WAN-LOCAL rule 20 protocol 'icmp'
set firewall name WAN-LOCAL rule 20 state new 'enable'
set firewall name WAN-LOCAL rule 30 action 'drop'
set firewall name WAN-LOCAL rule 30 destination port '22'
set firewall name WAN-LOCAL rule 30 protocol 'tcp'
set firewall name WAN-LOCAL rule 30 recent count '4'
set firewall name WAN-LOCAL rule 30 recent time 'minute'
set firewall name WAN-LOCAL rule 30 state new 'enable'
set firewall name WAN-LOCAL rule 31 action 'accept'
set firewall name WAN-LOCAL rule 31 destination port '22'
set firewall name WAN-LOCAL rule 31 protocol 'tcp'
set firewall name WAN-LOCAL rule 31 state new 'enable'
set firewall name WAN-LOCAL rule 40 action 'accept'
set firewall name WAN-LOCAL rule 40 protocol 'esp'
set firewall name WAN-LOCAL rule 41 action 'accept'
set firewall name WAN-LOCAL rule 41 destination port '500'
set firewall name WAN-LOCAL rule 41 protocol 'udp'
set firewall name WAN-LOCAL rule 42 action 'accept'
set firewall name WAN-LOCAL rule 42 destination port '4500'
set firewall name WAN-LOCAL rule 42 protocol 'udp'
set firewall name WAN-LOCAL rule 43 action 'accept'
set firewall name WAN-LOCAL rule 43 destination port '1701'
set firewall name WAN-LOCAL rule 43 ipsec match-ipsec
set firewall name WAN-LOCAL rule 43 protocol 'udp'
set firewall zone LAN default-action 'drop'
set firewall zone LAN from LOCAL firewall name 'LOCAL-LAN'
set firewall zone LAN from WAN firewall name 'WAN-LAN'
set firewall zone LAN interface 'eth1'
set firewall zone LOCAL default-action 'drop'
set firewall zone LOCAL from LAN firewall name 'LAN-LOCAL'
set firewall zone LOCAL from WAN firewall name 'WAN-LOCAL'
set firewall zone LOCAL local-zone
set firewall zone WAN default-action 'drop'
set firewall zone WAN from LAN firewall name 'LAN-WAN'
set firewall zone WAN from LOCAL firewall name 'LOCAL-WAN'
set firewall zone WAN interface 'pppoe0'
set interfaces ethernet eth0 hw-id 'xx:xx:xx:xx:xx:5a'
set interfaces ethernet eth1 address 'xxx.xxx.0.1/24'
set interfaces ethernet eth1 description 'LAN'
set interfaces ethernet eth1 hw-id 'xx:xx:xx:xx:xx:5b'
set interfaces ethernet eth1 ip adjust-mss '1452'
set interfaces ethernet eth2 hw-id 'xx:xx:xx:xx:xx:5c'
set interfaces ethernet eth3 hw-id 'xx:xx:xx:xx:xx:5d'
set interfaces loopback lo
set interfaces pppoe pppoe0 authentication password xxxxxx
set interfaces pppoe pppoe0 authentication user xxxxxx
set interfaces pppoe pppoe0 description 'WAN'
set interfaces pppoe pppoe0 dhcpv6-options pd 0 interface eth0 address '1'
set interfaces pppoe pppoe0 dhcpv6-options pd 0 interface eth0 sla-id '0'
set interfaces pppoe pppoe0 dhcpv6-options pd 0 interface eth1 address '100'
set interfaces pppoe pppoe0 dhcpv6-options pd 0 interface eth1 sla-id '0'
set interfaces pppoe pppoe0 dhcpv6-options pd 0 length '56'
set interfaces pppoe pppoe0 ip adjust-mss '1452'
set interfaces pppoe pppoe0 ipv6 address autoconf
set interfaces pppoe pppoe0 ipv6 adjust-mss 'clamp-mss-to-pmtu'
set interfaces pppoe pppoe0 mtu '1492'
set interfaces pppoe pppoe0 source-interface 'eth0'
set nat source rule 100 outbound-interface 'pppoe0'
set nat source rule 100 source address 'xxx.xxx.0.0/24'
set nat source rule 100 translation address 'masquerade'
set nat source rule 110 outbound-interface 'pppoe0'
set nat source rule 110 source address 'xxx.xxx.255.0/24'
set nat source rule 110 translation address 'masquerade'
set policy route MSS description 'TCP MSS Clamping for PPPoE'
set policy route MSS rule 5 protocol 'tcp'
set policy route MSS rule 5 set tcp-mss '1452'
set policy route MSS rule 5 tcp flags syn
set service dhcp-server shared-network-name xxxxxx subnet xxx.xxx.0.0/24 default-router 'xxx.xxx.0.1'
set service dhcp-server shared-network-name xxxxxx subnet xxx.xxx.0.0/24 domain-name xxxxxx
set service dhcp-server shared-network-name xxxxxx subnet xxx.xxx.0.0/24 lease '86400'
set service dhcp-server shared-network-name xxxxxx subnet xxx.xxx.0.0/24 name-server 'xxx.xxx.0.1'
set service dhcp-server shared-network-name xxxxxx subnet xxx.xxx.0.0/24 range 0 start 'xxx.xxx.0.9'
set service dhcp-server shared-network-name xxxxxx subnet xxx.xxx.0.0/24 range 0 stop 'xxx.xxx.0.254'
set service dns forwarding allow-from 'xxx.xxx.0.0/24'
set service dns forwarding cache-size '0'
set service dns forwarding listen-address 'xxx.xxx.0.1'
set service dns forwarding name-server 'xxx.xxx.0.1'
set service dns forwarding name-server 'xxx.xxx.1.1'
set service dns forwarding name-server 'xxx.xxx.4.4'
set service dns forwarding name-server 'xxx.xxx.8.8'
set service router-advert interface eth1 link-mtu '1492'
set service router-advert interface eth1 prefix ::/64 preferred-lifetime '2700'
set service router-advert interface eth1 prefix ::/64 valid-lifetime '5400'
set service ssh dynamic-protection
set service ssh port '22'
set system config-management commit-revisions '100'
set system conntrack modules ftp
set system conntrack modules h323
set system conntrack modules nfs
set system conntrack modules pptp
set system conntrack modules sip
set system conntrack modules sqlnet
set system conntrack modules tftp
set system console device ttyS0 speed '115200'
set system host-name xxxxxx
set system login user xxxxxx authentication encrypted-password xxxxxx
set system name-server 'xxx.xxx.0.1'
set system ntp server xxxxx.tld
set system ntp server xxxxx.tld
set system ntp server xxxxx.tld
set system syslog global facility all level 'info'
set system syslog global facility protocols level 'debug'
set system time-zone 'America/Sao_Paulo'
set vpn ipsec interface 'pppoe0'
set vpn l2tp remote-access authentication local-users username xxxxxx password xxxxxx
set vpn l2tp remote-access authentication mode 'local'
set vpn l2tp remote-access client-ip-pool start 'xxx.xxx.255.2'
set vpn l2tp remote-access client-ip-pool stop 'xxx.xxx.255.254'
set vpn l2tp remote-access ipsec-settings authentication mode 'pre-shared-secret'
set vpn l2tp remote-access ipsec-settings authentication pre-shared-secret xxxxxx
set vpn l2tp remote-access ipsec-settings lifetime '3600'
set vpn l2tp remote-access name-server 'xxx.xxx.8.8'
set vpn l2tp remote-access name-server 'xxx.xxx.1.1'
set vpn l2tp remote-access outside-address 'xxx.xxx.30.2'
set vrf

Viacheslav · December 7, 2022, 12:23am

It seems you didn’t attach ‘l2tp*’ interfaces to any zone
It should be in some “VPN” zone or “Lan/Wan/Dmz/etc”

felipesc · December 9, 2022, 3:44pm

Hi @Viacheslav,

Thank you for your reply. Although I didn't think it was possible, the rule configuration for the nonexistent interface is currently functional. I just made a new firewall zone and rule.

set firewall name VPN-WAN default-action 'accept'
set firewall name WAN-VPN default-action 'drop'
set firewall name WAN-VPN description 'Allow EST/Related Traffic'
set firewall name WAN-VPN rule 5 action 'accept'
set firewall name WAN-VPN rule 5 description 'Allow EST/Related Traffic'
set firewall name WAN-VPN rule 5 state established 'enable'
set firewall name WAN-VPN rule 5 state related 'enable'
set firewall name WAN-VPN rule 20 action 'accept'
set firewall name WAN-VPN rule 20 protocol 'icmp'
set firewall name WAN-VPN rule 20 state new 'enable'
set firewall zone VPN default-action 'drop'
set firewall zone VPN from WAN firewall name 'WAN-VPN'
set firewall zone VPN interface 'l2tp*'
set firewall zone WAN from VPN firewall name 'VPN-WAN'

In throughput tests, I’m noticing some packet loss with VPN connections; the CPU is functioning OK, but 4% of all packets are being lost.

Viacheslav · December 9, 2022, 4:18pm

Do you have any issues from the client to external address of the VyOS router without l2tp?
Do you have issues without firewall?

show int ethernet ethX queue
show int ethernet ethX statistics
show int ethernet ethX
sudo netstat -i
sudo column -t /proc/net/dev
sudo netstat -e -s

Also, try to dump traffic and check with wireshark if you have any retransmissions or dup

felipesc · January 23, 2023, 2:38pm

Hi @Viacheslav ,ddd563d

Please accept my apologies for the delay in responding. Recently, some jobs have demanded nearly all of my time.

I did some tests yesterday and got some updates.
At first, all IPSec/L2TP connections failed, and an upgrade (the last rolling ISO) or reboot was unable to restore the service. But when the VPN configuration was deleted and reconfigured, the service worked, but after some time it went back to the failed (failing to establish the tunnel) state.

vyos@vyos:~$ show int pppoe pppoe0
pppoe0: <POINTOPOINT,MULTICAST,NOARP,UP,LOWER_UP> mtu 1492 qdisc pfifo_fast state UNKNOWN group default qlen 3
    link/ppp
    inet xx.xx.xx.xx peer xx.xx.xx.xx/32 scope global pppoe0
       valid_lft forever preferred_lft forever
    inet6 fe80::f912:f1ff:fe44:506c/64 scope link
       valid_lft forever preferred_lft forever
    inet6 fe80::c592:d5f7:130d:8685 peer fe80::6212:3cff:feb0:f8a2/128 scope link
       valid_lft forever preferred_lft forever
    Description: WAN-Giganet

    RX:   bytes  packets  errors  dropped  overrun       mcast
         247834     1081       0        0        0           0
    TX:   bytes  packets  errors  dropped  carrier  collisions
         312343      904       0        0        0           0
vyos@vyos:~$ show int pppoe pppoe0 statistics
      IN   PACK VJCOMP  VJUNC  VJERR  |      OUT   PACK VJCOMP  VJUNC NON-VJ
  263703   1153      0      0      0  |   321719    965      0      0    965
vyos@vyos:~$ sudo netstat -i
Kernel Interface table
Iface      MTU    RX-OK RX-ERR RX-DRP RX-OVR    TX-OK TX-ERR TX-DRP TX-OVR Flg
br0       1500 15597461      0      3 0      24441965      0      0      0 BMRU
br2241    1500        0      0      0 0             0      0      0      0 BMU
eth0      1500  6367399      0    354 354     5644520      0      0      0 BMRU
eth1      1500 15598520      0   2998 1959   24441964      0      0      0 BMRU
eth2      1500        0      0      0 0             0      0      0      0 BMU
eth3      1500 18572050      0  14427 797    10025703      0      0      0 BMRU
eth0.224  1500        0      0      0 0             0      0      0      0 BMRU
lo       65536     2902      0      0 0          2902      0      0      0 LRU
pppoe0    1492     1262      0      0 0          1041      0      0      0 MOPRU
pppoe1    1492 18554924      0      0 0      10025342      0      0      0 MOPRU
tun0      1400       52      0      0 0           291      0      0      0 OPRU
vyos@vyos:~$ sudo column -t /proc/net/de
dev        dev_mcast  dev_snmp6/
vyos@vyos:~$ sudo column -t /proc/net/dev
Inter-|     Receive      |         Transmit
face        |bytes       packets   errs      drop   fifo  frame  compressed  multicast|bytes  packets      errs      drop  fifo  colls  carrier  compressed
lo:         287898       2902      0         0      0     0      0           0                287898       2902      0     0     0      0        0           0
eth0:       6857514427   6367624   0         354    354   0      0           0                6747901777   5644686   0     0     0      0        0           0
eth1:       18531825927  15598954  0         2999   1959  0      0           19648            31595200311  24442288  0     0     0      0        0           0
eth2:       0            0         0         0      0     0      0           0                0            0         0     0     0      0        0           0
eth3:       25001259587  18572212  0         14432  797   0      0           0                11910538119  10025904  0     0     0      0        0           0
br0:        18310629261  15597894  0         3      0     0      0           18582            31595200311  24442288  0     0     0      0        0           0
br2241:     0            0         0         0      0     0      0           0                0            0         0     0     0      0        0           0
eth0.2241:  0            0         0         0      0     0      0           0                0            0         0     0     0      0        0           0
sit0:       0            0         0         0      0     0      0           0                0            0         0     0     0      0        0           0
tun0:       5408         52        0         0      0     0      0           0                30144        291       0     0     0      0        0           0
pppoe1:     24590178776  18555080  0         0      0     0      0           0                11694594817  10025542  0     0     0      0        0           0
pppoe0:     323989       1485      0         0      0     0      0           0                378571       1205      0     0     0      0        0           0
vyos@vyos:~$ sudo netstat -e -s
Ip:
    Forwarding: 1
    40502771 total packets received
    15 with invalid headers
    39950738 forwarded
    226492 with unknown protocol
    0 incoming packets discarded
    96960 incoming packets delivered
    40103379 requests sent out
    6 outgoing packets dropped
    524 dropped because of missing route
    2 reassemblies required
    1 packets reassembled ok
    1 fragments received ok
    1 fragments failed
    2 fragments created
Icmp:
    30183 ICMP messages received
    11 input ICMP message failed
    ICMP input histogram:
        destination unreachable: 225
        timeout in transit: 168
        echo requests: 3166
        echo replies: 26624
    98398 ICMP messages sent
    0 ICMP messages failed
    ICMP output histogram:
        destination unreachable: 68571
        time exceeded: 15
        echo requests: 26646
        echo replies: 3166
IcmpMsg:
        InType0: 26624
        InType3: 225
        InType8: 3166
        InType11: 168
        OutType0: 3166
        OutType3: 68571
        OutType8: 26646
        OutType11: 15
Tcp:
    4 active connection openings
    889 passive connection openings
    146 failed connection attempts
    36 connection resets received
    4 connections established
    19786 segments received
    20690 segments sent out
    3747 segments retransmitted
    0 bad segments received
    42 resets sent
Udp:
    32920 packets received
    265 packets to unknown port received
    0 packet receive errors
    31230 packets sent
    0 receive buffer errors
    0 send buffer errors
    IgnoredMulti: 13138
UdpLite:
TcpExt:
    144 resets received for embryonic SYN_RECV sockets
    33 TCP sockets finished time wait in fast timer
    1 packetes rejected in established connections because of timestamp
    1571 delayed acks sent
    Quick ack mode was activated 13 times
    1360 packet headers predicted
    4514 acknowledgments not containing data payload received
    5419 predicted acknowledgments
    TCPSackRecovery: 4
    Detected reordering 2 times using SACK
    TCPDSACKUndo: 4
    2 congestion windows recovered without slow start after partial ack
    TCPLostRetransmit: 1204
    1 timeouts in loss state
    4 fast retransmits
    TCPTimeouts: 3448
    TCPLossProbes: 251
    TCPLossProbeRecovery: 12
    TCPDSACKOldSent: 15
    TCPDSACKRecv: 22
    12 connections reset due to unexpected data
    211 connections aborted due to timeout
    TCPDSACKIgnoredNoUndo: 3
    TCPSackShiftFallback: 10
    TCPDeferAcceptDrop: 187
    IPReversePathFilter: 5
    TCPRcvCoalesce: 518
    TCPOFOQueue: 41
    TCPChallengeACK: 1
    TCPAutoCorking: 679
    TCPSynRetrans: 2074
    TCPOrigDataSent: 12959
    TCPHystartTrainDetect: 1
    TCPHystartTrainCwnd: 33
    TCPACKSkippedSeq: 1
    TCPKeepAlive: 27
    TCPDelivered: 12645
    TcpTimeoutRehash: 1423
    TCPDSACKRecvSegs: 22
IpExt:
    InNoRoutes: 243
    InMcastPkts: 12754
    OutMcastPkts: 6
    InBcastPkts: 654
    InOctets: 49614652776
    OutOctets: 99125627730
    InMcastOctets: 2557448
    OutMcastOctets: 384
    InBcastOctets: 93071
    InNoECTPkts: 40502604
    InECT0Pkts: 167
MPTcpExt: