Regarding x710 I assume you already seen this?
Along with the usual suspects to try out different offloading options under interface settings in VyOS, note that not all offloading options turns out to be healthy so recommended to try one at a time.
Then to tweak conntrack you can adjust several settings in VyOS.
For example hash-size, table-size and expect-table-size but also the timers to make them as small as possible (otherwise many defaults in Linux kernel are like 2 weeks for established TTL which is a bit too much to waste RAM on).
Common things to check/evaluate (note that not all nic offloading options are healthy depending on hardware and drivers):
firewall {
global-options {
all-ping "enable"
broadcast-ping "disable"
ip-src-route "disable"
ipv6-receive-redirects "disable"
ipv6-source-validation "strict"
ipv6-src-route "disable"
log-martians "enable"
receive-redirects "disable"
resolver-cache
resolver-interval "60"
send-redirects "disable"
source-validation "strict"
syn-cookies "enable"
twa-hazards-protection "disable"
}
...
interfaces {
ethernet ethX {
ip {
arp-cache-timeout "240"
}
offload {
gro
gso
lro
rfs
rps
sg
tso
}
ring-buffer {
rx "4096"
tx "4096"
}
...
system {
conntrack {
expect-table-size "10485760"
hash-size "10485760"
log {
icmp {
new
}
other {
new
}
tcp {
new
}
udp {
new
}
}
table-size "10485760"
timeout {
icmp "10"
other "600"
tcp {
close "10"
close-wait "30"
established "600"
fin-wait "30"
last-ack "30"
syn-recv "30"
syn-sent "30"
time-wait "30"
}
udp {
other "600"
stream "600"
}
}
}
...
ip {
arp {
table-size "32768"
}
disable-directed-broadcast
multipath {
layer4-hashing
}
}
ipv6 {
disable-forwarding
multipath {
layer4-hashing
}
neighbor {
table-size "32768"
}
}
...
option {
ctrl-alt-delete "reboot"
http-client {
source-interface "ethX"
}
keyboard-layout "se-latin1"
performance "throughput"
reboot-on-panic
root-partition-auto-resize
ssh-client {
source-interface "ethX"
}
startup-beep
time-format "24-hour"
}
sysctl {
parameter kernel.core_uses_pid {
value "1"
}
parameter kernel.sysrq {
value "0"
}
parameter vm.swappiness {
value "1"
}
parameter vm.vfs_cache_pressure {
value "50"
}
}
...