Site to Site VPN with multiple vlan

hello, I’m trying to setup a site to site vpn connection to a site with pfsense.

Am i suppose to setup different vlan to different tunnel on both side?

below is my config and sa info, the remote site is an dynamic ip, I can only get the tunnel 1 up. 192.168.2.0/24 is vlan1 and 192.168.97.0/24 is vlan97. Does anyone have idea? Thanks

show vpn ipsec sa
Peer ID / IP Local ID / IP


X.X.X.X X.X.X.X

Tunnel  State  Bytes Out/In   Encrypt  Hash    NAT-T  A-Time  L-Time  Proto
------  -----  -------------  -------  ----    -----  ------  ------  -----
1       up     0.0/0.0        aes128   sha1_96 no     0       1800    all
vpn {
    ipsec {
        esp-group ESP-1 {
            lifetime 1800
            mode tunnel
            pfs disable
            proposal 1 {
                encryption aes128
                hash sha1
            }
        }
        ike-group IKE-1 {
            ikev2-reauth no
            key-exchange ikev2
            lifetime 3600
            proposal 1 {
                dh-group 2
                encryption aes256
                hash sha1
            }
        }
        ipsec-interfaces {
            interface eth1
        }
        nat-networks {
            allowed-network 0.0.0.0/0 {
            }
        }
        nat-traversal enable
        site-to-site {
            peer 0.0.0.0 {
                authentication {
                    mode pre-shared-secret
                    pre-shared-secret ****************
                }
                default-esp-group ESP-1
                ike-group IKE-1
                local-address X.X.X.X
                tunnel 1 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    esp-group ESP-1
                    local {
                        prefix 192.168.2.0/24
                    }
                    remote {
                        prefix 192.168.4.0/24
                    }
                }
                tunnel 2 {
                    allow-nat-networks disable
                    allow-public-networks disable
                    esp-group ESP-1
                    local {
                        prefix 192.168.97.0/24
                    }
                    remote {
                        prefix 192.168.4.0/24
                    }
                }
            }
        }