Page MenuHomeVyOS Platform
Create Task
Maniphest T5337

MPLS/BGP: Route leak does not happen from the VPNv4 table to specific vrf
Closed, ResolvedPublicBUG
Actions

Description

Description :

The route leak from bgp vpn4 table to specific vrf route table does not happen when physical interfaces(which are directly connected) are configured as neighbors but works when the loopback address configured in default vrf between cisco and vyos.

31.1.1.2/32 is defined as the loopback address in the cisco router which is seen in the vpnv4 table but not leaked to the defined vrf

VyOS:

set interfaces dummy dum0 address '11.11.11.1/32'
set interfaces dummy dum0 vrf 'blue-vrf'
set interfaces ethernet eth0 address 'dhcp'
set interfaces ethernet eth0 hw-id '50:0b:00:07:00:00'
set interfaces ethernet eth1 hw-id '50:0b:00:07:00:01'
set interfaces ethernet eth1 vif 2818 address '20.20.21.130/30'
set interfaces ethernet eth1 vif 2818 description 'B2Bconnection'
set interfaces ethernet eth2 address '88.2.2.1/24'
set interfaces ethernet eth2 hw-id '50:0b:00:07:00:02'
set interfaces ethernet eth2 vrf 'internet'
set interfaces ethernet eth3 hw-id '50:0b:00:07:00:03'
set interfaces loopback lo
set interfaces tunnel tun1 encapsulation 'gre'
set interfaces tunnel tun1 ip adjust-mss '1336'
set interfaces tunnel tun1 mtu '1376'
set interfaces tunnel tun1 remote '10.1.1.6'
set interfaces tunnel tun1 source-address '88.2.2.1'
set interfaces tunnel tun1 vrf 'blue-vrf'
set policy prefix-list OUT rule 10 action 'deny'
set policy prefix-list OUT rule 10 prefix '11.11.11.1/32'
set policy prefix-list OUT rule 20 action 'permit'
set policy prefix-list OUT rule 20 le '32'
set policy prefix-list OUT rule 20 prefix '0.0.0.0/0'
set policy route-map MAPOUT rule 10 action 'permit'
set policy route-map MAPOUT rule 10 match ip address prefix-list 'OUT'

<<protocols mpls ldp + iBGP for l3vpn>>

set protocols bgp neighbor 20.20.21.129 address-family ipv4-vpn nexthop-self
set protocols bgp neighbor 20.20.21.129 remote-as '65241'
set protocols bgp parameters log-neighbor-changes
set protocols bgp system-as '65241'
set protocols mpls interface 'eth1.2818'
set protocols mpls ldp discovery transport-ipv4-address '20.20.21.130'
set protocols mpls ldp interface 'eth1.2818'
set protocols mpls ldp router-id '20.20.21.130'
set protocols static route 10.1.1.0/24 next-hop 88.2.2.2 vrf 'internet'
set protocols static route 172.16.100.10/32 interface tun1 vrf 'blue-vrf'

<<vpn between vyos and hub >>
set vpn ipsec authentication psk vyos_cisco_l id '[email protected]'
set vpn ipsec authentication psk vyos_cisco_l id 'agg1.sdip.verizon.net'
set vpn ipsec authentication psk vyos_cisco_l secret 'sdip-2000'
set vpn ipsec esp-group e1 lifetime '3600'
set vpn ipsec esp-group e1 mode 'tunnel'
set vpn ipsec esp-group e1 pfs 'disable'
set vpn ipsec esp-group e1 proposal 1 encryption 'aes128'
set vpn ipsec esp-group e1 proposal 1 hash 'sha256'
set vpn ipsec ike-group i1 close-action 'none'
set vpn ipsec ike-group i1 key-exchange 'ikev2'
set vpn ipsec ike-group i1 lifetime '28800'
set vpn ipsec ike-group i1 proposal 1 dh-group '5'
set vpn ipsec ike-group i1 proposal 1 encryption 'aes256'
set vpn ipsec ike-group i1 proposal 1 hash 'sha256'
set vpn ipsec interface 'eth2'
set vpn ipsec log level '1'
set vpn ipsec options disable-route-autoinstall
set vpn ipsec options flexvpn
set vpn ipsec options interface 'tun1'
set vpn ipsec options virtual-ip
set vpn ipsec site-to-site peer cisco_hub authentication local-id '[email protected]'
set vpn ipsec site-to-site peer cisco_hub authentication mode 'pre-shared-secret'
set vpn ipsec site-to-site peer cisco_hub authentication remote-id 'agg1.sdip.verizon.net'
set vpn ipsec site-to-site peer cisco_hub connection-type 'initiate'
set vpn ipsec site-to-site peer cisco_hub default-esp-group 'e1'
set vpn ipsec site-to-site peer cisco_hub ike-group 'i1'
set vpn ipsec site-to-site peer cisco_hub local-address '88.2.2.1'
set vpn ipsec site-to-site peer cisco_hub remote-address '10.1.1.6'
set vpn ipsec site-to-site peer cisco_hub tunnel 1 local prefix '88.2.2.1/32'
set vpn ipsec site-to-site peer cisco_hub tunnel 1 protocol 'gre'
set vpn ipsec site-to-site peer cisco_hub tunnel 1 remote prefix '10.1.1.6/32'
set vpn ipsec site-to-site peer cisco_hub virtual-address '0.0.0.0'

<<l3vpn vrf >>
set vrf bind-to-all
set vrf name blue-vrf protocols bgp address-family ipv4-unicast export vpn
set vrf name blue-vrf protocols bgp address-family ipv4-unicast import vpn
set vrf name blue-vrf protocols bgp address-family ipv4-unicast label vpn export 'auto'
set vrf name blue-vrf protocols bgp address-family ipv4-unicast rd vpn export '11.11.11.1:1011'
set vrf name blue-vrf protocols bgp address-family ipv4-unicast redistribute connected
set vrf name blue-vrf protocols bgp address-family ipv4-unicast route-target vpn export '64512:4001'
set vrf name blue-vrf protocols bgp address-family ipv4-unicast route-target vpn import '64512:4001'
set vrf name blue-vrf protocols bgp neighbor 172.16.100.10 address-family ipv4-unicast route-map export 'MAPOUT'
set vrf name blue-vrf protocols bgp neighbor 172.16.100.10 ebgp-multihop '3'
set vrf name blue-vrf protocols bgp neighbor 172.16.100.10 local-as 65200
set vrf name blue-vrf protocols bgp neighbor 172.16.100.10 remote-as '65402'
set vrf name blue-vrf protocols bgp neighbor 172.16.100.10 update-source '11.11.11.1'
set vrf name blue-vrf protocols bgp system-as '65241'
set vrf name blue-vrf protocols ospf interface tun1 area '0'
set vrf name blue-vrf protocols ospf parameters router-id '11.11.11.1'
set vrf name blue-vrf protocols ospf redistribute connected
set vrf name blue-vrf protocols static route 172.16.100.10/32 interface tun1
set vrf name blue-vrf table '1002'
set vrf name internet protocols static route 10.1.1.0/24 next-hop 88.2.2.2
set vrf name internet table '1000'

Output:

vyos@vyos:~$ sh ip route vrf blue-vrf
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
       f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

VRF blue-vrf:
C>* 11.11.11.1/32 is directly connected, dum0, 00:12:34
C>* 172.16.1.0/24 is directly connected, eth3, 00:12:34
O   172.16.100.10/32 [110/2] via 172.16.100.10, tun1 onlink, weight 1, 00:11:5
S>* 172.16.100.10/32 [1/0] is directly connected, tun1, weight 1, 00:12:23
C>* 172.16.122.1/32 is directly connected, tun1, 00:12:11
B>  172.16.200.10/32 [20/0] via 172.16.100.10 (recursive), weight 1, 00:11:44
  *                           via 172.16.100.10, tun1 onlink, weight 1, 00:11:
B>  192.168.96.0/30 [20/0] via 172.16.100.10 (recursive), weight 1, 00:11:44
  *                          via 172.16.100.10, tun1 onlink, weight 1, 00:11:4
B>  192.168.200.0/30 [20/0] via 172.16.100.10 (recursive), weight 1, 00:11:44
  *                           via 172.16.100.10, tun1 onlink, weight 1, 00:11:


vyos@vyos:~$ sh bgp ipv4 vpn
BGP table version is 18, local router ID is 20.20.21.130, vrf id 0
Default local pref 100, local AS 65241
Status codes:  s suppressed, d damped, h history, * valid, > best, = multipath
               i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes:  i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found

    Network          Next Hop            Metric LocPrf Weight Path
Route Distinguisher: 64512:5
 *>i31.1.1.2/32      20.20.21.129             0    100      0 ?
    UN=20.20.21.129 EC{64512:4001} label=17 type=bgp, subtype=0
 *>i172.16.100.10/32 20.20.21.129             0    100      0 65200 65402 ?
    UN=20.20.21.129 EC{64512:4001} label=19 type=bgp, subtype=0
 *>i172.16.122.2/32  20.20.21.129             0    100      0 ?
    UN=20.20.21.129 EC{64512:4001} label=18 type=bgp, subtype=0
 *>i172.16.200.10/32 20.20.21.129             0    100      0 65200 65402 ?
    UN=20.20.21.129 EC{64512:4001} label=20 type=bgp, subtype=0
 *>i192.168.96.0/30  20.20.21.129             0    100      0 65200 65402 ?
    UN=20.20.21.129 EC{64512:4001} label=21 type=bgp, subtype=0
 *>i192.168.200.0/30 20.20.21.129             0    100      0 65200 65402 ?
    UN=20.20.21.129 EC{64512:4001} label=22 type=bgp, subtype=0
Route Distinguisher: 11.11.11.1:1011
 *> 11.11.11.1/32    0.0.0.0@6<               0         32768 ?
    UN=0.0.0.0 EC{64512:4001} label=144 type=bgp, subtype=5
 *> 172.16.1.0/24    0.0.0.0@6<               0         32768 ?
    UN=0.0.0.0 EC{64512:4001} label=144 type=bgp, subtype=5
    172.16.100.10/32 172.16.100.10@6<
                                             0             0 65200 65402 ?
    UN=172.16.100.10 EC{1222:1 64512:4001} label=144 type=bgp, subtype=5
 *> 172.16.122.1/32  0.0.0.0@6<               0         32768 ?
    UN=0.0.0.0 EC{64512:4001} label=144 type=bgp, subtype=5
 *> 172.16.200.10/32 172.16.100.10@6<
                                             0             0 65200 65402 ?
    UN=172.16.100.10 EC{1222:1 64512:4001} label=144 type=bgp, subtype=5
 *> 192.168.96.0/30  172.16.100.10@6<
                                             0             0 65200 65402 ?
    UN=172.16.100.10 EC{1222:1 64512:4001} label=144 type=bgp, subtype=5
 *> 192.168.200.0/30 172.16.100.10@6<
                                             0             0 65200 65402 ?
    UN=172.16.100.10 EC{1222:1 64512:4001} label=144 type=bgp, subtype=5

Displayed  13 routes and 13 total paths

vyos@vyos# run sh bgp ipv4 summary

IPv4 VPN Summary (VRF default):
BGP router identifier 20.20.21.130, local AS number 65241 vrf-id 0
BGP table version 0
RIB entries 3, using 576 bytes of memory
Peers 1, using 725 KiB of memory

Neighbor        V         AS   MsgRcvd   MsgSent   TblVer  InQ OutQ  Up/Down State/PfxRcd   PfxSnt Desc
20.20.21.129    4      65241       108        99        0    0    0 01:14:55            6        5 N/A

Total number of neighbors 1
[edit]
vyos@vyos# run sh mpls table
 Inbound Label  Type  Nexthop       Outbound Label
 ---------------------------------------------------
 16             LDP   20.20.21.129  implicit-null
 144            BGP   blue-vrf

When the neighbor configured with loopback interfaces:

vyos@vyos:~$ sh ip route vrf blue-vrf
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
       f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

VRF blue-vrf:
C>* 11.11.11.1/32 is directly connected, dum0, 22:12:30
B>  31.1.1.2/32 [200/0] via 31.1.1.10 (vrf default) (recursive), label 17, weight 1, 22:11:01
  *                       via 20.20.21.129, eth1.2818 (vrf default), label implicit-null/17, weight 1, 22:11:01
C>* 172.16.1.0/24 is directly connected, eth3, 22:12:29
B   172.16.100.10/32 [200/0] via 31.1.1.10 (vrf default) (recursive), label 18, weight 1, 22:11:01
                               via 20.20.21.129, eth1.2818 (vrf default), label implicit-null/18, weight 1, 22:11:01
O   172.16.100.10/32 [110/2] via 172.16.100.10, tun1 onlink, weight 1, 22:11:48
S>* 172.16.100.10/32 [1/0] is directly connected, tun1, weight 1, 22:12:17
C>* 172.16.122.1/32 is directly connected, tun1, 22:12:04
B>  172.16.122.2/32 [200/0] via 31.1.1.10 (vrf default) (recursive), label 19, weight 1, 22:11:01
  *                           via 20.20.21.129, eth1.2818 (vrf default), label implicit-null/19, weight 1, 22:11:01
B>  172.16.200.10/32 [20/0] via 172.16.100.10 (recursive), weight 1, 22:11:37
  *                           via 172.16.100.10, tun1 onlink, weight 1, 22:11:37
B>  192.168.96.0/30 [20/0] via 172.16.100.10 (recursive), weight 1, 22:11:37
  *                          via 172.16.100.10, tun1 onlink, weight 1, 22:11:37
B>  192.168.200.0/30 [20/0] via 172.16.100.10 (recursive), weight 1, 22:11:37
  *                           via 172.16.100.10, tun1 onlink, weight 1, 22:11:37

Cisco Output:

CiscoR1#sh ip route vrf lan-vrf

.
.
Gateway of last resort is not set

      31.0.0.0/32 is subnetted, 1 subnets
C        31.1.1.2 is directly connected, Loopback0
      172.16.0.0/16 is variably subnetted, 5 subnets, 2 masks
B        172.16.1.0/24 [200/0] via 20.20.21.130, 00:07:12
S        172.16.100.10/32 is directly connected, Tunnel0
B        172.16.122.1/32 [200/0] via 20.20.21.130, 00:07:12
C        172.16.122.2/32 is directly connected, Tunnel0
B        172.16.200.10/32 [20/0] via 172.16.100.10, 00:51:55
      192.168.96.0/30 is subnetted, 1 subnets
B        192.168.96.0 [20/0] via 172.16.100.10, 00:51:55
      192.168.200.0/30 is subnetted, 1 subnets
B        192.168.200.0 [20/0] via 172.16.100.10, 00:51:55

Topology(configuration shared between VyOS and CiscoR1):

image.png (311×702 px, 33 KB)

Details

Difficulty level
Unknown (require assessment)
Version
1.4-rolling-202305120317
Why the issue appeared?
Will be filled on close
Is it a breaking change?
Unspecified (possibly destroys the router)
Issue type
Unspecified (please specify)

Related Objects
Search...

Event Timeline

SrividyaA created this task.Jul 4 2023, 1:06 PM
pasik added a subscriber: pasik.Jul 4 2023, 1:15 PM
a.apostoliuk added a subscriber: a.apostoliuk.Jul 4 2023, 1:40 PM

It can work if you use

mpls bgp forwarding

under interface configuration in frr and eBGP instead iBGP

a.apostoliuk added a subtask: T5338: Add 'mpls bgp forwarding' feature.Jul 4 2023, 1:45 PM
fernando added a subscriber: fernando.Jul 4 2023, 10:53 PM
fernando added a comment.Jul 10 2023, 9:15 PM

I've done extra test , I confirm this behavior is associated transport label that can't be allocated when using interfaces directed connected to created ldp session without IGP protocols(ospf/isis) . let's see what is going on :

add a transport label static to the kernel :

#label static assign next-hop in the interface connected : 
ip route del 10.0.0.1/32 encap mpls 100 via 10.0.0.2 dev eth1

#zebra can reach the next-hop , so install the routes on RIB :

Jul 10 20:42:05 PE-01 bgpd[1149]: [K0WB7-QW9PS] VRF default(0): Rcvd NH update 10.0.0.1/32(0)0) - metric 0/0 #nhops 1/1 flags Valid Reg Notify
Jul 10 20:42:05 PE-01 bgpd[1149]: [HJ3XJ-8EHF0]     nhop via 10.0.0.2 if 3 (1 labels)
Jul 10 20:42:05 PE-01 bgpd[1149]: [K7JBS-J1T3G] NH update for 10.0.0.1/32(0)(0)(VRF default) - flags Valid Reg Notify Label Valid  chgflags Changed - evaluate paths
Jul 10 20:42:05 PE-01 bgpd[1149]: [JY1CY-BM14V] ... eval path 1/1 10.10.10.1/32 VRF blue-vrf flags 0x4000
Jul 10 20:42:05 PE-01 bgpd[1149]: [G0V2R-JTRQK] parse nexthop update(10.0.0.1/32(0)(VRF default)): bnc info not found for import check
Jul 10 20:42:05 PE-01 zebra[1130]: [TS3SH-1276M] default(0:254):10.0.0.1/32 update_from_ctx(): no fib nhg
Jul 10 20:42:05 PE-01 zebra[1130]: [HKQXC-4STSK] default(0:254):10.0.0.1/32 update_from_ctx(): rib nhg matched, changed 'true'
Jul 10 20:42:05 PE-01 zebra[1130]: [Z1MP1-RFGJA] (0:254):10.0.0.1/32(0): Redist update re 0x5650e40a08a0 (kernel), old 0x0 (None)
Jul 10 20:42:05 PE-01 zebra[1130]: [HW8ZM-FRFCQ] redistribute_update: client ldp 10.0.0.1/32(0:254), type=1, distance=0, metric=0
Jul 10 20:42:05 PE-01 zebra[1130]: [HFAXM-2790E] default(0):10.0.0.1/32: Evaluate RNH,
Jul 10 20:42:05 PE-01 zebra[1130]: [WP9YA-G5D8G] default(0):10.0.0.1/32: NH resolved over route 10.0.0.1/32
Jul 10 20:42:05 PE-01 zebra[1130]: [PE79W-J9QS0] default(0):10.0.0.1/32: Notifying client bgp about NH
Jul 10 20:42:05 PE-01 bgpd[1149]: [X11WM-VBA5R] bgp_zebra_announce: p=10.10.10.1/32, bgp_is_valid_label: 2
Jul 10 20:42:05 PE-01 bgpd[1149]: [V15FP-4CPVK] Tx route add VRF 6 10.10.10.1/32 metric 0 tag 0 count 1 nhg 0
Jul 10 20:42:05 PE-01 bgpd[1149]: [JQXM8-V0CKB]   nhop [1]: 10.0.0.1 if 0 VRF 0 wt 0 label 16
Jul 10 20:42:05 PE-01 bgpd[1149]: [JMVGR-VQ1M1] bgp_zebra_announce: 10.10.10.1/32: announcing to zebra (recursion set)
Jul 10 20:42:05 PE-01 zebra[1130]: [YXCJP-0WZWV] netlink_nexthop_msg_encode: ID (25): 10.0.0.2, via eth1(3) vrf default(0) label 200/16
Jul 10 20:42:05 PE-01 zebra[1130]: [R43C6-KYHWT] netlink_nexthop_msg_encode: RTM_NEWNEXTHOP, id=25
Jul 10 20:42:05 PE-01 zebra[1130]: [HYEHE-CQZ9G] nl_batch_send: netlink-dp (NS 0), batch size=72, msg cnt=1
Jul 10 20:42:05 PE-01 zebra[1130]: [YXPF5-B2CE0] netlink_route_multipath_msg_encode: RTM_NEWROUTE 10.10.10.1/32 vrf 6(1002)
Jul 10 20:42:05 PE-01 zebra[1130]: [J87BH-XW5PP] netlink_route_multipath_msg_encode: 10.10.10.1/32 nhg_id is 25
Jul 10 20:42:05 PE-01 zebra[1130]: [HYEHE-CQZ9G] nl_batch_send: netlink-dp (NS 0), batch size=60, msg cnt=1
Jul 10 20:42:05 PE-01 zebra[1130]: [MFYWV-KH3MC] process_subq_early_route_add: (6:?):10.10.10.1/32: Inserting route rn 0x5650e40a11b0, re 0x5650e40a05a0 (bgp) existing 0x0, same_count 0
Jul 10 20:42:05 PE-01 zebra[1130]: [ZFRN1-EZNJZ] blue-vrf(6:1002):10.10.10.1/32: Adding route rn 0x5650e40a11b0, re 0x5650e40a05a0 (bgp)
Jul 10 20:42:05 PE-01 zebra[1130]: [TS3SH-1276M] blue-vrf(6:1002):10.10.10.1/32 update_from_ctx(): no fib nhg
Jul 10 20:42:05 PE-01 zebra[1130]: [HKQXC-4STSK] blue-vrf(6:1002):10.10.10.1/32 update_from_ctx(): rib nhg matched, changed 'true'
Jul 10 20:42:05 PE-01 zebra[1130]: [Z1MP1-RFGJA] (6:1002):10.10.10.1/32(0): Redist update re 0x5650e40a05a0 (bgp), old 0x0 (None)

## vrf routes :

vyos@PE-01:~$  how bgp ipv4 vpn summary
BGP router identifier 172.16.50.48, local AS number 65000 vrf-id 0
BGP table version 0
RIB entries 3, using 576 bytes of memory
Peers 1, using 725 KiB of memory

Neighbor        V         AS   MsgRcvd   MsgSent   TblVer  InQ OutQ  Up/Down State/PfxRcd   PfxSnt Desc
10.0.0.1        4      65000        19        15        0    0    0 00:04:12            1        1 N/A

Total number oshow ip route vrf blue-vrf
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
       f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

VRF blue-vrf:
C>* 2.2.2.2/32 is directly connected, dum1, 00:08:56
B>  10.10.10.1/32 [200/0] via 10.0.0.1 (vrf default) (recursive), label 16, weight 1, 00:00:12
  *                         via 10.0.0.2, eth1 (vrf default), label 200/16, weight 1, 00:00:12
vyos@PE-01:~$

# default table where we see label static :

vyos@PE-01:~$ show ip route
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
       f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

C>* 10.0.0.0/30 is directly connected, eth1, 00:09:45
K>* 10.0.0.1/32 [0/0] via 10.0.0.2, eth1, label 200, 00:01:02
C>* 172.16.50.0/24 is directly connected, eth0, 00:09:44

The issue at hand is the need to install a static label on the Cisco device after the BGP session is terminated by the expiration of a 3-minute timer ,therefore , we need to talk with FRR if it's possible extend the capability to resolved the next-hop and install the transport label how is done on mpls bgp forwarding

fernando changed the task status from Open to Confirmed.Jul 10 2023, 9:27 PM
fernando added a comment.Jul 11 2023, 12:05 AM

well , I've had an idea how to make a workaround , I've used explicit-null label to add next-hop from the network-address connected sudo ip route add 10.0.0.1/32 encap mpls 0 via 10.0.0.2 dev eth1

it should show the following outputs :

vyos@PE-01:~$ show ip route vrf blue-vrf
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
       f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

VRF blue-vrf:
C>* 2.2.2.2/32 is directly connected, dum1, 03:15:30
B>  10.10.10.1/32 [200/0] via 10.0.0.1 (vrf default) (recursive), label 16, weight 1, 00:07:06
  *                         via 10.0.0.2, eth1 (vrf default), label IPv4 Explicit Null/16, weight 1, 00:07:06

#label binding :
vyos@PE-01:~$ show mpls ldp binding
AF   Destination          Nexthop         Local Label Remote Label  In Use
ipv4 10.0.0.0/30          10.0.0.1        -           imp-null          no
ipv4 10.0.0.1/32          0.0.0.0         16          -                 no

#ping  

vyos@PE-01:~$ ping 10.10.10.1  source-address 2.2.2.2 vrf blue-vrf
PING 10.10.10.1 (10.10.10.1) from 2.2.2.2 : 56(84) bytes of data.
64 bytes from 10.10.10.1: icmp_seq=1 ttl=254 time=0.405 ms
64 bytes from 10.10.10.1: icmp_seq=2 ttl=254 time=0.806 ms
64 bytes from 10.10.10.1: icmp_seq=3 ttl=254 time=0.752 ms

#log when prefix is added : 


Jul 10 23:41:45 PE-01 zebra[1130]: [YXCJP-0WZWV] netlink_nexthop_msg_encode: ID (27): 10.0.0.2, via eth1(3) vrf default(0) label 0
Jul 10 23:41:45 PE-01 zebra[1130]: [R43C6-KYHWT] netlink_nexthop_msg_encode: RTM_NEWNEXTHOP, id=27
Jul 10 23:41:45 PE-01 zebra[1130]: [HYEHE-CQZ9G] nl_batch_send: netlink-dp (NS 0), batch size=68, msg cnt=1
Jul 10 23:41:45 PE-01 bgpd[1149]: [X11WM-VBA5R] bgp_zebra_announce: p=10.10.10.1/32, bgp_is_valid_label: 2
Jul 10 23:41:45 PE-01 bgpd[1149]: [V15FP-4CPVK] Tx route add VRF 6 10.10.10.1/32 metric 0 tag 0 count 1 nhg 0
Jul 10 23:41:45 PE-01 bgpd[1149]: [JQXM8-V0CKB]   nhop [1]: 10.0.0.1 if 0 VRF 0 wt 0 label 16
Jul 10 23:41:45 PE-01 bgpd[1149]: [JMVGR-VQ1M1] bgp_zebra_announce: 10.10.10.1/32: announcing to zebra (recursion set)
Jul 10 23:41:45 PE-01 zebra[1130]: [YXCJP-0WZWV] netlink_nexthop_msg_encode: ID (30): 10.0.0.2, via eth1(3) vrf default(0) label 0/16
Jul 10 23:41:45 PE-01 zebra[1130]: [R43C6-KYHWT] netlink_nexthop_msg_encode: RTM_NEWNEXTHOP, id=30
Jul 10 23:41:45 PE-01 zebra[1130]: [HYEHE-CQZ9G] nl_batch_send: netlink-dp (NS 0), batch size=72, msg cnt=1
Jul 10 23:41:45 PE-01 zebra[1130]: [YXPF5-B2CE0] netlink_route_multipath_msg_encode: RTM_NEWROUTE 10.10.10.1/32 vrf 6(1002)
Jul 10 23:41:45 PE-01 zebra[1130]: [J87BH-XW5PP] netlink_route_multipath_msg_encode: 10.10.10.1/32 nhg_id is 30
Jul 10 23:41:45 PE-01 zebra[1130]: [HYEHE-CQZ9G] nl_batch_send: netlink-dp (NS 0), batch size=60, msg cnt=1
Jul 10 23:41:45 PE-01 zebra[1130]: [MFYWV-KH3MC] process_subq_early_route_add: (6:?):10.10.10.1/32: Inserting route rn 0x5650e40a1070, re 0x5650e40a0c70 (bgp) existing 0x0, same_count 0
Jul 10 23:41:45 PE-01 zebra[1130]: [ZFRN1-EZNJZ] blue-vrf(6:1002):10.10.10.1/32: Adding route rn 0x5650e40a1070, re 0x5650e40a0c70 (bgp)
Jul 10 23:41:45 PE-01 zebra[1130]: [TS3SH-1276M] blue-vrf(6:1002):10.10.10.1/32 update_from_ctx(): no fib nhg
Jul 10 23:41:45 PE-01 zebra[1130]: [HKQXC-4STSK] blue-vrf(6:1002):10.10.10.1/32 update_from_ctx(): rib nhg matched, changed 'true'
Jul 10 23:41:45 PE-01 zebra[1130]: [Z1MP1-RFGJA] (6:1002):10.10.10.1/32(0): Redist update re 0x5650e40a0c70 (bgp), old 0x0 (None)

Cisco PE :

PE-02#show bgp vpnv4 unicast all summary
BGP router identifier 10.0.0.1, local AS number 65000
BGP table version is 12, main routing table version 12
3 network entries using 768 bytes of memory
3 path entries using 384 bytes of memory
2/2 BGP path/bestpath attribute entries using 560 bytes of memory
1 BGP extended community entries using 24 bytes of memory
0 BGP route-map cache entries using 0 bytes of memory
0 BGP filter-list cache entries using 0 bytes of memory
BGP using 1736 total bytes of memory
BGP activity 17/14 prefixes, 25/22 paths, scan interval 60 secs

Neighbor        V           AS MsgRcvd MsgSent   TblVer  InQ OutQ Up/Down  State/PfxRcd
10.0.0.2        4        65000      33      38       12    0    0 00:30:41        1
PE-02#show ip route vrf lan-vrf

Routing Table: lan-vrf
Codes: L - local, C - connected, S - static, R - RIP, M - mobile, B - BGP
       D - EIGRP, EX - EIGRP external, O - OSPF, IA - OSPF inter area
       N1 - OSPF NSSA external type 1, N2 - OSPF NSSA external type 2
       E1 - OSPF external type 1, E2 - OSPF external type 2
       i - IS-IS, su - IS-IS summary, L1 - IS-IS level-1, L2 - IS-IS level-2
       ia - IS-IS inter area, * - candidate default, U - per-user static route
       o - ODR, P - periodic downloaded static route, H - NHRP, l - LISP
       a - application route
       + - replicated route, % - next hop override, p - overrides from PfR

Gateway of last resort is not set

      2.0.0.0/32 is subnetted, 1 subnets
B        2.2.2.2 [200/0] via 10.0.0.2, 00:30:58
      10.0.0.0/32 is subnetted, 1 subnets
C        10.10.10.1 is directly connected, Loopback1

Local      Outgoing   Prefix           Bytes Label   Outgoing   Next Hop
Label      Label      or Tunnel Id     Switched      interface
16         Pop Label  10.10.10.1/32[V] 0             aggregate/lan-vrf
	MAC/Encaps=0/0, MRU=0, Label Stack{}
	VPN route: lan-vrf
	No output feature configured
fernando added a comment.Jul 12 2023, 8:51 PM

I've summited an issues case on FRR , where explain the problematic :

https://github.com/FRRouting/frr/issues/13996

a.apostoliuk changed the status of subtask T5338: Add 'mpls bgp forwarding' feature from In progress to Needs testing.Jul 17 2023, 11:56 AM
a.apostoliuk changed the status of subtask T5338: Add 'mpls bgp forwarding' feature from Needs testing to In progress.Jul 17 2023, 12:06 PM
a.apostoliuk added a comment.Oct 16 2023, 10:54 AM

In this case we can use the next solution:

set policy route-map TEST rule 10 action 'permit'
set policy route-map TEST rule 10 set l3vpn-nexthop encapsulation gre
set protocols bgp neighbor 10.0.0.1 address-family ipv4-vpn route-map import 'TEST'
vyos@vyos:~$ show ip route vrf  blue-vrf
Codes: K - kernel route, C - connected, S - static, R - RIP,
       O - OSPF, I - IS-IS, B - BGP, E - EIGRP, N - NHRP,
       T - Table, v - VNC, V - VNC-Direct, A - Babel, F - PBR,
       f - OpenFabric,
       > - selected route, * - FIB route, q - queued, r - rejected, b - backup
       t - trapped, o - offload failure

VRF blue-vrf:
C>* 2.2.2.2/32 is directly connected, dum1, 00:12:26
B>* 10.10.10.1/32 [200/0] via 10.0.0.1, eth1 (vrf default), label 16, weight 1, 00:04:22
vyos@vyos:~$ show ip bgp vrf blue-vrf
BGP table version is 12, local router ID is 2.2.2.2, vrf id 7
Default local pref 100, local AS 65241
Status codes:  s suppressed, d damped, h history, * valid, > best, = multipath,
               i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes:  i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found

    Network          Next Hop            Metric LocPrf Weight Path
 *> 2.2.2.2/32       0.0.0.0                  0         32768 ?
 *> 10.10.10.1/32    10.0.0.1@0<              0    100      0 ?

Displayed  2 routes and 2 total paths
vyos@vyos:~$ show bgp ipv4 vpn
BGP table version is 2, local router ID is 1.1.1.1, vrf id 0
Default local pref 100, local AS 65000
Status codes:  s suppressed, d damped, h history, * valid, > best, = multipath,
               i internal, r RIB-failure, S Stale, R Removed
Nexthop codes: @NNN nexthop's vrf id, < announce-nh-self
Origin codes:  i - IGP, e - EGP, ? - incomplete
RPKI validation codes: V valid, I invalid, N Not found

    Network          Next Hop            Metric LocPrf Weight Path
Route Distinguisher: 64512:8
 *> 2.2.2.2/32       0.0.0.0@7<               0         32768 ?
    UN=0.0.0.0 EC{64512:4001} label=144 type=bgp, subtype=5
Route Distinguisher: 64512:1012
 *>i10.10.10.1/32    10.0.0.1                 0    100      0 ?
    UN=10.0.0.1 EC{64512:4001} label=16 type=bgp, subtype=0

Displayed  2 routes and 2 total paths
2023-10-16 10:33:33.281 [DEBG] bgpd: [W2H3A-DK97Z] vpn_leak_to_vrf_withdraw: entry: p=10.10.10.1/32, type=9, sub_type=0
2023-10-16 10:33:33.281 [DEBG] bgpd: [J9M43-9VGVR] vpn_leak_to_vrf_withdraw: start (path_vpn=0x55cb1ca472b0)
2023-10-16 10:33:33.281 [DEBG] bgpd: [QVYVY-AF4CA] vpn_leak_to_vrf_withdraw: from VRF default, skipping: import not set
2023-10-16 10:33:33.281 [DEBG] bgpd: [GY107-1TREX] vpn_leak_to_vrf_withdraw: withdrawing from vrf VRF blue-vrf
2023-10-16 10:33:33.281 [DEBG] bgpd: [PWJWM-XJTM5] vpn_leak_to_vrf_withdraw: deleting bpi 0x55cb1d1618e0
2023-10-16 10:33:35.284 [DEBG] bgpd: [WNKP5-SN018] Found existing bnc 10.0.0.1/32(0)(VRF default) flags 0xb ifindex 0 #paths 0 peer 0x55cb1d1441e0
2023-10-16 10:33:35.291 [DEBG] bgpd: [J9M43-9VGVR] vpn_leak_to_vrf_update: start (path_vpn=0x55cb1d143450)
2023-10-16 10:33:35.291 [DEBG] bgpd: [QW8EX-RYMHS] vpn_leak_to_vrf_update_onevrf: from vpn (VRF default) to vrf (VRF default), skipping: import not set
2023-10-16 10:33:35.291 [DEBG] bgpd: [X3NND-AS9MD] vpn_leak_to_vrf_update_onevrf: updating RD 64512:1012, 10.10.10.1/32 to VRF blue-vrf
2023-10-16 10:33:35.291 [DEBG] bgpd: [GAK9R-YBKNZ] vpn_leak_to_vrf_update_onevrf: pfx 10.10.10.1/32: num_labels 1
2023-10-16 10:33:35.291 [DEBG] bgpd: [RD1WY-YE9EC] leak_update: entry: leak-to=VRF blue-vrf, p=10.10.10.1/32, type=9, sub_type=0
2023-10-16 10:33:35.291 [DEBG] bgpd: [WNKP5-SN018] Found existing bnc 10.0.0.1/32(0)(VRF default) flags 0xb ifindex 0 #paths 0 peer 0x55cb1d1441e0
2023-10-16 10:33:35.291 [DEBG] bgpd: [VWC2R-4REXZ] leak_update_nexthop_valid: 10.10.10.1/32 nexthop is valid (in VRF default)
2023-10-16 10:33:35.291 [DEBG] bgpd: [Z3E24-0WZCT] leak_update: ->VRF blue-vrf: 10.10.10.1/32: Added new route
2023-10-16 10:33:35.291 [INFO] bgpd: [M59KS-A3ZXZ] bgp_update_receive: rcvd End-of-RIB for IPv4 VPN from 10.0.0.1 in vrf default
vyos@vyos:~$ ping 10.10.10.1 vrf blue-vrf source-address 2.2.2.2
PING 10.10.10.1 (10.10.10.1) from 2.2.2.2 : 56(84) bytes of data.
64 bytes from 10.10.10.1: icmp_seq=1 ttl=254 time=2.05 ms
64 bytes from 10.10.10.1: icmp_seq=2 ttl=254 time=1.69 ms
64 bytes from 10.10.10.1: icmp_seq=3 ttl=254 time=1.60 ms
64 bytes from 10.10.10.1: icmp_seq=4 ttl=254 time=1.54 ms
64 bytes from 10.10.10.1: icmp_seq=5 ttl=254 time=1.63 ms
64 bytes from 10.10.10.1: icmp_seq=6 ttl=254 time=1.61 ms
fernando closed this task as Resolved.Oct 23 2023, 7:51 PM
fernando triaged this task as Normal priority.

this case was resolved lasted configuration done .

a.apostoliuk closed subtask T5338: Add 'mpls bgp forwarding' feature as Resolved.Nov 22 2023, 10:48 AM