Uploaded image for project: 'Fast Datapath Product'
  1. Fast Datapath Product
  2. FDP-744

Changing NAT type=snat logical_ip to 0.0.0.0/0 breaks FIP to FIP communication

    • Icon: Bug Bug
    • Resolution: Done-Errata
    • Icon: Major Major
    • None
    • None
    • ovn24.09
    • 5
    • False
    • Hide

      None

      Show
      None
    • False
    • rhel-sst-network-fastdatapath-ovn
    • ssg_networking
    • +

      Scenario

      Simple OpenStack devnest install

      Two networks, public and private. FIPs assigned from the public network.

      Private: 10.0.0.0/26, Public: 172.24.4.0/24

      Two VMs with interfaces on the private network, and dnat_and_snat NAT entries for their FIPs and an snat rule for 10.0.0.0/26.

      Switching the snat rule from logical_ip 10.0.0.0/26 to 0.0.0.0/0 causes ping replies to fail when pinging from one VM to another's FIP. The ping arrives to the target VM, but the reply does not make its way back. VMs can still ping the  gateway address of 172.24.4.1 and the external_ip entry for the NAT snat entry.

      $ sudo ovn-nbctl list Logical_Switch
      _uuid               : dd0312dd-cfce-4baa-9a46-5948dfef1fd4
      acls                : []
      copp                : []
      dns_records         : []
      external_ids        : {"neutron:availability_zone_hints"="", "neutron:mtu"="1500", "neutron:network_name"=public, "neutron:provnet-network-type"=flat, "neutron:revision_number"="2"}
      forwarding_groups   : []
      load_balancer       : []
      load_balancer_group : []
      name                : neutron-a399b821-d25d-435b-9ddb-30fffabc185c
      other_config        : {broadcast-arps-to-all-routers="true", fdb_age_threshold="0", mcast_flood_unregistered="false", mcast_snoop="false", vlan-passthru="false"}
      ports               : [079d5124-0344-4bf1-afbf-b9ac6dbe30a3, 5b38d72e-2e45-45b9-97c8-7a7de59e4b0e, f86db0c6-4a33-4986-94a0-304ec9310d4f]
      qos_rules           : []

      _uuid               : 88929853-1b0f-4bd0-829f-e47b9419e880
      acls                : []
      copp                : []
      dns_records         : [cc657ec2-b079-4ef8-8b5a-93ab8d878d11]
      external_ids        : {"neutron:availability_zone_hints"="", "neutron:mtu"="1442", "neutron:network_name"=private, "neutron:provnet-network-type"=geneve, "neutron:revision_number"="2"}
      forwarding_groups   : []
      load_balancer       : []
      load_balancer_group : []
      name                : neutron-242fb9c0-fd4f-42fa-bfac-dc85547e1020
      other_config        : {mcast_flood_unregistered="false", mcast_snoop="false", vlan-passthru="false"}
      ports               : [1314c5c7-a506-4009-9b2f-4ce2be40f698, 1a2a30e3-0f58-4390-bb9a-df641981b599, 39bcb6a4-5718-430e-92f9-6e41b4856df5, 40b53329-cb92-4460-a9d8-82bd24587d64]
      qos_rules           : []

      $ sudo ovn-nbctl list Logical_Router
      _uuid               : f3270f5e-2c75-40bc-a3eb-f8f729626798
      copp                : []
      enabled             : true
      external_ids        : {"neutron:availability_zone_hints"="", "neutron:revision_number"="5", "neutron:router_name"=router1}
      load_balancer       : []
      load_balancer_group : []
      name                : neutron-7461ea1d-aad5-4c8b-a168-9e23f064ac58
      nat                 : [384a1f1f-dcf9-4494-80c9-b96c9bbdc9a2, 6a9fa53b-d361-421f-9752-f774f4fa0902, 90e6f0a7-d1cc-493c-a773-8fab2686202b]
      options             : {always_learn_from_arp_request="false", dynamic_neigh_routers="true", mac_binding_age_threshold="0"}
      policies            : []
      ports               : [0e82dc10-7b5f-4a55-a65b-b6d6a3aea569, 19296564-b0d8-466c-8867-fc6052d05063]
      static_routes       : [2dbf9da1-2256-491b-bc3b-44bea1f2a1cd]

      $ sudo ovn-nbctl list NAT
      _uuid               : 90e6f0a7-d1cc-493c-a773-8fab2686202b
      allowed_ext_ips     : []
      exempted_ext_ips    : []
      external_ids        : {"neutron:fip_external_mac"="fa:16:3e:5b:f7:9c", "neutron:fip_id"="5be30423-9444-46b9-a350-450a5e1f665a", "neutron:fip_network_id"="a399b821-d25d-435b-9ddb-30fffabc185c", "neutron:fip_port_id"="2b092446-df9b-4425-a6e1-65dc6559348a", "neutron:revision_number"="2", "neutron:router_name"=neutron-7461ea1d-aad5-4c8b-a168-9e23f064ac58}
      external_ip         : "172.24.4.214"
      external_mac        : []
      external_port_range : ""
      logical_ip          : "10.0.0.9"
      logical_port        : "2b092446-df9b-4425-a6e1-65dc6559348a"
      options             : {}
      type                : dnat_and_snat

      _uuid               : 384a1f1f-dcf9-4494-80c9-b96c9bbdc9a2
      allowed_ext_ips     : []
      exempted_ext_ips    : []
      external_ids        : {"neutron:fip_external_mac"="fa:16:3e:aa:59:55", "neutron:fip_id"="524c817d-0239-485f-b971-f61ddec25bd8", "neutron:fip_network_id"="a399b821-d25d-435b-9ddb-30fffabc185c", "neutron:fip_port_id"="2c72c92a-ca16-4fde-90d1-482c36876300", "neutron:revision_number"="2", "neutron:router_name"=neutron-7461ea1d-aad5-4c8b-a168-9e23f064ac58}
      external_ip         : "172.24.4.182"
      external_mac        : []
      external_port_range : ""
      logical_ip          : "10.0.0.47"
      logical_port        : "2c72c92a-ca16-4fde-90d1-482c36876300"
      options             : {}
      type                : dnat_and_snat

      _uuid               : 6a9fa53b-d361-421f-9752-f774f4fa0902
      allowed_ext_ips     : []
      exempted_ext_ips    : []
      external_ids        : {}
      external_ip         : "172.24.4.12"
      external_mac        : []
      external_port_range : ""
      logical_ip          : "0.0.0.0/0"
      logical_port        : []
      options             : {}
      type                : snat

      $ sudo ovn-nbctl list Logical_Router_Static_Route
      _uuid               : 2dbf9da1-2256-491b-bc3b-44bea1f2a1cd
      bfd                 : []
      external_ids        : {"neutron:is_ext_gw"="true", "neutron:subnet_id"="d5a7295b-f822-46cb-8cc5-9ec023f013bb"}
      ip_prefix           : "0.0.0.0/0"
      nexthop             : "172.24.4.1"
      options             : {}
      output_port         : []
      policy              : []
      route_table         : ""

            [FDP-744] Changing NAT type=snat logical_ip to 0.0.0.0/0 breaks FIP to FIP communication

            Since the problem described in this issue should be resolved in a recent advisory, it has been closed.

            For information on the advisory (ovn24.09 bug fix and enhancement update), and where to find the updated files, follow the link below.

            If the solution does not work for you, open a new bug report.
            https://access.redhat.com/errata/RHBA-2024:10898

            Errata Tool added a comment - Since the problem described in this issue should be resolved in a recent advisory, it has been closed. For information on the advisory (ovn24.09 bug fix and enhancement update), and where to find the updated files, follow the link below. If the solution does not work for you, open a new bug report. https://access.redhat.com/errata/RHBA-2024:10898

            Jianlin Shi mentioned this issue in a commit of Kernel QE / Kernel Tests on branch master:

            Merge branch 'jishi' into 'master'

            GitLab CEE Bot added a comment - Jianlin Shi mentioned this issue in a commit of Kernel QE / Kernel Tests on branch master : Merge branch 'jishi' into 'master'

            Jianlin Shi mentioned this issue in a merge request of Kernel QE / Kernel Tests on branch jishi:

            ovn: reproducer for FDP-744

            GitLab CEE Bot added a comment - Jianlin Shi mentioned this issue in a merge request of Kernel QE / Kernel Tests on branch jishi : ovn: reproducer for FDP-744

            Jianlin Shi added a comment -

            tested with following script:

            systemctl start openvswitch                          
            systemctl start ovn-northd
            ovn-nbctl set-connection ptcp:6641                                         
            ovn-sbctl set-connection ptcp:6642
            ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.86.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.86.25
            systemctl restart ovn-controller
                                                   
            ovn-nbctl ls-add ls1                         
            ovn-nbctl lsp-add ls1 ls1p1                                         
            ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:02 192.168.1.1 2001::1"
            ovn-nbctl lsp-add ls1 ls1p2                                              
            ovn-nbctl lsp-set-addresses ls1p2 "00:00:00:01:02:02 192.168.1.2 2001::2"
            ovn-nbctl lr-add lr1
            ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 2001::a/64
            ovn-nbctl lsp-add ls1 ls1-lr1
            ovn-nbctl lsp-set-addresses ls1-lr1 "00:00:00:00:00:01 192.168.1.254 2001::a"
            ovn-nbctl lsp-set-type ls1-lr1 router
            ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1ovn-nbctl ls-add public
            ovn-nbctl lrp-add lr1 rp-public 00:00:02:01:02:03 172.16.1.1/24 1000::a/64 \
                -- lrp-set-gateway-chassis rp-public hv1
            ovn-nbctl lsp-add public public-rp -- set Logical_Switch_Port public-rp \
                type=router options:router-port=rp-public \
                -- lsp-set-addresses public-rp router
                                           
            ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:00:02 192.168.2.254/24 2002::a/64
                   
            ovn-nbctl ls-add ls2
            ovn-nbctl lsp-add ls2 ls2-lr1                                            
            ovn-nbctl lsp-set-addresses ls2-lr1 "00:00:00:00:00:02 192.168.2.254 2002::a"
            ovn-nbctl lsp-set-type ls2-lr1 router                  
            ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2                                                                                                                                                      
            ovn-nbctl lsp-add ls2 ls2p1                                                        
            ovn-nbctl lsp-set-addresses ls2p1 "00:00:00:02:01:02 192.168.2.1 2002::1"          
                                       
            ovn-nbctl lsp-add ls1 ls1p3
            ovn-nbctl lsp-set-addresses ls1p3 "00:00:00:01:03:02 192.168.1.3 2001::3"ovn-nbctl lr-nat-add lr1 snat 172.16.1.50 0.0.0.0/0
            ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.16.1.11 192.168.1.1 ls1p1 00:00:ff:01:01:01
            ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.16.1.12 192.168.1.2 ls1p2 00:00:ff:01:01:02ovs-vsctl add-port br-int ls1p1 -- set interface ls1p1 type=internal external_ids:iface-id=ls1p1
            ip netns add ls1p1
            ip link set ls1p1 netns ls1p1
            ip netns exec ls1p1 ip link set lo up
            ip netns exec ls1p1 ip link set ls1p1 address 00:00:00:01:01:02
            ip netns exec ls1p1 ip link set ls1p1 up
            ip netns exec ls1p1 ip addr add 192.168.1.1/24 dev ls1p1
            ip netns exec ls1p1 ip addr add 2001::1/64 dev ls1p1
            ip netns exec ls1p1 ip route add default via 192.168.1.254
            ip netns exec ls1p1 ip -6 route add default via 2001::aovs-vsctl add-port br-int ls1p2 -- set interface ls1p2 type=internal external_ids:iface-id=ls1p2
            ip netns add ls1p2
            ip link set ls1p2 netns ls1p2
            ip netns exec ls1p2 ip link set lo up
            ip netns exec ls1p2 ip link set ls1p2 address 00:00:00:01:02:02
            ip netns exec ls1p2 ip link set ls1p2 up
            ip netns exec ls1p2 ip addr add 192.168.1.2/24 dev ls1p2
            ip netns exec ls1p2 ip addr add 2001::2/64 dev ls1p2
            ip netns exec ls1p2 ip route add default via 192.168.1.254
            ip netns exec ls1p2 ip -6 route add default via 2001::aovs-vsctl add-port br-int ls2p1 -- set interface ls2p1 type=internal external_ids:iface-id=ls2p1
            ip netns add ls2p1
            ip link set ls2p1 netns ls2p1
            ip netns exec ls2p1 ip link set lo up
            ip netns exec ls2p1 ip link set ls2p1 address 00:00:00:02:01:02
            ip netns exec ls2p1 ip link set ls2p1 up
            ip netns exec ls2p1 ip addr add 192.168.2.1/24 dev ls2p1
            ip netns exec ls2p1 ip addr add 2002::1/64 dev ls2p1
            ip netns exec ls2p1 ip route add default via 192.168.2.254
            ip netns exec ls2p1 ip -6 route add default via 2002::aovn-nbctl --wait=hv sync
            ip netns exec ls1p1 ping 172.16.1.12 -c 1 

            result on ovn24.09-24.09.0-33:

            + ovn-nbctl --wait=hv sync
            + ip netns exec ls1p1 ping 172.16.1.12 -c 1
            PING 172.16.1.12 (172.16.1.12) 56(84) bytes of data.
                                                                                                     
            --- 172.16.1.12 ping statistics ---                                                      
            1 packets transmitted, 0 received, 100% packet loss, time 0ms 
            
            <=== ping failed 

            result on ovn24.09-24.09.1-10:

            [root@wsfd-advnetlab18 FDP-744]# rpm -qa | grep -E "openvswitch3.2|ovn24.09"
            openvswitch3.2-3.2.0-99.el9fdp.x86_64
            ovn24.09-24.09.1-10.el9fdp.x86_64
            ovn24.09-central-24.09.1-10.el9fdp.x86_64
            ovn24.09-host-24.09.1-10.el9fdp.x86_64
            + ovn-nbctl --wait=hv sync                                                               
            + ip netns exec ls1p1 ping 172.16.1.12 -c 1                                              
            PING 172.16.1.12 (172.16.1.12) 56(84) bytes of data.                                     
            64 bytes from 172.16.1.12: icmp_seq=1 ttl=62 time=4.67 ms                                
                                                                                                     
            --- 172.16.1.12 ping statistics ---                                                      
            1 packets transmitted, 1 received, 0% packet loss, time 0ms                              
            rtt min/avg/max/mdev = 4.669/4.669/4.669/0.000 ms
            
            <=== ping passed 

            Jianlin Shi added a comment - tested with following script: systemctl start openvswitch                           systemctl start ovn-northd ovn-nbctl set-connection ptcp:6641                                          ovn-sbctl set-connection ptcp:6642 ovs-vsctl set open . external_ids:system-id=hv1 external_ids:ovn-remote=tcp:20.0.86.25:6642 external_ids:ovn-encap-type=geneve external_ids:ovn-encap-ip=20.0.86.25 systemctl restart ovn-controller                                         ovn-nbctl ls-add ls1                          ovn-nbctl lsp-add ls1 ls1p1                                          ovn-nbctl lsp-set-addresses ls1p1 "00:00:00:01:01:02 192.168.1.1 2001::1" ovn-nbctl lsp-add ls1 ls1p2                                               ovn-nbctl lsp-set-addresses ls1p2 "00:00:00:01:02:02 192.168.1.2 2001::2" ovn-nbctl lr-add lr1 ovn-nbctl lrp-add lr1 lr1-ls1 00:00:00:00:00:01 192.168.1.254/24 2001::a/64 ovn-nbctl lsp-add ls1 ls1-lr1 ovn-nbctl lsp-set-addresses ls1-lr1 "00:00:00:00:00:01 192.168.1.254 2001::a" ovn-nbctl lsp-set-type ls1-lr1 router ovn-nbctl lsp-set-options ls1-lr1 router-port=lr1-ls1ovn-nbctl ls-add public ovn-nbctl lrp-add lr1 rp- public 00:00:02:01:02:03 172.16.1.1/24 1000::a/64 \     -- lrp-set-gateway-chassis rp- public hv1 ovn-nbctl lsp-add public public -rp -- set Logical_Switch_Port public -rp \     type=router options:router-port=rp- public \     -- lsp-set-addresses public -rp router                                 ovn-nbctl lrp-add lr1 lr1-ls2 00:00:00:00:00:02 192.168.2.254/24 2002::a/64         ovn-nbctl ls-add ls2 ovn-nbctl lsp-add ls2 ls2-lr1                                             ovn-nbctl lsp-set-addresses ls2-lr1 "00:00:00:00:00:02 192.168.2.254 2002::a" ovn-nbctl lsp-set-type ls2-lr1 router                   ovn-nbctl lsp-set-options ls2-lr1 router-port=lr1-ls2                                                                                                                                                       ovn-nbctl lsp-add ls2 ls2p1                                                         ovn-nbctl lsp-set-addresses ls2p1 "00:00:00:02:01:02 192.168.2.1 2002::1"                                       ovn-nbctl lsp-add ls1 ls1p3 ovn-nbctl lsp-set-addresses ls1p3 "00:00:00:01:03:02 192.168.1.3 2001::3" ovn-nbctl lr-nat-add lr1 snat 172.16.1.50 0.0.0.0/0 ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.16.1.11 192.168.1.1 ls1p1 00:00:ff:01:01:01 ovn-nbctl lr-nat-add lr1 dnat_and_snat 172.16.1.12 192.168.1.2 ls1p2 00:00:ff:01:01:02ovs-vsctl add-port br- int ls1p1 -- set interface ls1p1 type=internal external_ids:iface-id=ls1p1 ip netns add ls1p1 ip link set ls1p1 netns ls1p1 ip netns exec ls1p1 ip link set lo up ip netns exec ls1p1 ip link set ls1p1 address 00:00:00:01:01:02 ip netns exec ls1p1 ip link set ls1p1 up ip netns exec ls1p1 ip addr add 192.168.1.1/24 dev ls1p1 ip netns exec ls1p1 ip addr add 2001::1/64 dev ls1p1 ip netns exec ls1p1 ip route add default via 192.168.1.254 ip netns exec ls1p1 ip -6 route add default via 2001::aovs-vsctl add-port br- int ls1p2 -- set interface ls1p2 type=internal external_ids:iface-id=ls1p2 ip netns add ls1p2 ip link set ls1p2 netns ls1p2 ip netns exec ls1p2 ip link set lo up ip netns exec ls1p2 ip link set ls1p2 address 00:00:00:01:02:02 ip netns exec ls1p2 ip link set ls1p2 up ip netns exec ls1p2 ip addr add 192.168.1.2/24 dev ls1p2 ip netns exec ls1p2 ip addr add 2001::2/64 dev ls1p2 ip netns exec ls1p2 ip route add default via 192.168.1.254 ip netns exec ls1p2 ip -6 route add default via 2001::aovs-vsctl add-port br- int ls2p1 -- set interface ls2p1 type=internal external_ids:iface-id=ls2p1 ip netns add ls2p1 ip link set ls2p1 netns ls2p1 ip netns exec ls2p1 ip link set lo up ip netns exec ls2p1 ip link set ls2p1 address 00:00:00:02:01:02 ip netns exec ls2p1 ip link set ls2p1 up ip netns exec ls2p1 ip addr add 192.168.2.1/24 dev ls2p1 ip netns exec ls2p1 ip addr add 2002::1/64 dev ls2p1 ip netns exec ls2p1 ip route add default via 192.168.2.254 ip netns exec ls2p1 ip -6 route add default via 2002::aovn-nbctl --wait=hv sync ip netns exec ls1p1 ping 172.16.1.12 -c 1 result on ovn24.09-24.09.0-33: + ovn-nbctl --wait=hv sync + ip netns exec ls1p1 ping 172.16.1.12 -c 1 PING 172.16.1.12 (172.16.1.12) 56(84) bytes of data.                                                                                           --- 172.16.1.12 ping statistics ---                                                       1 packets transmitted, 0 received, 100% packet loss, time 0ms  <=== ping failed result on ovn24.09-24.09.1-10: [root@wsfd-advnetlab18 FDP-744]# rpm -qa | grep -E "openvswitch3.2|ovn24.09" openvswitch3.2-3.2.0-99.el9fdp.x86_64 ovn24.09-24.09.1-10.el9fdp.x86_64 ovn24.09-central-24.09.1-10.el9fdp.x86_64 ovn24.09-host-24.09.1-10.el9fdp.x86_64 + ovn-nbctl --wait=hv sync                                                                + ip netns exec ls1p1 ping 172.16.1.12 -c 1                                               PING 172.16.1.12 (172.16.1.12) 56(84) bytes of data.                                      64 bytes from 172.16.1.12: icmp_seq=1 ttl=62 time=4.67 ms                                                                                                                           --- 172.16.1.12 ping statistics ---                                                       1 packets transmitted, 1 received, 0% packet loss, time 0ms                               rtt min/avg/max/mdev = 4.669/4.669/4.669/0.000 ms <=== ping passed

            OVN Team added a comment -

            This issue is fixed in ovn24.09-24.09.0-39.el9fdp.

            OVN Team added a comment - This issue is fixed in ovn24.09-24.09.0-39.el9fdp.

            OVN Team added a comment -

            A review mentioning this issue has been posted to https://patchwork.ozlabs.org/project/ovn/list/?series=421093.

            OVN Team added a comment - A review mentioning this issue has been posted to https://patchwork.ozlabs.org/project/ovn/list/?series=421093 .

            OVN Team added a comment -

            A review mentioning this issue has been posted to https://patchwork.ozlabs.org/project/ovn/list/?series=420264.

            OVN Team added a comment - A review mentioning this issue has been posted to https://patchwork.ozlabs.org/project/ovn/list/?series=420264 .

            Terry Wilson added a comment - - edited

            Testing showed this actually is a problem in 22.03 and 24.09, but not 23.09. The culprit patch, at least for the main branch (thanks amusil@redhat.com), seems to be https://github.com/ovn-org/ovn/commit/40136a2f.

            Terry Wilson added a comment - - edited Testing showed this actually is a problem in 22.03 and 24.09, but not 23.09. The culprit patch, at least for the main branch (thanks amusil@redhat.com ), seems to be https://github.com/ovn-org/ovn/commit/40136a2f .

            mmichelson this is related to a major escalation for RHOSP. Depending on whether this bug is solved and backported in time, this determines the RHOSP design for the nested SNAT support feature. I'd hope we can raise the priority here.

            Ihar Hrachyshka added a comment - mmichelson this is related to a major escalation for RHOSP. Depending on whether this bug is solved and backported in time, this determines the RHOSP design for the nested SNAT support feature. I'd hope we can raise the priority here.

            I'm prioritizing this as "Major" which means we likely will start work on this when we start our next sprint in a few weeks. If this work is blocking something that needs to be fixed before then, please let me know and I'll bump it to "Critical" instead.

            Mark Michelson added a comment - I'm prioritizing this as "Major" which means we likely will start work on this when we start our next sprint in a few weeks. If this work is blocking something that needs to be fixed before then, please let me know and I'll bump it to "Critical" instead.

              amusil@redhat.com Ales Musil
              twilson@redhat.com Terry Wilson
              Jianlin Shi Jianlin Shi
              Votes:
              0 Vote for this issue
              Watchers:
              8 Start watching this issue

                Created:
                Updated:
                Resolved: