Loading...

Type: Bug
Resolution: Done
Priority: Major
Fix Version/s: None
Affects Version/s: 4.14.z
Component/s: Networking / ovn-kubernetes
Labels:

Severity:
Moderate
Regression:
No
Blocked:
False
Blocked Reason:

Hide

None

Show
None
Release Note Type:
Release Note Not Required
Release Note Status:
In Progress
Target Version:

4.16

SFDC Cases Counter:
SFDC Cases Open:
SFDC Cases Links:

PX Impact Score:
PX Priority Data:
PX Technical Impact Notes:
workaround is to change sysctls via a machine config
PX Impact Range:
PX Review Complete:
PX Technical Impact:

When switching from ipForwarding: Global to Restricted, sysctl settings are not adjusted

Switch from:

# oc edit network.operator/cluster
apiVersion: operator.openshift.io/v1
kind: Network
metadata:
  annotations:
    networkoperator.openshift.io/ovn-cluster-initiator: 10.19.1.66
  creationTimestamp: "2023-11-22T12:14:46Z"
  generation: 207
  name: cluster
  resourceVersion: "235152"
  uid: 225d404d-4e26-41bf-8e77-4fc44948f239
spec:
  clusterNetwork:
  - cidr: 10.128.0.0/14
    hostPrefix: 23
  defaultNetwork:
    ovnKubernetesConfig:
      egressIPConfig: {}
      gatewayConfig:
        ipForwarding: Global
(...)

To:

# oc edit network.operator/cluster
apiVersion: operator.openshift.io/v1
kind: Network
metadata:
  annotations:
    networkoperator.openshift.io/ovn-cluster-initiator: 10.19.1.66
  creationTimestamp: "2023-11-22T12:14:46Z"
  generation: 207
  name: cluster
  resourceVersion: "235152"
  uid: 225d404d-4e26-41bf-8e77-4fc44948f239
spec:
  clusterNetwork:
  - cidr: 10.128.0.0/14
    hostPrefix: 23
  defaultNetwork:
    ovnKubernetesConfig:
      egressIPConfig: {}
      gatewayConfig:
        ipForwarding: Restricted

You'll see that the pods are updated:

# oc get pods -o yaml -n openshift-ovn-kubernetes ovnkube-node-fnl9z | grep sysctl -C10
      fi

      admin_network_policy_enabled_flag=
      if [[ "false" == "true" ]]; then
        admin_network_policy_enabled_flag="--enable-admin-network-policy"
      fi

      # If IP Forwarding mode is global set it in the host here.
      ip_forwarding_flag=
      if [ "Restricted" == "Global" ]; then
        sysctl -w net.ipv4.ip_forward=1
        sysctl -w net.ipv6.conf.all.forwarding=1
      else
        ip_forwarding_flag="--disable-forwarding"
      fi

      NETWORK_NODE_IDENTITY_ENABLE=
      if [[ "true" == "true" ]]; then
        NETWORK_NODE_IDENTITY_ENABLE="
          --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig
          --cert-dir=/etc/ovn/ovnkube-node-certs
          --cert-duration=24h

And that ovnkube correctly takes the settings:

# ps aux | grep disable-for
root       74963  0.3  0.0 8085828 153464 ?      Ssl  Nov22   3:38 /usr/bin/ovnkube --init-ovnkube-controller master1.site1.r450.org --init-node master1.site1.r450.org --config-file=/run/ovnkube-config/ovnkube.conf --ovn-empty-lb-events --loglevel 4 --inactivity-probe=180000 --gateway-mode shared --gateway-interface br-ex --metrics-bind-address 127.0.0.1:29103 --ovn-metrics-bind-address 127.0.0.1:29105 --metrics-enable-pprof --metrics-enable-config-duration --export-ovs-metrics --disable-snat-multiple-gws --enable-multi-network --enable-multicast --zone master1.site1.r450.org --enable-interconnect --acl-logging-rate-limit 20 --enable-multi-external-gateway=true --disable-forwarding --bootstrap-kubeconfig=/var/lib/kubelet/kubeconfig --cert-dir=/etc/ovn/ovnkube-node-certs --cert-duration=24h
root     2096007  0.0  0.0   3880  2144 pts/0    S+   10:07   0:00 grep --color=auto disable-for

But sysctls are never restricted:

[root@master1 ~]# sysctl -a | grep forward
net.ipv4.conf.0eca9d9e7fd3231.bc_forwarding = 0
net.ipv4.conf.0eca9d9e7fd3231.forwarding = 1
net.ipv4.conf.0eca9d9e7fd3231.mc_forwarding = 0
net.ipv4.conf.21a32cf76c3bcdf.bc_forwarding = 0
net.ipv4.conf.21a32cf76c3bcdf.forwarding = 1
net.ipv4.conf.21a32cf76c3bcdf.mc_forwarding = 0
net.ipv4.conf.22f9bca61beeaba.bc_forwarding = 0
net.ipv4.conf.22f9bca61beeaba.forwarding = 1
net.ipv4.conf.22f9bca61beeaba.mc_forwarding = 0
net.ipv4.conf.2ee438a7201c1f7.bc_forwarding = 0
net.ipv4.conf.2ee438a7201c1f7.forwarding = 1
net.ipv4.conf.2ee438a7201c1f7.mc_forwarding = 0
net.ipv4.conf.3560ce219f7b591.bc_forwarding = 0
net.ipv4.conf.3560ce219f7b591.forwarding = 1
net.ipv4.conf.3560ce219f7b591.mc_forwarding = 0
net.ipv4.conf.507c81eb9944c2e.bc_forwarding = 0
net.ipv4.conf.507c81eb9944c2e.forwarding = 1
net.ipv4.conf.507c81eb9944c2e.mc_forwarding = 0
net.ipv4.conf.6278633ca74482f.bc_forwarding = 0
net.ipv4.conf.6278633ca74482f.forwarding = 1
net.ipv4.conf.6278633ca74482f.mc_forwarding = 0
net.ipv4.conf.68b572ce18f3b82.bc_forwarding = 0
net.ipv4.conf.68b572ce18f3b82.forwarding = 1
net.ipv4.conf.68b572ce18f3b82.mc_forwarding = 0
net.ipv4.conf.7291c80dd47a6f3.bc_forwarding = 0
net.ipv4.conf.7291c80dd47a6f3.forwarding = 1
net.ipv4.conf.7291c80dd47a6f3.mc_forwarding = 0
net.ipv4.conf.76abdac44c6aee7.bc_forwarding = 0
net.ipv4.conf.76abdac44c6aee7.forwarding = 1
net.ipv4.conf.76abdac44c6aee7.mc_forwarding = 0
net.ipv4.conf.7f9abb486611f68.bc_forwarding = 0
net.ipv4.conf.7f9abb486611f68.forwarding = 1
net.ipv4.conf.7f9abb486611f68.mc_forwarding = 0
net.ipv4.conf.8cd86bfb8ea635f.bc_forwarding = 0
net.ipv4.conf.8cd86bfb8ea635f.forwarding = 1
net.ipv4.conf.8cd86bfb8ea635f.mc_forwarding = 0
net.ipv4.conf.8e87bd3f6ddc9f8.bc_forwarding = 0
net.ipv4.conf.8e87bd3f6ddc9f8.forwarding = 1
net.ipv4.conf.8e87bd3f6ddc9f8.mc_forwarding = 0
net.ipv4.conf.91079c8f5c1630f.bc_forwarding = 0
net.ipv4.conf.91079c8f5c1630f.forwarding = 1
net.ipv4.conf.91079c8f5c1630f.mc_forwarding = 0
net.ipv4.conf.92e754a12836f63.bc_forwarding = 0
net.ipv4.conf.92e754a12836f63.forwarding = 1
net.ipv4.conf.92e754a12836f63.mc_forwarding = 0
net.ipv4.conf.a5c01549a6070ab.bc_forwarding = 0
net.ipv4.conf.a5c01549a6070ab.forwarding = 1
net.ipv4.conf.a5c01549a6070ab.mc_forwarding = 0
net.ipv4.conf.a621d1234f0f25a.bc_forwarding = 0
net.ipv4.conf.a621d1234f0f25a.forwarding = 1
net.ipv4.conf.a621d1234f0f25a.mc_forwarding = 0
net.ipv4.conf.all.bc_forwarding = 0
net.ipv4.conf.all.forwarding = 1
net.ipv4.conf.all.mc_forwarding = 0
net.ipv4.conf.br-ex.bc_forwarding = 0
net.ipv4.conf.br-ex.forwarding = 1
net.ipv4.conf.br-ex.mc_forwarding = 0
net.ipv4.conf.br-int.bc_forwarding = 0
net.ipv4.conf.br-int.forwarding = 1
net.ipv4.conf.br-int.mc_forwarding = 0
net.ipv4.conf.c3f3da187245cf6.bc_forwarding = 0
net.ipv4.conf.c3f3da187245cf6.forwarding = 1
net.ipv4.conf.c3f3da187245cf6.mc_forwarding = 0
net.ipv4.conf.c7e518fff8ff973.bc_forwarding = 0
net.ipv4.conf.c7e518fff8ff973.forwarding = 1
net.ipv4.conf.c7e518fff8ff973.mc_forwarding = 0
net.ipv4.conf.d17c6fb6d3dd021.bc_forwarding = 0
net.ipv4.conf.d17c6fb6d3dd021.forwarding = 1
net.ipv4.conf.d17c6fb6d3dd021.mc_forwarding = 0
net.ipv4.conf.default.bc_forwarding = 0
net.ipv4.conf.default.forwarding = 1
net.ipv4.conf.default.mc_forwarding = 0
net.ipv4.conf.eno8303.bc_forwarding = 0
net.ipv4.conf.eno8303.forwarding = 1
net.ipv4.conf.eno8303.mc_forwarding = 0
net.ipv4.conf.eno8403.bc_forwarding = 0
net.ipv4.conf.eno8403.forwarding = 1
net.ipv4.conf.eno8403.mc_forwarding = 0
net.ipv4.conf.ens1f0.bc_forwarding = 0
net.ipv4.conf.ens1f0.forwarding = 1
net.ipv4.conf.ens1f0.mc_forwarding = 0
net.ipv4.conf.ens1f0/3516.bc_forwarding = 0
net.ipv4.conf.ens1f0/3516.forwarding = 1
net.ipv4.conf.ens1f0/3516.mc_forwarding = 0
net.ipv4.conf.ens1f0/3517.bc_forwarding = 0
net.ipv4.conf.ens1f0/3517.forwarding = 1
net.ipv4.conf.ens1f0/3517.mc_forwarding = 0
net.ipv4.conf.ens1f0/3518.bc_forwarding = 0
net.ipv4.conf.ens1f0/3518.forwarding = 1
net.ipv4.conf.ens1f0/3518.mc_forwarding = 0
net.ipv4.conf.ens1f1.bc_forwarding = 0
net.ipv4.conf.ens1f1.forwarding = 1
net.ipv4.conf.ens1f1.mc_forwarding = 0
net.ipv4.conf.ens3f0.bc_forwarding = 0
net.ipv4.conf.ens3f0.forwarding = 1
net.ipv4.conf.ens3f0.mc_forwarding = 0
net.ipv4.conf.ens3f1.bc_forwarding = 0
net.ipv4.conf.ens3f1.forwarding = 1
net.ipv4.conf.ens3f1.mc_forwarding = 0
net.ipv4.conf.fcb6e9468a65d70.bc_forwarding = 0
net.ipv4.conf.fcb6e9468a65d70.forwarding = 1
net.ipv4.conf.fcb6e9468a65d70.mc_forwarding = 0
net.ipv4.conf.fcd96084b7f5a9a.bc_forwarding = 0
net.ipv4.conf.fcd96084b7f5a9a.forwarding = 1
net.ipv4.conf.fcd96084b7f5a9a.mc_forwarding = 0
net.ipv4.conf.genev_sys_6081.bc_forwarding = 0
net.ipv4.conf.genev_sys_6081.forwarding = 1
net.ipv4.conf.genev_sys_6081.mc_forwarding = 0
net.ipv4.conf.lo.bc_forwarding = 0
net.ipv4.conf.lo.forwarding = 1
net.ipv4.conf.lo.mc_forwarding = 0
net.ipv4.conf.ovn-k8s-mp0.bc_forwarding = 0
net.ipv4.conf.ovn-k8s-mp0.forwarding = 1
net.ipv4.conf.ovn-k8s-mp0.mc_forwarding = 0
net.ipv4.conf.ovs-system.bc_forwarding = 0
net.ipv4.conf.ovs-system.forwarding = 1
net.ipv4.conf.ovs-system.mc_forwarding = 0
net.ipv4.ip_forward = 1
net.ipv4.ip_forward_update_priority = 1
net.ipv4.ip_forward_use_pmtu = 0
net.ipv6.conf.0eca9d9e7fd3231.forwarding = 1
net.ipv6.conf.0eca9d9e7fd3231.mc_forwarding = 0
net.ipv6.conf.21a32cf76c3bcdf.forwarding = 1
net.ipv6.conf.21a32cf76c3bcdf.mc_forwarding = 0
net.ipv6.conf.22f9bca61beeaba.forwarding = 1
net.ipv6.conf.22f9bca61beeaba.mc_forwarding = 0
net.ipv6.conf.2ee438a7201c1f7.forwarding = 1
net.ipv6.conf.2ee438a7201c1f7.mc_forwarding = 0
net.ipv6.conf.3560ce219f7b591.forwarding = 1
net.ipv6.conf.3560ce219f7b591.mc_forwarding = 0
net.ipv6.conf.507c81eb9944c2e.forwarding = 1
net.ipv6.conf.507c81eb9944c2e.mc_forwarding = 0
net.ipv6.conf.6278633ca74482f.forwarding = 1
net.ipv6.conf.6278633ca74482f.mc_forwarding = 0
net.ipv6.conf.68b572ce18f3b82.forwarding = 1
net.ipv6.conf.68b572ce18f3b82.mc_forwarding = 0
net.ipv6.conf.7291c80dd47a6f3.forwarding = 1
net.ipv6.conf.7291c80dd47a6f3.mc_forwarding = 0
net.ipv6.conf.76abdac44c6aee7.forwarding = 1
net.ipv6.conf.76abdac44c6aee7.mc_forwarding = 0
net.ipv6.conf.7f9abb486611f68.forwarding = 1
net.ipv6.conf.7f9abb486611f68.mc_forwarding = 0
net.ipv6.conf.8cd86bfb8ea635f.forwarding = 1
net.ipv6.conf.8cd86bfb8ea635f.mc_forwarding = 0
net.ipv6.conf.8e87bd3f6ddc9f8.forwarding = 1
net.ipv6.conf.8e87bd3f6ddc9f8.mc_forwarding = 0
net.ipv6.conf.91079c8f5c1630f.forwarding = 1
net.ipv6.conf.91079c8f5c1630f.mc_forwarding = 0
net.ipv6.conf.92e754a12836f63.forwarding = 1
net.ipv6.conf.92e754a12836f63.mc_forwarding = 0
net.ipv6.conf.a5c01549a6070ab.forwarding = 1
net.ipv6.conf.a5c01549a6070ab.mc_forwarding = 0
net.ipv6.conf.a621d1234f0f25a.forwarding = 1
net.ipv6.conf.a621d1234f0f25a.mc_forwarding = 0
net.ipv6.conf.all.forwarding = 1
net.ipv6.conf.all.mc_forwarding = 0
net.ipv6.conf.br-ex.forwarding = 1
net.ipv6.conf.br-ex.mc_forwarding = 0
net.ipv6.conf.br-int.forwarding = 1
net.ipv6.conf.br-int.mc_forwarding = 0
net.ipv6.conf.c3f3da187245cf6.forwarding = 1
net.ipv6.conf.c3f3da187245cf6.mc_forwarding = 0
net.ipv6.conf.c7e518fff8ff973.forwarding = 1
net.ipv6.conf.c7e518fff8ff973.mc_forwarding = 0
net.ipv6.conf.d17c6fb6d3dd021.forwarding = 1
net.ipv6.conf.d17c6fb6d3dd021.mc_forwarding = 0
net.ipv6.conf.default.forwarding = 1
net.ipv6.conf.default.mc_forwarding = 0
net.ipv6.conf.eno8303.forwarding = 1
net.ipv6.conf.eno8303.mc_forwarding = 0
net.ipv6.conf.eno8403.forwarding = 1
net.ipv6.conf.eno8403.mc_forwarding = 0
net.ipv6.conf.ens1f0.forwarding = 1
net.ipv6.conf.ens1f0.mc_forwarding = 0
net.ipv6.conf.ens1f0/3516.forwarding = 0
net.ipv6.conf.ens1f0/3516.mc_forwarding = 0
net.ipv6.conf.ens1f0/3517.forwarding = 0
net.ipv6.conf.ens1f0/3517.mc_forwarding = 0
net.ipv6.conf.ens1f0/3518.forwarding = 0
net.ipv6.conf.ens1f0/3518.mc_forwarding = 0
net.ipv6.conf.ens1f1.forwarding = 1
net.ipv6.conf.ens1f1.mc_forwarding = 0
net.ipv6.conf.ens3f0.forwarding = 1
net.ipv6.conf.ens3f0.mc_forwarding = 0
net.ipv6.conf.ens3f1.forwarding = 1
net.ipv6.conf.ens3f1.mc_forwarding = 0
net.ipv6.conf.fcb6e9468a65d70.forwarding = 1
net.ipv6.conf.fcb6e9468a65d70.mc_forwarding = 0
net.ipv6.conf.fcd96084b7f5a9a.forwarding = 1
net.ipv6.conf.fcd96084b7f5a9a.mc_forwarding = 0
net.ipv6.conf.genev_sys_6081.forwarding = 1
net.ipv6.conf.genev_sys_6081.mc_forwarding = 0
net.ipv6.conf.lo.forwarding = 1
net.ipv6.conf.lo.mc_forwarding = 0
net.ipv6.conf.ovn-k8s-mp0.forwarding = 1
net.ipv6.conf.ovn-k8s-mp0.mc_forwarding = 0
net.ipv6.conf.ovs-system.forwarding = 1
net.ipv6.conf.ovs-system.mc_forwarding = 0

It's logical that this is happening, because nowhere in the code is there a mechanism to tune the global sysctl back to 0 when the mode is switched from `Global` to `Restricted`. There's also no mechanism to sequentially reboot the nodes so that they'd reboot back to their defaults (= sysctl ip forward off).

clones

OCPBUGS-23758 When switching from ipForwarding: Global to Restricted, sysctl settings are not adjusted

Closed

depends on

OCPBUGS-23758 When switching from ipForwarding: Global to Restricted, sysctl settings are not adjusted

Closed

links to

openshift/ovn-kubernetes#2212: OCPBUGS-36145: Implementation required to enable Forwarding if it is already disabled

Details

Description

Attachments

Issue Links

Easy Agile Planning Poker

Activity

People

Dates