The runc crashes and causes init container to fail with go statements in logs

    OCP 4.12

    - Customer's environment

    1.
    2.
    3.
    

runtime: epollwait on fd 7 failed with 1
fatal error: runtime: netpoll failed

runtime stack:
runtime.throw({0x55c305839652?, 0x7ffc67fb4a20?})
/usr/lib/golang/src/runtime/panic.go:1047 +0x5f fp=0x7ffc67fb49d0 sp=0x7ffc67fb49a0 pc=0x55c3054516ff
runtime.netpoll(0xc0000071e0?)
/usr/lib/golang/src/runtime/netpoll_epoll.go:130 +0x34e fp=0x7ffc67fb5030 sp=0x7ffc67fb49d0 pc=0x55c30544d9ce
runtime.startTheWorldWithSema(0x0)
/usr/lib/golang/src/runtime/proc.go:1264 +0x45 fp=0x7ffc67fb5090 sp=0x7ffc67fb5030 pc=0x55c305456925
runtime.gcStart.func2()
/usr/lib/golang/src/runtime/mgc.go:730 +0x2d fp=0x7ffc67fb50b8 sp=0x7ffc67fb5090 pc=0x55c305435f4d
runtime.systemstack()
/usr/lib/golang/src/runtime/asm_amd64.s:492 +0x46 fp=0x7ffc67fb50c0 sp=0x7ffc67fb50b8 pc=0x55c3054809c6

goroutine 1 [running, locked to thread]:
runtime.systemstack_switch()
/usr/lib/golang/src/runtime/asm_amd64.s:459 fp=0xc000241618 sp=0xc000241610 pc=0x55c305480960
runtime.gcStart({0x7f6b05c36108?, 0xb68a?, 0x1f01?})
/usr/lib/golang/src/runtime/mgc.go:729 +0x425 fp=0xc0002416a0 sp=0xc000241618 pc=0x55c305435dc5
runtime.mallocgc(0xb68a, 0x55c305b30160, 0x1)
/usr/lib/golang/src/runtime/malloc.go:1148 +0x817 fp=0xc000241718 sp=0xc0002416a0 pc=0x55c3054291f7
runtime.makeslice(0xc000419c00?, 0x6?, 0x55c305429100?)
/usr/lib/golang/src/runtime/slice.go:103 +0x52 fp=0xc000241740 sp=0xc000241718 pc=0x55c305468d72
syscall.SlicePtrFromStrings({0xc000438000, 0x3ed, 0xc0002417e8?})
/usr/lib/golang/src/syscall/exec_unix.go:94 +0xf2 fp=0xc0002417c0 sp=0xc000241740 pc=0x55c3054c73d2
syscall.Exec({0xc000419c58?, 0x3ed?}, {0xc00016c4c0, 0x3, 0x4}, {0xc000438000, 0x3ed, 0x3ed})
/usr/lib/golang/src/syscall/exec_unix.go:279 +0xdb fp=0xc000241858 sp=0xc0002417c0 pc=0x55c3054c7d7b
github.com/opencontainers/runc/vendor/golang.org/x/sys/unix.Exec(...)
/builddir/build/BUILD/runc-1.1.6/GOPATH/src/github.com/opencontainers/runc/GOPATH/src/github.com/opencontainers/runc/vendor/golang.org/x/sys/unix/syscall_unix.go:559
github.com/opencontainers/runc/libcontainer/system.Exec({0xc000419c58, 0x7}, {0xc00016c4c0, 0x3, 0x4}, {0xc000438000, 0x3ed, 0x3ed})
/builddir/build/BUILD/runc-1.1.6/GOPATH/src/github.com/opencontainers/runc/GOPATH/src/github.com/opencontainers/runc/libcontainer/system/linux.go:45 +0x88 fp=0xc0002418b8 sp=0xc000241858 pc=0x55c3056cdba8
github.com/opencontainers/runc/libcontainer.(*linuxStandardInit).Init(0xc000178370)
/builddir/build/BUILD/runc-1.1.6/GOPATH/src/github.com/opencontainers/runc/GOPATH/src/github.com/opencontainers/runc/libcontainer/standard_init_linux.go:280 +0x13b4 fp=0xc000241bc8 sp=0xc0002418b8 pc=0x55c305773894
github.com/opencontainers/runc/libcontainer.(*LinuxFactory).StartInitialization(0x0?)
/builddir/build/BUILD/runc-1.1.6/GOPATH/src/github.com/opencontainers/runc/GOPATH/src/github.com/opencontainers/runc/libcontainer/factory_linux.go:361 +0x5f3 fp=0xc000241e00 sp=0xc000241bc8 pc=0x55c30575ed33
main.init.0()
/builddir/build/BUILD/runc-1.1.6/GOPATH/src/github.com/opencontainers/runc/GOPATH/src/github.com/opencontainers/runc/init.go:36 +0x174 fp=0xc000241e50 sp=0xc000241e00 pc=0x55c305812174
runtime.doInit(0x55c305ec1560)
/usr/lib/golang/src/runtime/proc.go:6331 +0x128 fp=0xc000241f80 sp=0xc000241e50 pc=0x55c305461268
runtime.main()
/usr/lib/golang/src/runtime/proc.go:233 +0x1d4 fp=0xc000241fe0 sp=0xc000241f80 pc=0x55c305453f54
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000241fe8 sp=0xc000241fe0 pc=0x55c305482c01

goroutine 2 [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:363 +0xd6 fp=0xc000144fb0 sp=0xc000144f90 pc=0x55c305454356
runtime.goparkunlock(...)
/usr/lib/golang/src/runtime/proc.go:369
runtime.forcegchelper()
/usr/lib/golang/src/runtime/proc.go:302 +0xad fp=0xc000144fe0 sp=0xc000144fb0 pc=0x55c3054541ed
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000144fe8 sp=0xc000144fe0 pc=0x55c305482c01
created by runtime.init.7
/usr/lib/golang/src/runtime/proc.go:290 +0x25

goroutine 3 [GC sweep wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:363 +0xd6 fp=0xc000145790 sp=0xc000145770 pc=0x55c305454356
runtime.goparkunlock(...)
/usr/lib/golang/src/runtime/proc.go:369
runtime.bgsweep(0x0?)
/usr/lib/golang/src/runtime/mgcsweep.go:278 +0x8e fp=0xc0001457c8 sp=0xc000145790 pc=0x55c3054408ae
runtime.gcenable.func1()
/usr/lib/golang/src/runtime/mgc.go:178 +0x26 fp=0xc0001457e0 sp=0xc0001457c8 pc=0x55c305435546
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001457e8 sp=0xc0001457e0 pc=0x55c305482c01
created by runtime.gcenable
/usr/lib/golang/src/runtime/mgc.go:178 +0x6b

goroutine 4 [runnable]:
runtime.gopark(0xc00002c070?, 0x55c3058e1848?, 0x1?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:363 +0xd6 fp=0xc000145f70 sp=0xc000145f50 pc=0x55c305454356
runtime.goparkunlock(...)
/usr/lib/golang/src/runtime/proc.go:369
runtime.(*scavengerState).park(0x55c305ed5160)
/usr/lib/golang/src/runtime/mgcscavenge.go:389 +0x53 fp=0xc000145fa0 sp=0xc000145f70 pc=0x55c30543e953
runtime.bgscavenge(0x0?)
/usr/lib/golang/src/runtime/mgcscavenge.go:617 +0x45 fp=0xc000145fc8 sp=0xc000145fa0 pc=0x55c30543ef25
runtime.gcenable.func2()
/usr/lib/golang/src/runtime/mgc.go:179 +0x26 fp=0xc000145fe0 sp=0xc000145fc8 pc=0x55c3054354e6
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000145fe8 sp=0xc000145fe0 pc=0x55c305482c01
created by runtime.gcenable
/usr/lib/golang/src/runtime/mgc.go:179 +0xaa

goroutine 18 [finalizer wait]:
runtime.gopark(0x55c305ed5920?, 0xc0001824e0?, 0x0?, 0x0?, 0xc000144770?)
/usr/lib/golang/src/runtime/proc.go:363 +0xd6 fp=0xc000144628 sp=0xc000144608 pc=0x55c305454356
runtime.goparkunlock(...)
/usr/lib/golang/src/runtime/proc.go:369
runtime.runfinq()
/usr/lib/golang/src/runtime/mfinal.go:180 +0x10f fp=0xc0001447e0 sp=0xc000144628 pc=0x55c30543464f
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001447e8 sp=0xc0001447e0 pc=0x55c305482c01
created by runtime.createfing
/usr/lib/golang/src/runtime/mfinal.go:157 +0x45

goroutine 6 [GC worker (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
/usr/lib/golang/src/runtime/proc.go:363 +0xd6 fp=0xc000146750 sp=0xc000146730 pc=0x55c305454356
runtime.gcBgMarkWorker()
/usr/lib/golang/src/runtime/mgc.go:1235 +0xf1 fp=0xc0001467e0 sp=0xc000146750 pc=0x55c3054376b1
runtime.goexit()
/usr/lib/golang/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc0001467e8 sp=0xc0001467e0 pc=0x55c305482c01
created by runtime.gcBgMarkStartWorkers
/usr/lib/golang/src/runtime/mgc.go:1159 +0x25
    

- Containers to run smoothly    

It seems to be a panic in netpoll_epoll, notably here:

https://cs.opensource.google/go/go/+/release-branch.go1.14:src/runtime/netpoll_epoll.go;l=119-123

This appears to happen due to the numbering of the file descriptors.  If runc init is called without FD 0,1,2 set, then it appears it can panic like so:

https://github.com/golang/go/issues/68333