Uploaded image for project: 'OpenShift Bugs'
  1. OpenShift Bugs
  2. OCPBUGS-3431

Failed to install Security Profiles operator due to a panic on a fips enabled cluster

XMLWordPrintable

    • Icon: Bug Bug
    • Resolution: Done
    • Icon: Critical Critical
    • None
    • 4.12
    • Security Profiles Operator
    • None
    • Critical
    • None
    • 3
    • CMP Sprint 55, CMP Sprint 56, CMP Sprint 57, CMP Sprint 58
    • 4
    • Proposed
    • False
    • Hide

      None

      Show
      None

      Description of problem:

       

      Failed to install Security Profiles operator due to a panic with a fips enabled cluster:
$ oc get ip
NAME            CSV                                 APPROVAL    APPROVED
install-p4mdx   security-profiles-operator.v0.5.0   Automatic   true
$ oc get csv
NAME                                DISPLAY                            VERSION   REPLACES   PHASE
elasticsearch-operator.v5.6.0       OpenShift Elasticsearch Operator   5.6.0                Succeeded
loki-operator.v5.6.0                Loki Operator                      5.6.0                Succeeded
security-profiles-operator.v0.5.0   Security Profiles Operator         0.5.0                Failed
$ oc get pod
NAME                                         READY   STATUS             RESTARTS         AGE
security-profiles-operator-f788775c9-gwj44   0/1     CrashLoopBackOff   42 (2m44s ago)   3h12m
security-profiles-operator-f788775c9-s4lts   0/1     CrashLoopBackOff   42 (2m8s ago)    3h12m
security-profiles-operator-f788775c9-sp52q   0/1     CrashLoopBackOff   42 (2m31s ago)   3h12m
$ oc get event
LAST SEEN   TYPE      REASON    OBJECT                                           MESSAGE
3m13s       Normal    Pulling   pod/security-profiles-operator-f788775c9-gwj44   Pulling image "registry.redhat.io/compliance/openshift-security-profiles-rhel8-operator@sha256:dd0906c7aa7296450b2ce146cd4b01bca660345102ccd8e039999132d1fdc10e"
18m         Warning   BackOff   pod/security-profiles-operator-f788775c9-gwj44   Back-off restarting failed container
23m         Normal    Pulling   pod/security-profiles-operator-f788775c9-s4lts   Pulling image "registry.redhat.io/compliance/openshift-security-profiles-rhel8-operator@sha256:dd0906c7aa7296450b2ce146cd4b01bca660345102ccd8e039999132d1fdc10e"
3m14s       Warning   BackOff   pod/security-profiles-operator-f788775c9-s4lts   Back-off restarting failed container
28m         Normal    Pulled    pod/security-profiles-operator-f788775c9-s4lts   (combined from similar events): Successfully pulled image "registry.redhat.io/compliance/openshift-security-profiles-rhel8-operator@sha256:dd0906c7aa7296450b2ce146cd4b01bca660345102ccd8e039999132d1fdc10e" in 1.047750602s
8m9s        Normal    Pulling   pod/security-profiles-operator-f788775c9-sp52q   Pulling image "registry.redhat.io/compliance/openshift-security-profiles-rhel8-operator@sha256:dd0906c7aa7296450b2ce146cd4b01bca660345102ccd8e039999132d1fdc10e"
3m13s       Warning   BackOff   pod/security-profiles-operator-f788775c9-sp52q   Back-off restarting failed container
13m         Normal    Pulled    pod/security-profiles-operator-f788775c9-sp52q   (combined from similar events): Successfully pulled image "registry.redhat.io/compliance/openshift-security-profiles-rhel8-operator@sha256:dd0906c7aa7296450b2ce146cd4b01bca660345102ccd8e039999132d1fdc10e" in 1.142894837s
$ oc logs pod/security-profiles-operator-f788775c9-gwj44 --all-containers 
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x7ff025ef92bc]
 
runtime stack:
runtime.throw({0x1c6b7ee?, 0x40b1950?})
runtime/panic.go:1047 +0x5d fp=0x7ffdf8a86d50 sp=0x7ffdf8a86d20 pc=0x43ac3d
runtime.sigpanic()
runtime/signal_unix.go:819 +0x369 fp=0x7ffdf8a86da0 sp=0x7ffdf8a86d50 pc=0x4515a9
 
goroutine 1 [syscall, locked to thread]:
runtime.cgocall(0x1707600, 0xc000131830)
runtime/cgocall.go:158 +0x5c fp=0xc000131808 sp=0xc0001317d0 pc=0x404bdc
vendor/github.com/golang-fips/openssl-fips/openssl._Cfunc__goboringcrypto_DLOPEN_OPENSSL()
_cgo_gotypes.go:309 +0x49 fp=0xc000131830 sp=0xc000131808 pc=0x548589
vendor/github.com/golang-fips/openssl-fips/openssl.init.0()
vendor/github.com/golang-fips/openssl-fips/openssl/openssl.go:53 +0x45 fp=0xc000131860 sp=0xc000131830 pc=0x557225
runtime.doInit(0x2e0cb40)
runtime/proc.go:6321 +0x126 fp=0xc000131990 sp=0xc000131860 pc=0x44a8e6
runtime.doInit(0x2dfcb20)
runtime/proc.go:6298 +0x71 fp=0xc000131ac0 sp=0xc000131990 pc=0x44a831
runtime.doInit(0x2e0ae80)
runtime/proc.go:6298 +0x71 fp=0xc000131bf0 sp=0xc000131ac0 pc=0x44a831
runtime.doInit(0x2e13d80)
runtime/proc.go:6298 +0x71 fp=0xc000131d20 sp=0xc000131bf0 pc=0x44a831
runtime.doInit(0x2e14040)
runtime/proc.go:6298 +0x71 fp=0xc000131e50 sp=0xc000131d20 pc=0x44a831
runtime.doInit(0x2e14480)
runtime/proc.go:6298 +0x71 fp=0xc000131f80 sp=0xc000131e50 pc=0x44a831
runtime.main()
runtime/proc.go:233 +0x1d3 fp=0xc000131fe0 sp=0xc000131f80 pc=0x43d513
runtime.goexit()
runtime/asm_amd64.s:1594 +0x1 fp=0xc000131fe8 sp=0xc000131fe0 pc=0x46ed61
 
goroutine 2 [force gc (idle)]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:363 +0xd6 fp=0xc000074fb0 sp=0xc000074f90 pc=0x43d916
runtime.goparkunlock(...)
runtime/proc.go:369
runtime.forcegchelper()
runtime/proc.go:302 +0xad fp=0xc000074fe0 sp=0xc000074fb0 pc=0x43d7ad
runtime.goexit()
runtime/asm_amd64.s:1594 +0x1 fp=0xc000074fe8 sp=0xc000074fe0 pc=0x46ed61
created by runtime.init.6
runtime/proc.go:290 +0x25
 
goroutine 18 [GC sweep wait]:
runtime.gopark(0x0?, 0x0?, 0x0?, 0x0?, 0x0?)
runtime/proc.go:363 +0xd6 fp=0xc000070790 sp=0xc000070770 pc=0x43d916
runtime.goparkunlock(...)
runtime/proc.go:369
runtime.bgsweep(0x0?)
runtime/mgcsweep.go:278 +0x8e fp=0xc0000707c8 sp=0xc000070790 pc=0x42806e
runtime.gcenable.func1()
runtime/mgc.go:178 +0x26 fp=0xc0000707e0 sp=0xc0000707c8 pc=0x41cd26
runtime.goexit()
runtime/asm_amd64.s:1594 +0x1 fp=0xc0000707e8 sp=0xc0000707e0 pc=0x46ed61
created by runtime.gcenable
runtime/mgc.go:178 +0x6b
 
goroutine 19 [GC scavenge wait]:
runtime.gopark(0xc0000b6000?, 0x1ec3220?, 0x1?, 0x0?, 0x0?)
runtime/proc.go:363 +0xd6 fp=0xc000070f70 sp=0xc000070f50 pc=0x43d916
runtime.goparkunlock(...)
runtime/proc.go:369
runtime.(*scavengerState).park(0x2e7b420)
runtime/mgcscavenge.go:389 +0x53 fp=0xc000070fa0 sp=0xc000070f70 pc=0x426113
runtime.bgscavenge(0x0?)
runtime/mgcscavenge.go:617 +0x45 fp=0xc000070fc8 sp=0xc000070fa0 pc=0x4266e5
runtime.gcenable.func2()
runtime/mgc.go:179 +0x26 fp=0xc000070fe0 sp=0xc000070fc8 pc=0x41ccc6
runtime.goexit()
runtime/asm_amd64.s:1594 +0x1 fp=0xc000070fe8 sp=0xc000070fe0 pc=0x46ed61
created by runtime.gcenable
runtime/mgc.go:179 +0xaa
 
goroutine 34 [finalizer wait]:
runtime.gopark(0x2e7c260?, 0xc0001041a0?, 0x0?, 0x0?, 0xc000074770?)
runtime/proc.go:363 +0xd6 fp=0xc000074628 sp=0xc000074608 pc=0x43d916
runtime.goparkunlock(...)
runtime/proc.go:369
runtime.runfinq()
runtime/mfinal.go:180 +0x10f fp=0xc0000747e0 sp=0xc000074628 pc=0x41bdaf
runtime.goexit()
runtime/asm_amd64.s:1594 +0x1 fp=0xc0000747e8 sp=0xc0000747e0 pc=0x46ed61
created by runtime.createfing
runtime/mfinal.go:157 +0x45

       

      Version-Release number of selected component (if applicable):
      4.12.0-0.nightly-2022-11-07-181244 + security-profiles-operator-bundle-container-0.5.0-17

      How reproducible:

      Always

      Steps to Reproduce:

      1. Install Security Profiles Operator on a fips enabled cluster

      Actual results:

      The installation for security profiles operator failed due to a panic:

      $ oc get csv
NAME                                DISPLAY                            VERSION   REPLACES   PHASE
elasticsearch-operator.v5.6.0       OpenShift Elasticsearch Operator   5.6.0                Succeeded
loki-operator.v5.6.0                Loki Operator                      5.6.0                Succeeded
security-profiles-operator.v0.5.0   Security Profiles Operator         0.5.0                Failed
$ oc get pod
NAME                                         READY   STATUS             RESTARTS         AGE
security-profiles-operator-f788775c9-gwj44   0/1     CrashLoopBackOff   42 (2m44s ago)   3h12m
security-profiles-operator-f788775c9-s4lts   0/1     CrashLoopBackOff   42 (2m8s ago)    3h12m
security-profiles-operator-f788775c9-sp52q   0/1     CrashLoopBackOff   42 (2m31s ago)   3h12m
 
$ oc logs pod/security-profiles-operator-f788775c9-gwj44 --all-containers 
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x7ff025ef92bc]
 
runtime stack:
runtime.throw({0x1c6b7ee?, 0x40b1950?})
runtime/panic.go:1047 +0x5d fp=0x7ffdf8a86d50 sp=0x7ffdf8a86d20 pc=0x43ac3d
runtime.sigpanic()
runtime/signal_unix.go:819 +0x369 fp=0x7ffdf8a86da0 sp=0x7ffdf8a86d50 pc=0x4515a9
 
goroutine 1 [syscall, locked to thread]:
runtime.cgocall(0x1707600, 0xc000131830)
runtime/cgocall.go:158 +0x5c fp=0xc000131808 sp=0xc0001317d0 pc=0x404bdc
vendor/github.com/golang-fips/openssl-fips/openssl._Cfunc__goboringcrypto_DLOPEN_OPENSSL()
_cgo_gotypes.go:309 +0x49 fp=0xc000131830 sp=0xc000131808 pc=0x548589
vendor/github.com/golang-fips/openssl-fips/openssl.init.0()
vendor/github.com/golang-fips/openssl-fips/openssl/openssl.go:53 +0x45 fp=0xc000131860 sp=0xc000131830 pc=0x557225
runtime.doInit(0x2e0cb40)
runtime/proc.go:6321 +0x126 fp=0xc000131990 sp=0xc000131860 pc=0x44a8e6
runtime.doInit(0x2dfcb20)
runtime/proc.go:6298 +0x71 fp=0xc000131ac0 sp=0xc000131990 pc=0x44a831
runtime.doInit(0x2e0ae80)
runtime/proc.go:6298 +0x71 fp=0xc000131bf0 sp=0xc000131ac0 pc=0x44a831
runtime.doInit(0x2e13d80)
runtime/proc.go:6298 +0x71 fp=0xc000131d20 sp=0xc000131bf0 pc=0x44a831
runtime.doInit(0x2e14040)
runtime/proc.go:6298 +0x71 fp=0xc000131e50 sp=0xc000131d20 pc=0x44a831
runtime.doInit(0x2e14480)
runtime/proc.go:6298 +0x71 fp=0xc000131f80 sp=0xc000131e50 pc=0x44a831
runtime.main()
runtime/proc.go:233 +0x1d3 fp=0xc000131fe0 sp=0xc000131f80 pc=0x43d513
runtime.goexit()
runtime/asm_amd64.s:1594 +0x1 fp=0xc000131fe8 sp=0xc000131fe0 pc=0x46ed61

      Expected results:

      Security Profiles Operator could be installed successfully.

      Additional info:

       

      This issue is not reproduced with a simple ipi-aws-fips-disabled cluster.
May related with cm security-profiles-operator-profile:
$ oc get cm security-profiles-operator-profile -o yaml
apiVersion: v1
data:
  security-profiles-operator.json: |
    {
      "defaultAction": " SCMP_ACT_LOG",
      "architectures": ["SCMP_ARCH_X86_64", "SCMP_ARCH_X86", "SCMP_ARCH_X32", "SCMP_ARCH_AARCH64"],
      "syscalls": [
        {
          "names": [
            "accept4",
            "access",
            "arch_prctl",
            "bind",
            "brk",
            "capget",
            "capset",
            "clone",
            "clone3",
            "close",
            "connect",
            "chdir",
            "epoll_create1",
            "epoll_ctl",
            "epoll_pwait",
            "epoll_wait",
            "execve",
            "exit",
            "exit_group",
            "fcntl",
            "fchown",
            "fstat",
            "fstatfs",
            "futex",
            "getcwd",
            "getdents64",
            "getgid",
            "getpeername",
            "getpgrp",
            "getpid",
            "getppid",
            "getrandom",
            "getsockname",
            "getsockopt",
            "gettid",
            "getuid",
            "inotify_add_watch",
            "inotify_init1",
            "listen",
            "madvise",
            "membarrier",
            "mkdirat",
            "mlock",
            "mmap",
            "mprotect",
            "nanosleep",
            "newfstatat",
            "open",
            "openat",
            "pipe2",
            "pread64",
            "prctl",
            "read",
            "readlinkat",
            "rt_sigaction",
            "rt_sigprocmask",
            "rt_sigreturn",
            "sched_getaffinity",
            "sched_yield",
            "setgid",
            "setgroups",
            "setsockopt",
            "set_tid_address",
            "setuid",
            "sigaltstack",
            "socket",
            "tgkill",
            "uname",
            "unlinkat",
            "write"
          ],
          "action": "SCMP_ACT_ALLOW"
        }
      ]
    }
  selinuxd.cil: |
    (block selinuxd
        (blockinherit container)
        (allow process process ( capability ( dac_override dac_read_search lease audit_write audit_control )))
 
        (allow process default_context_t ( dir ( add_name create getattr ioctl lock open read remove_name rmdir search setattr write )))
        (allow process default_context_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process default_context_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process default_context_t ( sock_file ( append getattr open read write )))
        (allow process etc_t ( dir ( add_name create getattr ioctl lock open read remove_name rmdir search setattr write watch )))
        (allow process etc_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process etc_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process etc_t ( sock_file ( append getattr open read write )))
        (allow process file_context_t ( dir ( add_name create getattr ioctl lock open read remove_name rmdir search setattr write )))
        (allow process file_context_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process file_context_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process file_context_t ( sock_file ( append getattr open read write )))
        (allow process security_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process security_t ( security ( load_policy )))
        (allow process selinux_config_t ( dir ( add_name create getattr ioctl lock open read remove_name rename rmdir search setattr write )))
        (allow process selinux_config_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process selinux_config_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process selinux_config_t ( sock_file ( append getattr open read write )))
        (allow process selinux_login_config_t ( dir ( add_name create getattr ioctl lock open read remove_name rmdir search setattr write )))
        (allow process selinux_login_config_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process selinux_login_config_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process selinux_login_config_t ( sock_file ( append getattr open read write )))
        (allow process semanage_read_lock_t ( dir ( add_name create getattr ioctl lock open read remove_name rmdir search setattr write )))
        (allow process semanage_read_lock_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process semanage_read_lock_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process semanage_read_lock_t ( sock_file ( append getattr open read write )))
        (allow process semanage_store_t ( dir ( add_name create getattr ioctl lock open read rename remove_name rmdir search setattr write )))
        (allow process semanage_store_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process semanage_store_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process semanage_store_t ( sock_file ( append getattr open read write )))
        (allow process semanage_trans_lock_t ( dir ( add_name create getattr ioctl lock open read remove_name rmdir search setattr write )))
        (allow process semanage_trans_lock_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process semanage_trans_lock_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process semanage_trans_lock_t ( sock_file ( append getattr open read write )))
        (allow process sysfs_t ( dir ( add_name create getattr ioctl lock open read remove_name rmdir search setattr write )))
        (allow process sysfs_t ( fifo_file ( getattr read write append ioctl lock open )))
        (allow process sysfs_t ( file ( append create getattr ioctl lock map open read rename setattr unlink write )))
        (allow process sysfs_t ( sock_file ( append getattr open read write )))
    )
  selinuxrecording.cil: |
    (block selinuxrecording
      (blockinherit container)
      (typepermissive process)
    )
kind: ConfigMap
metadata:
  creationTimestamp: "2022-11-09T04:06:35Z"
  labels:
    app: security-profiles-operator
    operators.coreos.com/security-profiles-operator.security-profiles-operator: ""
  name: security-profiles-operator-profile
  namespace: security-profiles-operator
  ownerReferences:
  - apiVersion: operators.coreos.com/v1alpha1
    blockOwnerDeletion: false
    controller: false
    kind: ClusterServiceVersion
    name: security-profiles-operator.v0.5.0
    uid: 3dcbf240-78c2-4a12-9dcb-6c51b8f9e501
  resourceVersion: "161916"
  uid: e445108e-ba5f-43b4-9eac-e07213a4a4ee

              jhrozek@redhat.com Jakub Hrozek (Inactive)
              xiyuan@redhat.com Xiaojie Yuan
              Xiaojie Yuan Xiaojie Yuan
              Votes:
              0 Vote for this issue
              Watchers:
              5 Start watching this issue

                Created:
                Updated:
                Resolved: