Loading...

Type: Bug
Resolution: Done
Priority: Blocker
Fix Version/s: 1.27.0
Affects Version/s: 1.27.0
Component/s: None
Labels:
None

Blocked:
False
Blocked Reason:

Hide

None

Show
None
Ready:
False
Git Pull Request:
https://github.com/openshift-knative/net-kourier/pull/44
Intelligence Requested:
Market:

SFDC Cases Counter:
SFDC Cases Open:
SFDC Cases Links:

The gateway throws this error and Pods don't start after upgrade to OCP 4.12:

message: 'pods "3scale-kourier-gateway-9d6d8fcf8-7959f" is forbidden: violates
      PodSecurity "restricted:v1.24": runAsNonRoot != true (container "kourier-gateway"
      must not set securityContext.runAsNonRoot=false)'

Apparently, the pod security policy got restricted for the knative-serving-ingress namespace after cluster upgrade:

❯ oc get deployment.apps/3scale-kourier-gateway -n knative-serving-ingress -oyaml
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: "6"
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"creationTimestamp":null,"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.6.0","networking.knative.dev/ingress-provider":"kourier","serving.knative.openshift.io/ownerName":"knative-serving","serving.knative.openshift.io/ownerNamespace":"knative-serving"},"name":"3scale-kourier-gateway","namespace":"knative-serving-ingress"},"spec":{"replicas":2,"selector":{"matchLabels":{"app":"3scale-kourier-gateway"}},"strategy":{"rollingUpdate":{"maxSurge":"100%","maxUnavailable":0},"type":"RollingUpdate"},"template":{"metadata":{"annotations":{"networking.knative.dev/poke":"v0.26"},"creationTimestamp":null,"labels":{"app":"3scale-kourier-gateway"}},"spec":{"containers":[{"args":["--base-id 1","-c /tmp/config/envoy-bootstrap.yaml","--log-level info"],"command":["/usr/local/bin/envoy"],"env":[{"name":"KUBERNETES_MIN_VERSION","value":"v1.0.0"}],"image":"registry.redhat.io/openshift-service-mesh/proxyv2-rhel8@sha256:5716a22874c9afa06159da127caf28809ae4f3c18a58605ee662021eb8c9099a","lifecycle":{"preStop":{"exec":{"command":["/bin/sh","-c","curl -X POST --unix /tmp/envoy.admin http://localhost/healthcheck/fail; sleep 15"]}}},"name":"kourier-gateway","ports":[{"containerPort":8080,"name":"http2-external","protocol":"TCP"},{"containerPort":8081,"name":"http2-internal","protocol":"TCP"},{"containerPort":8443,"name":"https-external","protocol":"TCP"},{"containerPort":8090,"name":"http-probe","protocol":"TCP"},{"containerPort":9443,"name":"https-probe","protocol":"TCP"}],"readinessProbe":{"httpGet":{"httpHeaders":[{"name":"Host","value":"internalkourier"}],"path":"/ready","port":8081,"scheme":"HTTP"},"initialDelaySeconds":10,"periodSeconds":5},"resources":{},"securityContext":{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]},"readOnlyRootFilesystem":false,"runAsNonRoot":false,"seccompProfile":{"type":"RuntimeDefault"}},"volumeMounts":[{"mountPath":"/tmp/config","name":"config-volume"}]}],"restartPolicy":"Always","volumes":[{"configMap":{"name":"kourier-bootstrap"},"name":"config-volume"}]}}},"status":{}}
  creationTimestamp: "2023-01-17T14:59:28Z"
  generation: 7
  labels:
    app.kubernetes.io/component: net-kourier
    app.kubernetes.io/name: knative-serving
    app.kubernetes.io/version: 1.6.0
    networking.knative.dev/ingress-provider: kourier
    serving.knative.openshift.io/ownerName: knative-serving
    serving.knative.openshift.io/ownerNamespace: knative-serving
  name: 3scale-kourier-gateway
  namespace: knative-serving-ingress
  resourceVersion: "391575"
  uid: b6acf87c-92b3-4246-915e-1a0fd6de08b4
spec:
  progressDeadlineSeconds: 600
  replicas: 2
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: 3scale-kourier-gateway
  strategy:
    rollingUpdate:
      maxSurge: 100%
      maxUnavailable: 0
    type: RollingUpdate
  template:
    metadata:
      annotations:
        networking.knative.dev/poke: v0.26
      creationTimestamp: null
      labels:
        app: 3scale-kourier-gateway
    spec:
      containers:
      - args:
        - --base-id 1
        - -c /tmp/config/envoy-bootstrap.yaml
        - --log-level info
        command:
        - /usr/local/bin/envoy
        env:
        - name: KUBERNETES_MIN_VERSION
          value: v1.0.0
        image: registry.redhat.io/openshift-service-mesh/proxyv2-rhel8@sha256:5716a22874c9afa06159da127caf28809ae4f3c18a58605ee662021eb8c9099a
        imagePullPolicy: IfNotPresent
        lifecycle:
          preStop:
            exec:
              command:
              - /bin/sh
              - -c
              - curl -X POST --unix /tmp/envoy.admin http://localhost/healthcheck/fail;
                sleep 15
        name: kourier-gateway
        ports:
        - containerPort: 8080
          name: http2-external
          protocol: TCP
        - containerPort: 8081
          name: http2-internal
          protocol: TCP
        - containerPort: 8443
          name: https-external
          protocol: TCP
        - containerPort: 8090
          name: http-probe
          protocol: TCP
        - containerPort: 9443
          name: https-probe
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            httpHeaders:
            - name: Host
              value: internalkourier
            path: /ready
            port: 8081
            scheme: HTTP
          initialDelaySeconds: 10
          periodSeconds: 5
          successThreshold: 1
          timeoutSeconds: 1
        resources: {}
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          readOnlyRootFilesystem: false
          runAsNonRoot: false
          seccompProfile:
            type: RuntimeDefault
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /tmp/config
          name: config-volume
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext: {}
      terminationGracePeriodSeconds: 30
      volumes:
      - configMap:
          defaultMode: 420
          name: kourier-bootstrap
        name: config-volume
status:
  conditions:
  - lastTransitionTime: "2023-01-17T14:59:28Z"
    lastUpdateTime: "2023-01-17T15:37:38Z"
    message: ReplicaSet "3scale-kourier-gateway-9d6d8fcf8" has successfully progressed.
    reason: NewReplicaSetAvailable
    status: "True"
    type: Progressing
  - lastTransitionTime: "2023-01-17T16:36:21Z"
    lastUpdateTime: "2023-01-17T16:36:21Z"
    message: Deployment does not have minimum availability.
    reason: MinimumReplicasUnavailable
    status: "False"
    type: Available
  - lastTransitionTime: "2023-01-17T16:36:21Z"
    lastUpdateTime: "2023-01-17T16:36:21Z"
    message: 'pods "3scale-kourier-gateway-9d6d8fcf8-7959f" is forbidden: violates
      PodSecurity "restricted:v1.24": runAsNonRoot != true (container "kourier-gateway"
      must not set securityContext.runAsNonRoot=false)'
    reason: FailedCreate
    status: "True"
    type: ReplicaFailure
  observedGeneration: 7
  unavailableReplicas: 2

The security context for the pod itself is empty: "securityContext: {}"


❯ oc get ns knative-serving-ingress -oyaml
apiVersion: v1
kind: Namespace
metadata:
  annotations:
    kubectl.kubernetes.io/last-applied-configuration: |
      {"apiVersion":"v1","kind":"Namespace","metadata":{"labels":{"app.kubernetes.io/component":"net-kourier","app.kubernetes.io/name":"knative-serving","app.kubernetes.io/version":"1.6.0","knative.openshift.io/part-of":"openshift-serverless","networking.knative.dev/ingress-provider":"kourier","serving.knative.openshift.io/ownerName":"knative-serving","serving.knative.openshift.io/ownerNamespace":"knative-serving"},"name":"knative-serving-ingress"}}
    openshift.io/sa.scc.mcs: s0:c38,c2
    openshift.io/sa.scc.supplemental-groups: 1001410000/10000
    openshift.io/sa.scc.uid-range: 1001410000/10000
  creationTimestamp: "2023-01-17T14:59:27Z"
  labels:
    app.kubernetes.io/component: net-kourier
    app.kubernetes.io/name: knative-serving
    app.kubernetes.io/version: 1.6.0
    knative.openshift.io/part-of: openshift-serverless
    kubernetes.io/metadata.name: knative-serving-ingress
    networking.knative.dev/ingress-provider: kourier
    pod-security.kubernetes.io/enforce: restricted
    pod-security.kubernetes.io/enforce-version: v1.24
    serving.knative.openshift.io/ownerName: knative-serving
    serving.knative.openshift.io/ownerNamespace: knative-serving
  name: knative-serving-ingress
  resourceVersion: "315173"
  uid: f505129f-e77d-4542-a6aa-f1a257f414a0
spec:
  finalizers:
  - kubernetes
status:
  phase: Active

Steps to reproduce:

install Serverless 1.26 on OCP 4.11
upgrade to Serverless 1.27
upgrade cluster to OCP 4.12.0 RC8
check 3scale-kourier-gateway (the Deployment is there but pods won't start, the result is unavailable Knative services)

BTW, serverless-operator E2E pass on OCP 4.12 as can be seen in this run, so it must be related to Serverless upgrade 1.26->1.27 followed by OCP upgrade.

Details

Description

Attachments

Easy Agile Planning Poker

Activity

People

Dates

Hide