-
Bug
-
Resolution: Unresolved
-
Major
-
None
-
4.13
-
Moderate
-
No
-
3
-
Rejected
-
False
-
Description of problem:
VerticalPodAutoscaler does create VerticalPodAutoscalerCheckpoint resources for each Container in a pod. Yet it does not reconcile the object in case the container name in the pod is changing, causing podMinCPUMillicores to be applied on the base of two containers in one pod instead of just one. $ oc get clusterversion NAME VERSION AVAILABLE PROGRESSING SINCE STATUS version 4.13.11 True False 32m Cluster version is 4.13.11 $ oc get VerticalPodAutoscalerController default -n openshift-vertical-pod-autoscaler default -o yaml apiVersion: v1 items: - apiVersion: autoscaling.openshift.io/v1 kind: VerticalPodAutoscalerController metadata: creationTimestamp: "2023-09-18T09:20:22Z" generation: 2 name: default namespace: openshift-vertical-pod-autoscaler resourceVersion: "37671" uid: ff0eeb9b-6b15-440b-b96b-abfdda0d4bc1 spec: minReplicas: 2 podMinCPUMillicores: 50 podMinMemoryMb: 250 recommendationOnly: false safetyMarginFraction: 0.15 status: {} - apiVersion: autoscaling.openshift.io/v1 kind: VerticalPodAutoscalerController metadata: creationTimestamp: "2023-09-18T09:20:22Z" generation: 2 name: default namespace: openshift-vertical-pod-autoscaler resourceVersion: "37671" uid: ff0eeb9b-6b15-440b-b96b-abfdda0d4bc1 spec: minReplicas: 2 podMinCPUMillicores: 50 podMinMemoryMb: 250 recommendationOnly: false safetyMarginFraction: 0.15 status: {} kind: List metadata: resourceVersion: "" $ oc get deployment curl -o yaml apiVersion: apps/v1 kind: Deployment metadata: annotations: deployment.kubernetes.io/revision: "1" kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app":"curl","component":"curl"},"name":"curl","namespace":"project-100"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"curl","component":"curl"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"curl","component":"curl"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["curl"]}]},"topologyKey":"kubernetes.io/hostname"}]}},"containers":[{"env":[{"name":"CURLHOST","value":"https://kubernetes.default.svc/version"}],"image":"quay.io/rhn_support_sreber/curl:latest","imagePullPolicy":"IfNotPresent","name":"curl-a","resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}} creationTimestamp: "2023-09-18T09:32:40Z" generation: 1 labels: app: curl component: curl name: curl namespace: project-100 resourceVersion: "38880" uid: 7fd1b371-fc73-40bc-a202-65414cd3a8b4 spec: progressDeadlineSeconds: 600 replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: app: curl component: curl strategy: rollingUpdate: maxSurge: 25% maxUnavailable: 25% type: RollingUpdate template: metadata: creationTimestamp: null labels: app: curl component: curl spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - curl topologyKey: kubernetes.io/hostname containers: - env: - name: CURLHOST value: https://kubernetes.default.svc/version image: quay.io/rhn_support_sreber/curl:latest imagePullPolicy: IfNotPresent name: curl-a resources: requests: cpu: 10m memory: 20Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: File dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 30 status: availableReplicas: 1 conditions: - lastTransitionTime: "2023-09-18T09:32:46Z" lastUpdateTime: "2023-09-18T09:32:46Z" message: Deployment has minimum availability. reason: MinimumReplicasAvailable status: "True" type: Available - lastTransitionTime: "2023-09-18T09:32:40Z" lastUpdateTime: "2023-09-18T09:32:46Z" message: ReplicaSet "curl-5988dd475f" has successfully progressed. reason: NewReplicaSetAvailable status: "True" type: Progressing observedGeneration: 1 readyReplicas: 1 replicas: 1 updatedReplicas: 1 $ oc get vpa vpa -o yaml apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}} creationTimestamp: "2023-09-18T09:34:40Z" generation: 1 name: vpa namespace: project-100 resourceVersion: "39495" uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c spec: targetRef: apiVersion: apps/v1 kind: Deployment name: curl updatePolicy: updateMode: Recreate $ oc get vpa vpa -o yaml apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}} creationTimestamp: "2023-09-18T09:34:40Z" generation: 2 name: vpa namespace: project-100 resourceVersion: "39803" uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c spec: targetRef: apiVersion: apps/v1 kind: Deployment name: curl updatePolicy: updateMode: Recreate status: conditions: - lastTransitionTime: "2023-09-18T09:35:28Z" status: "True" type: RecommendationProvided recommendation: containerRecommendations: - containerName: curl-a lowerBound: cpu: 50m memory: 262144k target: cpu: 50m memory: 262144k uncappedTarget: cpu: 50m memory: 262144k upperBound: cpu: 33143m memory: 16571500k $ oc get deployment curl -o yaml apiVersion: apps/v1 kind: Deployment metadata: annotations: deployment.kubernetes.io/revision: "2" kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app":"curl","component":"curl"},"name":"curl","namespace":"project-100"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"curl","component":"curl"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"curl","component":"curl"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["curl"]}]},"topologyKey":"kubernetes.io/hostname"}]}},"containers":[{"env":[{"name":"CURLHOST","value":"https://kubernetes.default.svc/version"}],"image":"quay.io/rhn_support_sreber/curl:latest","imagePullPolicy":"IfNotPresent","name":"curl-a","resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}} creationTimestamp: "2023-09-18T09:32:40Z" generation: 2 labels: app: curl component: curl name: curl namespace: project-100 resourceVersion: "43145" uid: 7fd1b371-fc73-40bc-a202-65414cd3a8b4 spec: progressDeadlineSeconds: 600 replicas: 1 revisionHistoryLimit: 10 selector: matchLabels: app: curl component: curl strategy: rollingUpdate: maxSurge: 25% maxUnavailable: 25% type: RollingUpdate template: metadata: creationTimestamp: null labels: app: curl component: curl spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - curl topologyKey: kubernetes.io/hostname containers: - env: - name: CURLHOST value: https://kubernetes.default.svc/version image: quay.io/rhn_support_sreber/curl:latest imagePullPolicy: IfNotPresent name: curl-b resources: requests: cpu: 10m memory: 20Mi terminationMessagePath: /dev/termination-log terminationMessagePolicy: File dnsPolicy: ClusterFirst restartPolicy: Always schedulerName: default-scheduler securityContext: {} terminationGracePeriodSeconds: 30 status: availableReplicas: 1 conditions: - lastTransitionTime: "2023-09-18T09:32:46Z" lastUpdateTime: "2023-09-18T09:32:46Z" message: Deployment has minimum availability. reason: MinimumReplicasAvailable status: "True" type: Available - lastTransitionTime: "2023-09-18T09:32:40Z" lastUpdateTime: "2023-09-18T09:40:44Z" message: ReplicaSet "curl-859b9db4f8" has successfully progressed. reason: NewReplicaSetAvailable status: "True" type: Progressing observedGeneration: 2 readyReplicas: 1 replicas: 1 updatedReplicas: 1 $ oc get vpa vpa -o yaml apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}} creationTimestamp: "2023-09-18T09:34:40Z" generation: 8 name: vpa namespace: project-100 resourceVersion: "44088" uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c spec: targetRef: apiVersion: apps/v1 kind: Deployment name: curl updatePolicy: updateMode: Recreate status: conditions: - lastTransitionTime: "2023-09-18T09:35:28Z" status: "True" type: RecommendationProvided recommendation: containerRecommendations: - containerName: curl-a lowerBound: cpu: 25m memory: 131072k target: cpu: 25m memory: 131072k uncappedTarget: cpu: 25m memory: 131072k upperBound: cpu: 5543m memory: 2771500k - containerName: curl-b lowerBound: cpu: 25m memory: 131072k target: cpu: 25m memory: 131072k uncappedTarget: cpu: 25m memory: 131072k upperBound: cpu: 25m memory: 131072k $ oc get pod NAME READY STATUS RESTARTS AGE curl-859b9db4f8-6c7pp 1/1 Running 0 96s $ oc get pod curl-859b9db4f8-6c7pp -o yaml apiVersion: v1 kind: Pod metadata: annotations: k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.2.23/23"],"mac_address":"0a:58:0a:80:02:17","gateway_ips":["10.128.2.1"],"ip_address":"10.128.2.23/23","gateway_ip":"10.128.2.1"}}' k8s.v1.cni.cncf.io/network-status: |- [{ "name": "ovn-kubernetes", "interface": "eth0", "ips": [ "10.128.2.23" ], "mac": "0a:58:0a:80:02:17", "default": true, "dns": {} }] openshift.io/scc: restricted-v2 seccomp.security.alpha.kubernetes.io/pod: runtime/default vpaObservedContainers: curl-b vpaUpdates: 'Pod resources updated by vpa: container 0: ' creationTimestamp: "2023-09-18T09:40:38Z" generateName: curl-859b9db4f8- labels: app: curl component: curl pod-template-hash: 859b9db4f8 name: curl-859b9db4f8-6c7pp namespace: project-100 ownerReferences: - apiVersion: apps/v1 blockOwnerDeletion: true controller: true kind: ReplicaSet name: curl-859b9db4f8 uid: 1a549426-e45c-4c34-a147-f2282e439ecd resourceVersion: "43135" uid: f8d799e3-aa9a-4ee1-b840-d0897c3774d1 spec: affinity: podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: matchExpressions: - key: app operator: In values: - curl topologyKey: kubernetes.io/hostname containers: - env: - name: CURLHOST value: https://kubernetes.default.svc/version image: quay.io/rhn_support_sreber/curl:latest imagePullPolicy: IfNotPresent name: curl-b resources: requests: cpu: 10m memory: 20Mi securityContext: allowPrivilegeEscalation: false capabilities: drop: - ALL runAsNonRoot: true runAsUser: 1000680000 terminationMessagePath: /dev/termination-log terminationMessagePolicy: File volumeMounts: - mountPath: /var/run/secrets/kubernetes.io/serviceaccount name: kube-api-access-ztkb2 readOnly: true dnsPolicy: ClusterFirst enableServiceLinks: true imagePullSecrets: - name: default-dockercfg-kz47w nodeName: ip-10-0-213-108.us-east-2.compute.internal preemptionPolicy: PreemptLowerPriority priority: 0 restartPolicy: Always schedulerName: default-scheduler securityContext: fsGroup: 1000680000 seLinuxOptions: level: s0:c26,c15 seccompProfile: type: RuntimeDefault serviceAccount: default serviceAccountName: default terminationGracePeriodSeconds: 30 tolerations: - effect: NoExecute key: node.kubernetes.io/not-ready operator: Exists tolerationSeconds: 300 - effect: NoExecute key: node.kubernetes.io/unreachable operator: Exists tolerationSeconds: 300 - effect: NoSchedule key: node.kubernetes.io/memory-pressure operator: Exists volumes: - name: kube-api-access-ztkb2 projected: defaultMode: 420 sources: - serviceAccountToken: expirationSeconds: 3607 path: token - configMap: items: - key: ca.crt path: ca.crt name: kube-root-ca.crt - downwardAPI: items: - fieldRef: apiVersion: v1 fieldPath: metadata.namespace path: namespace - configMap: items: - key: service-ca.crt path: service-ca.crt name: openshift-service-ca.crt status: conditions: - lastProbeTime: null lastTransitionTime: "2023-09-18T09:40:38Z" status: "True" type: Initialized - lastProbeTime: null lastTransitionTime: "2023-09-18T09:40:44Z" status: "True" type: Ready - lastProbeTime: null lastTransitionTime: "2023-09-18T09:40:44Z" status: "True" type: ContainersReady - lastProbeTime: null lastTransitionTime: "2023-09-18T09:40:38Z" status: "True" type: PodScheduled containerStatuses: - containerID: cri-o://0a65161c4cfd00743633961028982f9dc315c000a84992559d6137f06bd0c9d8 image: quay.io/rhn_support_sreber/curl:latest imageID: quay.io/rhn_support_sreber/curl@sha256:ddbd292531554049d7b86d19f6265028d405a163c6d648e22f8246aee7af9bf5 lastState: {} name: curl-b ready: true restartCount: 0 started: true state: running: startedAt: "2023-09-18T09:40:43Z" hostIP: 10.0.213.108 phase: Running podIP: 10.128.2.23 podIPs: - ip: 10.128.2.23 qosClass: Burstable startTime: "2023-09-18T09:40:38Z" $ oc get VerticalPodAutoscalerCheckpoint NAME AGE vpa-curl-a 7m47s vpa-curl-b 107s $ oc get VerticalPodAutoscalerCheckpoint -o yaml apiVersion: v1 items: - apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscalerCheckpoint metadata: creationTimestamp: "2023-09-18T09:35:28Z" generation: 8 name: vpa-curl-a namespace: project-100 resourceVersion: "44982" uid: 406d29c4-ea46-495c-8fbb-c32e2939b5b5 spec: containerName: curl-a vpaObjectName: vpa status: cpuHistogram: bucketWeights: "0": 10000 "1": 7498 referenceTimestamp: "2023-09-18T00:00:00Z" totalWeight: 0.92430828142862 firstSampleStart: "2023-09-18T09:34:28Z" lastSampleStart: "2023-09-18T09:40:28Z" lastUpdateTime: "2023-09-18T09:42:28Z" memoryHistogram: bucketWeights: "0": 10000 referenceTimestamp: "2023-09-19T00:00:00Z" totalWeight: 1.318534376414498 totalSamplesCount: 7 version: v3 - apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscalerCheckpoint metadata: creationTimestamp: "2023-09-18T09:41:28Z" generation: 2 name: vpa-curl-b namespace: project-100 resourceVersion: "44983" uid: 4720920e-cc14-4122-85da-fd16b6092065 spec: containerName: curl-b vpaObjectName: vpa status: cpuHistogram: bucketWeights: "1": 10000 referenceTimestamp: "2023-09-18T00:00:00Z" totalWeight: 0.1323621600589326 firstSampleStart: "2023-09-18T09:42:28Z" lastSampleStart: "2023-09-18T09:42:28Z" lastUpdateTime: "2023-09-18T09:42:28Z" memoryHistogram: referenceTimestamp: "2023-09-19T00:00:00Z" totalSamplesCount: 1 version: v3 kind: List metadata: resourceVersion: "" $ oc get vpa vpa -o yaml apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}} creationTimestamp: "2023-09-18T09:34:40Z" generation: 10 name: vpa namespace: project-100 resourceVersion: "45830" uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c spec: targetRef: apiVersion: apps/v1 kind: Deployment name: curl updatePolicy: updateMode: Recreate status: conditions: - lastTransitionTime: "2023-09-18T09:35:28Z" status: "True" type: RecommendationProvided recommendation: containerRecommendations: - containerName: curl-a lowerBound: cpu: 25m memory: 131072k target: cpu: 25m memory: 131072k uncappedTarget: cpu: 25m memory: 131072k upperBound: cpu: 5543m memory: 2771500k - containerName: curl-b lowerBound: cpu: 25m memory: 131072k target: cpu: 25m memory: 131072k uncappedTarget: cpu: 25m memory: 131072k upperBound: cpu: 33143m memory: 16571500k Above, it's visible that after the deployment is created it would like to set the CPU value to 50m. But when changing the container name in the deployment, suddenly, two VerticalPodAutoscalerCheckpoint are being reported and the cpu minimum is set to 25m instead of 50, which is not correct (as it only holds one container). $ oc get vpa vpa -o yaml apiVersion: autoscaling.k8s.io/v1 kind: VerticalPodAutoscaler metadata: annotations: kubectl.kubernetes.io/last-applied-configuration: | {"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}} creationTimestamp: "2023-09-18T09:34:40Z" generation: 17 name: vpa namespace: project-100 resourceVersion: "51119" uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c spec: resourcePolicy: containerPolicies: - containerName: '*' minAllowed: cpu: 50m targetRef: apiVersion: apps/v1 kind: Deployment name: curl updatePolicy: updateMode: Recreate status: conditions: - lastTransitionTime: "2023-09-18T09:35:28Z" status: "True" type: RecommendationProvided recommendation: containerRecommendations: - containerName: curl-a lowerBound: cpu: 50m memory: 131072k target: cpu: 50m memory: 131072k uncappedTarget: cpu: 25m memory: 131072k upperBound: cpu: 5543m memory: 2771500k - containerName: curl-b lowerBound: cpu: 50m memory: 131072k target: cpu: 50m memory: 131072k uncappedTarget: cpu: 25m memory: 131072k upperBound: cpu: 4754m memory: "2377214285" Now with ".spec.resourcePolicy.containerPolicies" it's possible to address or overcome this problem. Yet it's nothing that is documented or where awareness is being raised and hence can quickly and easily trigger problems when podMinCPUMillicores is set to 50% instead of 100%
Version-Release number of selected component (if applicable):
OpenShift Container Platform 4.13 and potentially before
How reproducible:
- Always
Steps to Reproduce:
1. See problem description
Actual results:
podMinCPUMillicores value is set to 50% on the respective deployment even though it has only one container and therefore should have 100% of podMinCPUMillicores set.
Expected results:
VerticalPodAutoscalerCheckpoint should be reconciled when the container name is changing to prevent a false image of the respective deployment (two containers but it's actually just 1)
Additional info:
- is documented by
-
OCPBUGS-32764 VerticalPodAutoscaler does not reconcile VerticalPodAutoscalerCheckpoint properly, causing to enforce wrong podMinCPUMillicores on pod
-
- Closed
-
- links to