-
Bug
-
Resolution: Unresolved
-
Major
-
None
-
4.13
-
Quality / Stability / Reliability
-
False
-
-
3
-
Moderate
-
No
-
None
-
None
-
Rejected
-
None
-
None
-
None
-
None
-
None
-
None
-
None
-
None
Description of problem:
VerticalPodAutoscaler does create VerticalPodAutoscalerCheckpoint resources for each Container in a pod. Yet it does not reconcile the object in case the container name in the pod is changing, causing podMinCPUMillicores to be applied on the base of two containers in one pod instead of just one.
$ oc get clusterversion
NAME VERSION AVAILABLE PROGRESSING SINCE STATUS
version 4.13.11 True False 32m Cluster version is 4.13.11
$ oc get VerticalPodAutoscalerController default -n openshift-vertical-pod-autoscaler default -o yaml
apiVersion: v1
items:
- apiVersion: autoscaling.openshift.io/v1
kind: VerticalPodAutoscalerController
metadata:
creationTimestamp: "2023-09-18T09:20:22Z"
generation: 2
name: default
namespace: openshift-vertical-pod-autoscaler
resourceVersion: "37671"
uid: ff0eeb9b-6b15-440b-b96b-abfdda0d4bc1
spec:
minReplicas: 2
podMinCPUMillicores: 50
podMinMemoryMb: 250
recommendationOnly: false
safetyMarginFraction: 0.15
status: {}
- apiVersion: autoscaling.openshift.io/v1
kind: VerticalPodAutoscalerController
metadata:
creationTimestamp: "2023-09-18T09:20:22Z"
generation: 2
name: default
namespace: openshift-vertical-pod-autoscaler
resourceVersion: "37671"
uid: ff0eeb9b-6b15-440b-b96b-abfdda0d4bc1
spec:
minReplicas: 2
podMinCPUMillicores: 50
podMinMemoryMb: 250
recommendationOnly: false
safetyMarginFraction: 0.15
status: {}
kind: List
metadata:
resourceVersion: ""
$ oc get deployment curl -o yaml
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "1"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app":"curl","component":"curl"},"name":"curl","namespace":"project-100"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"curl","component":"curl"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"curl","component":"curl"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["curl"]}]},"topologyKey":"kubernetes.io/hostname"}]}},"containers":[{"env":[{"name":"CURLHOST","value":"https://kubernetes.default.svc/version"}],"image":"quay.io/rhn_support_sreber/curl:latest","imagePullPolicy":"IfNotPresent","name":"curl-a","resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}}
creationTimestamp: "2023-09-18T09:32:40Z"
generation: 1
labels:
app: curl
component: curl
name: curl
namespace: project-100
resourceVersion: "38880"
uid: 7fd1b371-fc73-40bc-a202-65414cd3a8b4
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: curl
component: curl
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
app: curl
component: curl
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- curl
topologyKey: kubernetes.io/hostname
containers:
- env:
- name: CURLHOST
value: https://kubernetes.default.svc/version
image: quay.io/rhn_support_sreber/curl:latest
imagePullPolicy: IfNotPresent
name: curl-a
resources:
requests:
cpu: 10m
memory: 20Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2023-09-18T09:32:46Z"
lastUpdateTime: "2023-09-18T09:32:46Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
- lastTransitionTime: "2023-09-18T09:32:40Z"
lastUpdateTime: "2023-09-18T09:32:46Z"
message: ReplicaSet "curl-5988dd475f" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
observedGeneration: 1
readyReplicas: 1
replicas: 1
updatedReplicas: 1
$ oc get vpa vpa -o yaml
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}}
creationTimestamp: "2023-09-18T09:34:40Z"
generation: 1
name: vpa
namespace: project-100
resourceVersion: "39495"
uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: curl
updatePolicy:
updateMode: Recreate
$ oc get vpa vpa -o yaml
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}}
creationTimestamp: "2023-09-18T09:34:40Z"
generation: 2
name: vpa
namespace: project-100
resourceVersion: "39803"
uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: curl
updatePolicy:
updateMode: Recreate
status:
conditions:
- lastTransitionTime: "2023-09-18T09:35:28Z"
status: "True"
type: RecommendationProvided
recommendation:
containerRecommendations:
- containerName: curl-a
lowerBound:
cpu: 50m
memory: 262144k
target:
cpu: 50m
memory: 262144k
uncappedTarget:
cpu: 50m
memory: 262144k
upperBound:
cpu: 33143m
memory: 16571500k
$ oc get deployment curl -o yaml
apiVersion: apps/v1
kind: Deployment
metadata:
annotations:
deployment.kubernetes.io/revision: "2"
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"apps/v1","kind":"Deployment","metadata":{"annotations":{},"labels":{"app":"curl","component":"curl"},"name":"curl","namespace":"project-100"},"spec":{"progressDeadlineSeconds":600,"replicas":1,"revisionHistoryLimit":10,"selector":{"matchLabels":{"app":"curl","component":"curl"}},"strategy":{"rollingUpdate":{"maxSurge":"25%","maxUnavailable":"25%"},"type":"RollingUpdate"},"template":{"metadata":{"creationTimestamp":null,"labels":{"app":"curl","component":"curl"}},"spec":{"affinity":{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchExpressions":[{"key":"app","operator":"In","values":["curl"]}]},"topologyKey":"kubernetes.io/hostname"}]}},"containers":[{"env":[{"name":"CURLHOST","value":"https://kubernetes.default.svc/version"}],"image":"quay.io/rhn_support_sreber/curl:latest","imagePullPolicy":"IfNotPresent","name":"curl-a","resources":{"requests":{"cpu":"10m","memory":"20Mi"}},"terminationMessagePath":"/dev/termination-log","terminationMessagePolicy":"File"}],"dnsPolicy":"ClusterFirst","restartPolicy":"Always","schedulerName":"default-scheduler","securityContext":{},"terminationGracePeriodSeconds":30}}}}
creationTimestamp: "2023-09-18T09:32:40Z"
generation: 2
labels:
app: curl
component: curl
name: curl
namespace: project-100
resourceVersion: "43145"
uid: 7fd1b371-fc73-40bc-a202-65414cd3a8b4
spec:
progressDeadlineSeconds: 600
replicas: 1
revisionHistoryLimit: 10
selector:
matchLabels:
app: curl
component: curl
strategy:
rollingUpdate:
maxSurge: 25%
maxUnavailable: 25%
type: RollingUpdate
template:
metadata:
creationTimestamp: null
labels:
app: curl
component: curl
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- curl
topologyKey: kubernetes.io/hostname
containers:
- env:
- name: CURLHOST
value: https://kubernetes.default.svc/version
image: quay.io/rhn_support_sreber/curl:latest
imagePullPolicy: IfNotPresent
name: curl-b
resources:
requests:
cpu: 10m
memory: 20Mi
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
dnsPolicy: ClusterFirst
restartPolicy: Always
schedulerName: default-scheduler
securityContext: {}
terminationGracePeriodSeconds: 30
status:
availableReplicas: 1
conditions:
- lastTransitionTime: "2023-09-18T09:32:46Z"
lastUpdateTime: "2023-09-18T09:32:46Z"
message: Deployment has minimum availability.
reason: MinimumReplicasAvailable
status: "True"
type: Available
- lastTransitionTime: "2023-09-18T09:32:40Z"
lastUpdateTime: "2023-09-18T09:40:44Z"
message: ReplicaSet "curl-859b9db4f8" has successfully progressed.
reason: NewReplicaSetAvailable
status: "True"
type: Progressing
observedGeneration: 2
readyReplicas: 1
replicas: 1
updatedReplicas: 1
$ oc get vpa vpa -o yaml
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}}
creationTimestamp: "2023-09-18T09:34:40Z"
generation: 8
name: vpa
namespace: project-100
resourceVersion: "44088"
uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: curl
updatePolicy:
updateMode: Recreate
status:
conditions:
- lastTransitionTime: "2023-09-18T09:35:28Z"
status: "True"
type: RecommendationProvided
recommendation:
containerRecommendations:
- containerName: curl-a
lowerBound:
cpu: 25m
memory: 131072k
target:
cpu: 25m
memory: 131072k
uncappedTarget:
cpu: 25m
memory: 131072k
upperBound:
cpu: 5543m
memory: 2771500k
- containerName: curl-b
lowerBound:
cpu: 25m
memory: 131072k
target:
cpu: 25m
memory: 131072k
uncappedTarget:
cpu: 25m
memory: 131072k
upperBound:
cpu: 25m
memory: 131072k
$ oc get pod
NAME READY STATUS RESTARTS AGE
curl-859b9db4f8-6c7pp 1/1 Running 0 96s
$ oc get pod curl-859b9db4f8-6c7pp -o yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
k8s.ovn.org/pod-networks: '{"default":{"ip_addresses":["10.128.2.23/23"],"mac_address":"0a:58:0a:80:02:17","gateway_ips":["10.128.2.1"],"ip_address":"10.128.2.23/23","gateway_ip":"10.128.2.1"}}'
k8s.v1.cni.cncf.io/network-status: |-
[{
"name": "ovn-kubernetes",
"interface": "eth0",
"ips": [
"10.128.2.23"
],
"mac": "0a:58:0a:80:02:17",
"default": true,
"dns": {}
}]
openshift.io/scc: restricted-v2
seccomp.security.alpha.kubernetes.io/pod: runtime/default
vpaObservedContainers: curl-b
vpaUpdates: 'Pod resources updated by vpa: container 0: '
creationTimestamp: "2023-09-18T09:40:38Z"
generateName: curl-859b9db4f8-
labels:
app: curl
component: curl
pod-template-hash: 859b9db4f8
name: curl-859b9db4f8-6c7pp
namespace: project-100
ownerReferences:
- apiVersion: apps/v1
blockOwnerDeletion: true
controller: true
kind: ReplicaSet
name: curl-859b9db4f8
uid: 1a549426-e45c-4c34-a147-f2282e439ecd
resourceVersion: "43135"
uid: f8d799e3-aa9a-4ee1-b840-d0897c3774d1
spec:
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
- labelSelector:
matchExpressions:
- key: app
operator: In
values:
- curl
topologyKey: kubernetes.io/hostname
containers:
- env:
- name: CURLHOST
value: https://kubernetes.default.svc/version
image: quay.io/rhn_support_sreber/curl:latest
imagePullPolicy: IfNotPresent
name: curl-b
resources:
requests:
cpu: 10m
memory: 20Mi
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
runAsNonRoot: true
runAsUser: 1000680000
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /var/run/secrets/kubernetes.io/serviceaccount
name: kube-api-access-ztkb2
readOnly: true
dnsPolicy: ClusterFirst
enableServiceLinks: true
imagePullSecrets:
- name: default-dockercfg-kz47w
nodeName: ip-10-0-213-108.us-east-2.compute.internal
preemptionPolicy: PreemptLowerPriority
priority: 0
restartPolicy: Always
schedulerName: default-scheduler
securityContext:
fsGroup: 1000680000
seLinuxOptions:
level: s0:c26,c15
seccompProfile:
type: RuntimeDefault
serviceAccount: default
serviceAccountName: default
terminationGracePeriodSeconds: 30
tolerations:
- effect: NoExecute
key: node.kubernetes.io/not-ready
operator: Exists
tolerationSeconds: 300
- effect: NoExecute
key: node.kubernetes.io/unreachable
operator: Exists
tolerationSeconds: 300
- effect: NoSchedule
key: node.kubernetes.io/memory-pressure
operator: Exists
volumes:
- name: kube-api-access-ztkb2
projected:
defaultMode: 420
sources:
- serviceAccountToken:
expirationSeconds: 3607
path: token
- configMap:
items:
- key: ca.crt
path: ca.crt
name: kube-root-ca.crt
- downwardAPI:
items:
- fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
path: namespace
- configMap:
items:
- key: service-ca.crt
path: service-ca.crt
name: openshift-service-ca.crt
status:
conditions:
- lastProbeTime: null
lastTransitionTime: "2023-09-18T09:40:38Z"
status: "True"
type: Initialized
- lastProbeTime: null
lastTransitionTime: "2023-09-18T09:40:44Z"
status: "True"
type: Ready
- lastProbeTime: null
lastTransitionTime: "2023-09-18T09:40:44Z"
status: "True"
type: ContainersReady
- lastProbeTime: null
lastTransitionTime: "2023-09-18T09:40:38Z"
status: "True"
type: PodScheduled
containerStatuses:
- containerID: cri-o://0a65161c4cfd00743633961028982f9dc315c000a84992559d6137f06bd0c9d8
image: quay.io/rhn_support_sreber/curl:latest
imageID: quay.io/rhn_support_sreber/curl@sha256:ddbd292531554049d7b86d19f6265028d405a163c6d648e22f8246aee7af9bf5
lastState: {}
name: curl-b
ready: true
restartCount: 0
started: true
state:
running:
startedAt: "2023-09-18T09:40:43Z"
hostIP: 10.0.213.108
phase: Running
podIP: 10.128.2.23
podIPs:
- ip: 10.128.2.23
qosClass: Burstable
startTime: "2023-09-18T09:40:38Z"
$ oc get VerticalPodAutoscalerCheckpoint
NAME AGE
vpa-curl-a 7m47s
vpa-curl-b 107s
$ oc get VerticalPodAutoscalerCheckpoint -o yaml
apiVersion: v1
items:
- apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscalerCheckpoint
metadata:
creationTimestamp: "2023-09-18T09:35:28Z"
generation: 8
name: vpa-curl-a
namespace: project-100
resourceVersion: "44982"
uid: 406d29c4-ea46-495c-8fbb-c32e2939b5b5
spec:
containerName: curl-a
vpaObjectName: vpa
status:
cpuHistogram:
bucketWeights:
"0": 10000
"1": 7498
referenceTimestamp: "2023-09-18T00:00:00Z"
totalWeight: 0.92430828142862
firstSampleStart: "2023-09-18T09:34:28Z"
lastSampleStart: "2023-09-18T09:40:28Z"
lastUpdateTime: "2023-09-18T09:42:28Z"
memoryHistogram:
bucketWeights:
"0": 10000
referenceTimestamp: "2023-09-19T00:00:00Z"
totalWeight: 1.318534376414498
totalSamplesCount: 7
version: v3
- apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscalerCheckpoint
metadata:
creationTimestamp: "2023-09-18T09:41:28Z"
generation: 2
name: vpa-curl-b
namespace: project-100
resourceVersion: "44983"
uid: 4720920e-cc14-4122-85da-fd16b6092065
spec:
containerName: curl-b
vpaObjectName: vpa
status:
cpuHistogram:
bucketWeights:
"1": 10000
referenceTimestamp: "2023-09-18T00:00:00Z"
totalWeight: 0.1323621600589326
firstSampleStart: "2023-09-18T09:42:28Z"
lastSampleStart: "2023-09-18T09:42:28Z"
lastUpdateTime: "2023-09-18T09:42:28Z"
memoryHistogram:
referenceTimestamp: "2023-09-19T00:00:00Z"
totalSamplesCount: 1
version: v3
kind: List
metadata:
resourceVersion: ""
$ oc get vpa vpa -o yaml
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}}
creationTimestamp: "2023-09-18T09:34:40Z"
generation: 10
name: vpa
namespace: project-100
resourceVersion: "45830"
uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c
spec:
targetRef:
apiVersion: apps/v1
kind: Deployment
name: curl
updatePolicy:
updateMode: Recreate
status:
conditions:
- lastTransitionTime: "2023-09-18T09:35:28Z"
status: "True"
type: RecommendationProvided
recommendation:
containerRecommendations:
- containerName: curl-a
lowerBound:
cpu: 25m
memory: 131072k
target:
cpu: 25m
memory: 131072k
uncappedTarget:
cpu: 25m
memory: 131072k
upperBound:
cpu: 5543m
memory: 2771500k
- containerName: curl-b
lowerBound:
cpu: 25m
memory: 131072k
target:
cpu: 25m
memory: 131072k
uncappedTarget:
cpu: 25m
memory: 131072k
upperBound:
cpu: 33143m
memory: 16571500k
Above, it's visible that after the deployment is created it would like to set the CPU value to 50m. But when changing the container name in the deployment, suddenly, two VerticalPodAutoscalerCheckpoint are being reported and the cpu minimum is set to 25m instead of 50, which is not correct (as it only holds one container).
$ oc get vpa vpa -o yaml
apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"autoscaling.k8s.io/v1","kind":"VerticalPodAutoscaler","metadata":{"annotations":{},"name":"vpa","namespace":"project-100"},"spec":{"targetRef":{"apiVersion":"apps/v1","kind":"Deployment","name":"curl"},"updatePolicy":{"updateMode":"Recreate"}}}
creationTimestamp: "2023-09-18T09:34:40Z"
generation: 17
name: vpa
namespace: project-100
resourceVersion: "51119"
uid: 2a381ad5-a36b-4e48-86b6-6939fa80960c
spec:
resourcePolicy:
containerPolicies:
- containerName: '*'
minAllowed:
cpu: 50m
targetRef:
apiVersion: apps/v1
kind: Deployment
name: curl
updatePolicy:
updateMode: Recreate
status:
conditions:
- lastTransitionTime: "2023-09-18T09:35:28Z"
status: "True"
type: RecommendationProvided
recommendation:
containerRecommendations:
- containerName: curl-a
lowerBound:
cpu: 50m
memory: 131072k
target:
cpu: 50m
memory: 131072k
uncappedTarget:
cpu: 25m
memory: 131072k
upperBound:
cpu: 5543m
memory: 2771500k
- containerName: curl-b
lowerBound:
cpu: 50m
memory: 131072k
target:
cpu: 50m
memory: 131072k
uncappedTarget:
cpu: 25m
memory: 131072k
upperBound:
cpu: 4754m
memory: "2377214285"
Now with ".spec.resourcePolicy.containerPolicies" it's possible to address or overcome this problem. Yet it's nothing that is documented or where awareness is being raised and hence can quickly and easily trigger problems when podMinCPUMillicores is set to 50% instead of 100%
Version-Release number of selected component (if applicable):
OpenShift Container Platform 4.13 and potentially before
How reproducible:
- Always
Steps to Reproduce:
1. See problem description
Actual results:
podMinCPUMillicores value is set to 50% on the respective deployment even though it has only one container and therefore should have 100% of podMinCPUMillicores set.
Expected results:
VerticalPodAutoscalerCheckpoint should be reconciled when the container name is changing to prevent a false image of the respective deployment (two containers but it's actually just 1)
Additional info:
- is documented by
-
OCPBUGS-32764 VerticalPodAutoscaler does not reconcile VerticalPodAutoscalerCheckpoint properly, causing to enforce wrong podMinCPUMillicores on pod
-
- Closed
-
- links to