On the Azure HCP cluster when creating internal ingress controller we are getting authorization error

    4.19 and may be further versions

    create internal ingress controller in cluster bot or prowci created Azure HCP cluster

    1.Create a internal ingress controller
mjoseph@mjoseph-mac Downloads % oc get co
NAME                                       VERSION                              AVAILABLE   PROGRESSING   DEGRADED   SINCE   MESSAGE
console                                    4.19.0-0.nightly-2025-01-21-163021   True        False         False      107m    
csi-snapshot-controller                    4.19.0-0.nightly-2025-01-21-163021   True        False         False      120m    
dns                                        4.19.0-0.nightly-2025-01-21-163021   True        False         False      107m    
image-registry                             4.19.0-0.nightly-2025-01-21-163021   True        False         False      107m    
ingress                                    4.19.0-0.nightly-2025-01-21-163021   True        False         False      108m    
insights                                   4.19.0-0.nightly-2025-01-21-163021   True        False         False      109m    
kube-apiserver                             4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
kube-controller-manager                    4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
kube-scheduler                             4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
kube-storage-version-migrator              4.19.0-0.nightly-2025-01-21-163021   True        False         False      109m    
monitoring                                 4.19.0-0.nightly-2025-01-21-163021   True        False         False      102m    
network                                    4.19.0-0.nightly-2025-01-21-163021   True        False         False      120m    
node-tuning                                4.19.0-0.nightly-2025-01-21-163021   True        False         False      112m    
openshift-apiserver                        4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
openshift-controller-manager               4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
openshift-samples                          4.19.0-0.nightly-2025-01-21-163021   True        False         False      107m    
operator-lifecycle-manager                 4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
operator-lifecycle-manager-catalog         4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
operator-lifecycle-manager-packageserver   4.19.0-0.nightly-2025-01-21-163021   True        False         False      121m    
service-ca                                 4.19.0-0.nightly-2025-01-21-163021   True        False         False      109m    
storage                                    4.19.0-0.nightly-2025-01-21-163021   True        False         False      109m    
mjoseph@mjoseph-mac Downloads % oc get ingresses.config/cluster -o jsonpath={.spec.domain}
apps.93499d233a19644b81ad.qe.azure.devcluster.openshift.com%  

mjoseph@mjoseph-mac Downloads %  oc create -f New\ Folder\ With\ Items/internal_ingress_controller.yaml 
ingresscontroller.operator.openshift.io/internal created
mjoseph@mjoseph-mac Downloads % 
mjoseph@mjoseph-mac Downloads % 
mjoseph@mjoseph-mac Downloads % 
mjoseph@mjoseph-mac Downloads % cat New\ Folder\ With\ Items/internal_ingress_controller.yaml 
kind: IngressController
apiVersion: operator.openshift.io/v1
metadata:
  name: internal
  namespace: openshift-ingress-operator
spec:
  domain: internal.93499d233a19644b81ad.qe.azure.devcluster.openshift.com
  replicas: 1
  endpointPublishingStrategy:
    loadBalancer:
      scope: Internal
    type: LoadBalancerService

    2. Check the controller status
mjoseph@mjoseph-mac Downloads % oc -n openshift-ingress-operator get ingresscontroller 
NAME       AGE
default    139m
internal   29s
mjoseph@mjoseph-mac Downloads % oc get po -n openshift-ingress
NAME                              READY   STATUS    RESTARTS   AGE
router-default-5c4db6659b-7cq46   1/1     Running   0          128m
router-internal-6b6547cb9-hhtzq   1/1     Running   0          39s
mjoseph@mjoseph-mac Downloads % 
mjoseph@mjoseph-mac Downloads % 
mjoseph@mjoseph-mac Downloads % oc get co/ingress                                                      
NAME      VERSION                              AVAILABLE   PROGRESSING   DEGRADED   SINCE   MESSAGE
ingress   4.19.0-0.nightly-2025-01-21-163021   True        True          False      127m    Not all ingress controllers are available.

     3. Check the internal ingress controller status
mjoseph@mjoseph-mac Downloads % oc -n openshift-ingress-operator get ingresscontroller  internal -oyaml
apiVersion: operator.openshift.io/v1
kind: IngressController
metadata:
  creationTimestamp: "2025-01-23T07:46:15Z"
  finalizers:
  - ingresscontroller.operator.openshift.io/finalizer-ingresscontroller
  generation: 2
  name: internal
  namespace: openshift-ingress-operator
  resourceVersion: "29755"
  uid: 29244558-4d19-4ea4-a5b8-e98b9c07edb3
spec:
  clientTLS:
    clientCA:
      name: ""
    clientCertificatePolicy: ""
  domain: internal.93499d233a19644b81ad.qe.azure.devcluster.openshift.com
  endpointPublishingStrategy:
    loadBalancer:
      dnsManagementPolicy: Managed
      scope: Internal
    type: LoadBalancerService
  httpCompression: {}
  httpEmptyRequestsPolicy: Respond
  httpErrorCodePages:
    name: ""
  replicas: 1
  tuningOptions:
    reloadInterval: 0s
  unsupportedConfigOverrides: null
status:
  availableReplicas: 1
  conditions:
  - lastTransitionTime: "2025-01-23T07:46:15Z"
    reason: Valid
    status: "True"
    type: Admitted
  - lastTransitionTime: "2025-01-23T07:46:50Z"
    message: The deployment has Available status condition set to True
    reason: DeploymentAvailable
    status: "True"
    type: DeploymentAvailable
  - lastTransitionTime: "2025-01-23T07:46:50Z"
    message: Minimum replicas requirement is met
    reason: DeploymentMinimumReplicasMet
    status: "True"
    type: DeploymentReplicasMinAvailable
  - lastTransitionTime: "2025-01-23T07:46:50Z"
    message: All replicas are available
    reason: DeploymentReplicasAvailable
    status: "True"
    type: DeploymentReplicasAllAvailable
  - lastTransitionTime: "2025-01-23T07:46:50Z"
    message: Deployment is not actively rolling out
    reason: DeploymentNotRollingOut
    status: "False"
    type: DeploymentRollingOut
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: The endpoint publishing strategy supports a managed load balancer
    reason: WantedByEndpointPublishingStrategy
    status: "True"
    type: LoadBalancerManaged
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: |-
      The service-controller component is reporting SyncLoadBalancerFailed events like: Error syncing load balancer: failed to ensure load balancer: Retriable: false, RetryAfter: 0s, HTTPStatusCode: 403, RawError: {"error":{"code":"AuthorizationFailed","message":"The client '51b4e7f0-f41b-4b52-9bfc-412366b68308' with object id '51b4e7f0-f41b-4b52-9bfc-412366b68308' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/subnets/read' over scope '/subscriptions/53b8f551-f0fc-4bea-8cba-6d1fefd54c8a/resourceGroups/ci-ln-wqg34k2-c04e6-vnet-rg/providers/Microsoft.Network/virtualNetworks/ci-ln-wqg34k2-c04e6-vnet/subnets/ci-ln-wqg34k2-c04e6-subnet' or the scope is invalid. If access was recently granted, please refresh your credentials."}}
      The cloud-controller-manager logs may contain more details.
    reason: SyncLoadBalancerFailed
    status: "False"
    type: LoadBalancerReady
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: LoadBalancer is not progressing
    reason: LoadBalancerNotProgressing
    status: "False"
    type: LoadBalancerProgressing
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: DNS management is supported and zones are specified in the cluster DNS
      config.
    reason: Normal
    status: "True"
    type: DNSManaged
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: The wildcard record resource was not found.
    reason: RecordNotFound
    status: "False"
    type: DNSReady
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: |-
      One or more status conditions indicate unavailable: LoadBalancerReady=False (SyncLoadBalancerFailed: The service-controller component is reporting SyncLoadBalancerFailed events like: Error syncing load balancer: failed to ensure load balancer: Retriable: false, RetryAfter: 0s, HTTPStatusCode: 403, RawError: {"error":{"code":"AuthorizationFailed","message":"The client '51b4e7f0-f41b-4b52-9bfc-412366b68308' with object id '51b4e7f0-f41b-4b52-9bfc-412366b68308' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/subnets/read' over scope '/subscriptions/53b8f551-f0fc-4bea-8cba-6d1fefd54c8a/resourceGroups/ci-ln-wqg34k2-c04e6-vnet-rg/providers/Microsoft.Network/virtualNetworks/ci-ln-wqg34k2-c04e6-vnet/subnets/ci-ln-wqg34k2-c04e6-subnet' or the scope is invalid. If access was recently granted, please refresh your credentials."}}
      The cloud-controller-manager logs may contain more details.)
    reason: IngressControllerUnavailable
    status: "False"
    type: Available
  - lastTransitionTime: "2025-01-23T07:46:50Z"
    status: "False"
    type: Progressing
  - lastTransitionTime: "2025-01-23T07:47:46Z"
    message: |-
      One or more other status conditions indicate a degraded state: LoadBalancerReady=False (SyncLoadBalancerFailed: The service-controller component is reporting SyncLoadBalancerFailed events like: Error syncing load balancer: failed to ensure load balancer: Retriable: false, RetryAfter: 0s, HTTPStatusCode: 403, RawError: {"error":{"code":"AuthorizationFailed","message":"The client '51b4e7f0-f41b-4b52-9bfc-412366b68308' with object id '51b4e7f0-f41b-4b52-9bfc-412366b68308' does not have authorization to perform action 'Microsoft.Network/virtualNetworks/subnets/read' over scope '/subscriptions/53b8f551-f0fc-4bea-8cba-6d1fefd54c8a/resourceGroups/ci-ln-wqg34k2-c04e6-vnet-rg/providers/Microsoft.Network/virtualNetworks/ci-ln-wqg34k2-c04e6-vnet/subnets/ci-ln-wqg34k2-c04e6-subnet' or the scope is invalid. If access was recently granted, please refresh your credentials."}}
      The cloud-controller-manager logs may contain more details.)
    reason: DegradedConditions
    status: "True"
    type: Degraded
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: IngressController is upgradeable.
    reason: Upgradeable
    status: "True"
    type: Upgradeable
  - lastTransitionTime: "2025-01-23T07:46:16Z"
    message: No evaluation condition is detected.
    reason: NoEvaluationCondition
    status: "False"
    type: EvaluationConditionsDetected
  domain: internal.93499d233a19644b81ad.qe.azure.devcluster.openshift.com
  endpointPublishingStrategy:
    loadBalancer:
      dnsManagementPolicy: Managed
      scope: Internal
    type: LoadBalancerService
  observedGeneration: 2
  selector: ingresscontroller.operator.openshift.io/deployment-ingresscontroller=internal
  tlsProfile:
    ciphers:
    - ECDHE-ECDSA-AES128-GCM-SHA256
    - ECDHE-RSA-AES128-GCM-SHA256
    - ECDHE-ECDSA-AES256-GCM-SHA384
    - ECDHE-RSA-AES256-GCM-SHA384
    - ECDHE-ECDSA-CHACHA20-POLY1305
    - ECDHE-RSA-CHACHA20-POLY1305
    - DHE-RSA-AES128-GCM-SHA256
    - DHE-RSA-AES256-GCM-SHA384
    - TLS_AES_128_GCM_SHA256
    - TLS_AES_256_GCM_SHA384
    - TLS_CHACHA20_POLY1305_SHA256
    minTLSVersion: VersionTLS12
mjoseph@mjoseph-mac Downloads %      

mjoseph@mjoseph-mac Downloads % oc get co/ingress                                                      
NAME      VERSION                              AVAILABLE   PROGRESSING   DEGRADED   SINCE   MESSAGE
ingress   4.19.0-0.nightly-2025-01-21-163021   True        True          False      127m    Not all ingress controllers are available.    

    the internal controller should come up

 One more test scenario which is causing the similar error in the HCP cluster in internal LB

1. Create a web server with two services
mjoseph@mjoseph-mac Downloads % oc create -f New\ Folder\ With\ Items/webrc.yaml 
replicationcontroller/web-server-rc created
service/service-secure created
service/service-unsecure created
mjoseph@mjoseph-mac Downloads % oc get po
NAME                  READY   STATUS    RESTARTS   AGE
web-server-rc-q87rv   1/1     Running   0          40s
mjoseph@mjoseph-mac Downloads % oc get svc
oc geNAME                        TYPE           CLUSTER-IP       EXTERNAL-IP                            PORT(S)     AGE
kubernetes                  ClusterIP      172.31.0.1       <none>                                 443/TCP     152m
openshift                   ExternalName   <none>           kubernetes.default.svc.cluster.local   <none>      147m
openshift-apiserver         ClusterIP      172.31.165.239   <none>                                 443/TCP     150m
openshift-oauth-apiserver   ClusterIP      172.31.254.44    <none>                                 443/TCP     150m
packageserver               ClusterIP      172.31.131.10    <none>                                 443/TCP     150m
service-secure              ClusterIP      172.31.6.17      <none>                                 27443/TCP   46s
service-unsecure            ClusterIP      172.31.199.11    <none>                                 27017/TCP   46s

2. Add two lb services
mjoseph@mjoseph-mac Downloads % oc create -f ../Git/openshift-tests-private/test/extended/testdata/router/bug2013004-lb-services.yaml 
service/external-lb-57089 created
service/internal-lb-57089 created
mjoseph@mjoseph-mac Downloads % cat ../Git/openshift-tests-private/test/extended/testdata/router/bug2013004-lb-services.yaml
apiVersion: v1
kind: List
items:
- apiVersion: v1
  kind: Service
  metadata:
    name: external-lb-57089
  spec:
    ports:
    - name: https
      port: 28443
      protocol: TCP
      targetPort: 8443
    selector:
      name: web-server-rc
    type: LoadBalancer
- apiVersion: v1
  kind: Service
  metadata:
    name: internal-lb-57089
    annotations:
      service.beta.kubernetes.io/azure-load-balancer-internal: "true"
  spec:
    ports:
    - name: https
      port: 29443
      protocol: TCP
      targetPort: 8443
    selector:
      name: web-server-rc
    type: LoadBalancer


3. Check the external ip of the internal service, which is not yet assigned
mjoseph@mjoseph-mac Downloads % oc get svc -owide                                                                                    
NAME                        TYPE           CLUSTER-IP       EXTERNAL-IP                            PORT(S)           AGE    SELECTOR
external-lb-57089           LoadBalancer   172.31.248.177   20.83.73.54                            28443:30437/TCP   44s    name=web-server-rc
internal-lb-57089           LoadBalancer   172.31.156.88    <pending>                              29443:31885/TCP   44s    name=web-server-rc
kubernetes                  ClusterIP      172.31.0.1       <none>                                 443/TCP           153m   <none>
openshift                   ExternalName   <none>           kubernetes.default.svc.cluster.local   <none>            148m   <none>
openshift-apiserver         ClusterIP      172.31.165.239   <none>                                 443/TCP           151m   <none>
openshift-oauth-apiserver   ClusterIP      172.31.254.44    <none>                                 443/TCP           151m   <none>
packageserver               ClusterIP      172.31.131.10    <none>                                 443/TCP           151m   <none>
service-secure              ClusterIP      172.31.6.17      <none>                                 27443/TCP         112s   name=web-server-rc
service-unsecure            ClusterIP      172.31.199.11    <none>                                 27017/TCP         112s   name=web-server-rc