Issue:

Sssd: sssd-krb5-2.9.4-5.el8_10.2.x86_64
rhel: rhel8.10

The issue is because of upgrade of sssd-krb5 from sssd-krb5-2.9.4-5.el8_10.2.x86_64 to sssd-krb5-2.9.4-5.el8_10.3.x86_64
On hosts where we are running sssd-krb5-2.9.4-5.el8_10.2.x86_64, we dont see this problem.

The new version is adding this extra config in /var/lib/sss/pubconf/krb5.include.d/localauth_plugin to disable an2ln. While I understand this is for security reason, it is breaking Kerberos login.

How to replicate:
downgrade the version to 8_10.2 and delete the localauth_plugin file and restart sssd. It recreates the file without disable an2ln.
[plugins]
localauth =

{ disable = an2ln module = sssd:/usr/lib64/sssd/modules/sssd_krb5_localauth_plugin.so }

[root@example1 ~]$ cd /etc/krb5.keytabs/
[root@example1 krb5.keytabs]$ kinit -kt test@EXAMPLE.COM test
[root@example1 krb5.keytabs]$ ssh example1
root@example password:

currently we found it is failing to do ssh on recently build hosts compared to old existing hosts.

Also we noticed if we do comment out below line from /etc/krb5.conf then ssh works normally.
#includedir /etc/krb5.conf.d/
#includedir /var/lib/sss/pubconf/krb5.include.d/

[root@example1 ~]$ ssh -vv example2
OpenSSH_8.0p1, OpenSSL 1.1.1k FIPS 25 Mar 2021
debug1: Reading configuration data /etc/ssh/ssh_config
debug1: Reading configuration data /etc/ssh/ssh_config.d/05-redhat.conf
debug1: /etc/ssh/ssh_config.d/05-redhat.conf line 3: Applying options for *
debug2: resolving "example2" port 22
debug2: ssh_connect_direct
debug1: Connecting to example2 [10.x.x.x.x] port 22.
debug1: Connection established.
debug1: identity file /home/test/.ssh/id_rsa type -1
debug1: identity file /home/test/.ssh/id_rsa-cert type -1
debug1: identity file /home/test/.ssh/id_dsa type -1
debug1: identity file /home/test/.ssh/id_dsa-cert type -1
debug1: identity file /home/test/.ssh/id_ecdsa type -1
debug1: identity file /home/test/.ssh/id_ecdsa-cert type -1
debug1: identity file /home/test/.ssh/id_ed25519 type -1
debug1: identity file /home/test/.ssh/id_ed25519-cert type -1
debug1: identity file /home/test/.ssh/id_xmss type -1
debug1: identity file /home/test/.ssh/id_xmss-cert type -1
debug1: Local version string SSH-2.0-OpenSSH_8.0
debug1: Remote protocol version 2.0, remote software version OpenSSH_8.0
debug1: match: OpenSSH_8.0 pat OpenSSH* compat 0x04000000
debug2: fd 4 setting O_NONBLOCK
debug1: Authenticating to example2:22 as 'test'
debug1: SSH2_MSG_KEXINIT sent
debug1: SSH2_MSG_KEXINIT received
debug2: local client KEXINIT proposal
debug2: KEX algorithms: curve25519-sha256,curve25519-sha256@libssh.org,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256,diffie-hellman-group14-sha1,ext-info-c,kex-strict-c-v00@openssh.com
debug2: host key algorithms: rsa-sha2-512-cert-v01@openssh.com,rsa-sha2-256-cert-v01@openssh.com,ssh-rsa-cert-v01@openssh.com,rsa-sha2-512,rsa-sha2-256,ssh-rsa,ecdsa-sha2-nistp256-cert-v01@openssh.com,ecdsa-sha2-nistp384-cert-v01@openssh.com,ecdsa-sha2-nistp521-cert-v01@openssh.com,ssh-ed25519-cert-v01@openssh.com,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521,ssh-ed25519
debug2: ciphers ctos: chacha20-poly1305@openssh.com,aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm@openssh.com,aes256-gcm@openssh.com
debug2: ciphers stoc: chacha20-poly1305@openssh.com,aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm@openssh.com,aes256-gcm@openssh.com
debug2: MACs ctos: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
debug2: MACs stoc: umac-64-etm@openssh.com,umac-128-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,hmac-sha1-etm@openssh.com,umac-64@openssh.com,umac-128@openssh.com,hmac-sha2-256,hmac-sha2-512,hmac-sha1
debug2: compression ctos: none,zlib@openssh.com,zlib
debug2: compression stoc: none,zlib@openssh.com,zlib
debug2: languages ctos:
debug2: languages stoc:
debug2: first_kex_follows 0
debug2: reserved 0
debug2: peer server KEXINIT proposal
debug2: KEX algorithms: diffie-hellman-group14-sha1,diffie-hellman-group14-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group-exchange-sha256,ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,curve25519-sha256,curve25519-sha256@libssh.org,sntrup4591761x25519-sha512@tinyssh.org,kex-strict-s-v00@openssh.com
debug2: host key algorithms: rsa-sha2-512,rsa-sha2-256,ssh-rsa
debug2: ciphers ctos: rijndael-cbc@lysator.liu.se,aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm@openssh.com,aes256-gcm@openssh.com,chacha20-poly1305@openssh.com
debug2: ciphers stoc: rijndael-cbc@lysator.liu.se,aes128-ctr,aes192-ctr,aes256-ctr,aes128-gcm@openssh.com,aes256-gcm@openssh.com,chacha20-poly1305@openssh.com
debug2: MACs ctos: hmac-sha2-256,hmac-sha2-512,umac-128@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com
debug2: MACs stoc: hmac-sha2-256,hmac-sha2-512,umac-128@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com,umac-128-etm@openssh.com
debug2: compression ctos: none,zlib@openssh.com
debug2: compression stoc: none,zlib@openssh.com
debug2: languages ctos:
debug2: languages stoc:
debug2: first_kex_follows 0
debug2: reserved 0
debug1: kex: algorithm: curve25519-sha256
debug1: kex: host key algorithm: rsa-sha2-512
debug1: kex: server->client cipher: chacha20-poly1305@openssh.com MAC: <implicit> compression: none
debug1: kex: client->server cipher: chacha20-poly1305@openssh.com MAC: <implicit> compression: none
debug1: kex: curve25519-sha256 need=64 dh_need=64
debug1: kex: curve25519-sha256 need=64 dh_need=64
debug1: expecting SSH2_MSG_KEX_ECDH_REPLY
debug1: Server host key: ssh-rsa SHA256:tct5M/aqzk6isbng3Tomqr6ZGgyfTGMlX+BDETiyA7k
debug1: Host 'example2' is known and matches the RSA host key.
debug1: Found key in /home/test/.ssh/known_hosts:12
debug1: resetting send seqnr 3
debug2: set_newkeys: mode 1
debug1: rekey out after 134217728 blocks
debug1: SSH2_MSG_NEWKEYS sent
debug1: expecting SSH2_MSG_NEWKEYS
debug1: resetting read seqnr 3
debug1: SSH2_MSG_NEWKEYS received
debug2: set_newkeys: mode 0
debug1: rekey in after 134217728 blocks
debug1: Will attempt key: /home/test/.ssh/id_rsa
debug1: Will attempt key: /home/test/.ssh/id_dsa
debug1: Will attempt key: /home/test/.ssh/id_ecdsa
debug1: Will attempt key: /home/test/.ssh/id_ed25519
debug1: Will attempt key: /home/test/.ssh/id_xmss
debug2: pubkey_prepare: done
debug1: SSH2_MSG_EXT_INFO received
debug1: kex_input_ext_info: server-sig-algs=<ssh-ed25519,ssh-rsa,rsa-sha2-256,rsa-sha2-512,ssh-dss,ecdsa-sha2-nistp256,ecdsa-sha2-nistp384,ecdsa-sha2-nistp521>
debug2: service_accept: ssh-userauth
debug1: SSH2_MSG_SERVICE_ACCEPT received
debug1: Authentications that can continue: publickey,gssapi-keyex,gssapi-with-mic,password
debug1: Next authentication method: gssapi-with-mic
debug2: we sent a gssapi-with-mic packet, wait for reply
debug1: Authentications that can continue: publickey,gssapi-keyex,gssapi-with-mic,password
debug2: we sent a gssapi-with-mic packet, wait for reply
debug1: Authentications that can continue: publickey,gssapi-keyex,gssapi-with-mic,password
debug2: we did not send a packet, disable method
debug1: Next authentication method: publickey
debug1: Trying private key: /home/test/.ssh/id_rsa
debug1: Trying private key: /home/test/.ssh/id_dsa
debug1: Trying private key: /home/test/.ssh/id_ecdsa
debug1: Trying private key: /home/test/.ssh/id_ed25519
debug1: Trying private key: /home/test/.ssh/id_xmss
debug2: we did not send a packet, disable method
debug1: Next authentication method: password
test@example2's password:

From logs:

(2026-01-23 8:56:01): [nss] [cache_req_validate_domain_type] (0x2000): CID#4 Request type POSIX-only for domain NCD type POSIX is valid
(2026-01-23 8:56:01): [nss] [cache_req_set_domain] (0x0400): CID#4 CR #9: Using domain [NCD]
(2026-01-23 8:56:01): [nss] [cache_req_search_send] (0x0400): CID#4 CR #9: Looking up UID:3917078@NCD
(2026-01-23 8:56:01): [nss] [cache_req_search_ncache] (0x0400): CID#4 CR #9: Checking negative cache for [UID:3917078@NCD]
(2026-01-23 8:56:01): [nss] [sss_ncache_check_str] (0x2000): CID#4 Checking negative cache for [NCE/UID/NCD/3917078]
(2026-01-23 8:56:01): [nss] [sss_ncache_check_str] (0x2000): CID#4 Checking negative cache for [NCE/UID/3917078]
(2026-01-23 8:56:01): [nss] [cache_req_search_ncache] (0x0400): CID#4 CR #9: [UID:3917078@NCD] is not present in negative cache
(2026-01-23 8:56:01): [nss] [cache_req_search_cache] (0x0400): CID#4 CR #9: Looking up [UID:3917078@NCD] in cache
(2026-01-23 8:56:01): [nss] [cache_req_search_cache] (0x0400): CID#4 CR #9: Object [UID:3917078@NCD] was not found in cache
(2026-01-23 8:56:01): [nss] [sss_nss_get_object_send] (0x0400): CID#4 Client [0x555555800840][27]: sent cache request #9
(2026-01-23 8:56:01): [nss] [sss_ncache_check_str] (0x2000): CID#4 Checking negative cache for [NCE/DOM_LOCATE/NCE/UID/NCD/3917078]
(2026-01-23 8:56:01): [nss] [sss_ncache_set_str] (0x0400): CID#4 Adding [NCE/DOM_LOCATE/NCE/UID/NCD/3917078] to negative cache
(2026-01-23 8:56:01): [nss] [sbus_dispatch] (0x4000): Dispatching.
(2026-01-23 8:56:01): [nss] [sss_dp_get_account_domain_done] (0x2000): CID#4 Data Provider Error: 3, 1432158304 [GetAccountDomain() not supported]
(2026-01-23 8:56:01): [nss] [cache_req_common_get_acct_domain_recv] (0x0080): CID#4 CR #9: Could not get account domain [1432158304]: GetAccountDomain() not supported
(2026-01-23 8:56:01): [nss] [sss_ncache_set_str] (0x0400): CID#4 Adding [NCE/DOM_LOCATE_TYPE/NCD/User by ID] to negative cache permanently
(2026-01-23 8:56:01): [nss] [cache_req_locate_dom_done] (0x0100): CID#4 Disabled domain locating functionality for User by ID
(2026-01-23 8:56:01): [nss] [cache_req_validate_domain_type] (0x2000): CID#4 Request type POSIX-only for domain NCD type POSIX is valid
(2026-01-23 8:56:01): [nss] [cache_req_set_domain] (0x0400): CID#4 CR #9: Using domain [NCD]
(2026-01-23 8:56:01): [nss] [cache_req_search_send] (0x0400): CID#4 CR #9: Looking up UID:3917078@NCD
(2026-01-23 8:56:01): [nss] [cache_req_search_ncache] (0x0400): CID#4 CR #9: Checking negative cache for [UID:3917078@NCD]
(2026-01-23 8:56:01): [nss] [sss_ncache_check_str] (0x2000): CID#4 Checking negative cache for [NCE/UID/NCD/3917078]
(2026-01-23 8:56:01): [nss] [sss_ncache_check_str] (0x2000): CID#4 Checking negative cache for [NCE/UID/3917078]
(2026-01-23 8:56:01): [nss] [cache_req_search_ncache] (0x0400): CID#4 CR #9: [UID:3917078@NCD] is not present in negative cache
(2026-01-23 8:56:01): [nss] [cache_req_search_cache] (0x0400): CID#4 CR #9: Looking up [UID:3917078@NCD] in cache
(2026-01-23 8:56:01): [nss] [cache_req_search_cache] (0x0400): CID#4 CR #9: Object [UID:3917078@NCD] was not found in cache
(2026-01-23 8:56:01): [nss] [cache_req_search_dp] (0x0400): CID#4 CR #9: Looking up [UID:3917078@NCD] in data provider
(2026-01-23 8:56:01): [nss] [sss_dp_get_account_send] (0x0400): CID#4 Creating request for [NCD][0x1][BE_REQ_USER][idnumber=3917078:-]
(2026-01-23 8:56:01): [nss] [sbus_dispatch] (0x4000): Dispatching.
(2026-01-23 8:56:01): [nss] [sss_domain_get_state] (0x1000): CID#4 Domain NCD is Active
(2026-01-23 8:56:01): [nss] [cache_req_search_cache] (0x0400): CID#4 CR #9: Looking up [UID:3917078@NCD] in cache
(2026-01-23 8:56:01): [nss] [cache_req_search_ncache_filter] (0x0400): CID#4 CR #9: Filtering out results by negative cache
(2026-01-23 8:56:01): [nss] [sss_ncache_check_str] (0x2000): CID#4 Checking negative cache for [NCE/USER/NCD/mssql@ncd]
(2026-01-23 8:56:01): [nss] [cache_req_search_done] (0x0400): CID#4 CR #9: Returning updated object [UID:3917078@NCD]
(2026-01-23 8:56:01): [nss] [cache_req_create_and_add_result] (0x0400): CID#4 CR #9: Found 1 entries in domain NCD
(2026-01-23 8:56:01): [nss] [cache_req_done] (0x0400): CID#4 CR #9: Finished: Success
(2026-01-23 8:56:01): [nss] [sss_nss_protocol_done] (0x4000): CID#4 Sending reply: success
(2026-01-23 8:56:04): [nss] [client_idle_handler] (0x2000): Terminating idle client [0x5555557f49c0][28]
(2026-01-23 8:56:04): [nss] [client_close_fn] (0x2000): Terminated client [0x5555557f49c0][28]

configuration :

cat etc/krb5.conf
#Ansible managed
########################################################################

1. krb5.conf - Satellite deployment
2. modified from krb5.conf in SHINKYO build
   ########################################################################

includedir /etc/krb5.conf.d/
includedir /var/lib/sss/pubconf/krb5.include.d/

[libdefaults]
default_realm = EXAMPLE.COM
ticket_lifetime = 24000
dns_lookup_realm = false
dns_lookup_kdc = false
allow_weak_crypto = True
udp_preference_limit = 0

[realms]
EXAMPLE.COM =

{ kdc = eu-kdc.EXAMPLE.COM kdc = us-kdc.EXAMPLE.COM kdc = ap-kdc.EXAMPLE.COM admin_server = eu-kadmin.EXAMPLE.COM admin_server = us-kadmin.EXAMPLE.COM admin_server = ap-kadmin.EXAMPLE.COM kpasswd_server = eu-kadmin.EXAMPLE.COM kpasswd_server = us-kadmin.EXAMPLE.COM kpasswd_server = ap-kadmin.EXAMPLE.COM default_domain = EXAMPLE.COM kpasswd_protocol = SET_CHANGE }

example.net =

{ kdc = example.net admin_server = example.net default_domain = example.net }

example.in =

{ kdc = example.in admin_server = example.in default_domain = example.in }

test.com =

{ kdc = test.com admin_server = test.com default_domain = test.com }

[domain_realm]
EXAMPLE.COM = EXAMPLE.COM
.EXAMPLE.COM = EXAMPLE.COM

example.in = example.in
.example.in = example.in
test.com = test.com
.test.com = test.com
example.net = example.net
.example.net = example.net

[logging]
kdc_rotate =

{ period = 1d versions = 31 }

1. admin_server = FILE:/var/log/kadmind.log
2. default = FILE:/var/log/krb5libs.log
3. kdc = FILE:/var/log/krb5kdc.log

[appdefaults]
kinit =

{ renewable = true forwardable= true }

pam =

{ debug = false ticket_lifetime = 36000 renew_lifetime = 36000 forwardable = true krb4_convert = false }

---

• cat etc/sssd/sssd.conf *
  #Ansible managed

[autofs]
debug_level = 2

[domain/NCD]
auth_provider = krb5
chpass_provider = krb5
debug_level = 9
entry_cache_timeout = 300
enumerate = False
id_provider = ldap
krb5_backup_server = eu-kadmin.example.com, us-kadmin.example.com, ap-kadmin.example.com
krb5_kpasswd = eu-kadmin.example.com, us-kadmin.example.com, ap-kadmin.example.com
krb5_realm = example.com
krb5_server = eu-kadmin.example.com, us-kadmin.example.com, ap-kadmin.example.com
ldap_backup_uri = ldap://wkldap-test-prd-389_vs.example.com:389
ldap_connection_expire_timeout = 30
ldap_disable_paging = true
ldap_group_search_base = ou=groups,dc=example,dc=com??(!(cn=eun0510055))
ldap_netgroup_search_base = ou=unix,dc=example,dc=com??(!(cn=eun0510055))
ldap_schema = rfc2307
ldap_search_base = ou=unix,dc=example,dc=com??(!(cn=eun0510055))
ldap_search_timeout = 6
ldap_sudo_search_base = ou=SUDOers,ou=unix,dc=example,dc=com??(!(cn=eun0510055))
ldap_tls_reqcert = never
ldap_uri = ldap://ldap-test-prd.example.com:389
ldap_user_search_base = ou=users,dc=example,dc=com??(&(Unable to render embedded object: File ((nsaccountlock=true))() not found.(cn=eun0510055)))
sudo_provider = ldap

[nss]
debug_level = 2
entry_cache_nowait_percentage = 75
filter_groups = root, dbus, polkitd, ntp, postfix, rpc, nscd, git
filter_users = root, dbus, polkitd, ntp, postfix, rpc, nscd, git
reconnection_retries = 3

[pam]
debug_level = 9

[sssd]
config_file_version = 2
debug_level = 9
domains = NCD
reconnection_retries = 3
services = nss, pam, sudo

[sudo]
debug_level = 2