Uploaded image for project: 'RHEL'
  1. RHEL
  2. RHEL-137786

Upgrading IDM to latest version: 389-ds-base and ipa-server breaks replication

Linking RHIVOS CVEs to...Migration: Automation ...SWIFT: POC ConversionSync from "Extern...XMLWordPrintable

    • Yes
    • Critical
    • ZStream, Patch
    • rhel-idm-ds
    • None
    • None
    • Unspecified
    • Unspecified
    • Unspecified
    • None

      [BEFORE UPDATING the system]

      [root@rhel8a ~]# cat /etc/redhat-release 
Red Hat Enterprise Linux release 8.10 (Ootpa)
[BEFORE UPDATE] 
[root@rhel8a ~]# rpm -qa | grep ipa-server
ipa-server-common-4.9.13-20.module+el8.10.0+23534+744f3864.noarch
ipa-server-dns-4.9.13-20.module+el8.10.0+23534+744f3864.noarch
ipa-server-4.9.13-20.module+el8.10.0+23534+744f3864.x86_64
[root@rhel8a ~]# rpm -qa | grep 389-ds
389-ds-base-1.4.3.39-15.module+el8.10.0+23460+510532b7.x86_64
389-ds-base-libs-1.4.3.39-15.module+el8.10.0+23460+510532b7.x86_64

      [AFTER UPDATING and rebooting the system]

      [root@rhel8a ~]# rpm -qa | grep -i ipa-server
ipa-server-common-4.9.13-20.module+el8.10.0+23534+744f3864.noarch
ipa-server-dns-4.9.13-20.module+el8.10.0+23534+744f3864.noarch
ipa-server-4.9.13-20.module+el8.10.0+23610+7d6e87e5.x86_64           <-----------
ipa-server-4.9.13-20.module+el8.10.0+23534+744f3864.x86_64           <-----------
ipa-server-common-4.9.13-20.module+el8.10.0+23610+7d6e87e5.noarch
[root@rhel8a ~]# rpm -qa | grep -i 389-ds
389-ds-base-1.4.3.39-19.module+el8.10.0+23773+9fb87221.x86_64
389-ds-base-1.4.3.39-15.module+el8.10.0+23460+510532b7.x86_64
389-ds-base-libs-1.4.3.39-15.module+el8.10.0+23460+510532b7.x86_64    <-----------
389-ds-base-libs-1.4.3.39-19.module+el8.10.0+23773+9fb87221.x86_64    <-----------
[root@rhel8a ~]# ipa config-show
  Maximum username length: 32
  Maximum hostname length: 64
  Home directory base: /home
  Default shell: /bin/sh
  Default users group: ipausers
  Default e-mail domain: idm.example.local
  Search time limit: 2
  Search size limit: 100
  User search fields: uid,givenname,sn,telephonenumber,ou,title
  Group search fields: cn,description
  Enable migration mode: False
  Certificate Subject base: O=IDM.EXAMPLE.LOCAL
  Password Expiration Notification (days): 4
  Password plugin features: AllowNThash
  SELinux user map order: guest_u:s0$xguest_u:s0$user_u:s0$staff_u:s0-s0:c0.c1023$sysadm_u:s0-s0:c0.c1023$unconfined_u:s0-s0:c0.c1023
  Default SELinux user: unconfined_u:s0-s0:c0.c1023
  Default PAC types: MS-PAC, nfs:NONE
[root@rhel8a ~]# klist
Ticket cache: KCM:0
Default principal: admin@IDM.EXAMPLE.LOCAL
Valid starting       Expires              Service principal
12/26/2025 12:40:54  12/27/2025 12:21:14  krbtgt/IDM.EXAMPLE.LOCAL@IDM.EXAMPLE.LOCA
[root@rhel8a ~]# ipactl status
Directory Service: RUNNING
krb5kdc Service: RUNNING
kadmin Service: RUNNING
named Service: RUNNING
httpd Service: RUNNING
ipa-custodia Service: RUNNING
pki-tomcatd Service: RUNNING
ipa-otpd Service: RUNNING
ipa-dnskeysyncd Service: RUNNING
ipa: INFO: The ipactl command was successful
[root@rhel8a ~]# ipa-replica-manage list
Directory Manager password: 
Failed to read master data from 'rhel8a.idm.example.local': no matching entry found       <---------------------
[root@rhel8a ~]# systemctl status dirsrv@IDM-EXAMPLE-LOCAL.service 
● dirsrv@IDM-EXAMPLE-LOCAL.service - 389 Directory Server IDM-EXAMPLE-LOCAL.
   Loaded: loaded (/usr/lib/systemd/system/dirsrv@.service; enabled; vendor preset: disabled)
  Drop-In: /usr/lib/systemd/system/dirsrv@.service.d
           └─custom.conf
           /etc/systemd/system/dirsrv@IDM-EXAMPLE-LOCAL.service.d
           └─ipa-env.conf
   Active: active (running) since Fri 2025-12-26 10:40:44 -03; 59min ago
  Process: 3427 ExecStartPre=/usr/libexec/dirsrv/ds_selinux_restorecon.sh /etc/dirsrv/slapd-IDM-EXAMPLE-LOCAL/dse.ldif (code=exited, status=0/SUCCESS)
  Process: 3421 ExecStartPre=/usr/libexec/dirsrv/ds_systemd_ask_password_acl /etc/dirsrv/slapd-IDM-EXAMPLE-LOCAL/dse.ldif (code=exited, status=0/SUCCESS)
 Main PID: 3432 (ns-slapd)
   Status: "slapd started: Ready to process requests"
    Tasks: 40 (limit: 7789)
   Memory: 60.8M
   CGroup: /system.slice/system-dirsrv.slice/dirsrv@IDM-EXAMPLE-LOCAL.service
           └─3432 /usr/sbin/ns-slapd -D /etc/dirsrv/slapd-IDM-EXAMPLE-LOCAL -i /run/dirsrv/slapd-IDM-EXAMPLE-LOCAL.pid
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 2
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 1
Dec 26 11:39:04 rhel8a.idm.example.local ns-slapd[3432]: GSSAPI client step 2
[root@rhel8a ~]# ipa-healthcheck --failures-only
Unable to initialize ipahealthcheck.ipa: no matching entry found
Internal server error 'Link'
[
  {
    "source": "pki.server.healthcheck.clones.connectivity_and_data",
    "check": "ClonesConnectivyAndDataCheck",
    "result": "ERROR",
    "uuid": "b23e89c3-95ab-42d9-a002-a8d798ab0772",
    "when": "20251226155255Z",
    "duration": "0.529698",
    "kw": {
      "status": "ERROR:  pki-tomcat : Internal error testing CA clone. Host: rhel9b.idm.example.local Port: 443"
    }
  },
  {
    "source": "ipahealthcheck.ds.backends",
    "check": "BackendsCheck",
    "result": "CRITICAL",
    "uuid": "58e1b109-467c-426e-ba5e-5fba676cb7cb",
    "when": "20251226155256Z",
    "duration": "0.048382",
    "kw": {
      "key": "DSBLE0007",
      "items": [
        "cn=changelog"
      ],
      "msg": "System indexes are essential for proper directory server operation. Missing or\nincorrectly configured system indexes can lead to poor search performance, replication\nissues, and other operational problems.\n\nThe following system indexes should be present with correct configuration:\n- entryrdn: index type 'subtree'\n- parentid: index type 'eq' with matching rule 'integerOrderingMatch'\n- ancestorid: index type 'eq' with matching rule 'integerOrderingMatch'\n- objectClass: index type 'eq'\n- aci: index type 'pres'\n- nscpEntryDN: index type 'eq'\n- nsUniqueId: index type 'eq'\n- nsds5ReplConflict: index types 'eq', 'pres'\n- nsCertSubjectDN: index type 'eq'\n- numsubordinates: index type 'pres'\n- nsTombstoneCSN: index type 'eq'\n- targetuniqueid: index type 'eq'\n- changeNumber: index type 'eq' with matching rule 'integerOrderingMatch'\n- entryusn: index type 'eq' with matching rule 'integerOrderingMatch'\n\nCurrent discrepancies:\n- Index parentid missing matching rule: integerOrderingMatch\n- Index parentid missing fine grain definition of IDs limit: integerOrderingMatch\n"
    }
  },
  {
    "source": "ipahealthcheck.ds.backends",
    "check": "BackendsCheck",
    "result": "CRITICAL",
    "uuid": "487d0121-b13b-4e51-b57c-af469e97c6ce",
    "when": "20251226155256Z",
    "duration": "0.048391",
    "kw": {
      "key": "DSBLE0007",
      "items": [
        "o=ipaca"
      ],
      "msg": "System indexes are essential for proper directory server operation. Missing or\nincorrectly configured system indexes can lead to poor search performance, replication\nissues, and other operational problems.\n\nThe following system indexes should be present with correct configuration:\n- entryrdn: index type 'subtree'\n- parentid: index type 'eq' with matching rule 'integerOrderingMatch'\n- ancestorid: index type 'eq' with matching rule 'integerOrderingMatch'\n- objectClass: index type 'eq'\n- aci: index type 'pres'\n- nscpEntryDN: index type 'eq'\n- nsUniqueId: index type 'eq'\n- nsds5ReplConflict: index types 'eq', 'pres'\n- nsCertSubjectDN: index type 'eq'\n- numsubordinates: index type 'pres'\n- nsTombstoneCSN: index type 'eq'\n- targetuniqueid: index type 'eq'\n- entryusn: index type 'eq' with matching rule 'integerOrderingMatch'\n\nCurrent discrepancies:\n- Index parentid missing matching rule: integerOrderingMatch\n- Index parentid missing fine grain definition of IDs limit: integerOrderingMatch\n"
    }
  },
  {
    "source": "ipahealthcheck.ds.backends",
    "check": "BackendsCheck",
    "result": "CRITICAL",
    "uuid": "4b50fa0d-4fca-443c-87e6-5552f2d5295e",
    "when": "20251226155256Z",
    "duration": "0.048393",
    "kw": {
      "key": "DSBLE0007",
      "items": [
        "dc=idm,dc=example,dc=local"
      ],
      "msg": "System indexes are essential for proper directory server operation. Missing or\nincorrectly configured system indexes can lead to poor search performance, replication\nissues, and other operational problems.\n\nThe following system indexes should be present with correct configuration:\n- entryrdn: index type 'subtree'\n- parentid: index type 'eq' with matching rule 'integerOrderingMatch'\n- ancestorid: index type 'eq' with matching rule 'integerOrderingMatch'\n- objectClass: index type 'eq'\n- aci: index type 'pres'\n- nscpEntryDN: index type 'eq'\n- nsUniqueId: index type 'eq'\n- nsds5ReplConflict: index types 'eq', 'pres'\n- nsCertSubjectDN: index type 'eq'\n- numsubordinates: index type 'pres'\n- nsTombstoneCSN: index type 'eq'\n- targetuniqueid: index type 'eq'\n- entryusn: index type 'eq' with matching rule 'integerOrderingMatch'\n\nCurrent discrepancies:\n- Index parentid missing matching rule: integerOrderingMatch\n- Index parentid missing fine grain definition of IDs limit: integerOrderingMatch\n"
    }
  },
  {
    "source": "ipahealthcheck.ipa.idns",
    "check": "IPADNSSystemRecordsCheck",
    "result": "CRITICAL",
    "uuid": "6a8e70d8-4283-4e4a-8bf9-c31c99395fa7",
    "when": "20251226155301Z",
    "duration": "0.012373",
    "kw": {
      "exception": "no matching entry found",
      "traceback": "Traceback (most recent call last):\n  File "/usr/lib/python3.6/site-packages/ipahealthcheck/core/core.py", line 56, in run_plugin\n    for result in plugin.check():\n  File "/usr/lib/python3.6/site-packages/ipahealthcheck/core/plugin.py", line 18, in wrapper\n    for result in f(*args, **kwds):\n  File "/usr/lib/python3.6/site-packages/ipahealthcheck/ipa/idns.py", line 59, in check\n    system_records = IPASystemRecords(api)\n  File "/usr/lib/python3.6/site-packages/ipaserver/dns_data_management.py", line 97, in __init__\n    self.__init_data(all_servers=all_servers)\n  File "/usr/lib/python3.6/site-packages/ipaserver/dns_data_management.py", line 124, in __init_data\n    servers = self.api_instance.Command.server_find(**kwargs)\n  File "/usr/lib/python3.6/site-packages/ipalib/frontend.py", line 471, in __call__\n    return self.__do_call(*args, **options)\n  File "/usr/lib/python3.6/site-packages/ipalib/frontend.py", line 499, in __do_call\n    ret = self.run(*args, **options)\n  File "/usr/lib/python3.6/site-packages/ipalib/frontend.py", line 816, in run\n    return self.execute(*args, **options)\n  File "/usr/lib/python3.6/site-packages/ipaserver/plugins/baseldap.py", line 2155, in execute\n    self, ldap, filter, attrs_list, base_dn, scope, *args, **options)\n  File "/usr/lib/python3.6/site-packages/ipaserver/plugins/server.py", line 408, in pre_callback\n    ldap, options['servrole'])\n  File "/usr/lib/python3.6/site-packages/ipaserver/plugins/server.py", line 355, in _get_enabled_servrole_filter\n    servroles[0])\n  File "/usr/lib/python3.6/site-packages/ipaserver/plugins/server.py", line 348, in _get_masters_with_enabled_servrole\n    include_master=True,\n  File "/usr/lib/python3.6/site-packages/ipalib/frontend.py", line 471, in __call__\n    return self.__do_call(*args, **options)\n  File "/usr/lib/python3.6/site-packages/ipalib/frontend.py", line 499, in __do_call\n    ret = self.run(*args, **options)\n  File "/usr/lib/python3.6/site-packages/ipalib/frontend.py", line 816, in run\n    return self.execute(*args, **options)\n  File "/usr/lib/python3.6/site-packages/ipaserver/plugins/serverrole.py", line 161, in execute\n    status=status)\n  File "/usr/lib/python3.6/site-packages/ipaserver/plugins/serverroles.py", line 132, in server_role_search\n    role_status = found_role.status(self.api, server=server_server)\n  File "/usr/lib/python3.6/site-packages/ipaserver/servroles.py", line 563, in status\n    api_instance, server=server, attrs_list=('ipaConfigString', 'cn'))\n  File "/usr/lib/python3.6/site-packages/ipaserver/servroles.py", line 222, in status\n    self._fill_in_absent_masters(ldap2, api_instance, result))\n  File "/usr/lib/python3.6/site-packages/ipaserver/servroles.py", line 179, in _fill_in_absent_masters\n    attrs_list=attrs_list)\n  File "/usr/lib/python3.6/site-packages/ipapython/ipaldap.py", line 1466, in get_entries\n    **kwargs)\n  File "/usr/lib/python3.6/site-packages/ipapython/ipaldap.py", line 1607, in find_entries\n    raise errors.EmptyResult(reason='no matching entry found')\nipalib.errors.EmptyResult: no matching entry found\n"
    }
  }
]

      [From the replicas where I didn't update yet] 

      [root@rhel8client2 ~]# cat /etc/hostname 
rhel8client2.idm.example.local
[root@rhel8client2 ~]# cat /etc/redhat-release 
Red Hat Enterprise Linux release 8.10 (Ootpa)
[root@rhel8client2 ~]# rpm -qa | grep -i 389-ds
389-ds-base-1.4.3.39-15.module+el8.10.0+23460+510532b7.x86_64
389-ds-base-libs-1.4.3.39-15.module+el8.10.0+23460+510532b7.x86_64
[root@rhel8client2 ~]# rpm -qa | grep -i ipa-server
ipa-server-common-4.9.13-20.module+el8.10.0+23534+744f3864.noarch
ipa-server-4.9.13-20.module+el8.10.0+23534+744f3864.x86_64
ipa-server-dns-4.9.13-20.module+el8.10.0+23534+744f3864.noarch
[root@rhel8client2 ~]# ipa config-show
  Maximum username length: 32
  Maximum hostname length: 64
  Home directory base: /home
  Default shell: /bin/sh
  Default users group: ipausers
  Default e-mail domain: idm.example.local
  Search time limit: 2
  Search size limit: 100
  User search fields: uid,givenname,sn,telephonenumber,ou,title
  Group search fields: cn,description
  Enable migration mode: False
  Certificate Subject base: O=IDM.EXAMPLE.LOCAL
  Password Expiration Notification (days): 4
  Password plugin features: AllowNThash
  SELinux user map order: guest_u:s0$xguest_u:s0$user_u:s0$staff_u:s0-s0:c0.c1023$sysadm_u:s0-s0:c0.c1023$unconfined_u:s0-s0:c0.c1023
  Default SELinux user: unconfined_u:s0-s0:c0.c1023
  Default PAC types: MS-PAC, nfs:NONE
  IPA masters: rhel8a.idm.example.local, rhel8client2.idm.example.local, rhel9b.idm.example.local
  IPA master capable of PKINIT: rhel8a.idm.example.local, rhel8client2.idm.example.local, rhel9b.idm.example.local
  IPA CA servers: rhel8a.idm.example.local, rhel8client2.idm.example.local, rhel9b.idm.example.local
  IPA CA renewal master: rhel8a.idm.example.local    <---------------------------- still show my broken CA master here
  IPA DNS servers: rhel8a.idm.example.local, rhel8client2.idm.example.local, rhel9b.idm.example.local
[root@rhel8client2 ~]# ipa-replica-manage list
rhel8a.idm.example.local: master
rhel9b.idm.example.local: master
rhel8client2.idm.example.local: master
[root@rhel8client2 ~]# tail -n 10 /var/log/dirsrv/slapd-IDM-EXAMPLE-LOCAL/errors
[26/Dec/2025:12:49:12.768245181 -0300] - ERR - agmt="cn=rhel8client2.idm.example.local-to-rhel9b.idm.example.local" (rhel9b:389) - clcache_load_buffer - Can't locate CSN 694c3efe000400070000 in the changelog (DB rc=-30988). If replication stops, the consumer may need to be reinitialized.
[26/Dec/2025:12:49:12.790581109 -0300] - ERR - NSMMReplicationPlugin - send_updates - agmt="cn=rhel8client2.idm.example.local-to-rhel9b.idm.example.local" (rhel9b:389): Missing data encountered. If the error persists the replica must be reinitialized.
[26/Dec/2025:12:49:12.804173873 -0300] - ERR - agmt="cn=meTorhel8a.idm.example.local" (rhel8a:389) - clcache_load_buffer - Can't locate CSN 694c3efe000400070000 in the changelog (DB rc=-30988). If replication stops, the consumer may need to be reinitialized.
[26/Dec/2025:12:49:12.823477871 -0300] - ERR - NSMMReplicationPlugin - send_updates - agmt="cn=meTorhel8a.idm.example.local" (rhel8a:389): Missing data encountered. If the error persists the replica must be reinitialized.
[root@rhel8client2 ~]# ipa-healthcheck --failures-only
Internal server error 'Link'
[
  {
    "source": "pki.server.healthcheck.clones.connectivity_and_data",
    "check": "ClonesConnectivyAndDataCheck",
    "result": "ERROR",
    "uuid": "03a05303-b0cf-4fe2-9341-5554e59aa155",
    "when": "20251226155307Z",
    "duration": "0.171294",
    "kw": {
      "status": "ERROR:  pki-tomcat : Internal error testing CA clone. Host: rhel9b.idm.example.local Port: 443"
    }
  },
  {
    "source": "ipahealthcheck.ds.replication",
    "check": "ReplicationCheck",
    "result": "ERROR",
    "uuid": "b925df14-1d13-4f32-a5bb-4f7d29b3f4f4",
    "when": "20251226155309Z",
    "duration": "0.605525",
    "kw": {
      "key": "DSREPLLE0003",
      "items": [
        "Replication",
        "Agreement"
      ],
      "msg": "The replication agreement (metorhel8a.idm.example.local) under "dc=idm,dc=example,dc=local" is not in synchronization.\nStatus message: error (18) can't acquire replica (incremental update transient warning.  backing off, will retry update later.)"
    }
  },
  {
    "source": "ipahealthcheck.ds.replication",
    "check": "ReplicationCheck",
    "result": "ERROR",
    "uuid": "05822e74-c944-45ee-bf40-ae80eb8d6c91",
    "when": "20251226155309Z",
    "duration": "0.605538",
    "kw": {
      "key": "DSREPLLE0003",
      "items": [
        "Replication",
        "Agreement"
      ],
      "msg": "The replication agreement (rhel8client2.idm.example.local-to-rhel9b.idm.example.local) under "dc=idm,dc=example,dc=local" is not in synchronization.\nStatus message: error (18) can't acquire replica (incremental update transient warning.  backing off, will retry update later.)"
    }
  }
]

              idm-ds-dev-bugs IdM DS Dev
              rhn-support-jbotelho Jaqueline Botelho
              IdM DS Dev IdM DS Dev
              IdM DS QE IdM DS QE
              Evgenia Martyniuk Evgenia Martyniuk
              Votes:
              1 Vote for this issue
              Watchers:
              18 Start watching this issue

                Created:
                Updated: