Hello,

We rebooted our Primary FreeIPA server (ds01) and then it will not start pki-tomcatd, Kerberos will also not work, though it starts.
We realized that 2 certificates have expired. 
we tried stopped ipa, stopped NTP, going back to Dec 14th, 2018 and restarted certmonger, bring back date but still no luck.

this is our primary, and we do have 2 local and 2 remote FreeIPA server on them only one of the certificate (June 15th, 2018) is showing expired and others are good.

Do we have to go back on date before June 15th, 2018 on ds01?
Details are:

[root@ds01 ~]# cat /etc/centos-release
CentOS Linux release 7.4.1708 (Core)

[root@ds01 ~]# ipa ca-find
------------
1 CA matched
------------
  Name: ipa
  Description: IPA CA
  Authority ID: 606<...........SNIP..........>450
  Subject DN: CN=Certificate Authority,O=DOMAIN.COM
  Issuer DN: CN=Certificate Authority,O=DOMAIN.COM
----------------------------
Number of entries returned 1
----------------------------

[root@ds02 ~]# ipa ping
-------------------------------------------
IPA server version 4.5.0. API version 2.228

[root@ds01 ~]# KRB5_TRACE=/dev/stdout kinit admin
[5509] 1547598366.261229: Getting initial credentials for admin@DOMAIN.COM
[5509] 1547598366.267532: Sending request (171 bytes) to DOMAIN.COM
[5509] 1547598366.268593: Resolving hostname ds01.domain.com
[5509] 1547598366.269479: Sending initial UDP request to dgram 192.1xx.xxx.xxx:88
[5509] 1547598367.270712: Initiating TCP connection to stream 192.1xx.xxx.xxx:88
[5509] 1547598367.270884: Sending TCP request to stream 192.1xx.xxx.xxx:88
[5509] 1547598372.338780: Received answer (171 bytes) from dgram 192.1xx.xxx.xxx:88
[5509] 1547598372.338841: Terminating TCP connection to stream 192.1xx.xxx.xxx:88
[5509] 1547598372.338989: Response was from master KDC
[5509] 1547598372.339095: Received error from KDC: -1765328324/Generic error (see e-text)
kinit: Generic error (see e-text) while getting initial credentials


[root@ds01 ~]# getcert list
Number of certificates and requests being tracked: 9.
Request ID '20180228053337':
        status: MONITORING
        stuck: no
        key pair storage: type=FILE,location='/var/kerberos/krb5kdc/kdc.key'
        certificate: type=FILE,location='/var/kerberos/krb5kdc/kdc.crt'
        CA: SelfSign
        issuer: CN=ds01.domain.com,O=DOMAIN.COM
        subject: CN=ds01.domain.com,O=DOMAIN.COM
        expires: 2019-03-07 06:24:12 UTC
        principal name: krbtgt/DOMAIN.COM@DOMAIN.COM
        certificate template/profile: KDCs_PKINIT_Certs
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/renew_kdc_cert
        track: yes
        auto-renew: yes
Request ID '20180315021457':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='auditSigningCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='auditSigningCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=CA Audit,O=DOMAIN.COM
        expires: 2020-02-25 04:27:49 UTC
        key usage: digitalSignature,nonRepudiation
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "auditSigningCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180315021500':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='ocspSigningCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='ocspSigningCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=OCSP Subsystem,O=DOMAIN.COM
        expires: 2020-02-25 04:28:38 UTC
        eku: id-kp-OCSPSigning
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "ocspSigningCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180315021501':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='subsystemCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='subsystemCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=CA Subsystem,O=DOMAIN.COM
        expires: 2020-02-25 04:31:47 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "subsystemCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180315021502':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='caSigningCert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='caSigningCert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=Certificate Authority,O=DOMAIN.COM
        expires: 2038-03-07 03:47:46 UTC
        key usage: digitalSignature,nonRepudiation,keyCertSign,cRLSign
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "caSigningCert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180315021503':
        status: CA_UNREACHABLE
        ca-error: Internal error
        stuck: no
        key pair storage: type=FILE,location='/var/lib/ipa/ra-agent.key'
        certificate: type=FILE,location='/var/lib/ipa/ra-agent.pem'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=IPA RA,O=DOMAIN.COM
        expires: 2018-06-15 23:15:23 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command: /usr/libexec/ipa/certmonger/renew_ra_cert_pre
        post-save command: /usr/libexec/ipa/certmonger/renew_ra_cert
        track: yes
        auto-renew: yes
Request ID '20180315021504':
        status: CA_UNREACHABLE
        ca-error: Internal error
        stuck: no
        key pair storage: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB',pin set
        certificate: type=NSSDB,location='/etc/pki/pki-tomcat/alias',nickname='Server-Cert cert-pki-ca',token='NSS Certificate DB'
        CA: dogtag-ipa-ca-renew-agent
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=ds01.domain.com,O=DOMAIN.COM
        expires: 2018-12-16 21:02:44 UTC
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth,id-kp-emailProtection
        pre-save command: /usr/libexec/ipa/certmonger/stop_pkicad
        post-save command: /usr/libexec/ipa/certmonger/renew_ca_cert "Server-Cert cert-pki-ca"
        track: yes
        auto-renew: yes
Request ID '20180315021505':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/dirsrv/slapd-DOMAIN-COM',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/dirsrv/slapd-DOMAIN-COM/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/dirsrv/slapd-DOMAIN-COM',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=ds01.domain.com,O=DOMAIN.COM
        expires: 2020-03-07 08:49:36 UTC
        principal name: ldap/ds01.domain.com@DOMAIN.COM
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_dirsrv DOMAIN-COM
        track: yes
        auto-renew: yes
Request ID '20180315021510':
        status: MONITORING
        stuck: no
        key pair storage: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB',pinfile='/etc/httpd/alias/pwdfile.txt'
        certificate: type=NSSDB,location='/etc/httpd/alias',nickname='Server-Cert',token='NSS Certificate DB'
        CA: IPA
        issuer: CN=Certificate Authority,O=DOMAIN.COM
        subject: CN=ds01.domain.com,O=DOMAIN.COM
        expires: 2020-03-07 08:49:51 UTC
        principal name: HTTP/ds01.domain.com@DOMAIN.COM
        key usage: digitalSignature,nonRepudiation,keyEncipherment,dataEncipherment
        eku: id-kp-serverAuth,id-kp-clientAuth
        pre-save command:
        post-save command: /usr/libexec/ipa/certmonger/restart_httpd
        track: yes
        auto-renew: yes

thank you,
Bhavin