Hi,

On Fri, Jul 15, 2022 at 8:28 AM roy liang via FreeIPA-users <freeipa-users@lists.fedorahosted.org> wrote:
It's been stuck all day

#service certmonger  restart

getcert list | egrep 'status|Request'
Request ID '20200609161251':
        status: SUBMITTING
Request ID '20200609161252':
        status: SUBMITTING
Request ID '20200609161253':
        status: SUBMITTING
Request ID '20200609161254':
        status: MONITORING
Request ID '20200609161255':
        status: SUBMITTING
Request ID '20200609161256':
        status: SUBMITTING
Request ID '20200609161317':
        status: SUBMITTING
Request ID '20200609161342':
        status: SUBMITTING


To avoid renewing/restarting httpd and ldap at the same time, there is a file /var/run/ipa/renewal.lock that is used as cross-process lock and may block the renewal. When the lock is not taken, its content looks like the following:
# cat /var/run/ipa/renewal.lock
[lock]
locked = 0

The lock is taken for 1hour max and its expiration date is written in the file, with the owner process as well. This means that even if something went wrong (a renewal failing), your system will resume the renewal of the other certs after one hour.

We used to have an issue with renewals being blocked unnecessarily during certmonger restart (https://pagure.io/freeipa/issue/8425) and it has been fixed in IPA 4.8.9+. IIRC your version is older and you may be affected. In this case, the workaround is to delete the lock file.

flo

28172 root      20   0  228032  36424  12876 S   0.0  0.0   0:00.62 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28125 root      20   0  228028  36376  12828 S   0.0  0.0   0:00.59 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28132 root      20   0  228028  36364  12832 S   0.0  0.0   0:00.65 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28143 root      20   0  228028  36356  12808 S   0.8  0.0   0:00.68 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28170 root      20   0  228040  36356  12800 S   0.0  0.0   0:00.63 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28145 root      20   0  228028  36344  12804 S   0.0  0.0   0:00.67 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28142 root      20   0  228028  36324  12776 S   0.0  0.0   0:00.64 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28144 root      20   0  228028  36320  12784 S   0.0  0.0   0:00.64 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28146 root      20   0  228028  36300  12760 S   0.0  0.0   0:00.66 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28147 root      20   0  228028  36288  12744 S   0.0  0.0   0:00.67 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28126 root      20   0  228028  36280  12732 S   0.0  0.0   0:00.54 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28141 root      20   0  228028  36252  12708 S   0.0  0.0   0:00.64 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28148 root      20   0  228028  36232  12684 S   0.0  0.0   0:00.66 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/dogtag-ipa-renew-agent-submit                                                             
28129 root      20   0  228028  36212  12672 S   0.0  0.0   0:00.60 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28171 root      20   0  228024  36192  12636 S   0.0  0.0   0:00.62 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28127 root      20   0  228024  36172  12636 S   0.0  0.0   0:00.58 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28131 root      20   0  228028  36164  12620 S   0.0  0.0   0:00.62 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28130 root      20   0  228028  36152  12616 S   0.8  0.0   0:00.61 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit                                                                                 
28128 root      20   0  228028  36128  12584 S   0.0  0.0   0:00.57 /usr/bin/python2 -E /usr/lib/certmonger/ipa-server-guard /usr/lib/certmonger/ipa-submit


root@ipa-ca-65-197:/home/liangrui# strace -p 28130
strace: Process 28130 attached
select(0, NULL, NULL, NULL, {6, 426367}

) = 0 (Timeout)
open("/var/run/ipa/renewal.lock", O_RDWR|O_CREAT|O_APPEND, 0666) = 3
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
flock(3, LOCK_EX)                       = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
lseek(3, 0, SEEK_SET)                   = 0
read(3, "[lock]\nlocked = 1\nowner = ipa-se"..., 4096) = 81
read(3, "", 4096)                       = 0
close(3)                                = 0
select(0, NULL, NULL, NULL, {10, 0})    = 0 (Timeout)
open("/var/run/ipa/renewal.lock", O_RDWR|O_CREAT|O_APPEND, 0666) = 3
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
flock(3, LOCK_EX)                       = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
lseek(3, 0, SEEK_SET)                   = 0
read(3, "[lock]\nlocked = 1\nowner = ipa-se"..., 4096) = 81
read(3, "", 4096)                       = 0
close(3)                                = 0
select(0, NULL, NULL, NULL, {10, 0})    = 0 (Timeout)
open("/var/run/ipa/renewal.lock", O_RDWR|O_CREAT|O_APPEND, 0666) = 3
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
flock(3, LOCK_EX)                       = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
lseek(3, 0, SEEK_SET)                   = 0
read(3, "[lock]\nlocked = 1\nowner = ipa-se"..., 4096) = 81
read(3, "", 4096)                       = 0
close(3)                                = 0
select(0, NULL, NULL, NULL, {10, 0})    = 0 (Timeout)
open("/var/run/ipa/renewal.lock", O_RDWR|O_CREAT|O_APPEND, 0666) = 3
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
flock(3, LOCK_EX)                       = 0
fstat(3, {st_mode=S_IFREG|0600, st_size=81, ...}) = 0
lseek(3, 0, SEEK_SET)                   = 0
read(3, "[lock]\nlocked = 1\nowner = ipa-se"..., 4096) = 81
read(3, "", 4096)                       = 0
close(3)                                = 0
select(0, NULL, NULL, NULL, {10, 0}^Cstrace: Process 28130 detached
 <detached ...>
_______________________________________________
FreeIPA-users mailing list -- freeipa-users@lists.fedorahosted.org
To unsubscribe send an email to freeipa-users-leave@lists.fedorahosted.org
Fedora Code of Conduct: https://docs.fedoraproject.org/en-US/project/code-of-conduct/
List Guidelines: https://fedoraproject.org/wiki/Mailing_list_guidelines
List Archives: https://lists.fedorahosted.org/archives/list/freeipa-users@lists.fedorahosted.org
Do not reply to spam on the list, report it: https://pagure.io/fedora-infrastructure