New subject: [PATCH v4] selftest: kill VM reliably by recursively kill children processes

21 Jun 2021

qemu is launched in nested subprocess and can't be killed by simply
killing the job ids,
PID    Command
    2269634    │  ├─ sshd: root [priv]
    2269637    │  │  └─ sshd: root@pts/0
    2269638    │  │     └─ -bash
    2269744    │  │        └─ make test-run V=1
    2273117    │  │           └─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273712    │  │              ├─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273714    │  │              │  └─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273737    │  │              │     └─ timeout --foreground 10m /root/kexec-tools-300/tests/scripts/run-qemu -nodefaults -nographic -smp 2 -m 768M -monitor no
    2273738    │  │              │        └─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial stdio
    2273746    │  │              │           ├─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial std
    2273797    │  │              ├─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273798    │  │              │  └─ /bin/bash /root/kexec-tools-300/tests/scripts/run-test.sh
    2273831    │  │              │     └─ timeout --foreground 10m /root/kexec-tools-300/tests/scripts/run-qemu -nodefaults -nographic -smp 2 -m 768M -monitor no
    2273832    │  │              │        └─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial stdio
    2273840    │  │              │           ├─ /usr/bin/qemu-system-x86_64 -enable-kvm -cpu host -nodefaults -nographic -smp 2 -m 768M -monitor none -serial std
This led to the error "qemu-system-x86_64: can't bind ip=0.0.0.0 to
socket: Address already in use".
This patch will kill qemu by killing all the children of the job id.
Signed-off-by: Coiby Xu coxu@redhat.com
---
Thanks to Kairui for finding out an error and an issue about previous
version,
    - "$_job" should be used instead "$job"
    - "Ctrl-c" leads to the failure of parsing pstree result
This version stops parsing pstree and use pgrep to get the children of a
process and kill them recursively instead.
---
 tests/scripts/run-test.sh | 37 +++++++++++++++++++++++++++++++++----
 tests/scripts/test-lib.sh |  3 +--
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/tests/scripts/run-test.sh b/tests/scripts/run-test.sh
index a68504d..7113be8 100755
--- a/tests/scripts/run-test.sh
+++ b/tests/scripts/run-test.sh
@@ -1,9 +1,38 @@
 #!/bin/bash
-_kill_all_jobs() {
-	local _jobs=$(jobs -r -p)
+_kill_if_valid_pid() {
+    local _pid="$1"
+    if ps -p $_pid > /dev/null
+    then
+        kill $_pid
+    fi
+}
+
+_recursive_kill() {
+    local _pid="$1"
+    local _children _child
+
+    _children=$(pgrep -P $_pid)
+    if [ -n "$_children" ]; then
+        for _child in $_children
+        do
+            _recursive_kill $_child
+            _kill_if_valid_pid $_child
+        done
+    fi
+    _kill_if_valid_pid $_pid
+}
-	[ -n "$_jobs" ] && kill $_jobs
+_kill_all_jobs() {
+    local _jobs=$(jobs -r -p)
+    local _job
+
+    if [ -n "$_jobs" ]; then
+        for _job in $_jobs
+        do
+            _recursive_kill $_job
+        done
+    fi
 }
trap '
@@ -121,7 +150,7 @@ for test_case in $testcases; do
[ $? -ne 0 ] && ret=$(expr $ret + 1)
    results[$test_case]="$res"
-
+    _kill_all_jobs
    echo -e "-------- Test finished: $test_case $res --------"
    for script in $scripts; do
    	script="$testdir/$script"
diff --git a/tests/scripts/test-lib.sh b/tests/scripts/test-lib.sh
index f8a2249..8b24b2a 100644
--- a/tests/scripts/test-lib.sh
+++ b/tests/scripts/test-lib.sh
@@ -146,8 +146,7 @@ watch_test_outputs() {
    			ret=$?
if [ $ret -ne 255 ]; then
-					# Test finished, kill VMs
-					kill $(jobs -p)
+					# Test finished
    				break 2
    			fi
    		done
-- 
2.32.0

    