[master][PATCH] Run pylint in multiple processes
Vratislav Podzimek
vpodzime at redhat.com
Thu Nov 14 10:58:22 UTC 2013
<not-in-the-commit-msg>
NEEDS TESTING
</not-in-the-commit-msg>
This speeds up running runpylint.sh a lot
Signed-off-by: Vratislav Podzimek <vpodzime at redhat.com>
---
tests/pylint/pylint-one.sh | 38 +++++++++++++++++++++++++++++
tests/pylint/runpylint.sh | 61 +++++++++++++++++-----------------------------
2 files changed, 61 insertions(+), 38 deletions(-)
create mode 100755 tests/pylint/pylint-one.sh
diff --git a/tests/pylint/pylint-one.sh b/tests/pylint/pylint-one.sh
new file mode 100755
index 0000000..05532e2
--- /dev/null
+++ b/tests/pylint/pylint-one.sh
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# $1 -- python source to run pylint on
+#
+
+if [ $# -lt 1 ]; then
+ # no source, just exit
+ exit 1
+fi
+
+file_suffix="$(echo $1|sed s?/?_?g)"
+
+exec > pylint-out_$file_suffix
+
+pylint_output="$(pylint \
+ --msg-template='{msg_id}:{line:3d},{column}: {obj}: {msg}' \
+ -r n --disable=C,R --rcfile=/dev/null \
+ --dummy-variables-rgx=_ \
+ --ignored-classes=DefaultInstall,Popen,QueueFactory,TransactionSet \
+ --defining-attr-methods=__init__,_grabObjects,initialize,reset,start,setUp \
+ --load-plugins=intl \
+ $DISABLED_WARN_OPTIONS \
+ $DISABLED_ERR_OPTIONS \
+ $NON_STRICT_OPTIONS $1 2>&1 | \
+ egrep -v "$(tr '\n' '|' < "$FALSE_POSITIVES") \
+ ")"
+
+# I0011 is the informational "Locally disabling ...." message
+if [ -n "$(echo "$pylint_output" | fgrep -v '************* Module ' |\
+ grep -v '^I0011:')" ]; then
+ # Replace the Module line with the actual filename
+ pylint_output="$(echo "$pylint_output" | sed "s|\* Module .*|* Module $1|")"
+
+ if [ "$pylint_log" -ne 0 ]; then
+ echo "$pylint_output" > pylint-out_$file_suffix
+ fi
+ touch "pylint-$file_suffix-failed"
+fi
diff --git a/tests/pylint/runpylint.sh b/tests/pylint/runpylint.sh
index b0f82b7..050e58e 100755
--- a/tests/pylint/runpylint.sh
+++ b/tests/pylint/runpylint.sh
@@ -10,9 +10,9 @@
# If $top_srcdir is set, assume this is being run from automake and we don't
# need to keep a separate log
-pylint_log=0
+export pylint_log=0
if [ -z "$top_srcdir" ]; then
- pylint_log=1
+ export pylint_log=1
fi
# Unset TERM so that things that use readline don't output terminal garbage
@@ -27,15 +27,15 @@ fi
srcdir="${top_srcdir}/tests/pylint"
# Need to add the pylint module directory to PYTHONPATH as well.
-PYTHONPATH="${PYTHONPATH}:${srcdir}"
+export PYTHONPATH="${PYTHONPATH}:${srcdir}"
-FALSE_POSITIVES="${srcdir}"/pylint-false-positives
+export FALSE_POSITIVES="${srcdir}"/pylint-false-positives
# W0212 - Access to a protected member %s of a client class
-NON_STRICT_OPTIONS="--disable=W0212"
+export NON_STRICT_OPTIONS="--disable=W0212"
# E1103 - %s %r has no %r member (but some types could not be inferred)
-DISABLED_ERR_OPTIONS="--disable=E1103"
+export DISABLED_ERR_OPTIONS="--disable=E1103"
# W0110 - map/filter on lambda could be replaced by comprehension
# W0141 - Used builtin function %r
@@ -46,7 +46,7 @@ DISABLED_ERR_OPTIONS="--disable=E1103"
# W0604 - Using the global statement at the module level
# W0613 - Unused argument %r
# W0614 - Unused import %s from wildcard import
-DISABLED_WARN_OPTIONS="--disable=W0110,W0141,W0142,W0223,W0511,W0603,W0604,W0613,W0614"
+export DISABLED_WARN_OPTIONS="--disable=W0110,W0141,W0142,W0223,W0511,W0603,W0604,W0613,W0614"
usage () {
echo "usage: `basename $0` [--strict] [--help] [files...]"
@@ -57,7 +57,7 @@ FILES=
while [ $# -gt 0 ]; do
case $1 in
--strict)
- NON_STRICT_OPTIONS=
+ export NON_STRICT_OPTIONS=
;;
--help)
usage 0
@@ -76,9 +76,6 @@ if [ "`tail -c 1 $FALSE_POSITIVES`" == "`echo`" ]; then
fi
exit_status=0
-if [ "$pylint_log" -ne 0 ]; then
- > pylint-log
-fi
# run pylint one file / module at a time, otherwise it sometimes gets
# confused
@@ -87,35 +84,23 @@ if [ -z "$FILES" ]; then
# or contains #!/usr/bin/python in the first line.
FILES="$(find "${top_srcdir}"/{anaconda,pyanaconda,tests,widgets,utils,scripts} -type f \( -name '*.py' -o -exec awk -e 'NR==1 { if ($0 ~ /^#!\/usr\/bin\/python/) exit 0; else exit 1; }' -e 'END { if (NR == 0) exit 1; }' {} \; \) -print)"
fi
-for i in $FILES; do
- pylint_output="$(pylint \
- --msg-template='{msg_id}:{line:3d},{column}: {obj}: {msg}' \
- -r n --disable=C,R --rcfile=/dev/null \
- --dummy-variables-rgx=_ \
- --ignored-classes=DefaultInstall,Popen,QueueFactory,TransactionSet \
- --defining-attr-methods=__init__,_grabObjects,initialize,reset,start,setUp \
- --load-plugins=intl \
- $DISABLED_WARN_OPTIONS \
- $DISABLED_ERR_OPTIONS \
- $NON_STRICT_OPTIONS $i 2>&1 | \
- egrep -v "$(tr '\n' '|' < "$FALSE_POSITIVES") \
- ")"
-
- # I0011 is the informational "Locally disabling ...." message
- if [ -n "$(echo "$pylint_output" | fgrep -v '************* Module ' |\
- grep -v '^I0011:')" ]; then
- # Replace the Module line with the actual filename
- pylint_output="$(echo "$pylint_output" | sed "s|\* Module .*|* Module $i|")"
-
- if [ "$pylint_log" -ne 0 ]; then
- echo "$pylint_output" >> pylint-log
- else
- echo "$pylint_output"
- fi
- exit_status=1
- fi
+
+num_cpus=$(getconf _NPROCESSORS_ONLN)
+# run pylint in paralel
+echo $FILES | xargs --max-procs=$num_cpus -n 1 "$srcdir"/pylint-one.sh
+
+for file in $(find -name 'pylint-out*'); do
+ cat "$file" >> pylint-log
+ rm "$file"
done
+fails=$(find -name 'pylint*failed' -print -exec rm '{}' \;)
+if [ -z "$fails" ]; then
+ exit_status=0
+else
+ exit_status=1
+fi
+
if [ "$pylint_log" -ne 0 ]; then
if [ -s pylint-log ]; then
echo "pylint reports the following issues:"
--
1.8.4.2
More information about the anaconda-patches
mailing list