Change all tests that scan all files to scan only the list of files known to git if run from a git tree. Otherwise, scan all files, which fixes (for example) make distcheck for pylint. Included all of the files in the commit this time, and added filtering options to the shell and python functions.
David Shea (1): Make the lists of files to check consistent across all checks.
tests/Makefile.am | 1 + tests/cppcheck/runcppcheck.sh | 5 ++- tests/gettext/gettext_potfiles.py | 65 ++++++++++++++++++++------------------- tests/glade/run_glade_tests.sh | 5 ++- tests/lib/filelist.py | 49 +++++++++++++++++++++++++++++ tests/lib/testlib.sh | 37 ++++++++++++++++++++++ tests/pylint/runpylint.sh | 17 +++++----- 7 files changed, 136 insertions(+), 43 deletions(-) create mode 100644 tests/lib/filelist.py create mode 100644 tests/lib/testlib.sh
Add shell and python functions to provide a list of potential files to check. Fallback to a full file list if not run from a git tree. --- tests/Makefile.am | 1 + tests/cppcheck/runcppcheck.sh | 5 ++- tests/gettext/gettext_potfiles.py | 65 ++++++++++++++++++++------------------- tests/glade/run_glade_tests.sh | 3 +- tests/lib/filelist.py | 49 +++++++++++++++++++++++++++++ tests/lib/testlib.sh | 37 ++++++++++++++++++++++ tests/pylint/runpylint.sh | 17 +++++----- 7 files changed, 134 insertions(+), 43 deletions(-) create mode 100644 tests/lib/filelist.py create mode 100644 tests/lib/testlib.sh
diff --git a/tests/Makefile.am b/tests/Makefile.am index 479e68c..6ab99b8 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -43,6 +43,7 @@ dist_check_SCRIPTS = glade/accelerators/check_accelerators.py \ glade/format_string/check_format_string.py \ glade/run_glade_tests.sh \ $(srcdir)/lib/*.py \ + $(srcdir)/lib/*.sh \ nosetests.sh \ pylint/intl.py \ pylint/preconf.py \ diff --git a/tests/cppcheck/runcppcheck.sh b/tests/cppcheck/runcppcheck.sh index 7dc2f4d..589f604 100755 --- a/tests/cppcheck/runcppcheck.sh +++ b/tests/cppcheck/runcppcheck.sh @@ -6,6 +6,8 @@ if [ -z "$top_srcdir" ]; then . ${top_srcdir}/tests/testenv.sh fi
+. ${top_srcdir}/tests/lib/testlib.sh + if ! type cppcheck > /dev/null 2>&1 ; then echo "cppcheck must be installed" exit 99 @@ -13,10 +15,11 @@ fi
# If files were specified on the command line, use those. Otherwise, look # for all .c files +filelist= if [ "$#" -gt 0 ]; then filelist="$@" else - filelist="$(find "$top_srcdir" -name '*.c')" + filelist=$(findtestfiles -name '*.c') fi
# Disable unusedFunction in widgets since everything will show up as unused diff --git a/tests/gettext/gettext_potfiles.py b/tests/gettext/gettext_potfiles.py index 96daec8..5d80308 100755 --- a/tests/gettext/gettext_potfiles.py +++ b/tests/gettext/gettext_potfiles.py @@ -11,52 +11,55 @@ if "top_srcdir" not in os.environ: # This return code tells the automake test driver that the test setup failed sys.exit(99)
+# Ensure that tests/lib is in sys.path +testlibpath = os.path.abspath(os.path.join(os.environ["top_srcdir"], "tests/lib")) +if testlibpath not in sys.path: + sys.path.append(testlibpath) + +from filelist import testfilelist + success = True
-def check_potfiles(checklist, dirname, names): +def check_potfile(checkfile, potlist): global success
- potcheckfiles = [] + potcheckfile = None + if checkfile.endswith(".py"): + # Check whether the file imports the i18n module + if subprocess.call(["grep", "-q", "^from pyanaconda.i18n import", checkfile]) == 0: + potcheckfile = checkfile + elif checkfile.endswith(".c"): + # Check whether the file includes intl.h + if subprocess.call(["grep", "-q", "#include .intl\.h.", checkfile]) == 0: + potcheckfile = checkfile + elif checkfile.endswith(".glade"): + # Look for a "translatable=yes" attribute + if ET.parse(checkfile).findall(".//*[@translatable='yes']"): + potcheckfile = checkfile + elif checkfile.endswith(".desktop.in"): + # These are handled by intltool, make sure the .h version is present + potcheckfile = checkfile + ".h"
- # Skip the .git directory - if ".git" in names: - names.remove(".git") + if not potcheckfile: + return
- # Strip the "./" component from the path name - dirname = os.path.relpath(dirname, ".") + # Compute the path relative to top_srcdir + potcheckfile = os.path.relpath(potcheckfile, os.environ["top_srcdir"])
- for checkfile in (os.path.join(dirname, name) for name in names): - if checkfile.endswith(".py"): - # Check whether the file imports the i18n module - if subprocess.call(["grep", "-q", "^from pyanaconda.i18n import", checkfile]) == 0: - potcheckfiles.append(checkfile) - elif checkfile.endswith(".c"): - # Check whether the file includes intl.h - if subprocess.call(["grep", "-q", "#include .intl\.h.", checkfile]) == 0: - potcheckfiles.append(checkfile) - elif checkfile.endswith(".glade"): - # Look for a "translatable=yes" attribute - if ET.parse(checkfile).findall(".//*[@translatable='yes']"): - potcheckfiles.append(checkfile) - elif checkfile.endswith(".desktop.in"): - # These are handled by intltool, make sure the .h version is present - potcheckfiles.append(checkfile + ".h") - - difference = set(potcheckfiles) - checklist - for missing in difference: - sys.stderr.write("%s not in POTFILES.in\n" % missing) + if potcheckfile not in potlist: + sys.stderr.write("%s not in POTFILES.in\n" % potcheckfile) success = False
# Read in POTFILES.in, skip comments and blank lines -POTFILES = [] +POTFILES = set() with open(os.path.join(os.environ["top_srcdir"], "po", "POTFILES.in")) as f: for line in (line.strip() for line in f): if line and not line.startswith("#"): - POTFILES.append(line) + POTFILES.add(line)
# Walk the source tree and look for files with translatable strings -os.chdir(os.environ["top_srcdir"]) -os.path.walk(".", check_potfiles, set(POTFILES)) +for testfile in testfilelist(): + check_potfile(testfile, POTFILES)
if not success: sys.exit(1) diff --git a/tests/glade/run_glade_tests.sh b/tests/glade/run_glade_tests.sh index 6f34484..e197180 100755 --- a/tests/glade/run_glade_tests.sh +++ b/tests/glade/run_glade_tests.sh @@ -3,6 +3,7 @@ : "${top_srcdir:=$(dirname "$0")/../..}" . "${top_srcdir}/tests/testenv.sh" srcdir="${top_srcdir}/tests/glade" +. "${top_srcdir}/tests/lib/testlib.sh"
# If --translated was specified but not --podir, add --podir translate_set=0 @@ -21,7 +22,7 @@ fi
status=0 for check in ${srcdir}/*/check_*.py ; do - find "${top_srcdir}" -name '*.glade' | xargs "${check}" "$@" + findtestfiles -name '*.glade' | xargs "${check}" "$@" if [ "$?" -ne 0 ]; then status=1 fi diff --git a/tests/lib/filelist.py b/tests/lib/filelist.py new file mode 100644 index 0000000..9da5fce --- /dev/null +++ b/tests/lib/filelist.py @@ -0,0 +1,49 @@ +# +# filelist.py: method for determining which files to check +# +# Copyright (C) 2014 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# Author: David Shea dshea@redhat.com + +import os +import subprocess + +def testfilelist(filterfunc=None): + """A generator function for the list of file names to check. + + If the check is run from a git work tree, the method returns a list + of files known to git. Otherwise, it returns a list of every file + beneath $top_srcdir. + + top_srcdir must be set in the environment. + + filterfunc, if provided, is a function that takes a filename as an + argument and returns True for files that should be included in the + file list. For example, something like lambda x: fnmatch(x, '*.py') + could be used to match only filenames that end in .py. + """ + + if os.path.isdir(os.path.join(os.environ["top_srcdir"], ".git")): + output = subprocess.check_output(["git", "ls-files", "-c", os.environ["top_srcdir"]]) + filelist = output.split("\n") + else: + filelist = (os.path.join(path, testfile) \ + for path, _dirs, files in os.walk(os.environ["top_srcdir"]) \ + for testfile in files) + + for f in filelist: + if not filterfunc or filterfunc(f): + yield f diff --git a/tests/lib/testlib.sh b/tests/lib/testlib.sh new file mode 100644 index 0000000..896d884 --- /dev/null +++ b/tests/lib/testlib.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Shell functions for use by anaconda tests +# +# Copyright (C) 2014 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# Author: David Shea dshea@redhat.com + +# Print a list of files to test on stdout +# Takes filter arguments identical to the find utility, for example +# findtestfiles -name '*.py'. Note that pruning directories will not +# work since find is passed a list of filenames as the path arguments. +findtestfiles() +{ + # If the test is being run from a git work tree, use a list of all files + # known to git + if [ -d "${top_srcdir}/.git" ]; then + findpath=$(git ls-files -c "${top_srcdir}") + # Otherwise list everything under $top_srcdir + else + findpath="${top_srcdir} -type f" + fi + + find $findpath "$@" +} diff --git a/tests/pylint/runpylint.sh b/tests/pylint/runpylint.sh index 93896ad..971a8f9 100755 --- a/tests/pylint/runpylint.sh +++ b/tests/pylint/runpylint.sh @@ -34,6 +34,8 @@ if [ -z "$top_srcdir" ]; then . ${top_srcdir}/tests/testenv.sh fi
+. ${top_srcdir}/tests/lib/testlib.sh + srcdir="${top_srcdir}/tests/pylint" builddir="${top_builddir}/tests/pylint"
@@ -96,16 +98,11 @@ fi # run pylint one file / module at a time, otherwise it sometimes gets # confused if [ -z "$FILES" ]; then - # Find any file in the git working tree that either ends in .py - # or contains #!/usr/bin/python in the first line. - # Scan everything except old_tests - for testfile in $(git ls-files -c "${top_srcdir}" | egrep -v '(^|/)old_tests/') ; do - if [ -f "${testfile}" ] && \ - ( [ "${testfile%.py}" != "${testfile}" ] || \ - head -1 "${testfile}" | grep -q '^#!/usr/bin/python' ) ; then - FILES="$FILES $testfile" - fi - done + # Test any file that either ends in .py or contains #!/usr/bin/python in + # the first line. Scan everything except old_tests + FILES=$(findtestfiles ( -name '*.py' -o \ + -exec /bin/sh -c "head -1 {} | grep -q '#!/usr/bin/python'" ; ) -print | \ + egrep -v '(|/)old_tests/') fi
num_cpus=$(getconf _NPROCESSORS_ONLN)
diff --git a/tests/lib/testlib.sh b/tests/lib/testlib.sh new file mode 100644 index 0000000..896d884 --- /dev/null +++ b/tests/lib/testlib.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Shell functions for use by anaconda tests +# +# Copyright (C) 2014 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# Author: David Shea dshea@redhat.com
+# Print a list of files to test on stdout +# Takes filter arguments identical to the find utility, for example +# findtestfiles -name '*.py'. Note that pruning directories will not +# work since find is passed a list of filenames as the path arguments. +findtestfiles() +{
- # If the test is being run from a git work tree, use a list of all files
- # known to git
- if [ -d "${top_srcdir}/.git" ]; then
findpath=$(git ls-files -c "${top_srcdir}")
- # Otherwise list everything under $top_srcdir
- else
findpath="${top_srcdir} -type f"
- fi
- find $findpath "$@"
+}
Shouldn't we filter out dot directories?
This looks good to me with or without filtering dot directories.
On 03/19/2014 04:01 AM, Vratislav Podzimek wrote:
diff --git a/tests/lib/testlib.sh b/tests/lib/testlib.sh new file mode 100644 index 0000000..896d884 --- /dev/null +++ b/tests/lib/testlib.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# Shell functions for use by anaconda tests +# +# Copyright (C) 2014 Red Hat, Inc. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation; either version 2.1 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program. If not, see http://www.gnu.org/licenses/. +# +# Author: David Shea dshea@redhat.com
+# Print a list of files to test on stdout +# Takes filter arguments identical to the find utility, for example +# findtestfiles -name '*.py'. Note that pruning directories will not +# work since find is passed a list of filenames as the path arguments. +findtestfiles() +{
- # If the test is being run from a git work tree, use a list of all files
- # known to git
- if [ -d "${top_srcdir}/.git" ]; then
findpath=$(git ls-files -c "${top_srcdir}")
- # Otherwise list everything under $top_srcdir
- else
findpath="${top_srcdir} -type f"
- fi
- find $findpath "$@"
+}
Shouldn't we filter out dot directories?
This looks good to me with or without filtering dot directories.
I suppose we could. The only regular file we have in a dot directory is .tx/config. I think everything else will be things probably best skipped, like .pylint.d and all the directories of deps and object files that automake and libtool create. Adding " ! -name .*" to the else branch should do it.
On 03/19/2014 09:34 AM, David Shea wrote:
I suppose we could. The only regular file we have in a dot directory is .tx/config. I think everything else will be things probably best skipped, like .pylint.d and all the directories of deps and object files that automake and libtool create. Adding " ! -name .*" to the else branch should do it.
Actually I think I'll just leave that out, since it's kind of a pain to override once the condition is there. Including dot files and directories matches the old behavior of everything using find, so evidentially everything works ok with whatever automake spits out, and if we add something that doesn't it's easy to filter out the dot files and directories in the new tests.
On Tue, Mar 18, 2014 at 01:52:14PM -0400, David Shea wrote:
Change all tests that scan all files to scan only the list of files known to git if run from a git tree. Otherwise, scan all files, which fixes (for example) make distcheck for pylint. Included all of the files in the commit this time, and added filtering options to the shell and python functions.
David Shea (1): Make the lists of files to check consistent across all checks.
tests/Makefile.am | 1 + tests/cppcheck/runcppcheck.sh | 5 ++- tests/gettext/gettext_potfiles.py | 65 ++++++++++++++++++++------------------- tests/glade/run_glade_tests.sh | 5 ++- tests/lib/filelist.py | 49 +++++++++++++++++++++++++++++ tests/lib/testlib.sh | 37 ++++++++++++++++++++++ tests/pylint/runpylint.sh | 17 +++++----- 7 files changed, 136 insertions(+), 43 deletions(-) create mode 100644 tests/lib/filelist.py create mode 100644 tests/lib/testlib.sh
-- 1.9.0
Looks good to me.
anaconda-patches@lists.fedorahosted.org