From: "Brian C. Lane" bcl@redhat.com
Mostly logging, using .format() on the strings. --- .gitignore | 1 + docs/conf.py | 2 +- src/bin/image-minimizer | 3 --- src/pylorax/__init__.py | 13 ++++++------- src/pylorax/imgutils.py | 4 ++-- src/pylorax/ltmpl.py | 2 +- src/sbin/livemedia-creator | 32 ++++++++++++++++---------------- src/sbin/lorax | 4 ++-- 8 files changed, 29 insertions(+), 32 deletions(-)
diff --git a/.gitignore b/.gitignore index 9b92808..70111ab 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc src/pylorax/version.py* _build/ +tests/pylint/.pylint.d/ diff --git a/docs/conf.py b/docs/conf.py index 395f4c2..41183d0 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -50,7 +50,7 @@
# General information about the project. project = u'Lorax' -copyright = u'2015, Red Hat, Inc.' +copyright = u'2015, Red Hat, Inc.' # pylint: disable=redefined-builtin
# The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/src/bin/image-minimizer b/src/bin/image-minimizer index 42f8077..8c37ab4 100755 --- a/src/bin/image-minimizer +++ b/src/bin/image-minimizer @@ -192,9 +192,6 @@ def main(): sys.exit(e.code) except KeyboardInterrupt, e: print >> sys.stderr, "Aborted at user request" - except Exception, e: - print e - sys.exit(1)
if __name__ == "__main__": main() diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index 3d74a4b..72d55c8 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -190,15 +190,15 @@ def run(self, dbo, product, version, release, variant="", bugurl="", self.init_stream_logging() self.init_file_logging(logdir)
- logger.debug("version is {0}".format(vernum)) - logger.debug("using work directory {0.workdir}".format(self)) - logger.debug("using log directory {0}".format(logdir)) + logger.debug("version is %s", vernum) + logger.debug("using work directory %s", self.workdir) + logger.debug("using log directory %s", logdir)
# set up output directory self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.") if not os.path.isdir(self.outputdir): os.makedirs(self.outputdir) - logger.debug("using output directory {0.outputdir}".format(self)) + logger.debug("using output directory %s", self.outputdir)
# do we have root privileges? logger.info("checking for root privileges") @@ -228,7 +228,7 @@ def run(self, dbo, product, version, release, variant="", bugurl="", logger.critical("no dnf base object") sys.exit(1) self.inroot = dbo.conf.installroot - logger.debug("using install root: {0}".format(self.inroot)) + logger.debug("using install root: %s", self.inroot)
if not buildarch: buildarch = get_buildarch(dbo) @@ -246,8 +246,7 @@ def run(self, dbo, product, version, release, variant="", bugurl="",
# NOTE: if you change isolabel, you need to change pungi to match, or # the pungi images won't boot. - isolabel = volid or "{0.name}-{0.version}-{1.basearch}".format(self.product, - self.arch) + isolabel = volid or "%s-%s-%s" % (product, version, self.arch.basearch)
if len(isolabel) > 32: logger.fatal("the volume id cannot be longer than 32 characters") diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py index 232d91f..1617ca8 100644 --- a/src/pylorax/imgutils.py +++ b/src/pylorax/imgutils.py @@ -352,9 +352,9 @@ def __enter__(self): except CalledProcessError: logger.debug(traceback.format_exc()) if self.mount_dir: - logger.info("Partition mounted on {0} size={1}".format(self.mount_dir, self.mount_size)) + logger.info("Partition mounted on %s size=%s", self.mount_dir, self.mount_size) else: - logger.debug("Unable to mount anything from {0}".format(self.disk_img)) + logger.debug("Unable to mount anything from %s", self.disk_img) os.rmdir(mount_dir) return self
diff --git a/src/pylorax/ltmpl.py b/src/pylorax/ltmpl.py index 6a0d50c..117efd3 100644 --- a/src/pylorax/ltmpl.py +++ b/src/pylorax/ltmpl.py @@ -186,7 +186,7 @@ def _getsize(self, *files): def run(self, templatefile, **variables): for k,v in self.defaults.items() + self.builtins.items(): variables.setdefault(k,v) - logger.debug("executing {0} with variables={1}".format(templatefile, variables)) + logger.debug("executing %s with variables=%s", templatefile, variables) self.templatefile = templatefile t = LoraxTemplate(directories=[self.templatedir]) commands = t.parse(templatefile, variables) diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index 438d6b6..e60e091 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -128,7 +128,7 @@ class LogRequestHandler(SocketServer.BaseRequestHandler):
except socket.timeout: pass - except: + except Exception: # pylint: disable=broad-except break
def finish(self): @@ -434,7 +434,7 @@ class VirtualInstall(object):
Could use libvirt for this instead. """ - log.info("Shutting down {0}".format(self.virt_name)) + log.info("Shutting down %s", self.virt_name) subprocess.call(["virsh", "destroy", self.virt_name]) subprocess.call(["virsh", "undefine", self.virt_name])
@@ -503,12 +503,12 @@ def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024, if not (disk_img and template and outfile): return None
- log.info("Creating appliance definition using {0}".format(template)) + log.info("Creating appliance definition using %s", template)
if not arch: arch = "x86_64"
- log.info("Calculating SHA256 checksum of {0}".format(disk_img)) + log.info("Calculating SHA256 checksum of %s", disk_img) sha256 = hashlib.sha256() with open(disk_img) as f: while True: @@ -516,7 +516,7 @@ def make_appliance(disk_img, name, template, outfile, networks=None, ram=1024, if not data: break sha256.update(data) - log.info("SHA256 of {0} is {1}".format(disk_img, sha256.hexdigest())) + log.info("SHA256 of %s is %s", disk_img, sha256.hexdigest()) disk_info = DataHolder(name=os.path.basename(disk_img), format="raw", checksum_type="sha256", checksum=sha256.hexdigest()) try: @@ -596,7 +596,7 @@ def rebuild_initrds_for_live(opts, sys_root_dir, results_dir): dracut_args = [] for arg in opts.dracut_args: dracut_args += arg.split(" ", 1) - log.info("dracut args = {0}".format(dracut_args)) + log.info("dracut args = %s", dracut_args)
dracut = ["dracut", "--nomdadmconf", "--nolvmconf"] + dracut_args
@@ -729,7 +729,7 @@ def make_livecd(opts, mount_dir, work_dir): dracut_args = [] for arg in opts.dracut_args: dracut_args += arg.split(" ", 1) - log.info("dracut args = {0}".format(dracut_args)) + log.info("dracut args = %s", dracut_args) tb.rebuild_initrds(add_args=dracut_args) log.info("Building boot.iso") tb.build() @@ -874,8 +874,8 @@ def novirt_install(opts, disk_img, disk_size, repo_url): if not os.path.isdir(log_anaconda): os.mkdir(log_anaconda) for l in glob.glob("/tmp/*log")+glob.glob("/tmp/anaconda-tb-*"): - shutil.copy2(l, log_anaconda) - os.unlink(l) + shutil.copy2(l, log_anaconda) + os.unlink(l)
if opts.make_iso or opts.make_fsimage: umount(ROOT_PATH) @@ -1056,7 +1056,7 @@ def make_image(opts, ks):
virt_install(opts, install_log, disk_img, disk_size) except InstallError as e: - log.error("Install failed: {0}".format(e)) + log.error("Install failed: %s", e) if not opts.keep_image and os.path.exists(disk_img): log.info("Removing bad disk image") os.unlink(disk_img) @@ -1436,7 +1436,7 @@ def main(): result_dir = None if opts.make_iso: work_dir = tempfile.mkdtemp() - log.info("working dir is {0}".format(work_dir)) + log.info("working dir is %s", work_dir)
if (opts.fs_image or opts.no_virt) and not opts.disk_image: # Create iso from a filesystem image @@ -1469,7 +1469,7 @@ def main(): opts.vcpus, opts.arch, opts.title, opts.project, opts.releasever) elif opts.make_pxe_live: work_dir = tempfile.mkdtemp() - log.info("working dir is {0}".format(work_dir)) + log.info("working dir is %s", work_dir)
if (opts.fs_image or opts.no_virt) and not opts.disk_image: # Create pxe live images from a filesystem image @@ -1503,12 +1503,12 @@ def main():
log.info("SUMMARY") log.info("-------") - log.info("Logs are in {0}".format(os.path.abspath(os.path.dirname(opts.logfile)))) + log.info("Logs are in %s", os.path.abspath(os.path.dirname(opts.logfile))) if disk_img: - log.info("Disk image is at {0}".format(disk_img)) + log.info("Disk image is at %s", disk_img) if opts.make_appliance: - log.info("Appliance description is in {0}".format(opts.app_file)) - log.info("Results are in {0}".format(opts.result_dir)) + log.info("Appliance description is in %s", opts.app_file) + log.info("Results are in %s", opts.result_dir)
sys.exit( 0 )
diff --git a/src/sbin/lorax b/src/sbin/lorax index bd07bb3..7870d2b 100755 --- a/src/sbin/lorax +++ b/src/sbin/lorax @@ -307,7 +307,7 @@ def get_dnf_base_object(installroot, repositories, mirrorlists=None, # add the repositories for i, r in enumerate(repositories): if "SRPM" in r or "srpm" in r: - log.info("Skipping source repo: %s" % r) + log.info("Skipping source repo: %s", r) continue repo_name = "lorax-repo-%d" % i repo = dnf.repo.Repo(repo_name, cachedir) @@ -327,7 +327,7 @@ def get_dnf_base_object(installroot, repositories, mirrorlists=None, # add the mirrorlists for i, r in enumerate(mirrorlists): if "SRPM" in r or "srpm" in r: - log.info("Skipping source repo: %s" % r) + log.info("Skipping source repo: %s", r) continue repo_name = "lorax-mirrorlist-%d" % i repo = dnf.repo.Repo(repo_name, cachedir)
From: "Brian C. Lane" bcl@redhat.com
Fix up 2to3 complaints. I've decided to do with wrapping list comprehension inside list() to get the generators to run in several places instead of list(map( or list(filter( which seem less readable to me. --- src/pylorax/__init__.py | 11 +++++------ src/pylorax/base.py | 7 ++----- src/pylorax/buildstamp.py | 2 +- src/pylorax/decorators.py | 2 +- src/pylorax/discinfo.py | 2 +- src/pylorax/dnfhelper.py | 2 +- src/pylorax/executils.py | 31 +++++++++++++++-------------- src/pylorax/imgutils.py | 6 +++--- src/pylorax/ltmpl.py | 49 +++++++++++++++++++++++----------------------- src/pylorax/output.py | 2 +- src/pylorax/sysutils.py | 5 ++--- src/pylorax/treebuilder.py | 6 +++--- src/pylorax/treeinfo.py | 10 +++++----- tests/pylint/pylint-one.sh | 2 +- 14 files changed, 67 insertions(+), 70 deletions(-)
diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index 72d55c8..c8ee96d 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -1,7 +1,7 @@ # # __init__.py # -# Copyright (C) 2010-2014 Red Hat, Inc. +# Copyright (C) 2010-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@
import sys import os -import ConfigParser +import configparser import tempfile import locale from subprocess import CalledProcessError @@ -39,7 +39,6 @@ import dnf
from pylorax.sysutils import joinpaths, remove, linktree -from rpmUtils.arch import getBaseArch
from pylorax.treebuilder import RuntimeBuilder, TreeBuilder from pylorax.buildstamp import BuildStamp @@ -60,7 +59,7 @@ class ArchData(DataHolder): def __init__(self, buildarch): super(ArchData, self).__init__() self.buildarch = buildarch - self.basearch = getBaseArch(buildarch) + self.basearch = dnf.arch.basearch(buildarch) self.libdir = "lib64" if self.basearch in self.lib64_arches else "lib" self.bcj = self.bcj_arch.get(self.basearch)
@@ -81,7 +80,7 @@ def __init__(self): locale.setlocale(locale.LC_ALL, 'C')
def configure(self, conf_file="/etc/lorax/lorax.conf"): - self.conf = ConfigParser.SafeConfigParser() + self.conf = configparser.SafeConfigParser()
# set defaults self.conf.add_section("lorax") @@ -133,7 +132,7 @@ def configure(self, conf_file="/etc/lorax/lorax.conf"):
# remove some environmental variables that can cause problems with package scripts env_remove = ('DISPLAY', 'DBUS_SESSION_BUS_ADDRESS') - map(os.environ.pop, (k for k in env_remove if k in os.environ)) + list(os.environ.pop(k) for k in env_remove if k in os.environ)
self._configured = True
diff --git a/src/pylorax/base.py b/src/pylorax/base.py index 9fa6127..ae6813c 100644 --- a/src/pylorax/base.py +++ b/src/pylorax/base.py @@ -1,7 +1,7 @@ # # base.py # -# Copyright (C) 2009-2014 Red Hat, Inc. +# Copyright (C) 2009-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,10 +25,7 @@ import pylorax.output as output
-class BaseLoraxClass(object): - - __metaclass__ = ABCMeta - +class BaseLoraxClass(object, metaclass=ABCMeta): @abstractmethod def __init__(self): self.output = output.LoraxOutput() diff --git a/src/pylorax/buildstamp.py b/src/pylorax/buildstamp.py index cfa27ed..4219944 100644 --- a/src/pylorax/buildstamp.py +++ b/src/pylorax/buildstamp.py @@ -1,7 +1,7 @@ # # buildstamp.py # -# Copyright (C) 2010-2014 Red Hat, Inc. +# Copyright (C) 2010-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/pylorax/decorators.py b/src/pylorax/decorators.py index 72ed1aa..cb08c6a 100644 --- a/src/pylorax/decorators.py +++ b/src/pylorax/decorators.py @@ -1,7 +1,7 @@ # # decorators.py # -# Copyright (C) 2009-2014 Red Hat, Inc. +# Copyright (C) 2009-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/pylorax/discinfo.py b/src/pylorax/discinfo.py index 23dea53..9dad83b 100644 --- a/src/pylorax/discinfo.py +++ b/src/pylorax/discinfo.py @@ -1,7 +1,7 @@ # # discinfo.py # -# Copyright (C) 2010-2014 Red Hat, Inc. +# Copyright (C) 2010-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/pylorax/dnfhelper.py b/src/pylorax/dnfhelper.py index 8bf8230..8e35776 100644 --- a/src/pylorax/dnfhelper.py +++ b/src/pylorax/dnfhelper.py @@ -1,7 +1,7 @@ # # dnfhelper.py # -# Copyright (C) 2010-2014 Red Hat, Inc. +# Copyright (C) 2010-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index 6fbb271..b4c37db 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -1,7 +1,7 @@ # # executil.py - subprocess execution utility functions # -# Copyright (C) 1999-2014 +# Copyright (C) 1999-2015 # Red Hat, Inc. All rights reserved. # # This program is free software; you can redistribute it and/or modify @@ -21,6 +21,7 @@ #
import os, sys +import io import subprocess import threading from time import sleep @@ -98,7 +99,7 @@ def chroot (): stdin = sys.stdin.fileno() elif isinstance(stdin, int): pass - elif stdin is None or not isinstance(stdin, file): + elif stdin is None or not isinstance(stdin, io.IOBase): stdin = sys.stdin.fileno()
if isinstance(stdout, str): @@ -106,7 +107,7 @@ def chroot (): stdoutclose = lambda : os.close(stdout) elif isinstance(stdout, int): pass - elif stdout is None or not isinstance(stdout, file): + elif stdout is None or not isinstance(stdout, io.IOBase): stdout = sys.stdout.fileno()
if isinstance(stderr, str): @@ -114,7 +115,7 @@ def chroot (): stderrclose = lambda : os.close(stderr) elif isinstance(stderr, int): pass - elif stderr is None or not isinstance(stderr, file): + elif stderr is None or not isinstance(stderr, io.IOBase): stderr = sys.stderr.fileno()
program_log.info("Running... %s", " ".join([command] + argv)) @@ -220,7 +221,7 @@ def closefds (): stdin = sys.stdin.fileno() elif isinstance(stdin, int): pass - elif stdin is None or not isinstance(stdin, file): + elif stdin is None or not isinstance(stdin, io.IOBase): stdin = sys.stdin.fileno()
if isinstance(stderr, str): @@ -228,7 +229,7 @@ def closefds (): stderrclose = lambda : os.close(stderr) elif isinstance(stderr, int): pass - elif stderr is None or not isinstance(stderr, file): + elif stderr is None or not isinstance(stderr, io.IOBase): stderr = sys.stderr.fileno()
program_log.info("Running... %s", " ".join([command] + argv)) @@ -253,10 +254,10 @@ def closefds (): while True: (outStr, errStr) = proc.communicate() if outStr: - map(program_log.info, outStr.splitlines()) + list(program_log.info(line) for line in outStr.splitlines()) rc += outStr if errStr: - map(program_log.error, errStr.splitlines()) + list(program_log.error(line) for line in errStr.splitlines()) os.write(stderr, errStr)
if proc.returncode is not None: @@ -291,7 +292,7 @@ def closefds (): stdin = sys.stdin.fileno() elif isinstance(stdin, int): pass - elif stdin is None or not isinstance(stdin, file): + elif stdin is None or not isinstance(stdin, io.IOBase): stdin = sys.stdin.fileno()
if isinstance(stdout, str): @@ -299,7 +300,7 @@ def closefds (): stdoutclose = lambda : os.close(stdout) elif isinstance(stdout, int): pass - elif stdout is None or not isinstance(stdout, file): + elif stdout is None or not isinstance(stdout, io.IOBase): stdout = sys.stdout.fileno()
if isinstance(stderr, str): @@ -307,7 +308,7 @@ def closefds (): stderrclose = lambda : os.close(stderr) elif isinstance(stderr, int): pass - elif stderr is None or not isinstance(stderr, file): + elif stderr is None or not isinstance(stderr, io.IOBase): stderr = sys.stderr.fileno()
program_log.info("Running... %s", " ".join([command] + argv)) @@ -337,7 +338,7 @@ def closefds (): s = os.read(p[0], 1) except OSError as e: if e.errno != 4: - map(program_log.info, log_output.splitlines()) + list(program_log.info(line) for line in log_output.splitlines()) raise IOError(e.args)
if echo: @@ -359,7 +360,7 @@ def closefds (): if len(s) < 1: break
- map(program_log.info, log_output.splitlines()) + list(program_log.info(line) for line in log_output.splitlines())
log_errors = '' while 1: @@ -367,7 +368,7 @@ def closefds (): err = os.read(p_stderr[0], 128) except OSError as e: if e.errno != 4: - map(program_log.error, log_errors.splitlines()) + list(program_log.error(line) for line in log_errors.splitlines()) raise IOError(e.args) break log_errors += err @@ -375,7 +376,7 @@ def closefds (): break
os.write(stderr, log_errors) - map(program_log.error, log_errors.splitlines()) + list(program_log.error(line) for line in log_errors.splitlines()) os.close(p[0]) os.close(p_stderr[0])
diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py index 1617ca8..6594e0e 100644 --- a/src/pylorax/imgutils.py +++ b/src/pylorax/imgutils.py @@ -1,6 +1,6 @@ # imgutils.py - utility functions/classes for building disk images # -# Copyright (C) 2011-2014 Red Hat, Inc. +# Copyright (C) 2011-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -71,7 +71,7 @@ def compress(command, rootdir, outfile, compression="xz", compressargs=None): except OSError as e: logger.error(e) # Kill off any hanging processes - map(lambda p: p.kill(), (p for p in (find, archive, comp) if p)) + list(p.kill() for p in (find, archive, comp) if p) return 1
def mkcpio(rootdir, outfile, compression="xz", compressargs=None): @@ -259,7 +259,7 @@ def estimate_size(rootdir, graft=None, fstype=None, blocksize=4096, overhead=128 blocksize = 2048 getsize = lambda f: os.stat(f).st_size # no symlinks, count as copies total = overhead*blocksize - dirlist = graft.values() + dirlist = list(graft.values()) if rootdir: dirlist.append(rootdir) for root in dirlist: diff --git a/src/pylorax/ltmpl.py b/src/pylorax/ltmpl.py index 117efd3..df39e98 100644 --- a/src/pylorax/ltmpl.py +++ b/src/pylorax/ltmpl.py @@ -1,7 +1,7 @@ # # ltmpl.py # -# Copyright (C) 2009-2014 Red Hat, Inc. +# Copyright (C) 2009-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -40,7 +40,8 @@ import struct import dnf import multiprocessing -import Queue +import queue +import collections
class LoraxTemplate(object): def __init__(self, directories=None): @@ -62,17 +63,17 @@ def parse(self, template_file, variables):
# split, strip and remove empty lines lines = textbuf.splitlines() - lines = map(lambda line: line.strip(), lines) - lines = filter(lambda line: line, lines) + lines = [line.strip() for line in lines] + lines = [line for line in lines if line]
# remove comments - lines = filter(lambda line: not line.startswith("#"), lines) + lines = [line for line in lines if not line.startswith("#")]
# mako template now returns unicode strings - lines = map(lambda line: line.encode("utf8"), lines) + lines = [line.encode("utf8") for line in lines]
# split with shlex and perform brace expansion - lines = map(split_and_expand, lines) + lines = [split_and_expand(line) for line in lines]
return lines
@@ -184,7 +185,7 @@ def _getsize(self, *files): return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
def run(self, templatefile, **variables): - for k,v in self.defaults.items() + self.builtins.items(): + for k,v in list(self.defaults.items()) + list(self.builtins.items()): variables.setdefault(k,v) logger.debug("executing %s with variables=%s", templatefile, variables) self.templatefile = templatefile @@ -207,7 +208,7 @@ def _run(self, parsed_template): try: # grab the method named in cmd and pass it the given arguments f = getattr(self, cmd, None) - if cmd[0] == '_' or cmd == 'run' or not callable(f): + if cmd[0] == '_' or cmd == 'run' or not isinstance(f, collections.Callable): raise ValueError("unknown command %s" % cmd) f(*args) except Exception: # pylint: disable=broad-except @@ -506,18 +507,18 @@ def removepkg(self, *pkgs): else: logger.debug("removepkg %s: no files to remove!", p)
- def get_token_checked(self, process, queue): + def get_token_checked(self, process, token_queue): """Try to get token from queue checking that process is still alive"""
try: # wait at most a minute for the token - (token, msg) = queue.get(timeout=60) - except Queue.Empty: + (token, msg) = token_queue.get(timeout=60) + except queue.Empty: if process.is_alive(): try: # process still alive, give it 2 minutes more - (token, msg) = queue.get(timeout=120) - except Queue.Empty: + (token, msg) = token_queue.get(timeout=120) + except queue.Empty: # waited for 3 minutes and got nothing raise Exception("The transaction process got stuck somewhere (no message from it in 3 minutes)") else: @@ -532,13 +533,13 @@ def run_pkg_transaction(self): commands. '''
- def do_transaction(base, queue): + def do_transaction(base, token_queue): try: - display = LoraxRpmCallback(queue) + display = LoraxRpmCallback(token_queue) base.do_transaction(display=display) except BaseException as e: logger.error("The transaction process has ended abruptly: %s", e) - queue.put(('quit', str(e))) + token_queue.put(('quit', str(e)))
try: logger.info("Checking dependencies") @@ -560,17 +561,17 @@ def do_transaction(base, queue): raise
logger.info("Preparing transaction from installation source") - queue = multiprocessing.Queue() + token_queue = multiprocessing.Queue() msgout = output.LoraxOutput() - process = multiprocessing.Process(target=do_transaction, args=(self.dbo, queue)) + process = multiprocessing.Process(target=do_transaction, args=(self.dbo, token_queue)) process.start() - (token, msg) = self.get_token_checked(process, queue) + (token, msg) = self.get_token_checked(process, token_queue)
while token not in ('post', 'quit'): if token == 'install': logging.info("%s", msg) msgout.writeline(msg) - (token, msg) = self.get_token_checked(process, queue) + (token, msg) = self.get_token_checked(process, token_queue)
if token == 'quit': logger.error("Transaction failed.") @@ -608,7 +609,7 @@ def removefrom(self, pkg, *globs): matches = set() for g in globs: globs_re = re.compile(fnmatch.translate(g)) - m = filter(globs_re.match, filelist) + m = [f for f in filelist if globs_re.match(f)] if m: matches.update(m) else: @@ -670,7 +671,7 @@ def removekmod(self, *globs): matches = set() for g in keepglobs: globs_re = re.compile(fnmatch.translate("*"+g+"*")) - m = filter(globs_re.match, filelist) + m = [f for f in filelist if globs_re.match(f)] if m: matches.update(m) else: @@ -679,7 +680,7 @@ def removekmod(self, *globs):
if remove_files: logger.debug("removekmod: removing %d files", len(remove_files)) - map(remove, remove_files) + list(remove(f) for f in remove_files) else: logger.debug("removekmod %s: no files to remove!", cmd)
diff --git a/src/pylorax/output.py b/src/pylorax/output.py index 15746fe..6446f84 100644 --- a/src/pylorax/output.py +++ b/src/pylorax/output.py @@ -1,7 +1,7 @@ # # output.py # -# Copyright (C) 2009-2014 Red Hat, Inc. +# Copyright (C) 2009-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/src/pylorax/sysutils.py b/src/pylorax/sysutils.py index aa6bac4..7a291b4 100644 --- a/src/pylorax/sysutils.py +++ b/src/pylorax/sysutils.py @@ -1,7 +1,7 @@ # # sysutils.py # -# Copyright (C) 2009-2014 Red Hat, Inc. +# Copyright (C) 2009-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -43,8 +43,7 @@ def joinpaths(*args, **kwargs):
def touch(fname): - with open(fname, "w") as _: - pass + open(fname, "w")
def replace(fname, find, sub): diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py index c5ac269..efd5909 100644 --- a/src/pylorax/treebuilder.py +++ b/src/pylorax/treebuilder.py @@ -1,6 +1,6 @@ # treebuilder.py - handle arch-specific tree building stuff using templates # -# Copyright (C) 2011-2014 Red Hat, Inc. +# Copyright (C) 2011-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -324,7 +324,7 @@ def findkernels(root="/", kdir="boot"): udev_blacklist=' !"$%&'()*,/;<>?[\]^`{|}~' # ASCII printable, minus whitelist udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable def udev_escape(label): - out = u'' + out = '' for ch in label.decode('utf8'): - out += ch if ch not in udev_blacklist else u'\x%02x' % ord(ch) + out += ch if ch not in udev_blacklist else '\x%02x' % ord(ch) return out.encode('utf8') diff --git a/src/pylorax/treeinfo.py b/src/pylorax/treeinfo.py index 6f3f620..538a27e 100644 --- a/src/pylorax/treeinfo.py +++ b/src/pylorax/treeinfo.py @@ -1,7 +1,7 @@ # # treeinfo.py # -# Copyright (C) 2010-2014 Red Hat, Inc. +# Copyright (C) 2010-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -22,7 +22,7 @@ import logging logger = logging.getLogger("pylorax.treeinfo")
-import ConfigParser +import configparser import time
@@ -31,7 +31,7 @@ class TreeInfo(object): def __init__(self, product, version, variant, basearch, packagedir=""):
- self.c = ConfigParser.ConfigParser() + self.c = configparser.ConfigParser()
section = "general" data = {"timestamp": time.time(), @@ -43,13 +43,13 @@ def __init__(self, product, version, variant, basearch, "packagedir": packagedir}
self.c.add_section(section) - map(lambda (key, value): self.c.set(section, key, value), data.items()) + list(self.c.set(section, key, value) for key, value in data.items())
def add_section(self, section, data): if not self.c.has_section(section): self.c.add_section(section)
- map(lambda (key, value): self.c.set(section, key, value), data.items()) + list(self.c.set(section, key, value) for key, value in data.items())
def write(self, outfile): logger.info("writing .treeinfo file") diff --git a/tests/pylint/pylint-one.sh b/tests/pylint/pylint-one.sh index 2c9a40c..dc34e8e 100755 --- a/tests/pylint/pylint-one.sh +++ b/tests/pylint/pylint-one.sh @@ -10,7 +10,7 @@ fi
file_suffix="$(eval echo $$#|sed s?/?_?g)"
-pylint_output="$(pylint \ +pylint_output="$(python3-pylint \ --msg-template='{path}:{line}: {msg_id} {symbol} {msg}' \ -r n --disable=C,R --rcfile=/dev/null \ --dummy-variables-rgx=_ \
From: "Brian C. Lane" bcl@redhat.com
The previous ones were a bit of a kludge, the Anaconda ones have had a lot of work put into them so let's just use them with slight modifications. --- src/pylorax/executils.py | 533 +++++++++++++++-------------------------------- src/pylorax/imgutils.py | 4 +- 2 files changed, 173 insertions(+), 364 deletions(-)
diff --git a/src/pylorax/executils.py b/src/pylorax/executils.py index b4c37db..d127a61 100644 --- a/src/pylorax/executils.py +++ b/src/pylorax/executils.py @@ -17,401 +17,211 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see http://www.gnu.org/licenses/. # -# Author(s): Erik Troan ewt@redhat.com -#
-import os, sys -import io +import os import subprocess -import threading +import signal from time import sleep
import logging log = logging.getLogger("pylorax") program_log = logging.getLogger("program")
-class ExecProduct(object): - def __init__(self, rc, stdout, stderr): - self.rc = rc - self.stdout = stdout - self.stderr = stderr +from threading import Lock +program_log_lock = Lock()
-class tee(threading.Thread): - """ Python reimplementation of the shell tee process, so we can - feed the pipe output into two places at the same time - """ - def __init__(self, inputdesc, outputdesc, logmethod, command): - threading.Thread.__init__(self) - self.inputdesc = os.fdopen(inputdesc, "r") - self.outputdesc = outputdesc - self.logmethod = logmethod - self.running = True - self.command = command - - def run(self): - while self.running: - try: - data = self.inputdesc.readline() - except IOError: - self.logmethod("Can't read from pipe during a call to %s. " - "(program terminated suddenly?)" % self.command) - break - if data == "": - self.running = False - else: - self.logmethod(data.rstrip('\n')) - os.write(self.outputdesc, data) +_child_env = {}
- def stop(self): - self.running = False - return self +def setenv(name, value): + """ Set an environment variable to be used by child processes.
-def execWithRedirect(command, argv, stdin = None, stdout = None, - stderr = None, root = None, preexec_fn=None, cwd=None, - raise_err=False, callback=None): - """ Run an external program and redirect the output to a file. - @param command The command to run. - @param argv A list of arguments. - @param stdin The file descriptor to read stdin from. - @param stdout The file descriptor to redirect stdout to. - @param stderr The file descriptor to redirect stderr to. - @param root The directory to chroot to before running command. - @param preexec_fn function to pass to Popen - @param cwd working directory to pass to Popen - @param raise_err raise CalledProcessError when the returncode is not 0 - @param callback method to call while waiting for process to exit. - @return The return code of command. - - The callback is passed the Popen object. It should return False if - the polling loop should be exited. - """ - def chroot (): - os.chroot(root) + This method does not modify os.environ for the running process, which + is not thread-safe. If setenv has already been called for a particular + variable name, the old value is overwritten.
- stdinclose = stdoutclose = stderrclose = lambda : None + :param str name: The name of the environment variable + :param str value: The value of the environment variable + """
- argv = list(argv) - if isinstance(stdin, str): - if os.access(stdin, os.R_OK): - stdin = os.open(stdin, os.O_RDONLY) - stdinclose = lambda : os.close(stdin) - else: - stdin = sys.stdin.fileno() - elif isinstance(stdin, int): - pass - elif stdin is None or not isinstance(stdin, io.IOBase): - stdin = sys.stdin.fileno() - - if isinstance(stdout, str): - stdout = os.open(stdout, os.O_RDWR|os.O_CREAT) - stdoutclose = lambda : os.close(stdout) - elif isinstance(stdout, int): - pass - elif stdout is None or not isinstance(stdout, io.IOBase): - stdout = sys.stdout.fileno() - - if isinstance(stderr, str): - stderr = os.open(stderr, os.O_RDWR|os.O_CREAT) - stderrclose = lambda : os.close(stderr) - elif isinstance(stderr, int): - pass - elif stderr is None or not isinstance(stderr, io.IOBase): - stderr = sys.stderr.fileno() - - program_log.info("Running... %s", " ".join([command] + argv)) - - #prepare os pipes for feeding tee proceses - pstdout, pstdin = os.pipe() - perrout, perrin = os.pipe() + _child_env[name] = value
+def augmentEnv(): env = os.environ.copy() - env.update({"LC_ALL": "C"}) + env.update(_child_env) + return env
- if root: - preexec_fn = chroot - cwd = root - program_log.info("chrooting into %s", cwd) - elif cwd: - program_log.info("chdiring into %s", cwd) +class ExecProduct(object): + def __init__(self, rc, stdout, stderr): + self.rc = rc + self.stdout = stdout + self.stderr = stderr
+def startProgram(argv, root='/', stdin=None, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + env_prune=None, env_add=None, reset_handlers=True, reset_lang=True, **kwargs): + """ Start an external program and return the Popen object. + + The root and reset_handlers arguments are handled by passing a + preexec_fn argument to subprocess.Popen, but an additional preexec_fn + can still be specified and will be run. The user preexec_fn will be run + last. + + :param argv: The command to run and argument + :param root: The directory to chroot to before running command. + :param stdin: The file object to read stdin from. + :param stdout: The file object to write stdout to. + :param stderr: The file object to write stderr to. + :param env_prune: environment variables to remove before execution + :param env_add: environment variables to add before execution + :param reset_handlers: whether to reset to SIG_DFL any signal handlers set to SIG_IGN + :param reset_lang: whether to set the locale of the child process to C + :param kwargs: Additional parameters to pass to subprocess.Popen + :return: A Popen object for the running command. + """ + if env_prune is None: + env_prune = [] + + # Check for and save a preexec_fn argument + preexec_fn = kwargs.pop("preexec_fn", None) + + def preexec(): + # If a target root was specificed, chroot into it + if root and root != '/': + os.chroot(root) + os.chdir("/") + + # Signal handlers set to SIG_IGN persist across exec. Reset + # these to SIG_DFL if requested. In particular this will include the + # SIGPIPE handler set by python. + if reset_handlers: + for signum in range(1, signal.NSIG): + if signal.getsignal(signum) == signal.SIG_IGN: + signal.signal(signum, signal.SIG_DFL) + + # If the user specified an additional preexec_fn argument, run it + if preexec_fn is not None: + preexec_fn() + + with program_log_lock: + program_log.info("Running... %s", " ".join(argv)) + + env = augmentEnv() + for var in env_prune: + env.pop(var, None) + + if reset_lang: + env.update({"LC_ALL": "C"}) + + if env_add: + env.update(env_add) + + return subprocess.Popen(argv, + stdin=stdin, + stdout=stdout, + stderr=stderr, + close_fds=True, + preexec_fn=preexec, cwd=root, env=env, **kwargs) + +def _run_program(argv, root='/', stdin=None, stdout=None, env_prune=None, log_output=True, + binary_output=False, filter_stderr=False, raise_err=False, callback=None): + """ Run an external program, log the output and return it to the caller + :param argv: The command to run and argument + :param root: The directory to chroot to before running command. + :param stdin: The file object to read stdin from. + :param stdout: Optional file object to write the output to. + :param env_prune: environment variable to remove before execution + :param log_output: whether to log the output of command + :param binary_output: whether to treat the output of command as binary data + :param filter_stderr: whether to exclude the contents of stderr from the returned output + :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero + :param callback: method to call while waiting for process to finish, passed Popen object + :return: The return code of the command and the output + :raises: OSError or CalledProcessError + """ try: - #prepare tee proceses - proc_std = tee(pstdout, stdout, program_log.info, command) - proc_err = tee(perrout, stderr, program_log.error, command) - - #start monitoring the outputs - proc_std.start() - proc_err.start() + if filter_stderr: + stderr = subprocess.PIPE + else: + stderr = subprocess.STDOUT
- proc = subprocess.Popen([command] + argv, stdin=stdin, - stdout=pstdin, - stderr=perrin, - preexec_fn=preexec_fn, cwd=cwd, - env=env) + proc = startProgram(argv, root=root, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr, + env_prune=env_prune, universal_newlines=not binary_output)
if callback: while callback(proc) and proc.poll() is None: sleep(1) - proc.wait() - ret = proc.returncode - - #close the input ends of pipes so we get EOF in the tee processes - os.close(pstdin) - os.close(perrin) - - #wait for the output to be written and destroy them - proc_std.join() - del proc_std
- proc_err.join() - del proc_err - - stdinclose() - stdoutclose() - stderrclose() - except OSError as e: - errstr = "Error running %s: %s" % (command, e.strerror) - log.error(errstr) - program_log.error(errstr) - #close the input ends of pipes so we get EOF in the tee processes - os.close(pstdin) - os.close(perrin) - proc_std.join() - proc_err.join() - - stdinclose() - stdoutclose() - stderrclose() - raise RuntimeError(errstr) - - if ret and raise_err: - raise subprocess.CalledProcessError(ret, [command]+argv) - - return ret - -def execWithCapture(command, argv, stdin = None, stderr = None, root=None, - preexec_fn=None, cwd=None, raise_err=False): - """ Run an external program and capture standard out. - @param command The command to run. - @param argv A list of arguments. - @param stdin The file descriptor to read stdin from. - @param stderr The file descriptor to redirect stderr to. - @param root The directory to chroot to before running command. - @param preexec_fn function to pass to Popen - @param cwd working directory to pass to Popen - @param raise_err raise CalledProcessError when the returncode is not 0 - @return The output of command from stdout. - """ - def chroot(): - os.chroot(root) - - def closefds (): - stdinclose() - stderrclose() - - stdinclose = stderrclose = lambda : None - rc = "" - argv = list(argv) - - if isinstance(stdin, str): - if os.access(stdin, os.R_OK): - stdin = os.open(stdin, os.O_RDONLY) - stdinclose = lambda : os.close(stdin) - else: - stdin = sys.stdin.fileno() - elif isinstance(stdin, int): - pass - elif stdin is None or not isinstance(stdin, io.IOBase): - stdin = sys.stdin.fileno() - - if isinstance(stderr, str): - stderr = os.open(stderr, os.O_RDWR|os.O_CREAT) - stderrclose = lambda : os.close(stderr) - elif isinstance(stderr, int): - pass - elif stderr is None or not isinstance(stderr, io.IOBase): - stderr = sys.stderr.fileno() - - program_log.info("Running... %s", " ".join([command] + argv)) - - env = os.environ.copy() - env.update({"LC_ALL": "C"}) - - if root: - preexec_fn = chroot - cwd = root - program_log.info("chrooting into %s", cwd) - elif cwd: - program_log.info("chdiring into %s", cwd) - - try: - proc = subprocess.Popen([command] + argv, stdin=stdin, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - preexec_fn=preexec_fn, cwd=cwd, - env=env) - - while True: - (outStr, errStr) = proc.communicate() - if outStr: - list(program_log.info(line) for line in outStr.splitlines()) - rc += outStr - if errStr: - list(program_log.error(line) for line in errStr.splitlines()) - os.write(stderr, errStr) - - if proc.returncode is not None: - break - except OSError as e: - log.error("Error running %s: %s", command, e.strerror) - closefds() - raise RuntimeError("Error running %s: %s" % (command, e.strerror)) - - closefds() - if proc.returncode and raise_err: - raise subprocess.CalledProcessError(proc.returncode, [command]+argv) - - return rc + (output_string, err_string) = proc.communicate() + if output_string: + if binary_output: + output_lines = [output_string] + else: + if output_string[-1] != "\n": + output_string = output_string + "\n" + output_lines = output_string.splitlines(True)
-def execWithCallback(command, argv, stdin = None, stdout = None, - stderr = None, echo = True, callback = None, - callback_data = None, root = '/'): - def closefds (): - stdinclose() - stdoutclose() - stderrclose() + if log_output: + with program_log_lock: + for line in output_lines: + program_log.info(line.strip())
- stdinclose = stdoutclose = stderrclose = lambda : None + if stdout: + stdout.write(output_string)
- argv = list(argv) - if isinstance(stdin, str): - if os.access(stdin, os.R_OK): - stdin = os.open(stdin, os.O_RDONLY) - stdinclose = lambda : os.close(stdin) - else: - stdin = sys.stdin.fileno() - elif isinstance(stdin, int): - pass - elif stdin is None or not isinstance(stdin, io.IOBase): - stdin = sys.stdin.fileno() - - if isinstance(stdout, str): - stdout = os.open(stdout, os.O_RDWR|os.O_CREAT) - stdoutclose = lambda : os.close(stdout) - elif isinstance(stdout, int): - pass - elif stdout is None or not isinstance(stdout, io.IOBase): - stdout = sys.stdout.fileno() - - if isinstance(stderr, str): - stderr = os.open(stderr, os.O_RDWR|os.O_CREAT) - stderrclose = lambda : os.close(stderr) - elif isinstance(stderr, int): - pass - elif stderr is None or not isinstance(stderr, io.IOBase): - stderr = sys.stderr.fileno() - - program_log.info("Running... %s", " ".join([command] + argv)) - - p = os.pipe() - p_stderr = os.pipe() - childpid = os.fork() - if not childpid: - os.close(p[0]) - os.close(p_stderr[0]) - os.dup2(p[1], 1) - os.dup2(p_stderr[1], 2) - os.dup2(stdin, 0) - os.close(stdin) - os.close(p[1]) - os.close(p_stderr[1]) - - os.execvp(command, [command] + argv) - os._exit(1) - - os.close(p[1]) - os.close(p_stderr[1]) - - log_output = '' - while 1: - try: - s = os.read(p[0], 1) - except OSError as e: - if e.errno != 4: - list(program_log.info(line) for line in log_output.splitlines()) - raise IOError(e.args) - - if echo: - os.write(stdout, s) - log_output += s + # If stderr was filtered, log it separately + if filter_stderr and err_string and log_output: + err_lines = err_string.splitlines(True)
- if callback: - callback(s, callback_data=callback_data) - - # break out early if the sub-process changes status. - # no need to flush the stream if the process has exited - try: - (pid, status) = os.waitpid(childpid,os.WNOHANG) - if pid != 0: - break - except OSError as e: - log.critical("exception from waitpid: %s %s", e.errno, e.strerror) - - if len(s) < 1: - break - - list(program_log.info(line) for line in log_output.splitlines()) - - log_errors = '' - while 1: - try: - err = os.read(p_stderr[0], 128) - except OSError as e: - if e.errno != 4: - list(program_log.error(line) for line in log_errors.splitlines()) - raise IOError(e.args) - break - log_errors += err - if len(err) < 1: - break - - os.write(stderr, log_errors) - list(program_log.error(line) for line in log_errors.splitlines()) - os.close(p[0]) - os.close(p_stderr[0]) + with program_log_lock: + for line in err_lines: + program_log.info(line.strip())
- try: - #if we didn't already get our child's exit status above, do so now. - if not pid: - (pid, status) = os.waitpid(childpid, 0) except OSError as e: - log.critical("exception from waitpid: %s %s", e.errno, e.strerror) - - closefds() + with program_log_lock: + program_log.error("Error running %s: %s", argv[0], e.strerror) + raise
- rc = 1 - if os.WIFEXITED(status): - rc = os.WEXITSTATUS(status) - return ExecProduct(rc, log_output , log_errors) + with program_log_lock: + program_log.debug("Return code: %d", proc.returncode)
-def _pulseProgressCallback(data, callback_data=None): - if callback_data: - callback_data.pulse() + if proc.returncode and raise_err: + raise subprocess.CalledProcessError(proc.returncode, argv)
-def execWithPulseProgress(command, argv, stdin = None, stdout = None, - stderr = None, echo = True, progress = None, - root = '/'): - return execWithCallback(command, argv, stdin=stdin, stdout=stdout, - stderr=stderr, echo=echo, callback=_pulseProgressCallback, - callback_data=progress, root=root) + return (proc.returncode, output_string)
-## Run a shell. -def execConsole(): - try: - proc = subprocess.Popen(["/bin/sh"]) - proc.wait() - except OSError as e: - raise RuntimeError("Error running /bin/sh: %s" % e.strerror) +def execWithRedirect(command, argv, stdin=None, stdout=None, root='/', env_prune=None, + log_output=True, binary_output=False, raise_err=False, callback=None): + """ Run an external program and redirect the output to a file. + :param command: The command to run + :param argv: The argument list + :param stdin: The file object to read stdin from. + :param stdout: Optional file object to redirect stdout and stderr to. + :param root: The directory to chroot to before running command. + :param env_prune: environment variable to remove before execution + :param log_output: whether to log the output of command + :param binary_output: whether to treat the output of command as binary data + :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero + :param callback: method to call while waiting for process to finish, passed Popen object + :return: The return code of the command + """ + argv = [command] + list(argv) + return _run_program(argv, stdin=stdin, stdout=stdout, root=root, env_prune=env_prune, + log_output=log_output, binary_output=binary_output, raise_err=raise_err, callback=callback)[0] + +def execWithCapture(command, argv, stdin=None, root='/', log_output=True, filter_stderr=False, + raise_err=False, callback=None): + """ Run an external program and capture standard out and err. + :param command: The command to run + :param argv: The argument list + :param stdin: The file object to read stdin from. + :param root: The directory to chroot to before running command. + :param log_output: Whether to log the output of command + :param filter_stderr: Whether stderr should be excluded from the returned output + :param raise_err: whether to raise a CalledProcessError if the returncode is non-zero + :return: The output of the command + """ + argv = [command] + list(argv) + return _run_program(argv, stdin=stdin, root=root, log_output=log_output, filter_stderr=filter_stderr, + raise_err=raise_err, callback=callback)[1]
def runcmd(cmd, **kwargs): """ run execWithRedirect with raise_err=True @@ -424,4 +234,3 @@ def runcmd_output(cmd, **kwargs): """ kwargs["raise_err"] = True return execWithCapture(cmd[0], cmd[1:], **kwargs) - diff --git a/src/pylorax/imgutils.py b/src/pylorax/imgutils.py index 6594e0e..2625c98 100644 --- a/src/pylorax/imgutils.py +++ b/src/pylorax/imgutils.py @@ -220,8 +220,8 @@ def copytree(src, dest, preserve=True): raises CalledProcessError if copy fails.''' logger.debug("copytree %s %s", src, dest) cp = ["cp", "-a"] if preserve else ["cp", "-R", "-L"] - cp += [".", os.path.abspath(dest)] - runcmd(cp, cwd=src) + cp += [join(src, "."), os.path.abspath(dest)] + runcmd(cp)
def do_grafts(grafts, dest, preserve=True): '''Copy each of the items listed in grafts into dest.
From: "Brian C. Lane" bcl@redhat.com
--- src/sbin/lorax | 98 +++++++++++++++++++++++++--------------------------------- 1 file changed, 42 insertions(+), 56 deletions(-)
diff --git a/src/sbin/lorax b/src/sbin/lorax index 7870d2b..037a78b 100755 --- a/src/sbin/lorax +++ b/src/sbin/lorax @@ -1,8 +1,8 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 # # lorax # -# Copyright (C) 2009-2014 Red Hat, Inc. +# Copyright (C) 2009-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,8 +19,6 @@ # # Red Hat Author(s): Martin Gracik mgracik@redhat.com # -from __future__ import print_function - import logging log = logging.getLogger("lorax") program_log = logging.getLogger("program") @@ -31,7 +29,7 @@ dnf_log = logging.getLogger("dnf") import sys import os import tempfile -from optparse import OptionParser, OptionGroup +import argparse import shutil
import dnf @@ -80,91 +78,79 @@ def main(args): vernum = "devel"
version = "{0}-{1}".format(os.path.basename(args[0]), vernum) - usage = "%prog -p PRODUCT -v VERSION -r RELEASE -s REPOSITORY OUTPUTDIR" - - parser = OptionParser(usage=usage) + parser = argparse.ArgumentParser(description="Create the Anaconda boot.iso")
# required arguments for image creation - required = OptionGroup(parser, "required arguments") - required.add_option("-p", "--product", help="product name", - metavar="STRING") - required.add_option("-v", "--version", help="version identifier", - metavar="STRING") - required.add_option("-r", "--release", help="release information", - metavar="STRING") - required.add_option("-s", "--source", - help="source repository (may be listed multiple times)", - metavar="REPOSITORY", action="append", default=[]) + required = parser.add_argument_group("required arguments") + required.add_argument("-p", "--product", help="product name", required=True, metavar="STRING") + required.add_argument("-v", "--version", help="version identifier", required=True, metavar="STRING") + required.add_argument("-r", "--release", help="release information", required=True, metavar="STRING") + required.add_argument("-s", "--source", help="source repository (may be listed multiple times)", + metavar="REPOSITORY", action="append", default=[], required=True)
# optional arguments - optional = OptionGroup(parser, "optional arguments") - optional.add_option("-m", "--mirrorlist", + optional = parser.add_argument_group("optional arguments") + optional.add_argument("-m", "--mirrorlist", help="mirrorlist repository (may be listed multiple times)", metavar="REPOSITORY", action="append", default=[]) - optional.add_option("-t", "--variant", + optional.add_argument("-t", "--variant", help="variant name", metavar="STRING") - optional.add_option("-b", "--bugurl", + optional.add_argument("-b", "--bugurl", help="bug reporting URL for the product", metavar="URL", default="your distribution provided bug reporting tool") - optional.add_option("--isfinal", help="", + optional.add_argument("--isfinal", help="", action="store_true", default=False, dest="isfinal") - optional.add_option("-c", "--config", default="/etc/lorax/lorax.conf", + optional.add_argument("-c", "--config", default="/etc/lorax/lorax.conf", help="config file", metavar="STRING") - optional.add_option("--proxy", default=None, + optional.add_argument("--proxy", default=None, help="repo proxy url:port", metavar="STRING") - optional.add_option("-i", "--installpkgs", default=[], + optional.add_argument("-i", "--installpkgs", default=[], action="append", metavar="STRING", help="package glob to install before runtime-install.tmpl runs. (may be listed multiple times)") - optional.add_option("--buildarch", default=None, + optional.add_argument("--buildarch", default=None, help="build architecture", metavar="STRING") - optional.add_option("--volid", default=None, + optional.add_argument("--volid", default=None, help="volume id", metavar="STRING") - optional.add_option("--macboot", help="", + optional.add_argument("--macboot", help="", action="store_true", default=True, dest="domacboot") - optional.add_option("--nomacboot", help="", + optional.add_argument("--nomacboot", help="", action="store_false", dest="domacboot") - optional.add_option("--noupgrade", help="", + optional.add_argument("--noupgrade", help="", action="store_false", default=True, dest="doupgrade") - optional.add_option("--logfile", default="./lorax.log", + optional.add_argument("--logfile", default="./lorax.log", help="Path to logfile") - optional.add_option("--tmp", default="/var/tmp", + optional.add_argument("--tmp", default="/var/tmp", help="Top level temporary directory" ) - optional.add_option("--cachedir", default=None, + optional.add_argument("--cachedir", default=None, help="DNF cache directory. Default is a temporary dir.") - optional.add_option("--workdir", default=None, + optional.add_argument("--workdir", default=None, help="Work directory, overrides --tmp. Default is a temporary dir under /var/tmp") - optional.add_option("--force", default=False, action="store_true", + optional.add_argument("--force", default=False, action="store_true", help="Run even when the destination directory exists") - optional.add_option("--add-template", dest="add_templates", + optional.add_argument("--add-template", dest="add_templates", action="append", help="Additional template for runtime image", default=[]) - optional.add_option("--add-template-var", dest="add_template_vars", + optional.add_argument("--add-template-var", dest="add_template_vars", action="append", help="Set variable for runtime image template", default=[]) - optional.add_option("--add-arch-template", dest="add_arch_templates", + optional.add_argument("--add-arch-template", dest="add_arch_templates", action="append", help="Additional template for architecture-specific image", default=[]) - optional.add_option("--add-arch-template-var", dest="add_arch_template_vars", + optional.add_argument("--add-arch-template-var", dest="add_arch_template_vars", action="append", help="Set variable for architecture-specific image", default=[])
- # add the option groups to the parser - parser.add_option_group(required) - parser.add_option_group(optional) - # add the show version option - parser.add_option("-V", help="show program's version number and exit", - action="store_true", default=False, dest="showver") + parser.add_argument("-V", help="show program's version number and exit", + action="version", version=version)
- # parse the arguments - opts, args = parser.parse_args() + parser.add_argument("outputdir", help="Output directory", metavar="STRING")
- if opts.showver: - print(version) - sys.exit(0) + # parse the arguments + opts = parser.parse_args()
try: - outputdir = os.path.abspath(args[0]) + outputdir = os.path.abspath(opts.outputdir) except IndexError: parser.error("missing one or more required arguments")
@@ -270,12 +256,12 @@ def get_dnf_base_object(installroot, repositories, mirrorlists=None, mirrorlists = mirrorlists or []
# sanitize the repositories - repositories = map(sanitize_repo, repositories) - mirrorlists = map(sanitize_repo, mirrorlists) + repositories = list(sanitize_repo(r) for r in repositories) + mirrorlists = list(sanitize_repo(r) for r in mirrorlists)
# remove invalid repositories - repositories = filter(bool, repositories) - mirrorlists = filter(bool, mirrorlists) + repositories = list(r for r in repositories if r) + mirrorlists = list(r for r in mirrorlists if r)
if not cachedir: cachedir = os.path.join(tempdir, "dnf.cache")
From: "Brian C. Lane" bcl@redhat.com
Add a 'lower' filter to the templates to replace string.lower which no longer exists. Fix udev_escape, the strings are already unicode, and drop --chdir from runcmd. It wasn't ever used, and passing cwd to the new runcmd isn't supported. --- share/efi.tmpl | 1 - share/live/efi.tmpl | 1 - src/pylorax/ltmpl.py | 16 +++++----------- src/pylorax/treebuilder.py | 15 ++++++++++++--- src/pylorax/treeinfo.py | 2 +- 5 files changed, 18 insertions(+), 17 deletions(-)
diff --git a/share/efi.tmpl b/share/efi.tmpl index c82e1c7..e2585d8 100644 --- a/share/efi.tmpl +++ b/share/efi.tmpl @@ -1,6 +1,5 @@ <%page args="configdir, KERNELDIR, efiarch, isolabel"/> <% -from string import lower EFIBOOTDIR="EFI/BOOT" APPLE_EFI_ICON=inroot+"/usr/share/pixmaps/bootloader/fedora.icns" APPLE_EFI_DISKNAME=inroot+"/usr/share/pixmaps/bootloader/fedora-media.vol" diff --git a/share/live/efi.tmpl b/share/live/efi.tmpl index 54e1e15..009790c 100644 --- a/share/live/efi.tmpl +++ b/share/live/efi.tmpl @@ -1,6 +1,5 @@ <%page args="configdir, KERNELDIR, efiarch, isolabel"/> <% -from string import lower EFIBOOTDIR="EFI/BOOT" APPLE_EFI_ICON=inroot+"/usr/share/pixmaps/bootloader/fedora.icns" APPLE_EFI_DISKNAME=inroot+"/usr/share/pixmaps/bootloader/fedora-media.vol" diff --git a/src/pylorax/ltmpl.py b/src/pylorax/ltmpl.py index df39e98..93ecd14 100644 --- a/src/pylorax/ltmpl.py +++ b/src/pylorax/ltmpl.py @@ -69,9 +69,6 @@ def parse(self, template_file, variables): # remove comments lines = [line for line in lines if not line.startswith("#")]
- # mako template now returns unicode strings - lines = [line.encode("utf8") for line in lines] - # split with shlex and perform brace expansion lines = [split_and_expand(line) for line in lines]
@@ -302,7 +299,7 @@ def append(self, filename, data): append /etc/resolv.conf "" ''' with open(self._out(filename), "a") as fobj: - fobj.write(data.decode('string_escape')+"\n") + fobj.write(bytes(data, "utf8").decode('unicode_escape')+"\n")
def treeinfo(self, section, key, *valuetoks): ''' @@ -431,10 +428,8 @@ def log(self, msg): # TODO: add ssh-keygen, mkisofs(?), find, and other useful commands def runcmd(self, *cmdlist): ''' - runcmd CMD [--chdir=DIR] [ARG ...] + runcmd CMD [ARG ...] Run the given command with the given arguments. - If "--chdir=DIR" is given, change to the named directory - before executing the command.
NOTE: All paths given MUST be COMPLETE, ABSOLUTE PATHS to the file or files mentioned. ${root}/${inroot}/${outroot} are good for @@ -451,15 +446,14 @@ def runcmd(self, *cmdlist): remove ${f} %endfor ''' - cwd = None cmd = cmdlist logger.debug('running command: %s', cmd) if cmd[0].startswith("--chdir="): - cwd = cmd[0].split('=',1)[1] - cmd = cmd[1:] + logger.error("--chdir is no longer supported for runcmd.") + raise ValueError("--chdir is no longer supported for runcmd.")
try: - stdout = runcmd_output(cmd, cwd=cwd) + stdout = runcmd_output(cmd) if stdout: logger.debug('command output:\n%s', stdout) logger.debug("command finished successfully") diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py index efd5909..bea9bcb 100644 --- a/src/pylorax/treebuilder.py +++ b/src/pylorax/treebuilder.py @@ -193,7 +193,7 @@ def __init__(self, product, arch, inroot, outroot, runtime, isolabel, domacboot= inroot=inroot, outroot=outroot, basearch=arch.basearch, libdir=arch.libdir, isolabel=isolabel, udev=udev_escape, domacboot=domacboot, doupgrade=doupgrade, - workdir=workdir) + workdir=workdir, lower=string_lower) self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir) self._runner.defaults = self.vars self.add_templates = add_templates or [] @@ -325,6 +325,15 @@ def findkernels(root="/", kdir="boot"): udev_blacklist += ''.join(chr(i) for i in range(32)) # ASCII non-printable def udev_escape(label): out = '' - for ch in label.decode('utf8'): + for ch in label: out += ch if ch not in udev_blacklist else '\x%02x' % ord(ch) - return out.encode('utf8') + return out + +def string_lower(string): + """ Return a lowercase string. + + :param string: String to lowercase + + This is used as a filter in the templates. + """ + return string.lower() diff --git a/src/pylorax/treeinfo.py b/src/pylorax/treeinfo.py index 538a27e..4c84006 100644 --- a/src/pylorax/treeinfo.py +++ b/src/pylorax/treeinfo.py @@ -34,7 +34,7 @@ def __init__(self, product, version, variant, basearch, self.c = configparser.ConfigParser()
section = "general" - data = {"timestamp": time.time(), + data = {"timestamp": str(time.time()), "family": product, "version": version, "name": "%s-%s" % (product, version),
From: "Brian C. Lane" bcl@redhat.com
Just a couple prints. --- src/sbin/mkefiboot | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/src/sbin/mkefiboot b/src/sbin/mkefiboot index 38a4ce0..075e4f1 100755 --- a/src/sbin/mkefiboot +++ b/src/sbin/mkefiboot @@ -1,7 +1,7 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 # mkefiboot - a tool to make EFI boot images # -# Copyright (C) 2011-2014 Red Hat, Inc. +# Copyright (C) 2011-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -145,9 +145,9 @@ def main(): if os.getuid() > 0: parser.error("need root permissions") if opt.icon and not opt.imgtype == "apple": - print "Warning: --icon is only useful for Apple EFI images" + print("Warning: --icon is only useful for Apple EFI images") if opt.diskname and not opt.imgtype == "apple": - print "Warning: --diskname is only useful for Apple EFI images" + print("Warning: --diskname is only useful for Apple EFI images") # do the thing! if opt.imgtype == "apple": mkmacboot(opt.bootdir, opt.outfile, opt.label, opt.icon, opt.product,
From: "Brian C. Lane" bcl@redhat.com
Also switch to using argparse instead of optparse. --- src/bin/image-minimizer | 55 ++++++++++++++++++++++--------------------------- 1 file changed, 25 insertions(+), 30 deletions(-)
diff --git a/src/bin/image-minimizer b/src/bin/image-minimizer index 8c37ab4..16fc4b5 100755 --- a/src/bin/image-minimizer +++ b/src/bin/image-minimizer @@ -1,8 +1,8 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 # # image-minimizer: removes files and packages on the filesystem # -# Copyright 2007-2014 Red Hat, Inc. +# Copyright 2007-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,7 +17,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. import glob -import optparse +import argparse import os import sys import rpm @@ -53,7 +53,7 @@ class ImageMinimizer: def add_pattern(self, files, pattern): globs = glob.glob(pattern) if self.verbose and len(globs) == 0: - print "%s file not found" % pattern + print("%s file not found" % pattern) for g in globs: if os.path.isdir(g): self.add_directory(files, g) @@ -63,7 +63,7 @@ class ImageMinimizer: def add_pattern_rpm(self, rpms, pattern): if self.ts is None: if self.prefix is None: - raise Exception ('Must specify installation root for droprpm/keeprpm') + raise Exception('Must specify installation root for droprpm/keeprpm') self.ts = rpm.TransactionSet(self.prefix) mi = self.ts.dbMatch() mi.pattern('name', rpm.RPMMIRE_GLOB, pattern) @@ -72,7 +72,7 @@ class ImageMinimizer: not_found = False rpms.add(hdr['name']) if self.verbose and not_found: - print "%s package not found" % pattern + print("%s package not found" % pattern)
# Parses each line in the ifle def parse_line(self, line): @@ -107,7 +107,7 @@ class ImageMinimizer: elif command == 'droprpm': self.add_pattern_rpm(self.drops_rpm, pattern) else: - raise Exception ('Unknown Command: ' + command) + raise Exception('Unknown Command: ' + command)
def remove(self): for tag in sorted(self.drops, reverse=True): @@ -116,34 +116,34 @@ class ImageMinimizer: self.visited.add(tag) else: if self.dryrun: - print 'rm ' + tag + print('rm ' + tag) else: if self.verbose: - print 'rm ' + tag + print('rm ' + tag) os.remove(tag)
#remove all empty directory. Every 8k counts! for d in sorted(self.visited, reverse=True): if len(os.listdir(d)) == 0: if self.dryrun: - print 'rm -rf ' + d + print('rm -rf ' + d) else: if self.verbose: - print 'rm -rf ' + d + print('rm -rf ' + d) os.rmdir(d)
def remove_rpm(self):
def runCallback(reason, amount, total, key, client_data): if self.verbose and reason == rpm.RPMCALLBACK_UNINST_STOP: - print key, "erased" + print(key, "erased")
if len(self.drops_rpm) == 0: return
for pkg in self.drops_rpm: if self.dryrun: - print "erasing ", pkg + print("erasing ", pkg) else: self.ts.addErase(pkg) if not self.dryrun: @@ -158,40 +158,35 @@ class ImageMinimizer:
def parse_options(): - usage = "usage: %prog [options] filename" - parser = optparse.OptionParser(usage=usage) + parser = argparse.ArgumentParser(description="Image Minimizer")
parser.set_defaults(root=os.environ.get('INSTALL_ROOT', '/mnt/sysimage/'), dry_run=False)
- parser.add_option("-i", "--installroot", type="string", dest="root", + parser.add_argument("-i", "--installroot", metavar="STRING", dest="root", help="Root path to prepend to all file patterns and installation root for RPM " "operations. Defaults to INSTALL_ROOT or /mnt/sysimage/")
- parser.add_option("--dryrun", action="store_true", dest="dryrun", + parser.add_argument("--dryrun", metavar="BOOL", action="store_true", dest="dryrun", help="If set, no filesystem changes are made.")
- parser.add_option("-v", "--verbose", action="store_true", dest="verbose", + parser.add_argument("-v", "--verbose", metavar="BOOL", action="store_true", dest="verbose", help="Display every action as it is performed.")
- (options, args) = parser.parse_args() - if len(args) == 0: - parser.print_help() - sys.exit(1) + parser.add_argument("filename", metavar="STRING", help="Filename to process")
- return (options, args) + return parser.parse_args()
def main(): try: - (options, args) = parse_options() - filename = args[0] - minimizer = ImageMinimizer(filename, options.root, options.dryrun, - options.verbose) + args = parse_options() + minimizer = ImageMinimizer(args.filename, args.root, args.dryrun, + args.verbose) minimizer.filter() - except SystemExit, e: + except SystemExit as e: sys.exit(e.code) - except KeyboardInterrupt, e: - print >> sys.stderr, "Aborted at user request" + except KeyboardInterrupt as e: + print("Aborted at user request", file=sys.stderr)
if __name__ == "__main__": main()
From: "Brian C. Lane" bcl@redhat.com
--- utils/test-parse-template | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/utils/test-parse-template b/utils/test-parse-template index 796d770..80852d1 100755 --- a/utils/test-parse-template +++ b/utils/test-parse-template @@ -1,6 +1,6 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 # test-parse-template - parse and print (but don't execute!) a Lorax Template. -# Copyright (C) 2011 Red Hat, Inc. +# Copyright (C) 2011-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -46,14 +46,14 @@ def readtemplate(filename): if __name__ == '__main__': # check args if len(sys.argv) == 1: - print "usage: %s TEMPLATE [arg1=value1] [arg2=value2] ..." + print("usage: %s TEMPLATE [arg1=value1] [arg2=value2] ...")
# read template templatefile = sys.argv[1] - print "parsing %s" % templatefile + print("parsing %s" % templatefile) t = readtemplate(templatefile) reqargs = get_args(t) - print "required args: %s" % ' '.join(reqargs) + print("required args: %s" % ' '.join(reqargs))
# parse extra commandline args if len(sys.argv) > 2: @@ -66,15 +66,15 @@ if __name__ == '__main__': for a in reqargs.difference(data): data[a] = a.upper()
- print "rendering %s using values:" % templatefile + print("rendering %s using values:" % templatefile) for a in data: - print " %s=%s" % (a,repr(data[a])) + print(" %s=%s" % (a,repr(data[a])))
# render and print try: lines = t.render(**data).splitlines() for line in lines: - print line + print(line) except Exception: - print text_error_template().render() + print(text_error_template().render()) raise SystemExit(1)
From: "Brian C. Lane" bcl@redhat.com
--- src/sbin/livemedia-creator | 46 +++++++++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 21 deletions(-)
diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator index e60e091..91e54c7 100755 --- a/src/sbin/livemedia-creator +++ b/src/sbin/livemedia-creator @@ -1,8 +1,8 @@ -#!/usr/bin/python2 +#!/usr/bin/python3 # # Live Media Creator # -# Copyright (C) 2011-2014 Red Hat, Inc. +# Copyright (C) 2011-2015 Red Hat, Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,7 +31,7 @@ import tempfile import subprocess import socket import threading -import SocketServer +import socketserver import time from time import sleep import shutil @@ -57,6 +57,7 @@ from pylorax.sysutils import joinpaths, remove from pylorax.imgutils import PartitionMount, mksparse, mkext4img, loop_detach from pylorax.imgutils import get_loop_name, dm_detach, mount, umount, Mount from pylorax.imgutils import mksquashfs, mkqcow2, mktar, mkrootfsimg +from pylorax.imgutils import copytree from pylorax.executils import execWithRedirect, execWithCapture, runcmd
# no-virt mode doesn't need libvirt, so make it optional @@ -76,7 +77,7 @@ class InstallError(Exception): pass
-class LogRequestHandler(SocketServer.BaseRequestHandler): +class LogRequestHandler(socketserver.BaseRequestHandler): """ Handle monitoring and saving the logfiles from the virtual install
@@ -102,13 +103,14 @@ class LogRequestHandler(SocketServer.BaseRequestHandler):
Loops until self.server.kill is True """ + log.info("Processing logs from %s", self.client_address) line = "" while True: if self.server.kill: break
try: - data = self.request.recv(4096) + data = str(self.request.recv(4096), "utf8") if self.fp: self.fp.write(data) self.fp.flush() @@ -128,10 +130,12 @@ class LogRequestHandler(SocketServer.BaseRequestHandler):
except socket.timeout: pass - except Exception: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except + log.info("log processing killed by exception: %s", e) break
def finish(self): + log.info("Shutting down log processing") self.request.close() if self.fp: self.fp.close() @@ -166,7 +170,7 @@ class LogRequestHandler(SocketServer.BaseRequestHandler): return
-class LogServer(SocketServer.TCPServer): +class LogServer(socketserver.TCPServer): """A TCP Server that listens for log data"""
# Number of seconds to wait for a connection after startup @@ -185,7 +189,7 @@ class LogServer(SocketServer.TCPServer): self._timeout = kwargs.pop("timeout", None) if self._timeout: self._start_time = time.time() - SocketServer.TCPServer.__init__(self, *args, **kwargs) + socketserver.TCPServer.__init__(self, *args, **kwargs)
def log_check(self): """ @@ -421,7 +425,7 @@ class VirtualInstall(object): sys.stdout.write(".") sys.stdout.flush() sleep(10) - print + print()
if log_check(): log.info("Installation error detected. See logfile.") @@ -639,7 +643,7 @@ def rebuild_initrds_for_live(opts, sys_root_dir, results_dir):
new_initrd_path = joinpaths(results_dir, os.path.basename(kernel.initrd.path)) shutil.move(joinpaths(sys_root_dir, outfile), new_initrd_path) - os.chmod(new_initrd_path, 0644) + os.chmod(new_initrd_path, 0o644) shutil.copy2(joinpaths(sys_root_dir, kernel.path), results_dir)
os.unlink(joinpaths(sys_root_dir,"/proc/modules")) @@ -711,7 +715,7 @@ def make_livecd(opts, mount_dir, work_dir): fullpath = joinpaths(mount_dir, configdir_path) if os.path.exists(fullpath): remove(fullpath) - shutil.copytree(configdir, fullpath) + copytree(configdir, fullpath)
isolabel = opts.volid or "{0.name}-{0.version}-{1.basearch}".format(product, arch) if len(isolabel) > 32: @@ -1308,12 +1312,6 @@ def main(): errors.append("The results_dir (%s) should not exist, please delete or " "move its contents" % opts.result_dir)
- # Default to putting results into tmp - if not opts.result_dir: - opts.result_dir = opts.tmp - else: - os.makedirs(opts.result_dir) - if opts.iso and not os.path.exists(opts.iso): errors.append("The iso %s is missing." % opts.iso)
@@ -1331,7 +1329,7 @@ def main(): errors.append("the volume id cannot be longer than 32 characters")
if is_install and not opts.no_virt and not libvirt: - errors.append("virt install requires libvirt-python to be installed.") + errors.append("virt install requires libvirt-python3 to be installed.")
if is_install and not opts.no_virt \ and not os.path.exists("/usr/bin/virt-install"): @@ -1368,9 +1366,15 @@ def main(): errors.append("You need to run this as root")
if errors: - map(log.error, errors) + list(log.error(e) for e in errors) sys.exit(1)
+ # Default to putting results into tmp + if not opts.result_dir: + opts.result_dir = opts.tmp + else: + os.makedirs(opts.result_dir) + # AMI image is just a fsimage with an AMI label if opts.make_ami: opts.make_fsimage = True @@ -1422,7 +1426,7 @@ def main(): "multiple partitions. swap is allowed but not used.")
if errors: - map(log.error, errors) + list(log.error(e) for e in errors) sys.exit(1)
# Make the image. Output of this is either a partitioned disk image or a fsimage @@ -1498,7 +1502,7 @@ def main(): umount(mounted_sysroot_boot_dir)
if opts.result_dir != opts.tmp and result_dir: - shutil.copytree(result_dir, opts.result_dir) + copytree(result_dir, opts.result_dir) shutil.rmtree(result_dir)
log.info("SUMMARY")
From: "Brian C. Lane" bcl@redhat.com
Also cleanup some warnings. --- Makefile | 2 +- src/pylorax/__init__.py | 5 +- tests/pylint/pylint-false-positives | 3 - tests/pylint/pylint-one.sh | 32 --------- tests/pylint/runpylint.py | 38 ++++++++++ tests/pylint/runpylint.sh | 139 ------------------------------------ utils/filediff.py | 2 +- 7 files changed, 42 insertions(+), 179 deletions(-) delete mode 100644 tests/pylint/pylint-false-positives delete mode 100755 tests/pylint/pylint-one.sh create mode 100755 tests/pylint/runpylint.py delete mode 100755 tests/pylint/runpylint.sh
diff --git a/Makefile b/Makefile index d920669..017c008 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ install: all
check: @echo "*** Running pylint ***" - ./tests/pylint/runpylint.sh + PYTHONPATH=$(PYTHONPATH):./src/ ./tests/pylint/runpylint.py
clean: -rm -rf build src/pylorax/version.py diff --git a/src/pylorax/__init__.py b/src/pylorax/__init__.py index c8ee96d..e83635b 100644 --- a/src/pylorax/__init__.py +++ b/src/pylorax/__init__.py @@ -154,8 +154,7 @@ def run(self, dbo, product, version, release, variant="", bugurl="", add_templates=None, add_template_vars=None, add_arch_templates=None, - add_arch_template_vars=None, - template_tempdir=None): + add_arch_template_vars=None):
assert self._configured
@@ -295,7 +294,7 @@ def run(self, dbo, product, version, release, variant="", bugurl="", logger.info("creating the runtime image") runtime = "images/install.img" compression = self.conf.get("compression", "type") - compressargs = self.conf.get("compression", "args").split() + compressargs = self.conf.get("compression", "args").split() # pylint: disable=no-member if self.conf.getboolean("compression", "bcj"): if self.arch.bcj: compressargs += ["-Xbcj", self.arch.bcj] diff --git a/tests/pylint/pylint-false-positives b/tests/pylint/pylint-false-positives deleted file mode 100644 index 449a192..0000000 --- a/tests/pylint/pylint-false-positives +++ /dev/null @@ -1,3 +0,0 @@ -F0401 import-error Unable to import 'pylorax.version' -E0611 no-name-in-module No name 'version' in module 'pylorax' -E1101 no-member Module 'pylorax' has no 'version' member diff --git a/tests/pylint/pylint-one.sh b/tests/pylint/pylint-one.sh deleted file mode 100755 index dc34e8e..0000000 --- a/tests/pylint/pylint-one.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -# -# $1 -- python source to run pylint on -# - -if [ $# -lt 1 ]; then - # no source, just exit - exit 1 -fi - -file_suffix="$(eval echo $$#|sed s?/?_?g)" - -pylint_output="$(python3-pylint \ - --msg-template='{path}:{line}: {msg_id} {symbol} {msg}' \ - -r n --disable=C,R --rcfile=/dev/null \ - --dummy-variables-rgx=_ \ - --ignored-classes=Popen,TransactionSet \ - --defining-attr-methods=__init__,_grabObjects,initialize,reset,start,setUp \ - $DISABLED_WARN_OPTIONS \ - $DISABLED_ERR_OPTIONS \ - $NON_STRICT_OPTIONS "$@" 2>&1 | \ - egrep -v -f "$FALSE_POSITIVES" \ - )" - -# I0011 is the informational "Locally disabling ...." message -if [ -n "$(echo "$pylint_output" | fgrep -v '************* Module ' |\ - grep -v '^I0011:')" ]; then - # Replace the Module line with the actual filename - pylint_output="$(echo "$pylint_output" | sed "s|* Module .*|* Module $(eval echo $$#)|")" - echo "$pylint_output" > pylint-out_$file_suffix - touch "pylint-$file_suffix-failed" -fi diff --git a/tests/pylint/runpylint.py b/tests/pylint/runpylint.py new file mode 100755 index 0000000..a3e2cac --- /dev/null +++ b/tests/pylint/runpylint.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 + +import sys + +from pocketlint import FalsePositive, PocketLintConfig, PocketLinter + +class LoraxLintConfig(PocketLintConfig): + def __init__(self): + PocketLintConfig.__init__(self) + + self.falsePositives = [ FalsePositive(r"No name 'version' in module 'pylorax'"), + FalsePositive(r"Module 'pylorax' has no 'version' member"), + FalsePositive(r"Instance of 'int' has no .* member"), + FalsePositive(r"ImageMinimizer.remove_rpm.runCallback: Unused argument .*") + ] + + @property + def disabledOptions(self): + return [ "I0011", # Locally disabling %s + "W0511", # FIXME/TODO/XXX + "W0141", # Used builtin function %s + ] + + @property + def pylintPlugins(self): + retval = super(LoraxLintConfig, self).pylintPlugins + # Not using threads so we can skip these + retval.remove("pocketlint.checkers.environ") + retval.remove("pocketlint.checkers.eintr") + # No markup used + retval.remove("pocketlint.checkers.markup") + return retval + +if __name__ == "__main__": + conf = LoraxLintConfig() + linter = PocketLinter(conf) + rc = linter.run() + sys.exit(rc) diff --git a/tests/pylint/runpylint.sh b/tests/pylint/runpylint.sh deleted file mode 100755 index aab61d9..0000000 --- a/tests/pylint/runpylint.sh +++ /dev/null @@ -1,139 +0,0 @@ -#!/bin/bash -# This script will check the project for any pylint warning and errors using a set -# of options minimizing false positives, in combination with filtering of any -# warning regularexpressions listed in pylint-false-positives. -# -# If any warnings are found they will be stored in pylint-log and printed -# to stdout and this script will exit with a status of 1, if no (non filtered) -# warnings are found it exits with a status of 0 - -# XDG_RUNTIME_DIR is "required" to be set, so make one up in case something -# actually tries to do something with it -if [ -z "$XDG_RUNTIME_DIR" ]; then - export XDG_RUNTIME_DIR="$(mktemp -d)" - trap "rm -rf "$XDG_RUNTIME_DIR"" EXIT -fi - -# If $top_srcdir is set, assume this is being run from automake and we don't -# need to keep a separate log -export pylint_log=0 -if [ -z "$top_srcdir" ]; then - export pylint_log=1 -fi - -# Unset TERM so that things that use readline don't output terminal garbage -unset TERM - -# Don't try to connect to the accessibility socket -export NO_AT_BRIDGE=1 - -# Force the GDK backend to X11. Otherwise if no display can be found, Gdk -# tries every backend type, which includes "broadway," which prints an error -# and keeps changing the content of said error. -export GDK_BACKEND=x11 - -# If $top_srcdir has not been set by automake, import the test environment -if [ -z "$top_srcdir" ]; then - top_srcdir="$(dirname "$0")/../.." - . ${top_srcdir}/tests/testenv.sh -fi - -. ${top_srcdir}/tests/lib/testlib.sh - -srcdir="${top_srcdir}/tests/pylint" -builddir="${top_builddir}/tests/pylint" - -# Need to add the pylint module directory to PYTHONPATH as well. -export PYTHONPATH="${PYTHONPATH}:${srcdir}" - -# Save analysis data in the pylint directory -export PYLINTHOME="${builddir}/.pylint.d" -[ -d "$PYLINTHOME" ] || mkdir "$PYLINTHOME" - -export FALSE_POSITIVES="${srcdir}"/pylint-false-positives - -# W0212 - Access to a protected member %s of a client class -export NON_STRICT_OPTIONS="--disable=W0212" - -# E1103 - %s %r has no %r member (but some types could not be inferred) -export DISABLED_ERR_OPTIONS="--disable=E1103" - -# W0110 - map/filter on lambda could be replaced by comprehension -# W0123 - Use of eval -# W0141 - Used builtin function %r -# W0142 - Used * or ** magic -# W0511 - Used when a warning note as FIXME or XXX is detected. -# W0603 - Using the global statement -# W0613 - Unused argument %r -# W0614 - Unused import %s from wildcard import -# I0011 - Locally disabling %s (i.e., pylint: disable) -# I0012 - Locally enabling %s (i.e., pylint: enable) -# I0013 - Ignoring entire file (i.e., pylint: skip-file) -export DISABLED_WARN_OPTIONS="--disable=W0110,W0123,W0141,W0142,W0511,W0603,W0613,W0614,I0011,I0012,I0013" - -usage () { - echo "usage: `basename $0` [--strict] [--help] [files...]" - exit $1 -} - -# Separate the module parameters from the files list -ARGS= -FILES= -while [ $# -gt 0 ]; do - case $1 in - --strict) - export NON_STRICT_OPTIONS= - ;; - --help) - usage 0 - ;; - -*) - ARGS="$ARGS $1" - ;; - *) - FILES=$@ - break - esac - shift -done - -exit_status=0 - -if [ -s pylint-log ]; then - rm pylint-log -fi - -# run pylint one file / module at a time, otherwise it sometimes gets -# confused -if [ -z "$FILES" ]; then - # Test any file that either ends in .py or contains #!/usr/bin/python in - # the first line. Scan everything except old_tests - FILES=$(findtestfiles ( -name '*.py' -o \ - -exec /bin/sh -c "head -1 {} | grep -q '#!/usr/bin/python'" ; ) -print | \ - egrep -v '(|/)old_tests/') -fi - -num_cpus=$(getconf _NPROCESSORS_ONLN) -# run pylint in paralel -echo $FILES | xargs --max-procs=$num_cpus -n 1 "$srcdir"/pylint-one.sh $ARGS || exit 1 - -for file in $(find -name 'pylint-out*'); do - cat "$file" >> pylint-log - rm "$file" -done - -fails=$(find -name 'pylint*failed' -print -exec rm '{}' ;) -if [ -z "$fails" ]; then - exit_status=0 -else - exit_status=1 -fi - -if [ -s pylint-log ]; then - echo "pylint reports the following issues:" - cat pylint-log -elif [ -e pylint-log ]; then - rm pylint-log -fi - -exit "$exit_status" diff --git a/utils/filediff.py b/utils/filediff.py index b5543ac..37ce65c 100644 --- a/utils/filediff.py +++ b/utils/filediff.py @@ -2,7 +2,7 @@ import os import magic import difflib -import yum +import yum # pylint: disable=import-error import operator
From: "Brian C. Lane" bcl@redhat.com
This updates things for python3, and adds 2 subpackages to pull in dependencies needed by lmc when running with virt-install or --no-virt with anaconda. --- Makefile | 2 +- lorax.spec | 44 ++++++++++++++++++++++++++++++++++++-------- setup.py | 7 +++---- 3 files changed, 40 insertions(+), 13 deletions(-)
diff --git a/Makefile b/Makefile index 017c008..a0c8033 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON ?= /usr/bin/python +PYTHON ?= /usr/bin/python3 DESTDIR ?= /
PKGNAME = lorax diff --git a/lorax.spec b/lorax.spec index 11d162a..b859dcf 100644 --- a/lorax.spec +++ b/lorax.spec @@ -14,7 +14,8 @@ URL: https://github.com/rhinstaller/lorax # tito build --tgz Source0: %{name}-%{version}.tar.gz
-BuildRequires: python2-devel +BuildRequires: python3-devel +BuildRequires: python3-pocketlint >= 0.5
Requires: GConf2 Requires: cpio @@ -29,24 +30,27 @@ Requires: glibc Requires: glibc-common Requires: gzip Requires: isomd5sum -Requires: libselinux-python Requires: module-init-tools Requires: parted -Requires: python-mako Requires: squashfs-tools >= 4.2 Requires: util-linux Requires: xz Requires: pigz -Requires: python-kickstart Requires: dracut >= 030 -Requires: python-dnf + +# Python modules +Requires: libselinux-python3 +Requires: python3-mako +Requires: python3-kickstart +Requires: python3-dnf +
%if 0%{?fedora} # Fedora specific deps Requires: fedup-dracut Requires: fedup-dracut-plymouth %ifarch x86_64 -Requires: hfsplus-tools +Requires: hfsplus-tools %endif %endif
@@ -79,6 +83,26 @@ It also includes livemedia-creator which is used to create bootable livemedia, including live isos and disk images. It can use libvirtd for the install, or Anaconda's image install feature.
+%package lmc-virt +Summary: livemedia-creator libvirt dependencies +Requires: lorax = %{version}-%{release} +Requires: libvirt-python3 +Requires: virt-install + +%description lmc-virt +Additional dependencies required by livemedia-creator when using it with virt-install. + +%package lmc-novirt +Summary: livemedia-creator no-virt dependencies +Requires: lorax = %{version}-%{release} +Requires: anaconda-core +Requires: anaconda-tui + +%description lmc-novirt +Additional dependencies required by livemedia-creator when using it with --no-virt +to run Anaconda. + + %prep %setup -q -n %{name}-%{version}
@@ -93,8 +117,8 @@ make DESTDIR=$RPM_BUILD_ROOT mandir=%{_mandir} install %license COPYING %doc AUTHORS docs/livemedia-creator.rst docs/product-images.rst %doc docs/*ks -%{python_sitelib}/pylorax -%{python_sitelib}/*.egg-info +%{python3_sitelib}/pylorax +%{python3_sitelib}/*.egg-info %{_sbindir}/lorax %{_sbindir}/mkefiboot %{_sbindir}/livemedia-creator @@ -105,6 +129,10 @@ make DESTDIR=$RPM_BUILD_ROOT mandir=%{_mandir} install %{_datadir}/lorax/* %{_mandir}/man1/*.1*
+%files lmc-virt + +%files lmc-novirt + %changelog * Thu Apr 02 2015 Brian C. Lane bcl@redhat.com 23.8-1 - Include cryptsetup in the image (#1208214) (bcl@redhat.com) diff --git a/setup.py b/setup.py index 791ffb8..86b16bc 100644 --- a/setup.py +++ b/setup.py @@ -35,10 +35,9 @@ version=vernum, description="Lorax", long_description="", - author="Martin Gracik", - author_email="mgracik@redhat.com", - url="http://", - download_url="http://", + author="Martin Gracik, Will Woods wwoods@redhat.com, Brian C. Lane bcl@redhat.com", + url="http://www.github.com/rhinstaller/lorax/", + download_url="http://www.github.com/rhinstaller/lorax/releases/", license="GPLv2+", packages=["pylorax"], package_dir={"" : "src"},
anaconda-patches@lists.fedorahosted.org