[lorax 1/2] Cleanup manifest patch

Brian C. Lane bcl at redhat.com
Tue Sep 17 20:36:20 UTC 2013


From: "Brian C. Lane" <bcl at redhat.com>

Add error checking, don't fail the whole operation on just one part
failing.

Make sure execution happens inside chroot. Some of them were actually
running on the host's filesystem.

refactor deps gathering so rpm -qa is only run once.

(note that this patch will be squashed with the previous one and these
comments removed).
---
 src/pylorax/treebuilder.py | 140 +++++++++++++++++++++++++++++----------------
 src/sbin/livemedia-creator |   2 +-
 2 files changed, 91 insertions(+), 51 deletions(-)

diff --git a/src/pylorax/treebuilder.py b/src/pylorax/treebuilder.py
index 0311118..2581a1b 100644
--- a/src/pylorax/treebuilder.py
+++ b/src/pylorax/treebuilder.py
@@ -28,8 +28,8 @@ from shutil import copytree, copy2
 from base import DataHolder
 from ltmpl import LoraxTemplateRunner
 import imgutils
-from pylorax.executils import runcmd, runcmd_output
-from subprocess import Popen, PIPE, STDOUT
+from pylorax.executils import runcmd, runcmd_output, execWithCapture
+from subprocess import STDOUT, CalledProcessError
 
 templatemap = {
     'i386':    'x86.tmpl',
@@ -180,92 +180,132 @@ class TreeBuilder(object):
         self._runner = LoraxTemplateRunner(inroot, outroot, templatedir=templatedir)
         self._runner.defaults = self.vars
         self.templatedir = templatedir
+        self._manifest_path = joinpaths(self.vars.outroot, "isolinux")
 
     @property
     def kernels(self):
         return findkernels(root=self.vars.inroot)
 
     def collect_manifest(self, cmd, output_file, sort=False):
-        output = runcmd_output(cmd, root=self.vars.inroot)
-        with open(os.path.join(self.manifest_path, output_file), "w") as f:
-            if sort:
-                output = output.split("\n")
-                for line in sorted(set(output),key=str.lower):
-                    if line:
-                        f.write("%s\n" % line)
-            else:
-                f.write(output)
+        """ Collect the output of cmd into output_file, optionally sorting it.
+        """
+        try:
+            output = runcmd_output(cmd, root=self.vars.inroot)
+        except CalledProcessError as e:
+            logger.error(e)
+        else:
+            with open(joinpaths(self._manifest_path, output_file), "w") as f:
+                if sort:
+                    output = output.split("\n")
+                    for line in sorted(set(output), key=str.lower):
+                        if line:
+                            f.write("%s\n" % line)
+                else:
+                    f.write(output)
 
     def create_manifests(self):
-        self.manifest_path = os.path.join(self.vars.outroot, "isolinux")
-        env = os.environ.copy()
-        env['LC_ALL'] = 'C'
-        os.makedirs(self.manifest_path)
+        """ Create the manifest files
+
+            This gathers up various pieces of information about the
+            installed image and writes it to manifest_* files in
+            the isolinux directory.
+        """
+        os.makedirs(self._manifest_path)
         # collect installed packages information
+        logger.info("Collecting information on installed packages")
         cmd = ["rpm", "-qa", "--qf",
                "%{name}-%{version}-%{release}.%{arch} (%{SIGPGP:pgpsig})\n"]
         self.collect_manifest(cmd, "manifest-rpm.txt", True)
 
         # collect source rpms
+        logger.info("Collecting information on source packages")
         cmd = ["rpm", "-qa", "--qf", "%{sourcerpm}\n"]
         self.collect_manifest(cmd, "manifest-srpm.txt", True)
 
         # collect all included licenses rhbz#601927
+        logger.info("Collecting information on installed licenses")
         cmd = ["rpm", "-qa", "--qf", "%{license}\n"]
         self.collect_manifest(cmd, "manifest-license.txt", True)
 
         # get individial file sizes
+        logger.info("Collecting information on file sizes")
         cmd = ["du", "-akx", "--exclude=/var/cache/yum", "/"]
         self.collect_manifest(cmd, "manifest-file.txt", True)
 
         # get directory sizes
+        logger.info("Collecting information on directory sizes")
         cmd = ["du", "-x", "--exclude=/var/cache/yum", "/"]
         self.collect_manifest(cmd, "manifest-dir.txt", True)
 
         # find orphaned files
         file_list = []
-        cmd = ["rpm", "-qa"]
-        installed_rpms = runcmd_output(cmd).strip()
-
-        for pkg in installed_rpms.split("\n"):
-            cmd = ["rpm", "-ql", pkg]
-            pkg_files = runcmd_output(cmd).strip()
-            for f in pkg_files.split("\n"):
-                file_list.append("%s##%s" % (f, pkg))
-
-        cmd = ["find", "/", "-xdev"]
-        fs_list = runcmd_output(cmd).strip()
-
-        for f in fs_list.split("\n"):
-            file_list.append("%s##Not owned by any package." % f)
+        deps_list = []
+        installed_rpms = ""
+        try:
+            logger.info("Finding orphaned files and dependency list")
+            cmd = ["rpm", "-qa"]
+            installed_rpms = runcmd_output(cmd, root=self.vars.inroot).strip()
+        except CalledProcessError as e:
+            logger.error(e)
+        else:
+            for pkg in installed_rpms.split("\n"):
+                try:
+                    cmd = ["rpm", "-ql", pkg]
+                    pkg_files = runcmd_output(cmd, root=self.vars.inroot).strip()
+                except CalledProcessError as e:
+                    logger.error(e)
+                else:
+                    for f in pkg_files.split("\n"):
+                        file_list.append((f, pkg))
+
+                # Get the dependencies
+                try:
+                    cmd = ["rpm", "-e", "--test", pkg]
+                    dep_files = execWithCapture(cmd[0], cmd[1:], stderr=STDOUT,
+                                                root=self.vars.inroot)
+                except RuntimeError as e:
+                    logger.error(e)
+                else:
+                    if dep_files:
+                        for dep in dep_files.split("\n"):
+                            deps_list.append(dep)
+
+        deps_file = joinpaths(self._manifest_path, "manifest-deps.txt")
+        with open(deps_file, "w") as f:
+            for line in deps_list:
+                f.write("%s\n" % line)
+
+        try:
+            cmd = ["find", "/", "-xdev"]
+            fs_list = runcmd_output(cmd, root=self.vars.inroot).strip()
+        except CalledProcessError as e:
+            logger.error(e)
+        else:
+            for f in fs_list.split("\n"):
+                file_list.append((f, "Not owned by any package."))
 
         final_list = {}
-
-        for item in file_list:
-            f, owner = item.split("##")
+        for f, owner in file_list:
             if not f in final_list:
                 final_list[f] = owner
 
-        output_file = os.path.join(self.manifest_path, "manifest-owns.txt")
-        with open(output_file, "w") as t:
-            for x in final_list.keys():
-                t.write("%s\n%s\n" % (x, final_list[x]))
-
-        # get dependency list
-        cmd = ["rpm", "-qa"]
-        output = runcmd_output(cmd, root=self.vars.inroot)
-        piped_cmd = ["xargs", "-n1", "rpm", "-e", "--test"]
-        run_cmd = Popen(piped_cmd, stdout=PIPE, stdin=PIPE, stderr=PIPE)
-        stdout_data = run_cmd.communicate(input=output)[1]
-        deps_file = os.path.join(self.manifest_path, "manifest-deps.txt")
-        with open(deps_file, "w") as f:
-            f.write("%s\n" % stdout_data)
+        output_file = joinpaths(self._manifest_path, "manifest-owns.txt")
+        with open(output_file, "w") as f:
+            for x in final_list:
+                f.write("%s\n%s\n" % (x, final_list[x]))
+
 
-        # bzip large manufest files
+        # bzip large manifest files
+        logger.info("Compressing manifest files")
         for f in ["deps", "owns", "file", "dir" ]:
-            path = os.path.join(self.manifest_path, "manifest-%s.txt" % f)
-            cmd = ["bzip2", path]
-            runcmd(cmd)
+            path = joinpaths(self._manifest_path, "manifest-%s.txt" % f)
+            if not os.path.exists(path):
+                continue
+            try:
+                cmd = ["bzip2", path]
+                runcmd(cmd)
+            except CalledProcessError as e:
+                logger.error(e)
 
     def rebuild_initrds(self, add_args=[], backup="", prefix=""):
         '''Rebuild all the initrds in the tree. If backup is specified, each
diff --git a/src/sbin/livemedia-creator b/src/sbin/livemedia-creator
index 61c4a5e..d57655f 100755
--- a/src/sbin/livemedia-creator
+++ b/src/sbin/livemedia-creator
@@ -535,8 +535,8 @@ def make_livecd(opts, mount_dir, work_dir):
     log.info("dracut args = {0}".format(dracut_args))
     tb.rebuild_initrds(add_args=dracut_args)
     log.info("Building boot.iso")
-    log.info("Generating Manifests")
     if opts.create_manifests and opts.make_iso:
+        log.info("Generating Manifests")
         tb.create_manifests()
     tb.build()
 
-- 
1.8.3.1



More information about the anaconda-patches mailing list