Signed-off-by: Nikola Pajkovsky npajkovs@redhat.com --- .gitignore | 1 + configure.ac | 9 +- src/daemon/Makefile.am | 1 - src/daemon/abrtd.c | 2 - src/hooks/CCpp.conf | 4 + src/plugins/Makefile.am | 10 +- src/plugins/abrt-action-generate-backtrace.c | 27 +- src/plugins/abrt-action-install-debuginfo | 536 ------------------- .../abrt-action-install-debuginfo-to-abrt-cache.c | 28 +- src/plugins/abrt-action-install-debuginfo.in | 553 ++++++++++++++++++++ 10 files changed, 591 insertions(+), 580 deletions(-) delete mode 100644 src/plugins/abrt-action-install-debuginfo create mode 100644 src/plugins/abrt-action-install-debuginfo.in
diff --git a/.gitignore b/.gitignore index 8e96de3..d0df191 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,7 @@ src/plugins/abrt-action-install-debuginfo-to-abrt-cache src/plugins/abrt-action-trim-files src/plugins/abrt-dump-oops src/plugins/abrt-retrace-client +src/plugins/abrt-action-install-debuginfo src/hooks/abrt_exception_handler.py src/hooks/abrt-hook-ccpp src/applet/abrt-applet.desktop diff --git a/configure.ac b/configure.ac index ad32478..8debc2d 100644 --- a/configure.ac +++ b/configure.ac @@ -90,19 +90,11 @@ EVENTS_DIR='${sysconfdir}/libreport/events' EVENTS_CONF_DIR='${sysconfdir}/libreport/events.d' ENABLE_SOCKET_OR_DBUS='-DENABLE_DBUS=1'
-DEBUG_INFO_DIR='${localstatedir}/cache/${PACKAGE_NAME}-di' - -AC_ARG_WITH(debuginfosdir, - [AS_HELP_STRING([--with-debuginfodir=DIR], - [Directory where debuginfo is stored])], - [DEBUG_INFO_DIR="$withval"]) - AC_SUBST(CONF_DIR) AC_SUBST(VAR_RUN) AC_SUBST(PLUGINS_CONF_DIR) AC_SUBST(EVENTS_CONF_DIR) AC_SUBST(EVENTS_DIR) -AC_SUBST(DEBUG_INFO_DIR)
AC_CONFIG_HEADERS([config.h])
@@ -118,6 +110,7 @@ AC_CONFIG_FILES([ src/applet/Makefile src/gui-gtk/Makefile src/cli/Makefile + src/plugins/abrt-action-install-debuginfo po/Makefile.in icons/Makefile ]) diff --git a/src/daemon/Makefile.am b/src/daemon/Makefile.am index 2fc2b65..b6e4284 100644 --- a/src/daemon/Makefile.am +++ b/src/daemon/Makefile.am @@ -25,7 +25,6 @@ abrtd_CPPFLAGS = \ -I$(srcdir)/../include \ -I$(srcdir)/../lib \ -DVAR_RUN="$(VAR_RUN)" \ - -DDEBUG_INFO_DIR="$(DEBUG_INFO_DIR)" \ $(GLIB_CFLAGS) \ $(DBUS_CFLAGS) \ $(LIBREPORT_CFLAGS) \ diff --git a/src/daemon/abrtd.c b/src/daemon/abrtd.c index ed0add1..b6b06ee 100644 --- a/src/daemon/abrtd.c +++ b/src/daemon/abrtd.c @@ -601,8 +601,6 @@ static void sanitize_dump_dir_rights() /* 07000 bits are setuid, setgit, and sticky, and they must be unset */ /* 00777 bits are usual "rwxrwxrwx" access rights */ ensure_writable_dir(g_settings_dump_location, 0755, "abrt"); - /* debuginfo cache */ - ensure_writable_dir(DEBUG_INFO_DIR, 0775, "abrt"); /* temp dir */ ensure_writable_dir(VAR_RUN"/abrt", 0755, "root"); } diff --git a/src/hooks/CCpp.conf b/src/hooks/CCpp.conf index efd4ddd..05a21cf 100644 --- a/src/hooks/CCpp.conf +++ b/src/hooks/CCpp.conf @@ -7,3 +7,7 @@ MakeCompatCore = yes # Do you want a copy of crashed binary be saved? # (useful, for example, when _deleted binary_ segfaults) SaveBinaryImage = no + +# Specify where you want to store debuginfos (default: /var/cache/abrt-di) +# +#DebuginfoLocation = /var/cache/abrt-di diff --git a/src/plugins/Makefile.am b/src/plugins/Makefile.am index c997c6f..2f81c98 100644 --- a/src/plugins/Makefile.am +++ b/src/plugins/Makefile.am @@ -70,7 +70,7 @@ man1_MANS = ${MAN_TXT:%.txt=%.1} CLEANFILES = $(man1_MANS)
PYTHON_FILES = \ - abrt-action-install-debuginfo \ + abrt-action-install-debuginfo.in \ abrt-action-list-dsos \ abrt-action-analyze-core
@@ -162,12 +162,14 @@ abrt_action_generate_backtrace_CPPFLAGS = \ -I$(srcdir)/../include \ -I$(srcdir)/../lib \ -DLOCALSTATEDIR='"$(localstatedir)"' \ + -DPLUGINS_CONF_DIR="$(PLUGINS_CONF_DIR)" \ $(GLIB_CFLAGS) \ $(LIBREPORT_CFLAGS) \ -D_GNU_SOURCE \ -Wall -Wwrite-strings -Werror abrt_action_generate_backtrace_LDADD = \ - $(LIBREPORT_LIBS) + $(LIBREPORT_LIBS) \ + ../lib/libabrt.la
abrt_action_analyze_backtrace_SOURCES = \ abrt-action-analyze-backtrace.c @@ -191,7 +193,9 @@ abrt_action_install_debuginfo_to_abrt_cache_CPPFLAGS = \ -D_GNU_SOURCE \ $(LIBREPORT_CFLAGS) \ -Wall -Wwrite-strings -abrt_action_install_debuginfo_to_abrt_cache_LDADD = +abrt_action_install_debuginfo_to_abrt_cache_LDADD = \ + $(LIBREPORT_LIBS) \ + ../lib/libabrt.la
abrt_retrace_client_SOURCES = \ abrt-retrace-client.c diff --git a/src/plugins/abrt-action-generate-backtrace.c b/src/plugins/abrt-action-generate-backtrace.c index 9fb0ab6..170b29a 100644 --- a/src/plugins/abrt-action-generate-backtrace.c +++ b/src/plugins/abrt-action-generate-backtrace.c @@ -22,10 +22,7 @@ #include "abrtlib.h"
-#define DEBUGINFO_CACHE_DIR LOCALSTATEDIR"/cache/abrt-di" - static const char *dump_dir_name = "."; -static const char *debuginfo_dirs = DEBUGINFO_CACHE_DIR; /* 60 seconds was too limiting on slow machines */ static int exec_timeout_sec = 240;
@@ -122,7 +119,7 @@ static char* exec_vp(char **args, uid_t uid, int redirect_stderr, int *status) return strbuf_free_nobuf(buf_out); }
-static char *get_backtrace(struct dump_dir *dd) +static char *get_backtrace(struct dump_dir *dd, const char *debuginfo_dirs) { char *uid_str = dd_load_text_ext(dd, FILENAME_UID, DD_FAIL_QUIETLY_ENOENT | DD_LOAD_TEXT_RETURN_NULL_ON_FAILURE); uid_t uid = -1L; @@ -273,10 +270,21 @@ int main(int argc, char **argv)
export_abrt_envvars(0);
+ map_string_h *settings = new_map_string(); + if (!load_conf_file(PLUGINS_CONF_DIR"/CCpp.conf", settings, /*skip key w/o values:*/ false)) + error_msg("Can't open '%s'", PLUGINS_CONF_DIR"/CCpp.conf"); + + char *value = g_hash_table_lookup(settings, "DebuginfoLocation"); + char *debuginfo_location; + if (value) + debuginfo_location = xstrdup(value); + else + debuginfo_location = xstrdup(LOCALSTATEDIR"/cache/abrt-di"); + + free_map_string(settings); + char *debuginfo_dirs = NULL; if (i_opt) - { - debuginfo_dirs = xasprintf("%s:%s", DEBUGINFO_CACHE_DIR, i_opt); - } + debuginfo_dirs = xasprintf("%s:%s", debuginfo_location, i_opt);
struct dump_dir *dd = dd_opendir(dump_dir_name, /*flags:*/ 0); if (!dd) @@ -284,12 +292,15 @@ int main(int argc, char **argv)
/* Create gdb backtrace */ /* NB: get_backtrace() closes dd */ - char *backtrace = get_backtrace(dd); + char *backtrace = get_backtrace(dd, (debuginfo_dirs) ? debuginfo_dirs : debuginfo_location); + free(debuginfo_location); if (!backtrace) { backtrace = xstrdup(""); log("get_backtrace() returns NULL, broken core/gdb?"); } + free(debuginfo_dirs); + free_abrt_conf_data();
/* Store gdb backtrace */
diff --git a/src/plugins/abrt-action-install-debuginfo b/src/plugins/abrt-action-install-debuginfo deleted file mode 100644 index 4b01c7f..0000000 --- a/src/plugins/abrt-action-install-debuginfo +++ /dev/null @@ -1,536 +0,0 @@ -#! /usr/bin/python -u -# -*- coding: utf-8 -*- -# WARNING: python -u means unbuffered I/O without it the messages are -# passed to the parent asynchronously which looks bad in clients. - -PROGNAME = "abrt-action-install-debuginfo" - -import sys -import os -import errno -import time -import getopt -import shutil -import reportclient -from subprocess import Popen, PIPE -from yum import _, YumBase -from yum.callbacks import DownloadBaseCallback - -# everything was ok -RETURN_OK = 0 -# serious problem, should be logged somewhere -RETURN_FAILURE = 2 - - -GETTEXT_PROGNAME = "abrt" -import locale -import gettext - -_ = lambda x: gettext.lgettext(x) - -def init_gettext(): - try: - locale.setlocale(locale.LC_ALL, "") - except locale.Error: - os.environ['LC_ALL'] = 'C' - locale.setlocale(locale.LC_ALL, "") - gettext.bind_textdomain_codeset(GETTEXT_PROGNAME, locale.nl_langinfo(locale.CODESET)) - gettext.bindtextdomain(GETTEXT_PROGNAME, '/usr/share/locale') - gettext.textdomain(GETTEXT_PROGNAME) - - -verbose = 0 -def log(fmt, *args): - sys.stderr.write("%s\n" % (fmt % args)) - -def log1(fmt, *args): - """ prints log message if verbosity >= 1 """ - if verbose >= 1: - sys.stderr.write("%s\n" % (fmt % args)) - -def log2(fmt, *args): - """ prints log message if verbosity >= 2 """ - if verbose >= 2: - sys.stderr.write("%s\n" % (fmt % args)) - -def error_msg(fmt, *args): - sys.stderr.write("%s\n" % (fmt % args)) - -def error_msg_and_die(fmt, *args): - sys.stderr.write("%s\n" % (fmt % args)) - sys.exit(1) - - -old_stdout = -1 -def mute_stdout(): - if verbose < 2: - global old_stdout - old_stdout = sys.stdout - sys.stdout = open("/dev/null", "w") - -def unmute_stdout(): - if verbose < 2: - if old_stdout != -1: - sys.stdout = old_stdout - else: - print "ERR: unmute called without mute?" - -# TODO: unpack just required debuginfo and not entire rpm? -# ..that can lead to: foo.c No such file and directory -# files is not used... -def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm): - package_full_path = tmp_dir + "/" + package_file_name - log1("Extracting %s to %s", package_full_path, destdir) - log2("%s", files) - print _("Extracting cpio from %s") % (package_full_path) - unpacked_cpio_path = tmp_dir + "/unpacked.cpio" - try: - unpacked_cpio = open(unpacked_cpio_path, 'wb') - except IOError, ex: - print _("Can't write to '%s': %s") % (unpacked_cpio_path, ex) - return RETURN_FAILURE - rpm2cpio = Popen(["rpm2cpio", package_full_path], - stdout = unpacked_cpio, bufsize = -1) - retcode = rpm2cpio.wait() - - if retcode == 0: - log1("cpio written OK") - if not keeprpm: - log1("keeprpms = False, removing %s", package_full_path) - #print _("Removing temporary rpm file") - os.unlink(package_full_path) - else: - unpacked_cpio.close() - print _("Can't extract package '%s'") % package_full_path - return RETURN_FAILURE - - # close the file - unpacked_cpio.close() - # and open it for reading - unpacked_cpio = open(unpacked_cpio_path, 'rb') - - print _("Caching files from %s made from %s") % ("unpacked.cpio", package_file_name) - cpio = Popen(["cpio", "-idu", "--quiet"], - stdin=unpacked_cpio, cwd=destdir, bufsize=-1) - retcode = cpio.wait() - - if retcode == 0: - log1("files extracted OK") - #print _("Removing temporary cpio file") - os.unlink(unpacked_cpio_path) - else: - print _("Can't extract files from '%s'") % unpacked_cpio_path - return RETURN_FAILURE - -class MyDownloadCallback(DownloadBaseCallback): - def __init__(self, total_pkgs): - self.total_pkgs = total_pkgs - self.downloaded_pkgs = 0 - self.last_pct = 0 - self.last_time = 0 - DownloadBaseCallback.__init__(self) - - def updateProgress(self, name, frac, fread, ftime): - pct = int(frac * 100) - if pct == self.last_pct: - log2("percentage is the same, not updating progress") - return - - self.last_pct = pct - # if run from terminal we can have fancy output - if sys.stdout.isatty(): - sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u" - % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) - ) - if pct == 100: - print (_("Downloading (%i of %i) %s: %3u%%") - % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) - ) - # but we want machine friendly output when spawned from abrt-server - else: - t = time.time() - if self.last_time == 0: - self.last_time = t - # update only every 5 seconds - if pct == 100 or self.last_time > t or t - self.last_time >= 5: - print (_("Downloading (%i of %i) %s: %3u%%") - % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) - ) - self.last_time = t - if pct == 100: - self.last_time = 0 - - sys.stdout.flush() - -class DebugInfoDownload(YumBase): - def __init__(self, cache, tmp, keep_rpms=False): - self.cachedir = cache - self.tmpdir = tmp - self.keeprpms = keep_rpms - YumBase.__init__(self) - mute_stdout() - #self.conf.cache = os.geteuid() != 0 - # Setup yum (Ts, RPM db, Repo & Sack) - # doConfigSetup() takes some time, let user know what we are doing - print _("Initializing yum") - try: - # Saw this exception here: - # cannot open Packages index using db3 - Permission denied (13) - # yum.Errors.YumBaseError: Error: rpmdb open failed - self.doConfigSetup() - except Exception, e: - unmute_stdout() - print _("Error initializing yum (YumBase.doConfigSetup): '%s'") % str(e) - #return 1 - can't do this in constructor - exit(1) - unmute_stdout() - - # return value will be used as exitcode. So 0 = ok, !0 - error - def download(self, files): - """ @files - """ - installed_size = 0 - total_pkgs = 0 - todownload_size = 0 - downloaded_pkgs = 0 - # nothing to download? - if not files: - return - - #if verbose == 0: - # # this suppress yum messages about setting up repositories - # mute_stdout() - - # make yumdownloader work as non root user - if not self.setCacheDir(): - self.logger.error("Error: can't make cachedir, exiting") - exit(50) - - # disable all not needed - for repo in self.repos.listEnabled(): - repo.close() - self.repos.disableRepo(repo.id) - - # This takes some time, let user know what we are doing - print _("Setting up yum repositories") - # setting-up repos one-by-one, so we can skip the broken ones... - # this helps when users are using 3rd party repos like rpmfusion - # in rawhide it results in: Can't find valid base url... - for r in self.repos.findRepos(pattern="*debuginfo*"): - try: - rid = self.repos.enableRepo(r.id) - self.repos.doSetup(thisrepo=str(r.id)) - log1("enabled repo %s", rid) - setattr(r, "skip_if_unavailable", True) - except Exception, ex: - print _("Can't setup %s: %s, disabling") % (r.id, ex) - self.repos.disableRepo(r.id) - - # This is somewhat "magic", it unpacks the metadata making it usable. - # Looks like this is the moment when yum talks to remote servers, - # which takes time (sometimes minutes), let user know why - # we have "paused": - print _("Looking for needed packages in repositories") - try: - self.repos.populateSack(mdtype='metadata', cacheonly=1) - except Exception, e: - print _("Error retrieving metadata: '%s'") % str(e) - return 1 - try: - # Saw this exception here: - # raise Errors.NoMoreMirrorsRepoError, errstr - # NoMoreMirrorsRepoError: failure: - # repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2 - # from fedora-debuginfo: [Errno 256] No more mirrors to try. - self.repos.populateSack(mdtype='filelists', cacheonly=1) - except Exception, e: - print _("Error retrieving filelists: '%s'") % str(e) - return 1 - - #if verbose == 0: - # # re-enable the output to stdout - # unmute_stdout() - - not_found = [] - package_files_dict = {} - for debuginfo_path in files: - log2("yum whatprovides %s", debuginfo_path) - pkg = self.pkgSack.searchFiles(debuginfo_path) - # sometimes one file is provided by more rpms, we can use either of - # them, so let's use the first match - if pkg: - if pkg[0] in package_files_dict.keys(): - package_files_dict[pkg[0]].append(debuginfo_path) - else: - package_files_dict[pkg[0]] = [debuginfo_path] - todownload_size += float(pkg[0].size) - installed_size += float(pkg[0].installedsize) - total_pkgs += 1 - - log2("found pkg for %s: %s", debuginfo_path, pkg[0]) - else: - log2("not found pkg for %s", debuginfo_path) - not_found.append(debuginfo_path) - - # connect our progress update callback - dnlcb = MyDownloadCallback(total_pkgs) - self.repos.setProgressBar(dnlcb) - - if verbose != 0 or len(not_found) != 0: - print _("Can't find packages for %u debuginfo files") % len(not_found) - if verbose != 0 or total_pkgs != 0: - print _("Packages to download: %u") % total_pkgs - question = _("Downloading %.2fMb, installed size: %.2fMb. Continue?") % ( - todownload_size / (1024**2), - installed_size / (1024**2) - ) - if noninteractive == False and not reportclient.ask_yes_no(question): - print _("Download cancelled by user") - return RETURN_OK - - for pkg, files in package_files_dict.iteritems(): - dnlcb.downloaded_pkgs = downloaded_pkgs - repo.cache = 0 - remote = pkg.returnSimple('relativepath') - local = os.path.basename(remote) - if not os.path.exists(self.tmpdir): - os.makedirs(self.tmpdir) - if not os.path.exists(self.cachedir): - os.makedirs(self.cachedir) - local = os.path.join(self.tmpdir, local) - pkg.localpath = local # Hack: to set the localpath we want - err = self.downloadPkgs(pkglist=[pkg]) - # normalize the name - # just str(pkg) doesn't work because it can have epoch - pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch - package_file_name = pkg_nvra + ".rpm" - if err: - # I observed a zero-length file left on error, - # which prevents cleanup later. Fix it: - try: - os.unlink(self.tmpdir + "/" + package_file_name) - except: - pass - print (_("Downloading package %s failed") % pkg) - else: - unpack_result = unpack_rpm(package_file_name, files, self.tmpdir, - self.cachedir, keeprpms) - if unpack_result == RETURN_FAILURE: - # recursively delete the temp dir on failure - print _("Unpacking failed, aborting download...") - clean_up() - return RETURN_FAILURE - - downloaded_pkgs += 1 - - if not self.keeprpms and os.path.exists(self.tmpdir): - # Was: "All downloaded packages have been extracted, removing..." - # but it was appearing even if no packages were in fact extracted - # (say, when there was one package, and it has download error). - print (_("Removing %s") % self.tmpdir) - try: - os.rmdir(self.tmpdir) - except OSError: - error_msg(_("Can't remove %s, probably contains an error log"), self.tmpdir) - -def build_ids_to_path(build_ids): - """ - build_id1=${build_id:0:2} - build_id2=${build_id:2} - file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug" - """ - return ["/usr/lib/debug/.build-id/%s/%s.debug" % (b_id[:2], b_id[2:]) for b_id in build_ids] - -# beware this finds only missing libraries, but not the executable itself .. - -def filter_installed_debuginfos(build_ids, cache_dirs): - files = build_ids_to_path(build_ids) - missing = [] - - # 1st pass -> search in /usr/lib - for debuginfo_path in files: - log2("looking: %s", debuginfo_path) - if os.path.exists(debuginfo_path): - log2("found: %s", debuginfo_path) - continue - log2("not found: %s", debuginfo_path) - missing.append(debuginfo_path) - - if missing: - files = missing - missing = [] - else: # nothing is missing, we can stop looking - return missing - - for cache_dir in cache_dirs: - log2("looking in %s" % cache_dir) - for debuginfo_path in files: - cache_debuginfo_path = cache_dir + debuginfo_path - log2("looking: %s", cache_debuginfo_path) - if os.path.exists(cache_debuginfo_path): - log2("found: %s", cache_debuginfo_path) - continue - log2("not found: %s", debuginfo_path) - missing.append(debuginfo_path) - # in next iteration look only for files missing - # from previous iterations - if missing: - files = missing - missing = [] - else: # nothing is missing, we can stop looking - return missing - - return files - -tmpdir = None -def clean_up(): - if tmpdir: - try: - shutil.rmtree(tmpdir) - except OSError, ex: - if ex.errno != errno.ENOENT: - error_msg(_("Can't remove '%s': %s"), tmpdir, ex) - -def sigterm_handler(signum, frame): - clean_up() - exit(RETURN_OK) - -def sigint_handler(signum, frame): - clean_up() - print "\n", _("Exiting on user command") - sys.stdout.flush() - # ??! without "sys.", I am getting segv! - sys.exit(RETURN_OK) - -import signal - -if __name__ == "__main__": - # abrt-server can send SIGTERM to abort the download - signal.signal(signal.SIGTERM, sigterm_handler) - # ctrl-c - signal.signal(signal.SIGINT, sigint_handler) - fbuild_ids = "build_ids" - cachedirs = [] - size_mb = 4096 - tmpdir = None - keeprpms = False - noninteractive = False - b_ids = [] - - # localization - init_gettext() - - ABRT_VERBOSE = os.getenv("ABRT_VERBOSE") - if (ABRT_VERBOSE): - try: - verbose = int(ABRT_VERBOSE) - except: - pass - - PROGNAME = os.path.basename(sys.argv[0]) - # ____________________________________________________________________________________ 7 - # ______01234567890123456789012345678901234567890123456789012345678901234567890123456789 - help_text = _( - "Usage: %s [-vy] [--ids=BUILD_IDS_FILE]\n" - " [--tmpdir=TMPDIR] [--cache=CACHEDIR[:DEBUGINFODIR1:DEBUGINFODIR2...]] [--size_mb=SIZE]\n" - "\n" - "Installs debuginfos for all build-ids listed in BUILD_IDS_FILE\n" - "to CACHEDIR, using TMPDIR as temporary staging area.\n" - "Old files in CACHEDIR are deleted until it is smaller than SIZE.\n" - "\n" - " -v Be verbose\n" - " -y Noninteractive, assume 'Yes' to all questions\n" - " --ids Default: build_ids\n" - " --tmpdir Default: /tmp/abrt-tmp-debuginfo-RANDOM_SUFFIX\n" - " --cache Default: /var/cache/abrt-di\n" - " --size_mb Default: 4096\n" - # --keeprpms is not documented yet because it's a NOP so far - ) % PROGNAME - - try: - opts, args = getopt.getopt(sys.argv[1:], "vyh", - ["help", "ids=", "cache=", "size_mb=", "tmpdir=", "keeprpms"]) - except getopt.GetoptError, err: - print str(err) # prints something like "option -a not recognized" - exit(RETURN_FAILURE) - - for opt, arg in opts: - if opt in ("-h", "--help"): - print help_text - exit(0) - elif opt == "-v": - verbose += 1 - elif opt == "-y": - noninteractive = True - elif opt == "--ids": - fbuild_ids = arg - elif opt == "--cache": - cachedirs = arg.split(':') - elif opt == "--size_mb": - try: - size_mb = int(arg) - except: - pass - elif opt == "--tmpdir": - tmpdir = arg - elif opt == "--keeprpms": - keeprpms = True - - if not cachedirs: - cachedirs = ["/var/cache/abrt-di"] - if not tmpdir: - # security people prefer temp subdirs in app's private dir, like /var/run/abrt - # for now, we use /tmp... - tmpdir = "/tmp/abrt-tmp-debuginfo-%s.%u" % (time.strftime("%Y-%m-%d-%H:%M:%S"), os.getpid()) - - fin = sys.stdin - if fbuild_ids != "-": - try: - fin = open(fbuild_ids, "r") - except IOError, ex: - error_msg_and_die(_("Can't open %s: %s"), fbuild_ids, ex) - for line in fin.readlines(): - b_ids.append(line.strip('\n')) - - if not b_ids: - exit(RETURN_FAILURE) - - # Delete oldest/biggest files from cachedir. - # (Note that we need to do it before we check for missing debuginfos) - # - # We can do it as a separate step in abrt_event.conf, but this - # would require setuid'ing abrt-action-trim-files to abrt:abrt. - # Since we (via abrt-action-install-debuginfo-to-abrt-cache) - # are already running setuid, - # it makes sense to NOT setuid abrt-action-trim-files too, - # but instead run it as our child: - sys.stdout.flush() - try: - pid = os.fork() - if pid == 0: - log1("abrt-action-trim-files -f %um:%s", size_mb, cachedirs[0]); - os.execlp("abrt-action-trim-files", "abrt-action-trim-files", "-f", "%um:%s" % (size_mb, cachedirs[0])); - if pid > 0: - os.waitpid(pid, 0); - except Exception, e: - error_msg("Can't execute abrt-action-trim-files: %s", e); - - missing = filter_installed_debuginfos(b_ids, cachedirs) - if missing: - log2("%s", missing) - print _("Coredump references %u debuginfo files, %u of them are not installed") % (len(b_ids), len(missing)) - - # TODO: should we pass keep_rpms=keeprpms to DebugInfoDownload here?? - downloader = DebugInfoDownload(cache=cachedirs[0], tmp=tmpdir) - try: - result = downloader.download(missing) - except Exception, ex: - error_msg_and_die("Can't download debuginfos: %s", ex) - - missing = filter_installed_debuginfos(b_ids, cachedirs) - for bid in missing: - print _("Missing debuginfo file: %s") % bid - - if not missing: - print _("All %u debuginfo files are available") % len(b_ids) - - exit(RETURN_OK) diff --git a/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c b/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c index 8e96f78..24d9150 100644 --- a/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c +++ b/src/plugins/abrt-action-install-debuginfo-to-abrt-cache.c @@ -16,26 +16,10 @@ with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -#include <sys/types.h> -#include <unistd.h> -#include <stdlib.h> -#include <string.h>
-#define EXECUTABLE "abrt-action-install-debuginfo" - -static void error_msg_and_die(const char *msg, const char *arg) -{ - write(2, msg, strlen(msg)); - if (arg) - { - write(2, " '", 2); - write(2, arg, strlen(arg)); - write(2, "'", 1); - } - write(2, "\n", 1); - exit(1); -} +#include "abrtlib.h"
+#define EXECUTABLE "abrt-action-install-debuginfo"
/* A binary wrapper is needed around python scripts if we want * to run them in sgid/suid mode. @@ -58,11 +42,11 @@ int main(int argc, char **argv) continue;
if (strncmp(arg, "--cache", 7) == 0) - error_msg_and_die("bad option", arg); + error_msg_and_die("bad option %s", arg); if (strncmp(arg, "--tmpdir", 8) == 0) - error_msg_and_die("bad option", arg); + error_msg_and_die("bad option %s", arg); if (strncmp(arg, "--ids", 5) == 0) - error_msg_and_die("bad option", arg); + error_msg_and_die("bad option %s", arg); }
/* Switch real user/group to effective ones. @@ -104,5 +88,5 @@ int main(int argc, char **argv) }
execvp(EXECUTABLE, argv); - error_msg_and_die("Can't execute", EXECUTABLE); + error_msg_and_die("Can't execute %s", EXECUTABLE); } diff --git a/src/plugins/abrt-action-install-debuginfo.in b/src/plugins/abrt-action-install-debuginfo.in new file mode 100644 index 0000000..3e8a1c5 --- /dev/null +++ b/src/plugins/abrt-action-install-debuginfo.in @@ -0,0 +1,553 @@ +#! /usr/bin/python -u +# -*- coding: utf-8 -*- +# WARNING: python -u means unbuffered I/O without it the messages are +# passed to the parent asynchronously which looks bad in clients. + +PROGNAME = "abrt-action-install-debuginfo" + +import sys +import os +import errno +import time +import getopt +import shutil +import reportclient +from subprocess import Popen, PIPE +from yum import _, YumBase +from yum.callbacks import DownloadBaseCallback + +# everything was ok +RETURN_OK = 0 +# serious problem, should be logged somewhere +RETURN_FAILURE = 2 + + +GETTEXT_PROGNAME = "abrt" +import locale +import gettext + +_ = lambda x: gettext.lgettext(x) + +def init_gettext(): + try: + locale.setlocale(locale.LC_ALL, "") + except locale.Error: + os.environ['LC_ALL'] = 'C' + locale.setlocale(locale.LC_ALL, "") + gettext.bind_textdomain_codeset(GETTEXT_PROGNAME, locale.nl_langinfo(locale.CODESET)) + gettext.bindtextdomain(GETTEXT_PROGNAME, '/usr/share/locale') + gettext.textdomain(GETTEXT_PROGNAME) + + +verbose = 0 +def log(fmt, *args): + sys.stderr.write("%s\n" % (fmt % args)) + +def log1(fmt, *args): + """ prints log message if verbosity >= 1 """ + if verbose >= 1: + sys.stderr.write("%s\n" % (fmt % args)) + +def log2(fmt, *args): + """ prints log message if verbosity >= 2 """ + if verbose >= 2: + sys.stderr.write("%s\n" % (fmt % args)) + +def error_msg(fmt, *args): + sys.stderr.write("%s\n" % (fmt % args)) + +def error_msg_and_die(fmt, *args): + sys.stderr.write("%s\n" % (fmt % args)) + sys.exit(1) + + +old_stdout = -1 +def mute_stdout(): + if verbose < 2: + global old_stdout + old_stdout = sys.stdout + sys.stdout = open("/dev/null", "w") + +def unmute_stdout(): + if verbose < 2: + if old_stdout != -1: + sys.stdout = old_stdout + else: + print "ERR: unmute called without mute?" + +# TODO: unpack just required debuginfo and not entire rpm? +# ..that can lead to: foo.c No such file and directory +# files is not used... +def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm): + package_full_path = tmp_dir + "/" + package_file_name + log1("Extracting %s to %s", package_full_path, destdir) + log2("%s", files) + print _("Extracting cpio from %s") % (package_full_path) + unpacked_cpio_path = tmp_dir + "/unpacked.cpio" + try: + unpacked_cpio = open(unpacked_cpio_path, 'wb') + except IOError, ex: + print _("Can't write to '%s': %s") % (unpacked_cpio_path, ex) + return RETURN_FAILURE + rpm2cpio = Popen(["rpm2cpio", package_full_path], + stdout = unpacked_cpio, bufsize = -1) + retcode = rpm2cpio.wait() + + if retcode == 0: + log1("cpio written OK") + if not keeprpm: + log1("keeprpms = False, removing %s", package_full_path) + #print _("Removing temporary rpm file") + os.unlink(package_full_path) + else: + unpacked_cpio.close() + print _("Can't extract package '%s'") % package_full_path + return RETURN_FAILURE + + # close the file + unpacked_cpio.close() + # and open it for reading + unpacked_cpio = open(unpacked_cpio_path, 'rb') + + print _("Caching files from %s made from %s") % ("unpacked.cpio", package_file_name) + cpio = Popen(["cpio", "-idu", "--quiet"], + stdin=unpacked_cpio, cwd=destdir, bufsize=-1) + retcode = cpio.wait() + + if retcode == 0: + log1("files extracted OK") + #print _("Removing temporary cpio file") + os.unlink(unpacked_cpio_path) + else: + print _("Can't extract files from '%s'") % unpacked_cpio_path + return RETURN_FAILURE + +class MyDownloadCallback(DownloadBaseCallback): + def __init__(self, total_pkgs): + self.total_pkgs = total_pkgs + self.downloaded_pkgs = 0 + self.last_pct = 0 + self.last_time = 0 + DownloadBaseCallback.__init__(self) + + def updateProgress(self, name, frac, fread, ftime): + pct = int(frac * 100) + if pct == self.last_pct: + log2("percentage is the same, not updating progress") + return + + self.last_pct = pct + # if run from terminal we can have fancy output + if sys.stdout.isatty(): + sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u" + % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) + ) + if pct == 100: + print (_("Downloading (%i of %i) %s: %3u%%") + % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) + ) + # but we want machine friendly output when spawned from abrt-server + else: + t = time.time() + if self.last_time == 0: + self.last_time = t + # update only every 5 seconds + if pct == 100 or self.last_time > t or t - self.last_time >= 5: + print (_("Downloading (%i of %i) %s: %3u%%") + % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct) + ) + self.last_time = t + if pct == 100: + self.last_time = 0 + + sys.stdout.flush() + +class DebugInfoDownload(YumBase): + def __init__(self, cache, tmp, keep_rpms=False): + self.cachedir = cache + self.tmpdir = tmp + self.keeprpms = keep_rpms + YumBase.__init__(self) + mute_stdout() + #self.conf.cache = os.geteuid() != 0 + # Setup yum (Ts, RPM db, Repo & Sack) + # doConfigSetup() takes some time, let user know what we are doing + print _("Initializing yum") + try: + # Saw this exception here: + # cannot open Packages index using db3 - Permission denied (13) + # yum.Errors.YumBaseError: Error: rpmdb open failed + self.doConfigSetup() + except Exception, e: + unmute_stdout() + print _("Error initializing yum (YumBase.doConfigSetup): '%s'") % str(e) + #return 1 - can't do this in constructor + exit(1) + unmute_stdout() + + # return value will be used as exitcode. So 0 = ok, !0 - error + def download(self, files): + """ @files - """ + installed_size = 0 + total_pkgs = 0 + todownload_size = 0 + downloaded_pkgs = 0 + # nothing to download? + if not files: + return + + #if verbose == 0: + # # this suppress yum messages about setting up repositories + # mute_stdout() + + # make yumdownloader work as non root user + if not self.setCacheDir(): + self.logger.error("Error: can't make cachedir, exiting") + exit(50) + + # disable all not needed + for repo in self.repos.listEnabled(): + repo.close() + self.repos.disableRepo(repo.id) + + # This takes some time, let user know what we are doing + print _("Setting up yum repositories") + # setting-up repos one-by-one, so we can skip the broken ones... + # this helps when users are using 3rd party repos like rpmfusion + # in rawhide it results in: Can't find valid base url... + for r in self.repos.findRepos(pattern="*debuginfo*"): + try: + rid = self.repos.enableRepo(r.id) + self.repos.doSetup(thisrepo=str(r.id)) + log1("enabled repo %s", rid) + setattr(r, "skip_if_unavailable", True) + except Exception, ex: + print _("Can't setup %s: %s, disabling") % (r.id, ex) + self.repos.disableRepo(r.id) + + # This is somewhat "magic", it unpacks the metadata making it usable. + # Looks like this is the moment when yum talks to remote servers, + # which takes time (sometimes minutes), let user know why + # we have "paused": + print _("Looking for needed packages in repositories") + try: + self.repos.populateSack(mdtype='metadata', cacheonly=1) + except Exception, e: + print _("Error retrieving metadata: '%s'") % str(e) + return 1 + try: + # Saw this exception here: + # raise Errors.NoMoreMirrorsRepoError, errstr + # NoMoreMirrorsRepoError: failure: + # repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2 + # from fedora-debuginfo: [Errno 256] No more mirrors to try. + self.repos.populateSack(mdtype='filelists', cacheonly=1) + except Exception, e: + print _("Error retrieving filelists: '%s'") % str(e) + return 1 + + #if verbose == 0: + # # re-enable the output to stdout + # unmute_stdout() + + not_found = [] + package_files_dict = {} + for debuginfo_path in files: + log2("yum whatprovides %s", debuginfo_path) + pkg = self.pkgSack.searchFiles(debuginfo_path) + # sometimes one file is provided by more rpms, we can use either of + # them, so let's use the first match + if pkg: + if pkg[0] in package_files_dict.keys(): + package_files_dict[pkg[0]].append(debuginfo_path) + else: + package_files_dict[pkg[0]] = [debuginfo_path] + todownload_size += float(pkg[0].size) + installed_size += float(pkg[0].installedsize) + total_pkgs += 1 + + log2("found pkg for %s: %s", debuginfo_path, pkg[0]) + else: + log2("not found pkg for %s", debuginfo_path) + not_found.append(debuginfo_path) + + # connect our progress update callback + dnlcb = MyDownloadCallback(total_pkgs) + self.repos.setProgressBar(dnlcb) + + if verbose != 0 or len(not_found) != 0: + print _("Can't find packages for %u debuginfo files") % len(not_found) + if verbose != 0 or total_pkgs != 0: + print _("Packages to download: %u") % total_pkgs + question = _("Downloading %.2fMb, installed size: %.2fMb. Continue?") % ( + todownload_size / (1024**2), + installed_size / (1024**2) + ) + if noninteractive == False and not reportclient.ask_yes_no(question): + print _("Download cancelled by user") + return RETURN_OK + + for pkg, files in package_files_dict.iteritems(): + dnlcb.downloaded_pkgs = downloaded_pkgs + repo.cache = 0 + remote = pkg.returnSimple('relativepath') + local = os.path.basename(remote) + if not os.path.exists(self.tmpdir): + os.makedirs(self.tmpdir) + if not os.path.exists(self.cachedir): + os.makedirs(self.cachedir) + local = os.path.join(self.tmpdir, local) + pkg.localpath = local # Hack: to set the localpath we want + err = self.downloadPkgs(pkglist=[pkg]) + # normalize the name + # just str(pkg) doesn't work because it can have epoch + pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch + package_file_name = pkg_nvra + ".rpm" + if err: + # I observed a zero-length file left on error, + # which prevents cleanup later. Fix it: + try: + os.unlink(self.tmpdir + "/" + package_file_name) + except: + pass + print (_("Downloading package %s failed") % pkg) + else: + unpack_result = unpack_rpm(package_file_name, files, self.tmpdir, + self.cachedir, keeprpms) + if unpack_result == RETURN_FAILURE: + # recursively delete the temp dir on failure + print _("Unpacking failed, aborting download...") + clean_up() + return RETURN_FAILURE + + downloaded_pkgs += 1 + + if not self.keeprpms and os.path.exists(self.tmpdir): + # Was: "All downloaded packages have been extracted, removing..." + # but it was appearing even if no packages were in fact extracted + # (say, when there was one package, and it has download error). + print (_("Removing %s") % self.tmpdir) + try: + os.rmdir(self.tmpdir) + except OSError: + error_msg(_("Can't remove %s, probably contains an error log"), self.tmpdir) + +def build_ids_to_path(build_ids): + """ + build_id1=${build_id:0:2} + build_id2=${build_id:2} + file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug" + """ + return ["/usr/lib/debug/.build-id/%s/%s.debug" % (b_id[:2], b_id[2:]) for b_id in build_ids] + +# beware this finds only missing libraries, but not the executable itself .. + +def filter_installed_debuginfos(build_ids, cache_dirs): + files = build_ids_to_path(build_ids) + missing = [] + + # 1st pass -> search in /usr/lib + for debuginfo_path in files: + log2("looking: %s", debuginfo_path) + if os.path.exists(debuginfo_path): + log2("found: %s", debuginfo_path) + continue + log2("not found: %s", debuginfo_path) + missing.append(debuginfo_path) + + if missing: + files = missing + missing = [] + else: # nothing is missing, we can stop looking + return missing + + for cache_dir in cache_dirs: + log2("looking in %s" % cache_dir) + for debuginfo_path in files: + cache_debuginfo_path = cache_dir + debuginfo_path + log2("looking: %s", cache_debuginfo_path) + if os.path.exists(cache_debuginfo_path): + log2("found: %s", cache_debuginfo_path) + continue + log2("not found: %s", debuginfo_path) + missing.append(debuginfo_path) + # in next iteration look only for files missing + # from previous iterations + if missing: + files = missing + missing = [] + else: # nothing is missing, we can stop looking + return missing + + return files + +tmpdir = None +def clean_up(): + if tmpdir: + try: + shutil.rmtree(tmpdir) + except OSError, ex: + if ex.errno != errno.ENOENT: + error_msg(_("Can't remove '%s': %s"), tmpdir, ex) + +def sigterm_handler(signum, frame): + clean_up() + exit(RETURN_OK) + +def sigint_handler(signum, frame): + clean_up() + print "\n", _("Exiting on user command") + sys.stdout.flush() + # ??! without "sys.", I am getting segv! + sys.exit(RETURN_OK) + +import signal + +if __name__ == "__main__": + # abrt-server can send SIGTERM to abort the download + signal.signal(signal.SIGTERM, sigterm_handler) + # ctrl-c + signal.signal(signal.SIGINT, sigint_handler) + fbuild_ids = "build_ids" + cachedirs = [] + size_mb = 4096 + tmpdir = None + keeprpms = False + noninteractive = False + b_ids = [] + + # localization + init_gettext() + + ABRT_VERBOSE = os.getenv("ABRT_VERBOSE") + if (ABRT_VERBOSE): + try: + verbose = int(ABRT_VERBOSE) + except: + pass + + PROGNAME = os.path.basename(sys.argv[0]) + # ____________________________________________________________________________________ 7 + # ______01234567890123456789012345678901234567890123456789012345678901234567890123456789 + help_text = _( + "Usage: %s [-vy] [--ids=BUILD_IDS_FILE]\n" + " [--tmpdir=TMPDIR] [--cache=CACHEDIR[:DEBUGINFODIR1:DEBUGINFODIR2...]] [--size_mb=SIZE]\n" + "\n" + "Installs debuginfos for all build-ids listed in BUILD_IDS_FILE\n" + "to CACHEDIR, using TMPDIR as temporary staging area.\n" + "Old files in CACHEDIR are deleted until it is smaller than SIZE.\n" + "\n" + " -v Be verbose\n" + " -y Noninteractive, assume 'Yes' to all questions\n" + " --ids Default: build_ids\n" + " --tmpdir Default: /tmp/abrt-tmp-debuginfo-RANDOM_SUFFIX\n" + " --cache Default: /var/cache/abrt-di\n" + " --size_mb Default: 4096\n" + # --keeprpms is not documented yet because it's a NOP so far + ) % PROGNAME + + try: + opts, args = getopt.getopt(sys.argv[1:], "vyh", + ["help", "ids=", "cache=", "size_mb=", "tmpdir=", "keeprpms"]) + except getopt.GetoptError, err: + print str(err) # prints something like "option -a not recognized" + exit(RETURN_FAILURE) + + for opt, arg in opts: + if opt in ("-h", "--help"): + print help_text + exit(0) + elif opt == "-v": + verbose += 1 + elif opt == "-y": + noninteractive = True + elif opt == "--ids": + fbuild_ids = arg + elif opt == "--cache": + cachedirs = arg.split(':') + elif opt == "--size_mb": + try: + size_mb = int(arg) + except: + pass + elif opt == "--tmpdir": + tmpdir = arg + elif opt == "--keeprpms": + keeprpms = True + + if not cachedirs: + try: + fp = open("@sysconfdir@/@PACKAGE_NAME@/plugins/CCpp.conf", "r") + for line in fp: + stripped = line.lstrip() + if len(stripped) == 0: + continue + if stripped[0] == "#": + continue + if stripped[:len("DebuginfoLocation")] != "DebuginfoLocation": + continue + + cachedirs = stripped[len("DebuginfoLocation"):].lstrip(" =\n") + fp.close() + except IOError as (errno, strerror): + print "I/O error({0}): {1}".format(errno, strerror) + + if not cachedirs: + cachedirs = ["/var/cache/abrt-di"] + if not tmpdir: + # security people prefer temp subdirs in app's private dir, like /var/run/abrt + # for now, we use /tmp... + tmpdir = "/tmp/abrt-tmp-debuginfo-%s.%u" % (time.strftime("%Y-%m-%d-%H:%M:%S"), os.getpid()) + + fin = sys.stdin + if fbuild_ids != "-": + try: + fin = open(fbuild_ids, "r") + except IOError, ex: + error_msg_and_die(_("Can't open %s: %s"), fbuild_ids, ex) + for line in fin.readlines(): + b_ids.append(line.strip('\n')) + + if not b_ids: + exit(RETURN_FAILURE) + + # Delete oldest/biggest files from cachedir. + # (Note that we need to do it before we check for missing debuginfos) + # + # We can do it as a separate step in abrt_event.conf, but this + # would require setuid'ing abrt-action-trim-files to abrt:abrt. + # Since we (via abrt-action-install-debuginfo-to-abrt-cache) + # are already running setuid, + # it makes sense to NOT setuid abrt-action-trim-files too, + # but instead run it as our child: + sys.stdout.flush() + try: + pid = os.fork() + if pid == 0: + log1("abrt-action-trim-files -f %um:%s", size_mb, cachedirs[0]); + os.execlp("abrt-action-trim-files", "abrt-action-trim-files", "-f", "%um:%s" % (size_mb, cachedirs[0])); + if pid > 0: + os.waitpid(pid, 0); + except Exception, e: + error_msg("Can't execute abrt-action-trim-files: %s", e); + + missing = filter_installed_debuginfos(b_ids, cachedirs) + if missing: + log2("%s", missing) + print _("Coredump references %u debuginfo files, %u of them are not installed") % (len(b_ids), len(missing)) + + # TODO: should we pass keep_rpms=keeprpms to DebugInfoDownload here?? + downloader = DebugInfoDownload(cache=cachedirs[0], tmp=tmpdir) + try: + result = downloader.download(missing) + except Exception, ex: + error_msg_and_die("Can't download debuginfos: %s", ex) + + missing = filter_installed_debuginfos(b_ids, cachedirs) + for bid in missing: + print _("Missing debuginfo file: %s") % bid + + if not missing: + print _("All %u debuginfo files are available") % len(b_ids) + + exit(RETURN_OK)