Signed-off-by: Jakub Filak <jfilak(a)redhat.com>
---
configure.ac | 1 +
gnome-abrt.spec.in | 1 +
pylintrc | 2 +-
src/gnome-abrt | 7 ++
src/gnome_abrt/Makefile.am | 2 +-
src/gnome_abrt/problems.py | 41 +++++++--
src/gnome_abrt/url/Makefile.am | 3 +
src/gnome_abrt/url/__init__.py | 29 ++++++
src/gnome_abrt/url/gliburltitle.py | 178 +++++++++++++++++++++++++++++++++++++
src/gnome_abrt/url/urltitle.py | 139 +++++++++++++++++++++++++++++
src/gnome_abrt/views.py | 8 +-
11 files changed, 398 insertions(+), 13 deletions(-)
create mode 100644 src/gnome_abrt/url/Makefile.am
create mode 100644 src/gnome_abrt/url/__init__.py
create mode 100644 src/gnome_abrt/url/gliburltitle.py
create mode 100644 src/gnome_abrt/url/urltitle.py
diff --git a/configure.ac b/configure.ac
index 4a924ce..8f1a567 100644
--- a/configure.ac
+++ b/configure.ac
@@ -100,6 +100,7 @@ AC_CONFIG_FILES([
src/Makefile
src/gnome_abrt/Makefile
src/gnome_abrt/wrappers/Makefile
+ src/gnome_abrt/url/Makefile
po/Makefile.in
])
diff --git a/gnome-abrt.spec.in b/gnome-abrt.spec.in
index 5ed9a17..df30ac5 100644
--- a/gnome-abrt.spec.in
+++ b/gnome-abrt.spec.in
@@ -89,6 +89,7 @@ gtk-update-icon-cache %{_datadir}/icons/hicolor &>/dev/null || :
%files -f %{name}.lang
%doc COPYING
%{python_sitearch}/gnome_abrt
+%{python_sitearch}/gnome_abrt/url
%{_datadir}/%{name}
%{_bindir}/%{name}
%{_datadir}/applications/*
diff --git a/pylintrc b/pylintrc
index b5d55c5..fdb32d3 100644
--- a/pylintrc
+++ b/pylintrc
@@ -33,7 +33,7 @@ load-plugins=
# can either give multiple identifier separated by comma (,) or put this option
# multiple time (only on the command line, not in the configuration file where
# it should appear only once).
-disable=C0111,W0212,R0201,R0903,I0011,W0603,W0622,I0001
+disable=C0111,W0212,R0201,R0903,R0904,I0011,W0603,W0622,I0001
[REPORTS]
diff --git a/src/gnome-abrt b/src/gnome-abrt
index 633eb3a..d3417d2 100755
--- a/src/gnome-abrt
+++ b/src/gnome-abrt
@@ -34,6 +34,7 @@ from gi.repository import Gio
# gnome-abrt
import gnome_abrt
+import gnome_abrt.url
from gnome_abrt.views import OopsWindow
from gnome_abrt.controller import Controller
from gnome_abrt.signals import glib_sigchld_signal_handler
@@ -45,6 +46,8 @@ from gnome_abrt.l10n import _
from gnome_abrt.wrappers import show_events_list_dialog
from gnome_abrt.config import get_configuration
from gnome_abrt.dialogs import show_report_problem_with_abrt
+from gnome_abrt.url.gliburltitle import (GetURLTitleSourcePool,
+ GetURLTitleSourceCache)
# dbus
import dbus
@@ -54,6 +57,7 @@ from dbus.mainloop.glib import DBusGMainLoop
GNOME_ABRT_APPLICATION_ID = 'org.freedesktop.GnomeAbrt'
GNOME_ABRT_INTERFACE = 'org.freedesktop.GnomeAbrt'
GNOME_ABRT_OBJECT_PATH = '/org/freedesktop/GnomeAbrt'
+GNOME_ABRT_URL_POOL_CAPACITY = 10
# because of
https://bugzilla.gnome.org/show_bug.cgi?id=682331
@@ -148,6 +152,9 @@ class OopsApplication(Gtk.Application):
self.set_flags(Gio.ApplicationFlags.HANDLES_COMMAND_LINE)
self.gcontext = None
self.source = None
+ self._url_pool = GetURLTitleSourcePool(GNOME_ABRT_URL_POOL_CAPACITY)
+ self._url_cache = GetURLTitleSourceCache(self._url_pool)
+ gnome_abrt.url.set_async_worker(self._url_cache.get_url_title_async)
#pylint: disable=W0613
def _parse_command_line(self, sender, gcmdargs):
diff --git a/src/gnome_abrt/Makefile.am b/src/gnome_abrt/Makefile.am
index 60496f9..0f0e557 100644
--- a/src/gnome_abrt/Makefile.am
+++ b/src/gnome_abrt/Makefile.am
@@ -1,4 +1,4 @@
-SUBDIRS = wrappers
+SUBDIRS = wrappers url
gnome_abrtexecdir = $(pyexecdir)/gnome_abrt
diff --git a/src/gnome_abrt/problems.py b/src/gnome_abrt/problems.py
index 4bf7695..1d3974e 100644
--- a/src/gnome_abrt/problems.py
+++ b/src/gnome_abrt/problems.py
@@ -19,6 +19,7 @@ import datetime
import logging
# gnome-abrt
+import gnome_abrt.url
from gnome_abrt.application import find_application
from gnome_abrt.errors import (InvalidProblem,
UnavailableSource,
@@ -70,10 +71,40 @@ class Problem:
MSG = "MSG"
BTHASH = "BTHASH"
- def __init__(self, title, rtype, data):
- self.title = title
- self.rtype = rtype
- self.data = data
+ def __init__(self, problem, name, rtype, data):
+ self._problem = problem
+ self._name = name
+ self._title = name
+ self._rtype = rtype
+ self._data = data
+ self._url_done = False
+
+ def _update_title_async_cb(self, result, unused_userdata):
+ if result[1]:
+ self._title = result[1]
+ self._problem.source.notify(ProblemSource.CHANGED_PROBLEM,
+ self._problem)
+
+ @property
+ def name(self):
+ return self._name
+
+ @property
+ def rtype(self):
+ return self._rtype
+
+ @property
+ def data(self):
+ return self._data
+
+ @property
+ def title(self):
+ if self._rtype == Problem.Submission.URL and not self._url_done:
+ self._url_done = True
+ gnome_abrt.url.get_url_title_async(self._data,
+ self._update_title_async_cb, None)
+
+ return self._title
def __init__(self, problem_id, source):
@@ -226,7 +257,7 @@ class Problem:
i += 1
self.submission.append(
- Problem.Submission(pfx, typ, line[i:]))
+ Problem.Submission(self, pfx, typ, line[i:]))
return self.submission
diff --git a/src/gnome_abrt/url/Makefile.am b/src/gnome_abrt/url/Makefile.am
new file mode 100644
index 0000000..195ebec
--- /dev/null
+++ b/src/gnome_abrt/url/Makefile.am
@@ -0,0 +1,3 @@
+gnome_abrt_url_PYTHON = __init__.py urltitle.py gliburltitle.py
+
+gnome_abrt_urldir = $(pyexecdir)/gnome_abrt/url
diff --git a/src/gnome_abrt/url/__init__.py b/src/gnome_abrt/url/__init__.py
new file mode 100644
index 0000000..5b77759
--- /dev/null
+++ b/src/gnome_abrt/url/__init__.py
@@ -0,0 +1,29 @@
+# coding=UTF-8
+
+## Copyright (C) 2013 ABRT team <crash-catcher(a)lists.fedorahosted.org>
+## Copyright (C) 2013 Red Hat, Inc.
+
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
+
+from gnome_abrt.url import gliburltitle
+
+__ASYNC_WORKER = gliburltitle.get_url_title_async
+
+def set_async_worker(worker):
+ global __ASYNC_WORKER
+ __ASYNC_WORKER = worker
+
+def get_url_title_async(url, callback, user_data):
+ __ASYNC_WORKER(url, callback, user_data)
diff --git a/src/gnome_abrt/url/gliburltitle.py b/src/gnome_abrt/url/gliburltitle.py
new file mode 100644
index 0000000..9a6f6e6
--- /dev/null
+++ b/src/gnome_abrt/url/gliburltitle.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+# coding=UTF-8
+
+## Copyright (C) 2013 ABRT team <crash-catcher(a)lists.fedorahosted.org>
+## Copyright (C) 2013 Red Hat, Inc.
+
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
+
+import sys
+from Queue import Queue
+
+# pygobject
+#pylint: disable=E0611
+from gi.repository import GLib
+
+from gnome_abrt.url.urltitle import GetURLTitleThread
+
+
+def get_url_title_async(url, readycallback, userdata, context=None):
+ """Gets url's title assynchronously
+ """
+
+ if context is None:
+ context = GLib.main_context_default()
+
+ source = GetURLTitleThreadSource(url, readycallback, userdata)
+ #pylint: disable=E1101
+ source.attach(context)
+
+
+class GetURLTitleThreadSource(GLib.Source):
+ """Runs thread and waits until the thread does not return the
url's title
+ """
+
+ #pylint: disable=W0613
+ def __new__(cls, *args):
+ return GLib.Source.__new__(cls)
+
+ def __init__(self, url, readycallback, userdata):
+ GLib.Source.__init__(self)
+
+ self._queue = Queue()
+ self._readycallback = readycallback
+ self._userdata = userdata
+ self._resolver = GetURLTitleThread(url, self._queue)
+ self._resolver.start()
+
+ def _is_ready(self):
+ return not self._queue.empty()
+
+ def prepare(self, *args):
+ return (self._is_ready(), 10)
+
+ def check(self, *args):
+ return self._is_ready()
+
+ def dispatch(self, *args):
+ if not self._is_ready():
+ raise RuntimeError("GetURLHTMLTitleThreadSource.dispatch()"
+ " called but result is not ready yet, or already consumed.")
+
+ # An attempty to destroy the resolver's thread ASAP
+ self._resolver = None
+
+ self._readycallback(self._queue.get(), self._userdata)
+
+ # Destroy this source
+ return False
+
+ def finalize(self, *args):
+ pass
+
+
+class GetURLTitleSourcePool(object):
+ """Pool for reducing number of running threads at time
+ """
+
+ def __init__(self, capacity):
+ self._capacity = capacity
+ self._running = 0
+ self._defered = list()
+
+ def get_url_title_async(self, url, readycallback, userdata):
+ if self._running == self._capacity:
+ self._defer_request(url, readycallback, userdata)
+ else:
+ self._start_resolving(url, readycallback, userdata)
+
+ def _defer_request(self, url, readycallback, userdata):
+ self._defered.append((url, readycallback, userdata))
+
+ def _start_resolving(self, url, readycallback, userdata):
+ self._running += 1
+ get_url_title_async(url, self._resolving_done,
+ (readycallback, userdata))
+
+ def _resolving_done(self, result, userdata):
+ try:
+ userdata[0](result, userdata[1])
+ finally:
+ self._running -= 1
+ if len(self._defered):
+ url, readycallback, userdata = self._defered.pop()
+ self._start_resolving(url, readycallback, userdata)
+
+
+class GetURLTitleSourceCache(object):
+ """Asynchronous cache for URL titles
+ """
+
+ def __init__(self, worker, max_size=1024):
+ self._worker = worker
+ self._max_size = max_size
+ self._cache = dict()
+ # The oldest is on the top
+ self._stamp = []
+
+ def get_url_title_async(self, url, readycallback, userdata):
+ if url in self._cache:
+ # Move url at the end
+ self._stamp.remove(url)
+ self._stamp.append(url)
+ # Callers expect that result will be delivered asynchronously
+ GLib.timeout_add(1,
+ lambda unused: readycallback((url, self._cache[url]), userdata),
+ None)
+ return
+
+ self._worker.get_url_title_async(url, self._resolved,
+ (readycallback, userdata))
+
+ def _resolved(self, result, userdata):
+ url, title = result
+ if title:
+ if len(self._cache.keys()) >= self._max_size:
+ # Remove the url on the top (the oldest)
+ del self._cache[self._stamp.pop()]
+
+ self._cache[url] = title
+ self._stamp.append(url)
+
+ readycallback, data = userdata
+ readycallback(result, data)
+
+
+if __name__ == "__main__":
+ def ready(result, userdata):
+ print result[1] or result[0]
+
+ userdata["count"] = userdata["count"] - 1
+ if userdata["count"] == 0:
+ userdata["gloop"].quit()
+
+ URLS = ["http://docs.python.org/2/library/htmlparser.html"]
+
+ if len(sys.argv) > 1:
+ URLS = sys.argv[1:]
+
+ MAINLOOP = GLib.MainLoop()
+ WORKER = GetURLTitleSourcePool(2)
+ CONSUMER = {"count":len(URLS), "gloop":MAINLOOP}
+
+ for u in URLS:
+ WORKER.get_url_title_async(u, ready, CONSUMER)
+
+ MAINLOOP.run()
diff --git a/src/gnome_abrt/url/urltitle.py b/src/gnome_abrt/url/urltitle.py
new file mode 100644
index 0000000..369b1af
--- /dev/null
+++ b/src/gnome_abrt/url/urltitle.py
@@ -0,0 +1,139 @@
+#!/usr/bin/env python
+# coding=UTF-8
+
+## Copyright (C) 2013 ABRT team <crash-catcher(a)lists.fedorahosted.org>
+## Copyright (C) 2013 Red Hat, Inc.
+
+## This program is free software; you can redistribute it and/or modify
+## it under the terms of the GNU General Public License as published by
+## the Free Software Foundation; either version 2 of the License, or
+## (at your option) any later version.
+
+## This program is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+## GNU General Public License for more details.
+
+## You should have received a copy of the GNU General Public License
+## along with this program; if not, write to the Free Software
+## Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA
+
+import sys
+import threading
+import logging
+from urllib import urlopen
+from HTMLParser import HTMLParser, HTMLParseError
+
+
+def get_url_title(url):
+ """Returns a title of the url
+
+ Function expects that the url points to a HTML page with the title element
+ in the head element. Tries to download the page and return a value of the
+ title.
+ """
+
+ try:
+ urlio = urlopen(url)
+ except IOError as ex:
+ logging.debug("{1} ('{0}')".format(url, str(ex)))
+ return None
+
+ encoding = urlio.info().getparam('charset')
+ # Can't find any documentation but try except needed.
+ data = urlio.read()
+ if encoding:
+ try:
+ data = data.decode(encoding)
+ except UnicodeError as ex:
+ logging.debug("{1} ('{0}')".format(url, str(ex)))
+ return None
+
+ return HTMLTitleGetter.parse(data)
+
+
+class HTMLTitleGetter(HTMLParser):
+ def __init__(self):
+ HTMLParser.__init__(self)
+
+ self._tag_head = False
+ self._tag_title = False
+ self._title = None
+
+ def handle_starttag(self, tag, attrs):
+ if not self._tag_head:
+ self._tag_head = tag.lower() == "head"
+
+ if self._tag_head and not self._tag_title:
+ self._tag_title = tag.lower() == "title"
+
+ def handle_endtag(self, tag):
+ if self._tag_head:
+ self._tag_head = tag.lower() != "head"
+
+ if self._tag_head and self._tag_title:
+ self._tag_title = tag.lower() != "title"
+
+ def handle_data(self, data):
+ if self._tag_title:
+ if not self._title:
+ self._title = data
+ else:
+ self._title += data
+
+ @staticmethod
+ def parse(data):
+ tgt = HTMLTitleGetter()
+ try:
+ tgt.feed(data)
+ except HTMLParseError as ex:
+ logging.debug("{1} ('{0}')".format(data, str(ex)))
+ # Hopefully title is parsed correctly
+
+ return tgt._title
+
+
+class GetURLTitleThread(threading.Thread):
+ """Simple thread which puts url's title into a Queue
+ """
+
+ def __init__(self, url, que):
+ super(GetURLTitleThread, self).__init__()
+
+ self._url = url
+ self._que = que
+
+ def run(self):
+ title = get_url_title(self._url)
+ self._que.put((self._url, title))
+
+
+if __name__ == "__main__":
+ URLS = ["http://docs.python.org/2/library/htmlparser.html"]
+
+ THREADS = False
+ if len(sys.argv) > 1:
+ i = 1
+ if "--threads" == sys.argv[i]:
+ THREADS = True
+ i += 1
+ URLS = sys.argv[i:]
+
+ if THREADS:
+ from Queue import Queue
+ QUEUE = Queue()
+ THREADS = []
+ for u in URLS:
+ t = GetURLTitleThread(u, QUEUE)
+ t.start()
+ THREADS.append(t)
+
+ for t in THREADS:
+ t.join()
+
+ while not QUEUE.empty():
+ ORIG_URL, GOT_TITLE = QUEUE.get()
+ print GOT_TITLE or ORIG_URL
+ else:
+ for u in URLS:
+ print get_url_title(u) or u
diff --git a/src/gnome_abrt/views.py b/src/gnome_abrt/views.py
index 5c1f6e2..6c2e5a9 100644
--- a/src/gnome_abrt/views.py
+++ b/src/gnome_abrt/views.py
@@ -256,8 +256,7 @@ class OopsWindow(Gtk.ApplicationWindow):
except errors.InvalidProblem as ex:
self._remove_problem_from_storage(ex.problem_id)
logging.debug(ex.message)
-
- return
+ return
try:
if problem in self._get_selected(self._builder.tvs_problems):
@@ -266,8 +265,6 @@ class OopsWindow(Gtk.ApplicationWindow):
self._remove_problem_from_storage(ex.problem_id)
logging.debug(ex.message)
- return
-
def _reload_problems(self, source):
self._reloading = True
old_selection = self._get_selected(self._builder.tvs_problems)
@@ -318,7 +315,6 @@ class OopsWindow(Gtk.ApplicationWindow):
self._builder.tvs_problems.select_iter(pit)
self._builder.tv_problems.scroll_to_cell(path)
-
def _show_problem_links(self, submissions):
need_align = False
for sbm in submissions:
@@ -388,7 +384,7 @@ class OopsWindow(Gtk.ApplicationWindow):
if problem['not-reportable']:
self._show_problem_message(problem['not-reportable'])
elif (not problem['is_reported']
- or not any((s.title == "Bugzilla"
+ or not any((s.name == "Bugzilla"
for s in problem['submission']))):
self._show_problem_message(
_("This problem hasn't been reported to <i>Bugzilla</i> yet.
"
--
1.8.1.4