development.ini | 5 ++-
moksha/api/streams/datastream.py | 2 -
moksha/apps/feeds/moksha/apps/feeds/consumer.py | 4 ++
moksha/apps/feeds/moksha/apps/feeds/streams.py | 36 ++++++++++++++++++++----
moksha/apps/knowledge/pavement.py | 32 ---------------------
moksha/apps/knowledge/pavement.py.disabled | 32 +++++++++++++++++++++
moksha/hub/hub.py | 3 +-
7 files changed, 72 insertions(+), 42 deletions(-)
New commits:
commit dca67789bacbe6da03dc3191aa4d7cbdcae3f740
Author: Luke Macken <lmacken(a)redhat.com>
Date: Thu Jan 27 17:27:00 2011 -0500
When using stomp, resort to passing the default message body the consumer if we
can't decode it from JSON
diff --git a/moksha/hub/hub.py b/moksha/hub/hub.py
index 4fc155c..219abc1 100644
--- a/moksha/hub/hub.py
+++ b/moksha/hub/hub.py
@@ -146,7 +146,8 @@ class MokshaHub(StompHub, AMQPHub):
body = json.decode(message['body'])
except Exception, e:
log.warning('Cannot decode message from JSON: %s' % e)
- body = {}
+ #body = {}
+ body = message['body']
# feed all of our consumers
for callback in self.topics.get(topic, []):
commit da650ad74e649d33e9f38457a55c4e6ea8c246a6
Author: Luke Macken <lmacken(a)redhat.com>
Date: Thu Jan 27 17:25:43 2011 -0500
Allow for pluggable feed 'post-processors' in our feed app.
This allows people to plug in a custom method into the
`moksha.feeds.post_processor` entry-point, which will be called by the Feed
Streamer with each feed entry before it is sent to the message broker. This
allows people to customize the data in each feed entry.
diff --git a/moksha/apps/feeds/moksha/apps/feeds/streams.py
b/moksha/apps/feeds/moksha/apps/feeds/streams.py
index 3f89f0f..f892da5 100644
--- a/moksha/apps/feeds/moksha/apps/feeds/streams.py
+++ b/moksha/apps/feeds/moksha/apps/feeds/streams.py
@@ -17,6 +17,7 @@
import logging
import feedparser
+import pkg_resources
import time
import os
@@ -130,6 +131,10 @@ class FeederProtocol(object):
def __init__(self):
self.parsed = 1
self.hub = MokshaHub()
+ self.post_processors = []
+ for entry in
pkg_resources.iter_entry_points('moksha.feeds.post_processor'):
+ log.info('Registering feed post-processor: %s' % entry.name)
+ self.post_processors.append(entry.load())
def is_cached(self, site):
already_got = feed_storage.get(site)
@@ -215,8 +220,19 @@ class FeederProtocol(object):
channel_link = entry.get('channel', {'link':
addr})['link']
if entry['title'] not in oldtitles:
log.info('New feed entry found: %s' %
entry['title'])
- self.hub.send_message('moksha.feeds.%s' % channel_link,
- {'title': entry_title, 'link':
entry.get('link')})
+ if self.post_processors:
+ for processor in self.post_processors:
+ entry = processor(entry)
+ try:
+ self.hub.send_message('moksha.feeds.%s' %
channel_link, entry)
+ except Exception, e: # Usually JSON encoding issues.
+ log.error(str(e))
+ log.debug('Sending just the title and link instead')
+ self.hub.send_message('moksha.feeds.%s' %
channel_link,
+ {'title': entry_title, 'link':
entry.get('link')})
+ else:
+ self.hub.send_message('moksha.feeds.%s' % channel_link,
+ {'title': entry_title, 'link':
entry.get('link')})
def get_page(self, data, args):
return conditional_get_page(args, timeout=self.timeout)
commit fc454a46f4e874c472a4b8cdc9d26d168aee9354
Author: Luke Macken <lmacken(a)redhat.com>
Date: Thu Jan 27 17:25:24 2011 -0500
If we can't figure out the timestamp, try stripping off the timezone
diff --git a/moksha/apps/feeds/moksha/apps/feeds/streams.py
b/moksha/apps/feeds/moksha/apps/feeds/streams.py
index c78535f..3f89f0f 100644
--- a/moksha/apps/feeds/moksha/apps/feeds/streams.py
+++ b/moksha/apps/feeds/moksha/apps/feeds/streams.py
@@ -82,7 +82,13 @@ class ConditionalHTTPClientFactory(HTTPClientFactory):
self.deferred = defer.Deferred()
def lastModified(self, modtime):
- t = time.mktime(time.strptime(modtime[0], '%a, %d %b %Y %H:%M:%S %Z'))
+ try:
+ t = time.mktime(time.strptime(modtime[0], '%a, %d %b %Y %H:%M:%S
%Z'))
+ except ValueError:
+ # Try stripping off the timezone?
+ t = time.mktime(time.strptime(' '.join(modtime[0].split()[:-1]),
+ '%a, %d %b %Y %H:%M:%S'))
+
parsed_feed = {}
if self.url in feed_storage:
commit bb9767e4c71549072f928d04621abd160f2b48bf
Author: Luke Macken <lmacken(a)redhat.com>
Date: Thu Jan 27 17:25:08 2011 -0500
Use the simple feed cache by default
diff --git a/development.ini b/development.ini
index 4e9ff99..d856ef0 100644
--- a/development.ini
+++ b/development.ini
@@ -61,12 +61,11 @@ moksha.socket.notify = True
# Number of seconds between polling feeds
feed.poll_frequency = 35
# Where to store the feed caches.
-feed_cache = sqlite:///%(here)s/feeds.db
+#feed_cache = sqlite:///%(here)s/feeds.db
# Max age (in seconds) of each feed in the cache
feed.max_age = 300
# Timeout in seconds for the web request
feed.timeout = 60
-
# The number of simultaneous connections
feed.deferred_groups = 3
diff --git a/moksha/apps/feeds/moksha/apps/feeds/consumer.py
b/moksha/apps/feeds/moksha/apps/feeds/consumer.py
index 57a833b..6af349d 100644
--- a/moksha/apps/feeds/moksha/apps/feeds/consumer.py
+++ b/moksha/apps/feeds/moksha/apps/feeds/consumer.py
@@ -37,7 +37,9 @@ class MokshaFeedConsumer(Consumer):
def __init__(self):
super(MokshaFeedConsumer, self).__init__()
- self.feed_storage = Shove(config['feed_cache'], compress=True)
+ self.feed_storage = Shove(config.get('feed_store', 'simple://'),
+ config.get('feed_cache', 'simple://'),
+ compress=True)
def consume(self, message):
self.log.debug('MokshaFeedConsumer.consume(%s)' % message)
diff --git a/moksha/apps/feeds/moksha/apps/feeds/streams.py
b/moksha/apps/feeds/moksha/apps/feeds/streams.py
index 67a85a8..c78535f 100644
--- a/moksha/apps/feeds/moksha/apps/feeds/streams.py
+++ b/moksha/apps/feeds/moksha/apps/feeds/streams.py
@@ -39,7 +39,7 @@ from moksha.api.streams import PollingDataStream
log = logging.getLogger('moksha.hub')
-feed_storage = Shove(config['feed_cache'], compress=True)
+feed_storage = Shove(config.get('feed_cache', 'simple://'),
compress=True)
feed_cache = Cache(feed_storage)
class ConditionalHTTPPageGetter(HTTPPageGetter):
commit d8f01ab3e947b3c1eb28eb3c505e4a19da6d1abb
Author: Luke Macken <lmacken(a)redhat.com>
Date: Thu Jan 27 17:24:27 2011 -0500
Make our feed polling frequency configurable via the 'feeds.poll_frequency'
options
diff --git a/development.ini b/development.ini
index 0ff234a..4e9ff99 100644
--- a/development.ini
+++ b/development.ini
@@ -58,6 +58,8 @@ moksha.csrf_protection = True
# jGrowl notifications for socket state changes
moksha.socket.notify = True
+# Number of seconds between polling feeds
+feed.poll_frequency = 35
# Where to store the feed caches.
feed_cache = sqlite:///%(here)s/feeds.db
# Max age (in seconds) of each feed in the cache
diff --git a/moksha/api/streams/datastream.py b/moksha/api/streams/datastream.py
index 8ee6d13..5c10e0e 100644
--- a/moksha/api/streams/datastream.py
+++ b/moksha/api/streams/datastream.py
@@ -69,7 +69,7 @@ class PollingDataStream(DataStream):
(self.frequency.microseconds / 1000000.0)
else:
seconds = self.frequency
- log.debug("Setting a %s second timers" % seconds)
+ log.debug("Setting a %s second timer" % seconds)
self.timer.start(seconds, now=self.now)
def poll(self):
diff --git a/moksha/apps/feeds/moksha/apps/feeds/streams.py
b/moksha/apps/feeds/moksha/apps/feeds/streams.py
index 38f261f..67a85a8 100644
--- a/moksha/apps/feeds/moksha/apps/feeds/streams.py
+++ b/moksha/apps/feeds/moksha/apps/feeds/streams.py
@@ -271,9 +271,13 @@ class MokshaFeedStream(PollingDataStream):
then Moksha will automatically handle polling it. Upon new entries,
AMQP messages will be sent to the `feeds.$URL` queue.
"""
- frequency = timedelta(minutes=15)
+ #frequency = timedelta(minutes=1)
now = True
+ def __init__(self):
+ self.frequency = int(config.get('feed.poll_frequency', 900))
+ super(MokshaFeedStream, self).__init__()
+
def poll(self):
""" Poll all feeds in our feed cache """
log.debug('FeedStream.poll()')
commit 81db213fc2676d49ad7a7d9cb16e7fa141969170
Author: Luke Macken <lmacken(a)redhat.com>
Date: Thu Jan 27 11:32:43 2011 -0500
Disable our knowledge app from being installed automatically when people do
./start-moksha
diff --git a/moksha/apps/knowledge/pavement.py b/moksha/apps/knowledge/pavement.py
deleted file mode 100644
index 6133bcd..0000000
--- a/moksha/apps/knowledge/pavement.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# -*- coding: utf-8 -*-
-from paver.easy import *
-from paver.setuputils import (setup, find_package_data, find_packages,
- install_distutils_tasks)
-install_distutils_tasks()
-from moksha.lib.paver_tasks import *
-
-options(
- setup=Bunch(
- name="moksha.knowledge",
- version="0.1",
- release="1",
- url="http://moksha.fedorahosted.org",
- description="Describe your package here",
- license="ASL 2.0",
- long_description="",
- author="",
- author_email="",
- rpm_name='moksha-apps-knowledge',
- packages=find_packages(),
- package_data=find_package_data(),
- namespace_packages=[
- 'moksha',
- 'moksha.apps',
- ],
- install_requires=["Moksha", "tgext.crud"],
- entry_points="""
- [moksha.application]
- moksha.apps.knowledge =
moksha.apps.knowledge.controllers.root:RootController
- """,
- ),
-)
diff --git a/moksha/apps/knowledge/pavement.py.disabled
b/moksha/apps/knowledge/pavement.py.disabled
new file mode 100644
index 0000000..6133bcd
--- /dev/null
+++ b/moksha/apps/knowledge/pavement.py.disabled
@@ -0,0 +1,32 @@
+# -*- coding: utf-8 -*-
+from paver.easy import *
+from paver.setuputils import (setup, find_package_data, find_packages,
+ install_distutils_tasks)
+install_distutils_tasks()
+from moksha.lib.paver_tasks import *
+
+options(
+ setup=Bunch(
+ name="moksha.knowledge",
+ version="0.1",
+ release="1",
+ url="http://moksha.fedorahosted.org",
+ description="Describe your package here",
+ license="ASL 2.0",
+ long_description="",
+ author="",
+ author_email="",
+ rpm_name='moksha-apps-knowledge',
+ packages=find_packages(),
+ package_data=find_package_data(),
+ namespace_packages=[
+ 'moksha',
+ 'moksha.apps',
+ ],
+ install_requires=["Moksha", "tgext.crud"],
+ entry_points="""
+ [moksha.application]
+ moksha.apps.knowledge =
moksha.apps.knowledge.controllers.root:RootController
+ """,
+ ),
+)