Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit 69f65530119bbc6577dd298c71f2ef164aa95f7a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Wed Feb 25 18:17:49 2015 +0100
[frontend][backend] [rhbz:#1185959] - RFE: Present statistics about project
popularity. A few more counters for downloads from backend's result directory.
---------------------------------------------------------------
backend/conf/logstash/copr_backend.conf | 60 ++++++++++
backend/conf/logstash/lighttpd.pattern | 1 +
backend/copr-backend.spec | 12 ++
frontend/conf/logstash.conf | 37 ++++---
frontend/copr-frontend.spec | 5 +
frontend/coprs_frontend/coprs/__init__.py | 10 +-
frontend/coprs_frontend/coprs/helpers.py | 43 +++++++-
frontend/coprs_frontend/coprs/logic/stat_logic.py | 44 ++++++--
frontend/coprs_frontend/coprs/rmodels.py | 117 ++++++++++++++++++++
.../coprs/templates/coprs/detail/overview.html | 10 ++-
.../coprs/views/coprs_ns/coprs_general.py | 21 +++-
frontend/coprs_frontend/coprs/views/misc.py | 2 +-
.../coprs/views/stats_ns/stats_receiver.py | 21 +---
frontend/coprs_frontend/tests/test_rmodels.py | 53 +++++++++
14 files changed, 387 insertions(+), 49 deletions(-)
diff --git a/backend/conf/logstash/copr_backend.conf
b/backend/conf/logstash/copr_backend.conf
new file mode 100644
index 0000000..e6a96e2
--- /dev/null
+++ b/backend/conf/logstash/copr_backend.conf
@@ -0,0 +1,60 @@
+input {
+ #file {
+ # path => "/var/log/copr/backend.log"
+ # type => "copr.backend.main"
+ #}
+ file {
+ path => "/var/log/lighttpd/access.log"
+ type => "lighttpd-access"
+ }
+}
+
+filter {
+ mutate {
+ add_tag => [ "backend" ]
+ }
+ if [type] == 'lighttpd-access' {
+ grok {
+ patterns_dir => "/usr/share/logstash/patterns"
+ pattern => "%{LIGHTTPD}"
+ }
+ date {
+ match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+ }
+ if "repodata/repomd.xml" in [request] and
"devel/repodata/repomd.xml" not in [request] {
+ mutate { add_tag => "repomdxml" }
+ mutate { add_tag => "publish_stat" }
+ grok {
+ match => ["request",
"/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/repodata/repomd.xml"]
+ }
+ }
+ if [request] =~ "rpm$" {
+ mutate { add_tag => "rpm" }
+ mutate { add_tag => "publish_stat" }
+ grok {
+ match => ["request",
"/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/%{USERNAME:copr_build_dir}/%{USERNAME:copr_rpm}"]
+ }
+ }
+ }
+}
+
+output {
+ if "publish_stat" in [tags] {
+ http {
+ url =>
"http://copr-fe-dev.cloud.fedoraproject.org/stats_rcv/from_logstash"
+ format => "json"
+ http_method => "post"
+ }
+ }
+
+ file {
+ path => "/var/log/logstash/all.log"
+ codec => "rubydebug"
+ }
+
+ file {
+ path => "/tmp/logstashall.log"
+ codec => "rubydebug"
+ }
+
+}
diff --git a/backend/conf/logstash/lighttpd.pattern
b/backend/conf/logstash/lighttpd.pattern
new file mode 100644
index 0000000..7dce2a0
--- /dev/null
+++ b/backend/conf/logstash/lighttpd.pattern
@@ -0,0 +1 @@
+LIGHTTPD %{IPORHOST:clientip} %{IPORHOST:httphost} %{USER:auth} \[%{HTTPDATE:timestamp}\]
"(?:%{WORD:verb} %{URIPATHPARAM:request}(?: HTTP/%{NUMBER:httpversion})|-)"
%{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent}
diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec
index ec3dc85..bb39b63 100644
--- a/backend/copr-backend.spec
+++ b/backend/copr-backend.spec
@@ -72,6 +72,7 @@ Requires: fedmsg
Requires: gawk
Requires: crontabs
Requires: python-paramiko
+Requires: logstash
Requires(post): systemd
Requires(preun): systemd
@@ -145,6 +146,13 @@ touch %{buildroot}%{_var}/run/copr-backend/copr-be.pid
install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/
install -m 0644 conf/copr.sudoers.d %{buildroot}%{_sysconfdir}/sudoers.d/copr
+
+install -d %{buildroot}%{_sysconfdir}/logstash.d
+cp -a conf/logstash/copr_backend.conf
%{buildroot}%{_sysconfdir}/logstash.d/copr_backend.conf
+install -d %{buildroot}%{_datadir}/logstash/patterns/
+cp -a conf/logstash/lighttpd.pattern
%{buildroot}%{_datadir}/logstash/patterns/lighttpd.pattern
+
+
#doc
cp -a documentation/python-doc %{buildroot}%{_pkgdocdir}/
cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/
@@ -163,6 +171,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user"
copr
%post
%systemd_post copr-backend.service
+%systemd_post logstash.service
%preun
%systemd_preun copr-backend.service
@@ -197,6 +206,9 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user"
copr
%{_bindir}/*
%config(noreplace) %{_sysconfdir}/cron.daily/copr-backend
+%config(noreplace) %{_sysconfdir}/logstash.d/copr_backend.conf
+%{_datadir}/logstash/patterns/lighttpd.pattern
+
%config(noreplace) %attr(0600, root, root) %{_sysconfdir}/sudoers.d/copr
diff --git a/frontend/conf/logstash.conf b/frontend/conf/logstash.conf
index 82aa01d..36cb8cd 100644
--- a/frontend/conf/logstash.conf
+++ b/frontend/conf/logstash.conf
@@ -10,32 +10,39 @@ input {
}
filter {
- grok {
- type => "httpd-access"
- pattern => "%{COMBINEDAPACHELOG}"
+ mutate { add_tag => "frontend" }
+ if [type] == "httpd-access" {
+ if "POST /stats_rcv/from_logstash HTTP" in [message] {
+ drop{}
+ }
+ grok {
+ type => "httpd-access"
+ pattern => "%{COMBINEDAPACHELOG}"
+ }
+
+ if [request] =~ ".repo$" {
+ mutate { add_tag => "repo_dl" }
+ grok {
+ match => ["request",
"/coprs/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/repo/%{USERNAME:copr_name_release}/%{USERNAME:copr_repo_file}"]
+ }
+ }
+
}
}
output {
if [type] == "httpd-access" {
- if ".repo" in [request] and "stats_rcv" not in [request] {
+ if "repo_dl" in [tags] {
http {
url => "http://127.0.0.1/stats_rcv/from_logstash"
format => "json"
http_method => "post"
- message => "foobar"
}
}
- #file {
- # path => "/var/log/logstash/web.log"
- # codec => rubydebug {}
- #
- #}
}
- # file {
- # path => "/var/log/logstash/all.log"
- # codec => rubydebug {}
- # }
-
+ file {
+ path => "/var/log/logstash/all.log"
+ codec => rubydebug {}
+ }
}
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index bf83869..4439e30 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -59,6 +59,9 @@ Requires: python-mock
Requires: python-decorator
Requires: yum
Requires: logstash
+Requires: redis
+Requires: python-redis
+Requires: python-dateutil
%if 0%{?rhel} < 7 && 0%{?rhel} > 0
BuildRequires: python-argparse
%endif
@@ -72,6 +75,8 @@ BuildRequires: python-flask-whooshee
BuildRequires: python-pylibravatar
BuildRequires: python-flask-wtf
BuildRequires: python-netaddr
+BuildRequires: python-redis
+BuildRequires: python-dateutil
BuildRequires: pytest
BuildRequires: yum
BuildRequires: python-flexmock
diff --git a/frontend/coprs_frontend/coprs/__init__.py
b/frontend/coprs_frontend/coprs/__init__.py
index 492dec3..e86bb64 100644
--- a/frontend/coprs_frontend/coprs/__init__.py
+++ b/frontend/coprs_frontend/coprs/__init__.py
@@ -3,9 +3,9 @@ from __future__ import with_statement
import os
import flask
-from flask.ext.sqlalchemy import SQLAlchemy
-from flask.ext.openid import OpenID
-from flask.ext.whooshee import Whooshee
+from flask_sqlalchemy import SQLAlchemy
+from flask_openid import OpenID
+from flask_whooshee import Whooshee
app = flask.Flask(__name__)
@@ -25,12 +25,16 @@ oid = OpenID(app, app.config["OPENID_STORE"],
safe_roots=[])
db = SQLAlchemy(app)
whooshee = Whooshee(app)
+
import coprs.filters
import coprs.log
from coprs.log import setup_log
import coprs.models
import coprs.whoosheers
+from coprs.helpers import RedisConnectionProvider
+rcp = RedisConnectionProvider(config=app.config)
+
from coprs.views import admin_ns
from coprs.views.admin_ns import admin_general
from coprs.views import api_ns
diff --git a/frontend/coprs_frontend/coprs/helpers.py
b/frontend/coprs_frontend/coprs/helpers.py
index f4f8efa..811a481 100644
--- a/frontend/coprs_frontend/coprs/helpers.py
+++ b/frontend/coprs_frontend/coprs/helpers.py
@@ -4,6 +4,10 @@ import string
import urlparse
import flask
+from dateutil import parser as dt_parser
+
+from redis import StrictRedis
+
from coprs import constants
from coprs import app
@@ -21,7 +25,10 @@ def generate_api_token(size=30):
return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
-REPO_DL_STAT_FMT = "{user}@{copr}:{name_release}"
+REPO_DL_STAT_FMT =
"repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
+CHROOT_REPO_MD_DL_STAT_FMT =
"chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
+CHROOT_RPMS_DL_STAT_FMT =
"chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
+PROJECT_RPMS_DL_STAT_FMT =
"project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
class CounterStatType(object):
@@ -268,3 +275,37 @@ class Serializer(object):
@property
def serializable_attributes(self):
return map(lambda x: x.name, self.__table__.columns)
+
+
+class RedisConnectionProvider(object):
+ def __init__(self, config):
+ self.host = config.get("redis_host", "127.0.0.1")
+ self.port = int(config.get("redis_port", "6379"))
+
+ def get_connection(self):
+ return StrictRedis(host=self.host, port=self.port)
+
+
+def get_redis_connection():
+ """
+ Creates connection to redis, now we use default instance at localhost, no config
needed
+ """
+ return StrictRedis()
+
+
+def dt_to_unixtime(dt):
+ """
+ Converts datetime to unixtime
+ :param dt: DateTime instance
+ :rtype: float
+ """
+ return float(dt.strftime('%s'))
+
+
+def string_dt_to_unixtime(dt_string):
+ """
+ Converts datetime to unixtime from string
+ :param dt_string: datetime string
+ :rtype: str
+ """
+ return dt_to_unixtime(dt_parser.parse(dt_string))
diff --git a/frontend/coprs_frontend/coprs/logic/stat_logic.py
b/frontend/coprs_frontend/coprs/logic/stat_logic.py
index c1b2e0f..a2e4062 100644
--- a/frontend/coprs_frontend/coprs/logic/stat_logic.py
+++ b/frontend/coprs_frontend/coprs/logic/stat_logic.py
@@ -3,6 +3,8 @@ import json
import os
import pprint
import time
+
+
from sqlalchemy import or_
from sqlalchemy import and_
@@ -13,11 +15,11 @@ from coprs import db
from coprs import exceptions
from coprs.models import CounterStat
from coprs import helpers
-from coprs.helpers import REPO_DL_STAT_FMT
+from coprs.helpers import REPO_DL_STAT_FMT, CHROOT_REPO_MD_DL_STAT_FMT, dt_to_unixtime,
string_dt_to_unixtime, \
+ CHROOT_RPMS_DL_STAT_FMT, PROJECT_RPMS_DL_STAT_FMT
from coprs import signals
from coprs.helpers import CounterStatType
-
-
+from coprs.rmodels import TimedStatEvents
class CounterStatLogic(object):
@@ -65,9 +67,9 @@ class CounterStatLogic(object):
chroot_by_stat_name = {}
for chroot in copr.active_chroots:
kwargs = {
- "user": copr.owner.name,
- "copr": copr.name,
- "name_release": chroot.name_release
+ "copr_user": copr.owner.name,
+ "copr_project_name": copr.name,
+ "copr_name_release": chroot.name_release
}
chroot_by_stat_name[REPO_DL_STAT_FMT.format(**kwargs)] = chroot.name_release
@@ -83,6 +85,30 @@ class CounterStatLogic(object):
return repo_dl_stats
-
-
-
+def handle_logstash(rc, ls_data):
+ """
+ :param rc: connection to redis
+ :type rc: StrictRedis
+
+ :param ls_data: log stash record
+ :type ls_data: dict
+ """
+ dt_unixtime = string_dt_to_unixtime(ls_data["@timestamp"])
+ app.logger.debug("got ls_data: {}".format(ls_data))
+
+ if "tags" in ls_data:
+ tags = set(ls_data["tags"])
+ if "frontend" in tags and "repo_dl":
+ name = REPO_DL_STAT_FMT.format(**ls_data)
+ CounterStatLogic.incr(name=name, counter_type=CounterStatType.REPO_DL)
+ db.session.commit()
+
+ if "backend" in tags and "repomdxml" in tags:
+ key = CHROOT_REPO_MD_DL_STAT_FMT.format(**ls_data)
+ TimedStatEvents.add_event(rc, key, timestamp=dt_unixtime)
+
+ if "backend" in tags and "rpm" in tags:
+ key_chroot = CHROOT_RPMS_DL_STAT_FMT.format(**ls_data)
+ key_project = PROJECT_RPMS_DL_STAT_FMT.format(**ls_data)
+ TimedStatEvents.add_event(rc, key_chroot, timestamp=dt_unixtime)
+ TimedStatEvents.add_event(rc, key_project, timestamp=dt_unixtime)
diff --git a/frontend/coprs_frontend/coprs/rmodels.py
b/frontend/coprs_frontend/coprs/rmodels.py
new file mode 100644
index 0000000..43734ae
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/rmodels.py
@@ -0,0 +1,117 @@
+# coding: utf-8
+
+""" Models to redis entities """
+import time
+from math import ceil
+from datetime import datetime, timedelta
+from redis import StrictRedis
+
+
+class GenericRedisModel(object):
+ _KEY_BASE = "copr:generic"
+
+ @classmethod
+ def _get_key(cls, name, prefix=None):
+ if prefix:
+ return "{}:{}:{}".format(prefix, cls._KEY_BASE, name)
+ else:
+ return "{}:{}".format(cls._KEY_BASE, name)
+
+
+class TimedStatEvents(GenericRedisModel):
+ """
+ Wraps hset structure, where:
+ **key** - name of event, fix prefix specifying events type
+ **member** - bucket representing one day
+ **score** - events count
+ """
+ _KEY_BASE = "copr:tse"
+
+ @staticmethod
+ def timestamp_to_day(ut):
+ """
+ :param ut: unix timestamp
+ :type ut: float
+ :return: name for the day bucket
+ """
+ td = timedelta(days=1).total_seconds()
+ return int(ceil(ut / td))
+
+ @classmethod
+ def gen_days_interval(cls, min_ts, max_ts):
+ """
+ Generate list of days bucket names which contains
+ all events between `min_ts` and `max_ts`
+ :param min_ts: min unix timestamp
+ :param max_ts: max unix timestamp
+ :rtype: list
+ """
+ start_ut = cls.timestamp_to_day(min_ts)
+ end_ut = cls.timestamp_to_day(max_ts)
+
+ return range(start_ut, end_ut + 1)
+
+ @classmethod
+ def add_event(cls, rconnect, name, timestamp, count=1, prefix=None):
+ """
+ Stoted new event to redist
+ :param rconnect: Connection to a redis
+ :type rconnect: StrictRedis
+ :param name: statistics name
+ :param timestamp: timestamp of event
+ :param count: number of events, default=1
+ :param prefix: prefix for statistics, default is None
+ """
+ count = int(count)
+ ut_day = cls.timestamp_to_day(timestamp)
+
+ key = cls._get_key(name, prefix)
+
+ rconnect.hincrby(key, ut_day, count)
+
+ @classmethod
+ def get_count(cls, rconnect, name, day_min=None, prefix=None, day_max=None):
+ """
+ Count total event occurency between day_min and day_max
+ :param rconnect: Connection to a redis
+ :type rconnect: StrictRedis
+ :param name: statistics name
+ :param day_min: default: seven days ago
+ :param day_max: default: tomorrow
+ :param prefix: prefix for statistics, default is None
+
+ :rtype: int
+ """
+ key = cls._get_key(name, prefix)
+ if day_min is None:
+ day_min = time.time() - timedelta(days=7).total_seconds()
+
+ if day_max is None:
+ day_max = time.time() + timedelta(days=1).total_seconds()
+
+ interval = cls.gen_days_interval(day_min, day_max)
+ if len(interval) == 0:
+ return 0
+
+ res = rconnect.hmget(key, interval)
+ return sum(int(amount) for amount in res if amount is not None)
+
+
+ @classmethod
+ def trim_before(cls, rconnect, name, threshold_timestamp,
+ prefix=None):
+ """
+ Removes all records occured before `threshold_timestamp`
+ :param rconnect: StrictRedis
+ :param name: statistics name
+ :param threshold_timestamp: int
+ :param prefix: prefix for statistics, default is None
+ """
+
+ key = cls._get_key(name, prefix)
+
+ threshold_day = cls.timestamp_to_day(threshold_timestamp) + 1
+ all_members = rconnect.hgetall(key)
+ to_del = [mb for mb in all_members.keys() if int(mb) < threshold_day]
+
+ rconnect.hdel(key, *to_del)
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index 5c9629a..96c76e6 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -35,7 +35,15 @@
{{ repo.arch_list|join(", ") }}
</td>
<td>
- <center> {{ repo.dl_stat }} </center>
+ <center> {{ repo.dl_stat }} </center> <br />
+ <!-- --->
+ # of repo dl last week<br />
+ {% for arch in repo.arch_list %}
+
+ {{arch}}: {{ repo.rpm_dl_stat[arch] }} <br />
+
+ {% endfor %}
+ <!-- -->
</td>
<td class="rightmost">
<a href="{{
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index de05b68..2eb7312 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -12,18 +12,20 @@ from itertools import groupby
from coprs import app
from coprs import db
+from coprs import rcp
from coprs import exceptions
from coprs import forms
from coprs import helpers
from coprs import models
from coprs.logic.stat_logic import CounterStatLogic
+from coprs.rmodels import TimedStatEvents
from coprs.views.misc import login_required, page_not_found
from coprs.views.coprs_ns import coprs_ns
from coprs.logic import builds_logic, coprs_logic, actions_logic
-from coprs.helpers import parse_package_name, generate_repo_url
+from coprs.helpers import parse_package_name, generate_repo_url, CHROOT_RPMS_DL_STAT_FMT,
CHROOT_REPO_MD_DL_STAT_FMT
@coprs_ns.route("/", defaults={"page": 1})
@@ -189,16 +191,31 @@ def copr_detail(username, coprname):
repos_info = {}
for chroot in copr.active_chroots:
+ chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format(
+ copr_user=copr.owner.name,
+ copr_project_name=copr.name,
+ copr_chroot=chroot.name,
+ )
+ chroot_rpms_dl_stat = TimedStatEvents.get_count(
+ rconnect=rcp.get_connection(),
+ name=chroot_rpms_dl_stat_key,
+ )
+
if chroot.name_release not in repos_info:
repos_info[chroot.name_release] = {
"name_release": chroot.name_release,
"name_release_human": chroot.name_release_human,
"arch_list": [chroot.arch],
"repo_file": "{}-{}-{}.repo".format(copr.owner.name,
copr.name, chroot.name_release),
- "dl_stat": repo_dl_stat[chroot.name_release]
+ "dl_stat": repo_dl_stat[chroot.name_release],
+ "rpm_dl_stat": {
+ chroot.arch: chroot_rpms_dl_stat
+ }
}
else:
repos_info[chroot.name_release]["arch_list"].append(chroot.arch)
+ repos_info[chroot.name_release]["rpm_dl_stat"][chroot.arch] =
chroot_rpms_dl_stat
+
repos_info_list = sorted(repos_info.values(), key=lambda rec:
rec["name_release"])
return flask.render_template("coprs/detail/overview.html",
diff --git a/frontend/coprs_frontend/coprs/views/misc.py
b/frontend/coprs_frontend/coprs/views/misc.py
index 6714f78..fd90267 100644
--- a/frontend/coprs_frontend/coprs/views/misc.py
+++ b/frontend/coprs_frontend/coprs/views/misc.py
@@ -261,7 +261,7 @@ def intranet_required(f):
def decorated_function(*args, **kwargs):
ip_addr = IPAddress(flask.request.remote_addr)
if not any(ip_addr in IPNetwork(addr_or_net)
- for addr_or_net in app.config.get(["INTRANET_IPS"],
"127.0.0.1")):
+ for addr_or_net in app.config.get("INTRANET_IPS",
["127.0.0.1",])):
return ("Stats can be update only from intranet hosts, "
"not {}, check
config\n".format(flask.request.remote_addr)), 403
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
index 9b40e29..d1f5f15 100644
--- a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
@@ -1,12 +1,14 @@
# coding: utf-8
import flask
+from coprs import rcp
from coprs import app
from coprs import db
from coprs.helpers import REPO_DL_STAT_FMT, CounterStatType
from ..misc import intranet_required
from . import stats_rcv_ns
-from ...logic.stat_logic import CounterStatLogic
+from ...logic.stat_logic import CounterStatLogic, handle_logstash
+
@stats_rcv_ns.route("/")
def ping():
@@ -27,22 +29,7 @@ def increment(counter_type, name):
@intranet_required
def logstash_handler():
try:
- json = flask.request.json
- if "request" in json:
- # 0 1 2 3 4 5
- # "request":
"/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo",
- req_split = json["request"].split("/")
- kwargs = dict(
- user=req_split[2],
- copr=req_split[3],
- name_release=req_split[5]
- )
- name = REPO_DL_STAT_FMT.format(**kwargs)
- app.logger.debug("kwargs: {}; name: {}".format(kwargs, name))
-
- CounterStatLogic.incr(name=name,
- counter_type=CounterStatType.REPO_DL)
- db.session.commit()
+ handle_logstash(rcp.get_connection(), flask.request.json)
except Exception as err:
app.logger.exception(err)
diff --git a/frontend/coprs_frontend/tests/test_rmodels.py
b/frontend/coprs_frontend/tests/test_rmodels.py
new file mode 100644
index 0000000..939426c
--- /dev/null
+++ b/frontend/coprs_frontend/tests/test_rmodels.py
@@ -0,0 +1,53 @@
+# coding: utf-8
+
+import time
+import pytest
+
+from redis import StrictRedis
+
+from coprs.rmodels import TimedStatEvents
+
+
+class TestRModels(object):
+
+ def setup_method(self, method):
+ self.rc = StrictRedis()
+ self.prefix = "copr:test:r_models"
+
+ self.time_now = time.time()
+
+ def teardown_method(self, method):
+ keys = self.rc.keys('{}*'.format(self.prefix))
+ if keys:
+ self.rc.delete(*keys)
+
+ def test_nop(self):
+ pass
+
+ def test_timed_stats_events(self):
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now, )
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar",
prefix=self.prefix,) == 1
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now, count=2)
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar",
prefix=self.prefix,) == 3
+
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now - 1000000, count=2)
+ TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix,
+ timestamp=self.time_now - 3000000, count=3)
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar",
prefix=self.prefix,) == 3
+ assert TimedStatEvents.get_count(self.rc, name="foobar",
prefix=self.prefix,
+ day_min=self.time_now - 2000000) == 5
+ assert TimedStatEvents.get_count(self.rc, name="foobar",
prefix=self.prefix,
+ day_min=self.time_now - 5000000) == 8
+
+ TimedStatEvents.trim_before(self.rc, name="foobar",
+ prefix=self.prefix, threshold_timestamp=self.time_now
- 200000)
+
+ assert TimedStatEvents.get_count(self.rc, name="foobar",
prefix=self.prefix,
+ day_min=self.time_now - 5000000) == 3
+