Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : master
---------------------------------------------------------------
commit c1a5794e39b3d1ea433cccc1c1cb55e52b72fd4a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Feb 19 23:17:38 2015 +0100
[frontend] [rhbz:#1185959] RFE: Present statistics about project popularity (WIP todo:
package logstash config)
---------------------------------------------------------------
.../450fe5f7942d_added_table_counterstat.py | 31 +++++++
frontend/coprs_frontend/config/copr_devel.conf | 3 +-
frontend/coprs_frontend/coprs/__init__.py | 2 +
frontend/coprs_frontend/coprs/config.py | 1 +
frontend/coprs_frontend/coprs/helpers.py | 7 ++
frontend/coprs_frontend/coprs/logic/stat_logic.py | 88 ++++++++++++++++++++
frontend/coprs_frontend/coprs/models.py | 19 ++++
.../coprs/templates/coprs/detail/overview.html | 6 +-
.../coprs/views/coprs_ns/coprs_general.py | 8 ++-
frontend/coprs_frontend/coprs/views/misc.py | 18 ++++-
.../coprs/views/stats_ns/__init__.py | 4 +
.../coprs/views/stats_ns/stats_receiver.py | 46 ++++++++++
.../tests/test_logic/test_stat_logic.py | 36 ++++++++
13 files changed, 264 insertions(+), 5 deletions(-)
diff --git
a/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py
b/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py
new file mode 100644
index 0000000..dae6c40
--- /dev/null
+++ b/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py
@@ -0,0 +1,31 @@
+"""added table CounterStat
+
+Revision ID: 450fe5f7942d
+Revises: bd0dab2e478
+Create Date: 2015-02-19 23:40:08.934834
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '450fe5f7942d'
+down_revision = 'bd0dab2e478'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+ ### commands auto generated by Alembic - please adjust! ###
+ op.create_table('counter_stat',
+ sa.Column('name', sa.String(length=127), nullable=False),
+ sa.Column('counter_type', sa.String(length=30), nullable=True),
+ sa.Column('counter', sa.Integer(), server_default='0',
nullable=True),
+ sa.PrimaryKeyConstraint('name')
+ )
+ ### end Alembic commands ###
+
+
+def downgrade():
+ ### commands auto generated by Alembic - please adjust! ###
+ op.drop_table('counter_stat')
+ ### end Alembic commands ###
diff --git a/frontend/coprs_frontend/config/copr_devel.conf
b/frontend/coprs_frontend/config/copr_devel.conf
index 9f26dc2..d6c5615 100644
--- a/frontend/coprs_frontend/config/copr_devel.conf
+++ b/frontend/coprs_frontend/config/copr_devel.conf
@@ -26,7 +26,7 @@ SQLALCHEMY_DATABASE_URI =
'sqlite:////var/lib/copr/data/copr.db'
#LOGGING_LEVEL = logging.ERROR
DEBUG = True
-SQLALCHEMY_ECHO = True
+SQLALCHEMY_ECHO = False
#CSRF_ENABLED = True
# as of Flask-WTF 0.9+
@@ -44,3 +44,4 @@ ENFORCE_PROTOCOL_FOR_FRONTEND_URL = "http"
PUBLIC_COPR_HOSTNAME = "copr-fe-dev.cloud.fedoraproject.org"
LOG_FILENAME="/tmp/copr_frontend.log"
+INTRANET_IPS = ["127.0.0.1", "192.168.1.102"]
diff --git a/frontend/coprs_frontend/coprs/__init__.py
b/frontend/coprs_frontend/coprs/__init__.py
index de5fc5d..492dec3 100644
--- a/frontend/coprs_frontend/coprs/__init__.py
+++ b/frontend/coprs_frontend/coprs/__init__.py
@@ -46,6 +46,7 @@ from coprs.views import status_ns
from coprs.views.status_ns import status_general
from coprs.views import recent_ns
from coprs.views.recent_ns import recent_general
+from coprs.views.stats_ns import stats_receiver
from .context_processors import include_banner
@@ -58,5 +59,6 @@ app.register_blueprint(misc.misc)
app.register_blueprint(backend_ns.backend_ns)
app.register_blueprint(status_ns.status_ns)
app.register_blueprint(recent_ns.recent_ns)
+app.register_blueprint(stats_receiver.stats_rcv_ns)
app.add_url_rule("/", "coprs_ns.coprs_show",
coprs_general.coprs_show)
diff --git a/frontend/coprs_frontend/coprs/config.py
b/frontend/coprs_frontend/coprs/config.py
index 557dcc6..b8bb37a 100644
--- a/frontend/coprs_frontend/coprs/config.py
+++ b/frontend/coprs_frontend/coprs/config.py
@@ -42,6 +42,7 @@ class Config(object):
# primary log file
LOG_FILENAME = "/var/log/copr/frontend.log"
+ INTRANET_IPS = ["127.0.0.1"]
class ProductionConfig(Config):
diff --git a/frontend/coprs_frontend/coprs/helpers.py
b/frontend/coprs_frontend/coprs/helpers.py
index dfa5137..f4f8efa 100644
--- a/frontend/coprs_frontend/coprs/helpers.py
+++ b/frontend/coprs_frontend/coprs/helpers.py
@@ -21,6 +21,13 @@ def generate_api_token(size=30):
return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
+REPO_DL_STAT_FMT = "{user}@{copr}:{name_release}"
+
+
+class CounterStatType(object):
+ REPO_DL = "repo_dl"
+
+
class EnumType(type):
def __call__(self, attr):
diff --git a/frontend/coprs_frontend/coprs/logic/stat_logic.py
b/frontend/coprs_frontend/coprs/logic/stat_logic.py
new file mode 100644
index 0000000..c1b2e0f
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/logic/stat_logic.py
@@ -0,0 +1,88 @@
+from collections import defaultdict
+import json
+import os
+import pprint
+import time
+from sqlalchemy import or_
+from sqlalchemy import and_
+
+from sqlalchemy.orm.exc import NoResultFound
+
+from coprs import app
+from coprs import db
+from coprs import exceptions
+from coprs.models import CounterStat
+from coprs import helpers
+from coprs.helpers import REPO_DL_STAT_FMT
+from coprs import signals
+from coprs.helpers import CounterStatType
+
+
+
+
+class CounterStatLogic(object):
+
+ @classmethod
+ def get(cls, name):
+ """
+ :param name: counter name
+ :return:
+ """
+ return CounterStat.query.filter(CounterStat.name == name)
+
+ @classmethod
+ def get_multiply_same_type(cls, counter_type, names_list):
+ return (
+ CounterStat.query
+ .filter(CounterStat.counter_type == counter_type)
+ .filter(CounterStat.name.in_(names_list))
+ )
+
+ @classmethod
+ def add(cls, name, counter_type):
+ csl = CounterStat(name=name, counter_type=counter_type)
+ db.session.add(csl)
+ return csl
+
+ @classmethod
+ def incr(cls, name, counter_type):
+ """
+ Warning: dirty method: does commit if missing stat record.
+ """
+ try:
+ csl = CounterStatLogic.get(name).one()
+ csl.counter = CounterStat.counter + 1
+ except NoResultFound:
+ csl = CounterStatLogic.add(name, counter_type)
+ csl.counter = 1
+
+ db.session.add(csl)
+ return csl
+
+ @classmethod
+ def get_copr_repo_dl_stat(cls, copr):
+ # chroot -> stat_name
+ chroot_by_stat_name = {}
+ for chroot in copr.active_chroots:
+ kwargs = {
+ "user": copr.owner.name,
+ "copr": copr.name,
+ "name_release": chroot.name_release
+ }
+ chroot_by_stat_name[REPO_DL_STAT_FMT.format(**kwargs)] = chroot.name_release
+
+ # [{counter: <value>, name: <stat_name>}, ...]
+ stats = cls.get_multiply_same_type(counter_type=helpers.CounterStatType.REPO_DL,
+ names_list=chroot_by_stat_name.keys())
+
+ # need: {chroot -> value, ... }
+ repo_dl_stats = defaultdict(int)
+ for stat in stats:
+ repo_dl_stats[chroot_by_stat_name[stat.name]] = stat.counter
+
+ return repo_dl_stats
+
+
+
+
+
diff --git a/frontend/coprs_frontend/coprs/models.py
b/frontend/coprs_frontend/coprs/models.py
index ef8ad0e..d6cfe5d 100644
--- a/frontend/coprs_frontend/coprs/models.py
+++ b/frontend/coprs_frontend/coprs/models.py
@@ -423,6 +423,13 @@ class MockChroot(db.Model, helpers.Serializer):
return "{0}-{1}-{2}".format(self.os_release, self.os_version,
self.arch)
@property
+ def name_release(self):
+ """
+ Textual representation of name of this or release
+ """
+ return "{}-{}".format(self.os_release, self.os_version)
+
+ @property
def os(self):
"""
Textual representation of the operating system name
@@ -578,3 +585,15 @@ class Krb5Login(db.Model, helpers.Serializer):
primary = db.Column(db.String(80), nullable=False, primary_key=True)
user = db.relationship("User",
backref=db.backref("krb5_logins"))
+
+
+class CounterStat(db.Model, helpers.Serializer):
+ """
+ Generic store for simple statistics.
+ """
+
+ name = db.Column(db.String(127), primary_key=True)
+ counter_type = db.Column(db.String(30))
+
+ counter = db.Column(db.Integer, default=0, server_default="0")
+
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
index 7336814..6063a33 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
+++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html
@@ -22,8 +22,10 @@
<tr>
<th class="leftmost">Release</th>
<th>Architecture</th>
- <th class="rightmost">Yum Repo</th>
+
+ <th class="rightmost">Yum Repo
<small>[Downloads]</small></th>
</tr>
+ <!-- TODO: remove complex logic from html, group by os-release in the python view
-->
{% for mock_chroot in copr.active_chroots %}
{% if loop.index < copr.active_chroots|length %}
{% if mock_chroot.os_release != copr.active_chroots[loop.index].os_release or
@@ -67,8 +69,10 @@
)|fix_url_https_frontend}}">
{{ copr.owner.name }}-{{ copr.name
}}-{{mock_chroot.os_release+"-"+mock_chroot.os_version}}.repo
</a>
+ <small>[ {{
repo_dl_stat[mock_chroot.name_release] }} ]</small>
{% endif %}
</td>
+
</tr>
{% else %}
<tr colspan="2"><td>No active releases</td></tr>
diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
index be91b69..75ade2f 100644
--- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
+++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py
@@ -16,6 +16,7 @@ from coprs import exceptions
from coprs import forms
from coprs import helpers
from coprs import models
+from coprs.logic.stat_logic import CounterStatLogic
from coprs.views.misc import login_required, page_not_found
@@ -183,10 +184,13 @@ def copr_detail(username, coprname):
except sqlalchemy.orm.exc.NoResultFound:
return page_not_found(
"Copr with name {0} does not exist.".format(coprname))
-
+ from collections import defaultdict
+ repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr)
return flask.render_template("coprs/detail/overview.html",
copr=copr,
- form=form)
+ form=form,
+ repo_dl_stat=repo_dl_stat
+ )
@coprs_ns.route("/<username>/<coprname>/permissions/")
diff --git a/frontend/coprs_frontend/coprs/views/misc.py
b/frontend/coprs_frontend/coprs/views/misc.py
index 7d6138a..db29c29 100644
--- a/frontend/coprs_frontend/coprs/views/misc.py
+++ b/frontend/coprs_frontend/coprs/views/misc.py
@@ -249,7 +249,23 @@ def backend_authenticated(f):
def decorated_function(*args, **kwargs):
auth = flask.request.authorization
if not auth or auth.password != app.config["BACKEND_PASSWORD"]:
- return "You have to provide the correct password", 401
+ return "You have to provide the correct password\n", 401
+
+ return f(*args, **kwargs)
+ return decorated_function
+
+
+def intranet_required(f):
+ @functools.wraps(f)
+ def decorated_function(*args, **kwargs):
+ # app.logger.debug("MY: {}".format(flask.request.remote_addr))
+ # Use,
+ # >>> import ipaddress
+ # >>> ipaddress.ip_address('192.168.0.1') in
ipaddress.ip_network('192.168.0.0/24')
+ if not app.config["DEBUG"]:
+ if flask.request.remote_addr not in app.config["INTRANET_IPS"]:
+ return ("Stats can be update only from intranet hosts, "
+ "not {}, check
config\n".format(flask.request.remote_addr)), 403
return f(*args, **kwargs)
return decorated_function
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py
b/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py
new file mode 100644
index 0000000..33c09b2
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py
@@ -0,0 +1,4 @@
+# coding: utf-8
+import flask
+
+stats_rcv_ns = flask.Blueprint("stats_rcv_ns", __name__,
url_prefix="/stats_rcv")
diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
new file mode 100644
index 0000000..caf3039
--- /dev/null
+++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py
@@ -0,0 +1,46 @@
+# coding: utf-8
+
+import flask
+from coprs import app
+from coprs import db
+from coprs.helpers import REPO_DL_STAT_FMT, CounterStatType
+from ..misc import intranet_required
+from . import stats_rcv_ns
+from ...logic.stat_logic import CounterStatLogic
+
+(a)stats_rcv_ns.route("/")
+def ping():
+ return "OK", 200
+
+
+(a)stats_rcv_ns.route("/<counter_type>/<name>/",
methods=['POST'])
+@intranet_required
+def increment(counter_type, name):
+ app.logger.debug(flask.request.remote_addr)
+
+ CounterStatLogic.incr(name, counter_type)
+ db.session.commit()
+ return "", 201
+
+(a)stats_rcv_ns.route("/from_logstash", methods=['POST'])
+#@intranet_required
+def logstash_handler():
+ # import ipdb; ipdb.set_trace()
+ json = flask.request.json
+ if "request" in json:
+ # 0 1 2 3 4 5
+ # "request":
"/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo",
+ req_split = json["request"].split("/")
+ kwargs = dict(
+ user=req_split[2],
+ copr=req_split[3],
+ name_release=req_split[5]
+ )
+ name = REPO_DL_STAT_FMT.format(**kwargs)
+ app.logger.debug("kwargs: {}; name: {}".format(kwargs, name))
+
+ CounterStatLogic.incr(name=name,
+ counter_type=CounterStatType.REPO_DL)
+ db.session.commit()
+
+ return "", 201
diff --git a/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py
b/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py
new file mode 100644
index 0000000..14adb49
--- /dev/null
+++ b/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py
@@ -0,0 +1,36 @@
+# coding: utf-8
+import pytest
+
+from coprs.exceptions import ActionInProgressException
+from coprs.helpers import ActionTypeEnum
+from coprs.logic.coprs_logic import CoprsLogic
+from coprs.logic.stat_logic import CounterStatLogic
+from coprs.helpers import CounterStatType
+from coprs import models
+from tests.coprs_test_case import CoprsTestCase
+
+
+class TestStatLogic(CoprsTestCase):
+
+ def setup_method(self, method):
+ super(TestStatLogic, self).setup_method(method)
+
+ self.counter_type = CounterStatType.REPO_DL
+ self.counter_name = "{}:user/copr".format(CounterStatType.REPO_DL)
+
+ def test_counter_basic(self):
+ CounterStatLogic.add(self.counter_name, self.counter_type)
+ self.db.session.commit()
+ CounterStatLogic.incr(self.counter_name, self.counter_type)
+ self.db.session.commit()
+ csl = CounterStatLogic.get(self.counter_name).one()
+ assert csl.counter == 1
+
+ def test_new_by_incr(self):
+ with pytest.raises(Exception):
+ CounterStatLogic.get(self.counter_name).one()
+
+ CounterStatLogic.incr(self.counter_name, self.counter_type)
+ self.db.session.commit()
+ csl = CounterStatLogic.get(self.counter_name).one()
+ assert csl.counter == 1