[copr] master: [copr] backend: script fixes, dropped create_repo cli script (84fac62)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 84fac622202468aea8edb3c024d1a472c05ccc1b
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Mon Feb 16 14:50:05 2015 +0100
[copr] backend: script fixes, dropped create_repo cli script
>---------------------------------------------------------------
backend/conf/copr-be.conf.example | 2 -
backend/conf/copr-be.local.conf | 2 -
backend/run/copr_create_repo.py | 50 --------------------
backend/run/copr_prune_results.py | 10 +++-
backend/run/copr_sign_unsigned.py | 16 ++++++-
backend/tests/test_run_copr_create_repo.py | 68 ----------------------------
6 files changed, 21 insertions(+), 127 deletions(-)
diff --git a/backend/conf/copr-be.conf.example b/backend/conf/copr-be.conf.example
index edfe14d..70e78ac 100644
--- a/backend/conf/copr-be.conf.example
+++ b/backend/conf/copr-be.conf.example
@@ -89,8 +89,6 @@ worker_logdir=/var/log/copr/workers/
# minimum age for builds to be pruned
prune_days=14
-# path to executable script to clean old build
-prune_script=/usr/share/copr/copr_prune_old_builds.sh
# Spawn builder in advance, before we get task?
# It save time, but consume resources even when
diff --git a/backend/conf/copr-be.local.conf b/backend/conf/copr-be.local.conf
index 4fc5f47..0eeb2bf 100644
--- a/backend/conf/copr-be.local.conf
+++ b/backend/conf/copr-be.local.conf
@@ -90,8 +90,6 @@ verbose=true
# minimum age for builds to be pruned
prune_days=14
-# path to executable script to clean old build
-prune_script=./copr_prune_old_builds.sh
[builder]
# default is 1800
diff --git a/backend/run/copr_create_repo.py b/backend/run/copr_create_repo.py
deleted file mode 100644
index f0d3a45..0000000
--- a/backend/run/copr_create_repo.py
+++ /dev/null
@@ -1,50 +0,0 @@
-from __future__ import print_function
-
-import sys
-
-from backend.createrepo import createrepo
-from backend.helpers import SortedOptParser
-
-
-def main(args):
-
- parser = SortedOptParser(
- "mockremote -u user_name -p copr_project repo_dir")
-
- parser.add_option("-u", "--user", dest="user",
- help="copr project owner username")
-
- parser.add_option("-p", "--project", dest="project",
- help="copr project name")
-
- parser.add_option("-f", "--front_url", dest="front_url",
- help="copr frontend url")
-
- opts, args = parser.parse_args(args)
-
- if not opts.user:
- print("No user was specified, exiting", file=sys.stderr)
- sys.exit(1)
-
- if not opts.project:
- print("No project was specified, exiting", file=sys.stderr)
- sys.exit(1)
-
- if not opts.front_url:
- print("No front url was specified, exiting", file=sys.stderr)
- sys.exit(1)
-
- if not args:
- print("No directory with repo was specified, exiting", file=sys.stderr)
- sys.exit(1)
-
- createrepo(path=args[0], front_url=opts.front_url,
- username=opts.user, projectname=opts.project)
-
-
-if __name__ == "__main__":
- """
- Provides cli interface for conditional execution of createrepo_c
- depending on user setting `auto_createrepo`
- """
- main(sys.argv[1:])
diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py
index bc79816..70bc20c 100644
--- a/backend/run/copr_prune_results.py
+++ b/backend/run/copr_prune_results.py
@@ -1,14 +1,19 @@
+#!/usr/bin/python
+
from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
import os
+import sys
import logging
from subprocess import Popen, PIPE
from copr.client.exceptions import CoprException, CoprRequestException
+sys.path.append("/usr/share/copr/")
+
from backend.helpers import BackendConfigReader, get_auto_createrepo_status
from backend.createrepo import createrepo_unsafe
@@ -16,7 +21,7 @@ log = logging.getLogger(__name__)
log.addHandler(logging.NullHandler())
DEF_DAYS = 14
-DEF_PRUNE_SCRIPT = "/usr/share/copr/copr_prune_old_builds.sh"
+DEF_PRUNE_SCRIPT = "/usr/bin/copr_prune_old_builds.sh"
def list_subdir(path):
@@ -39,9 +44,8 @@ def prune_project(opts, path, username, projectname):
# run prune project sh
days = getattr(opts, "prune_days", DEF_DAYS)
- prune_script = getattr(opts, "prune_script", DEF_PRUNE_SCRIPT)
- cmd = map(str, [prune_script, path, days])
+ cmd = map(str, [DEF_PRUNE_SCRIPT, path, days])
handle = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = handle.communicate()
diff --git a/backend/run/copr_sign_unsigned.py b/backend/run/copr_sign_unsigned.py
index fc690eb..7845beb 100755
--- a/backend/run/copr_sign_unsigned.py
+++ b/backend/run/copr_sign_unsigned.py
@@ -115,14 +115,26 @@ def check_pubkey(pubkey_path, user, project, opts):
def main():
- shutil.rmtree("/tmp/users_failed.txt", ignore_errors=True)
- shutil.rmtree("/tmp/users_done.txt", ignore_errors=True)
+ # shutil.rmtree("/tmp/users_failed.txt", ignore_errors=True)
+ # shutil.rmtree("/tmp/users_done.txt", ignore_errors=True)
+ users_done_old = set()
+ try:
+ with open("/tmp/users_done.txt") as handle:
+ for line in handle:
+ users_done_old.add(line.strip())
+ except Exception as err:
+ log.exception(err)
+ log.debug("error during read old users done")
opts = BackendConfigReader().read()
log.info("Starting pubkey fill, destdir: {}".format(opts.destdir))
log.debug("list dir: {}".format(os.listdir(opts.destdir)))
for user_name in os.listdir(opts.destdir):
+ if user_name in users_done_old:
+ log.info("skipping user: {}".format(user_name))
+ continue
+
failed = False
log.info("Started processing user dir: {}".format(user_name))
user_dir = os.path.join(opts.destdir, user_name)
diff --git a/backend/tests/test_run_copr_create_repo.py b/backend/tests/test_run_copr_create_repo.py
deleted file mode 100644
index 506dacd..0000000
--- a/backend/tests/test_run_copr_create_repo.py
+++ /dev/null
@@ -1,68 +0,0 @@
-import sys
-
-import pytest
-import six
-
-if six.PY3:
- from unittest import mock
- from unittest.mock import MagicMock
-else:
- import mock
- from mock import MagicMock
-
-
-sys.path.insert(0, "../run")
-
-from copr_create_repo import main as createrepo_main
-
-
-(a)mock.patch("copr_create_repo.createrepo")
-class TestArgParser(object):
-
- def test_arg_parser_good_input(self, mc_createrepo, capsys):
- args = ['-u', 'foo', '-p', 'bar', '-f', 'http://example.com/api/', '/tmp']
-
- mc_createrepo.return_value = 0, "", ""
- createrepo_main(args)
- assert mc_createrepo.call_args == mock.call(username='foo', projectname='bar',
- front_url='http://example.com/api/', path='/tmp')
-
- def test_arg_parser_missing_path(self, mc_main, capsys):
- args = ['-u', 'foo', '-p', 'bar', '-f', 'http://example.com/api/']
-
- with pytest.raises(SystemExit) as err:
- createrepo_main(args)
-
- assert err.value.code == 1
- stdout, stderr = capsys.readouterr()
- assert "No directory" in stderr
-
- def test_arg_parser_missing_user(self, mc_main, capsys):
- args = ['-p', 'bar', '-f', 'http://example.com/api/', '/tmp']
-
- with pytest.raises(SystemExit) as err:
- createrepo_main(args)
-
- assert err.value.code == 1
- stdout, stderr = capsys.readouterr()
- assert "No user" in stderr
-
- def test_arg_parser_missing_project(self, mc_main, capsys):
- args = ['-u', 'foo', '-f', 'http://example.com/api/', '/tmp']
-
- with pytest.raises(SystemExit) as err:
- createrepo_main(args)
-
- assert err.value.code == 1
- stdout, stderr = capsys.readouterr()
- assert "No project" in stderr
-
- def test_arg_parser_missing_front(self, mc_main, capsys):
- args = ['-u', 'foo', '-p', 'bar', '/tmp']
-
- with pytest.raises(SystemExit) as err:
- createrepo_main(args)
-
- assert err.value.code == 1
- stdout, stderr = capsys.readouterr()
- assert "No front url" in stderr
9 years, 2 months
[copr] master: more file descriptors on builder (76127a9)
by asamalik@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 76127a97238a44f2e39a224ecc6e282adb534708
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Mon Feb 16 12:29:20 2015 +0100
more file descriptors on builder
fixes RHBZ 1102788
>---------------------------------------------------------------
backend/conf/playbooks/builderpb-x86_64.yml | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/backend/conf/playbooks/builderpb-x86_64.yml b/backend/conf/playbooks/builderpb-x86_64.yml
index deccdc4..e9c6954 100644
--- a/backend/conf/playbooks/builderpb-x86_64.yml
+++ b/backend/conf/playbooks/builderpb-x86_64.yml
@@ -97,4 +97,5 @@
- fedora-20-i386.cfg
- epel-7-x86_64.cfg
- - lineinfile: dest=/root/.bashrc line="ulimit -n 10240" insertafter=EOF
+ - lineinfile: dest=/etc/security/limits.conf line="* soft nofile 10240" insertafter=EOF
+ - lineinfile: dest=/etc/security/limits.conf line="* hard nofile 10240" insertafter=EOF
9 years, 2 months
[copr] master: Update split_multi_src_builds.py (e1d2e1a)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit e1d2e1a603b463bb8dbc605fb4510cd8e88e59a7
Author: Valentin Gologuzov <evilkost(a)users.noreply.github.com>
Date: Thu Feb 5 19:05:13 2015 +0100
Update split_multi_src_builds.py
[frontend] Cleanup
>---------------------------------------------------------------
frontend/coprs_frontend/split_multi_src_builds.py | 20 +-------------------
1 files changed, 1 insertions(+), 19 deletions(-)
diff --git a/frontend/coprs_frontend/split_multi_src_builds.py b/frontend/coprs_frontend/split_multi_src_builds.py
index 9dd7955..9207d92 100755
--- a/frontend/coprs_frontend/split_multi_src_builds.py
+++ b/frontend/coprs_frontend/split_multi_src_builds.py
@@ -5,33 +5,15 @@ from __future__ import print_function
from __future__ import unicode_literals
from __future__ import division
from __future__ import absolute_import
-from flask_sqlalchemy import SQLAlchemy
"""
This script splits all builds which contains multiply src_packages
into the more builds, each of them containing exactly one src_pkg.
-Script modifies only db records, now actual rpms are touched.
+Script modifies only db records, actual rpms are not touched.
"""
-
-import argparse
-import os
-import subprocess
-import datetime
-
-import flask
-from flask_script import Manager, Command, Option, Group
-
-
-
-
-import shutil
-
-import sys
-import os
import logging
-import pwd
logging.basicConfig(
9 years, 3 months
[copr] master: [frontend] script to split builds with multiply source packages. (5f7ea88)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 5f7ea881ad4a619dfdb5455c2a447ea05e6bf52b
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Feb 5 19:01:13 2015 +0100
[frontend] script to split builds with multiply source packages.
>---------------------------------------------------------------
frontend/coprs_frontend/split_multi_src_builds.py | 132 +++++++++++++++++++++
1 files changed, 132 insertions(+), 0 deletions(-)
diff --git a/frontend/coprs_frontend/split_multi_src_builds.py b/frontend/coprs_frontend/split_multi_src_builds.py
new file mode 100755
index 0000000..9dd7955
--- /dev/null
+++ b/frontend/coprs_frontend/split_multi_src_builds.py
@@ -0,0 +1,132 @@
+#!/usr/bin/python
+# coding: utf-8
+
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division
+from __future__ import absolute_import
+from flask_sqlalchemy import SQLAlchemy
+
+
+"""
+This script splits all builds which contains multiply src_packages
+into the more builds, each of them containing exactly one src_pkg.
+Script modifies only db records, now actual rpms are touched.
+"""
+
+
+import argparse
+import os
+import subprocess
+import datetime
+
+import flask
+from flask_script import Manager, Command, Option, Group
+
+
+
+
+import shutil
+
+import sys
+import os
+import logging
+import pwd
+
+
+logging.basicConfig(
+# filename="/var/log/copr/split_multi_src_builds.log",
+ format='[%(asctime)s][%(levelname)6s]: %(message)s',
+ level=logging.DEBUG)
+log = logging.getLogger(__name__)
+
+
+def split_one(orig_build_id):
+ from coprs import db
+ from coprs import models
+ from coprs.logic.builds_logic import BuildsLogic
+ from coprs.models import BuildChroot
+
+ def clone_build(build):
+ other = models.Build(
+ pkgs=build.pkgs,
+ built_packages=build.built_packages,
+ pkg_version=build.pkg_version,
+ canceled=build.canceled,
+ repos=build.repos,
+
+ submitted_on=build.submitted_on,
+ started_on=build.started_on,
+ ended_on=build.ended_on,
+
+ results=build.results,
+
+ memory_reqs=build.memory_reqs,
+ timeout=build.timeout,
+ enable_net=build.enable_net,
+
+ user=build.user,
+ copr=build.copr,
+ )
+ return other
+
+ build = BuildsLogic.get(orig_build_id).one()
+ log.info("Start splitting build: {}, pkgs: {}".format(build.id, build.pkgs))
+
+ src_pkg_list = []
+ for mb_pkg in build.pkgs.strip().split(" "):
+ #if mb_pkg.endswith(".src.rpm"):
+ src_pkg_list.append(mb_pkg)
+
+ if len(src_pkg_list) == 0:
+ log.error("> Got build with empty pkgs: id={}".format(build.id))
+ return
+ if len(src_pkg_list) == 1:
+ log.debug("> Got build with one pkg in pkgs, id={}, pkgs={}".format(build.id, build.pkgs))
+ return
+
+ new_builds = []
+ new_build_chroots = []
+
+ for src_pkg in src_pkg_list:
+ log.info("> Processing {} package".format(src_pkg))
+ new_build = clone_build(build)
+ new_build.pkgs = src_pkg
+
+ for bc in build.build_chroots:
+ log.info("> > Copying chroot {}".format(bc.name))
+ new_bc = BuildChroot(
+ build=new_build,
+ mock_chroot=bc.mock_chroot,
+ status=bc.status
+ )
+ new_build_chroots.append(new_bc)
+
+ new_builds.append(new_build)
+
+ log.info("> Finished build split for id: {}. Doing commit".format(build.id))
+
+ db.session.rollback() # some dirty state in SQLalchemy, no idea how to do correctly
+ db.session.add_all(new_build_chroots)
+ db.session.add_all(new_builds)
+ db.session.commit()
+ log.info("> New build objects were created ")
+ for bc in build.build_chroots:
+ db.session.delete(bc)
+ db.session.delete(build)
+ db.session.commit()
+
+ log.info("> Build {} deleted ".format(build.id))
+
+
+def main():
+
+ from coprs import models
+
+ query = models.Build.query.filter(models.Build.pkgs.contains(" "))
+ for build_id in [build.id for build in query.all()]:
+ split_one(build_id)
+
+
+if __name__ == "__main__":
+ main()
9 years, 3 months
[copr] master: Merge branch 'hotfix' (15063bc)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : master
>---------------------------------------------------------------
commit 15063bc84a20ffc2266f3d0867fef2bab11f2e91
Merge: f114c72 5086dc0
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Feb 5 17:10:23 2015 +0100
Merge branch 'hotfix'
>---------------------------------------------------------------
frontend/copr-frontend.spec | 6 +++++-
rel-eng/packages/copr-frontend | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index cd50598..49b2b8d 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -5,7 +5,7 @@
%endif
Name: copr-frontend
-Version: 1.52
+Version: 1.53
Release: 1%{?dist}
Summary: Frontend for Copr
@@ -180,6 +180,10 @@ service httpd condrestart
%doc documentation/python-doc
%changelog
+* Thu Feb 05 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.53-1
+- [frontend] enabled `gpgcheck=1` in .repo template
+- [frontend] correct url for pubkey in .repo
+
* Fri Jan 23 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.52-1
- add url to gpg pubkey in .repo files
- [rhbz:#1183702] Interrupted builds aren't re-added to the
diff --git a/rel-eng/packages/copr-frontend b/rel-eng/packages/copr-frontend
index 9d79b41..66745e9 100644
--- a/rel-eng/packages/copr-frontend
+++ b/rel-eng/packages/copr-frontend
@@ -1 +1 @@
-1.52-1 frontend/
+1.53-1 frontend/
9 years, 3 months
[copr] hotfix: Automatic commit of package [copr-frontend] release [1.53-1]. (5086dc0)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : hotfix
>---------------------------------------------------------------
commit 5086dc06fb3b7c5eb67b87cca83d48aba9e41ba0
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Feb 5 15:37:56 2015 +0100
Automatic commit of package [copr-frontend] release [1.53-1].
>---------------------------------------------------------------
frontend/copr-frontend.spec | 6 +++++-
rel-eng/packages/copr-frontend | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec
index 43e88ac..bd42bf3 100644
--- a/frontend/copr-frontend.spec
+++ b/frontend/copr-frontend.spec
@@ -5,7 +5,7 @@
%endif
Name: copr-frontend
-Version: 1.52
+Version: 1.53
Release: 1%{?dist}
Summary: Frontend for Copr
@@ -168,6 +168,10 @@ service httpd condrestart
%doc documentation/python-doc
%changelog
+* Thu Feb 05 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.53-1
+- [frontend] enabled `gpgcheck=1` in .repo template
+- [frontend] correct url for pubkey in .repo
+
* Fri Jan 23 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.52-1
- add url to gpg pubkey in .repo files
- [rhbz:#1183702] Interrupted builds aren't re-added to the
diff --git a/rel-eng/packages/copr-frontend b/rel-eng/packages/copr-frontend
index 9d79b41..66745e9 100644
--- a/rel-eng/packages/copr-frontend
+++ b/rel-eng/packages/copr-frontend
@@ -1 +1 @@
-1.52-1 frontend/
+1.53-1 frontend/
9 years, 3 months
[copr] hotfix: [frontend] enabled `gpgcheck=1` in .repo template (6c79c22)
by vgologuz@fedoraproject.org
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : hotfix
>---------------------------------------------------------------
commit 6c79c220eab9d87b86d17851f389c0f1bca0a73a
Author: Valentin Gologuzov <vgologuz(a)redhat.com>
Date: Thu Feb 5 14:47:31 2015 +0100
[frontend] enabled `gpgcheck=1` in .repo template
>---------------------------------------------------------------
.../coprs_frontend/coprs/templates/coprs/copr.repo | 2 +-
1 files changed, 1 insertions(+), 1 deletions(-)
diff --git a/frontend/coprs_frontend/coprs/templates/coprs/copr.repo b/frontend/coprs_frontend/coprs/templates/coprs/copr.repo
index d316ae7..07b2762 100644
--- a/frontend/coprs_frontend/coprs/templates/coprs/copr.repo
+++ b/frontend/coprs_frontend/coprs/templates/coprs/copr.repo
@@ -2,7 +2,7 @@
name=Copr repo for {{ copr.name }} owned by {{ copr.owner.name }}
baseurl={{ url | fix_url_https_backend }}
skip_if_unavailable=True
-gpgcheck=0
+gpgcheck=1
gpgkey={{ pubkey_url | fix_url_https_backend }}
enabled=1
9 years, 3 months