Repository :
http://git.fedorahosted.org/cgit/copr.git
On branch : dist-git
---------------------------------------------------------------
commit bfa97caeb735674f5d2c8b70669247f03c2f93b4
Author: Adam Samalik <asamalik(a)redhat.com>
Date: Thu Jun 25 15:08:43 2015 +0200
[dist-git] basic rpm package
---------------------------------------------------------------
{backend => dist-git}/LICENSE | 0
dist-git/conf/copr-dist-git.conf.example | 8 +
.../__init__.py => dist-git/copr-dist-git.service | 0
dist-git/copr-dist-git.spec | 57 +++++++
dist-git/dist-git/__init__.py | 1 +
dist-git/dist-git/dist_git_importer.py | 162 ++++++++++++++++++++
dist-git/dist-git/helpers.py | 61 ++++++++
dist-git/dit_git_import.py | 91 -----------
8 files changed, 289 insertions(+), 91 deletions(-)
diff --git a/backend/LICENSE b/dist-git/LICENSE
similarity index 100%
copy from backend/LICENSE
copy to dist-git/LICENSE
diff --git a/dist-git/conf/copr-dist-git.conf.example
b/dist-git/conf/copr-dist-git.conf.example
new file mode 100644
index 0000000..fec0824
--- /dev/null
+++ b/dist-git/conf/copr-dist-git.conf.example
@@ -0,0 +1,8 @@
+[dist-git]
+
+# default is
http://coprs/rest/api
+frontend_base_url=http://copr-fe-dev.cloud.fedoraproject.org
+
+# must have same value as BACKEND_PASSWORD from have frontend in /etc/copr/copr.conf
+# default is PASSWORDHERE but you really should change it. really.
+frontend_auth=backend_password_from_fe_config
diff --git a/frontend/coprs_frontend/coprs/logic/__init__.py
b/dist-git/copr-dist-git.service
similarity index 100%
copy from frontend/coprs_frontend/coprs/logic/__init__.py
copy to dist-git/copr-dist-git.service
diff --git a/dist-git/copr-dist-git.spec b/dist-git/copr-dist-git.spec
new file mode 100644
index 0000000..1930eb4
--- /dev/null
+++ b/dist-git/copr-dist-git.spec
@@ -0,0 +1,57 @@
+Name: copr-dist-git
+Version: 0.1
+Release: 1%{?dist}
+Summary: Copr services for Dist Git server
+
+Group: Applications/Productivity
+License: GPLv2+
+URL:
https://fedorahosted.org/copr/
+# Source is created by
+# git clone
https://git.fedorahosted.org/git/copr.git
+# cd copr/dist-git
+# tito build --tgz
+Source0: %{name}-%{version}.tar.gz
+
+BuildArch: noarch
+
+Requires: dist-git
+Requires: python-bunch
+Requires: python-requests
+
+%description
+COPR is lightweight build system. It allows you to create new project in WebUI
+and submit new builds and COPR will create yum repository from latest builds.
+
+This package contains Copr services for Dist Git server
+
+
+%prep
+%setup -q
+
+
+%build
+
+
+%install
+
+install -d %{buildroot}%{_datadir}/copr/dist-git
+install -d %{buildroot}%{_sysconfdir}/copr
+
+cp -a dist-git/* %{buildroot}%{_datadir}/copr/dist-git
+cp -a conf/copr-dist-git.conf.example %{buildroot}%{_sysconfdir}/copr/copr-dist-git.conf
+
+
+%check
+
+
+%files
+%license LICENSE
+
+%{_datadir}/copr/*
+%dir %{_sysconfdir}/copr
+%config(noreplace) %attr(0640, root, copr) %{_sysconfdir}/copr/copr-dist-git.conf
+
+
+%changelog
+* Thu Jun 25 2015 Adam Samalik <asamalik(a)redhat.com> 0.1
+- basic package
diff --git a/dist-git/dist-git/__init__.py b/dist-git/dist-git/__init__.py
new file mode 100644
index 0000000..f7f1a59
--- /dev/null
+++ b/dist-git/dist-git/__init__.py
@@ -0,0 +1 @@
+# Copr Dist Git
diff --git a/dist-git/dist-git/dist_git_importer.py
b/dist-git/dist-git/dist_git_importer.py
new file mode 100644
index 0000000..060d126
--- /dev/null
+++ b/dist-git/dist-git/dist_git_importer.py
@@ -0,0 +1,162 @@
+import os
+import json
+import time
+import types
+import urllib
+import shutil
+import tempfile
+from requests import get
+from requests import post
+from pyrpkg import Commands
+from subprocess import call
+from pyrpkg.errors import rpkgError
+
+from helpers import DistGitConfigReader
+
+# Example usage:
+#
+# user = "asamalik"
+# project = "project-for-dist-git"
+# pkg_name = "devtoolset-3"
+# branch = "f20"
+# filepath = "/tmp/rh-php56-php-5.6.5-5.el7.src.rpm"
+#
+# git_hash = import_srpm(user, project, pkg_name, branch, filepath)
+
+
+def _my_upload(repo_dir, reponame, filename, filehash):
+ """
+ This is a replacement function for uploading sources.
+ Rpkg uses upload.cgi for uploading which doesn't make sense
+ on the local machine.
+ """
+ lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/"
+ source = os.path.join(repo_dir, filename)
+ destination = os.path.join(lookaside, reponame, filename, filehash, filename)
+ if not os.path.exists(destination):
+ os.makedirs(os.path.dirname(destination))
+ shutil.copyfile(source, destination)
+
+def import_srpm(user, project, pkg, branch, filepath):
+ """
+ Imports a source rpm file into local dist git.
+ Repository name is in the Copr Style: user/project/package
+ filepath is a srpm file locally downloaded somewhere
+ """
+ # I need to use git via SSH because of gitolite as it manages
+ # permissions with it's hook that relies on gitolite console
+ # which is a default shell on SSH
+ gitbaseurl = "ssh://copr-dist-git@localhost/%(module)s"
+ tmp = tempfile.mkdtemp()
+ try:
+ repo_dir = os.path.join(tmp, pkg)
+
+ # use rpkg for importing the source rpm
+ commands = Commands(path =repo_dir,
+ lookaside ="",
+ lookasidehash ="md5",
+ lookaside_cgi ="",
+ gitbaseurl =gitbaseurl,
+ anongiturl ="",
+ branchre ="",
+ kojiconfig ="",
+ build_client ="")
+
+ # rpkg gets module_name as a basename of git url
+ # we use module_name as "username/projectname/packagename"
+ # basename is not working here - so I'm setting it manually
+ module = "{}/{}/{}".format(user, project, pkg)
+ commands.module_name = module
+ # rpkg calls upload.cgi script on the dist git server
+ # here, I just copy the source files manually with custom function
+ # I also add one parameter "repo_dir" to that function with this hack
+ commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir)
+
+ # clone the pkg repository into tmp directory
+ #module = "{}/{}/{}.git".format(user, project, pkg)
+ #giturl = gitbaseurl % {'module': module}
+ #cmd = ['git', 'clone', "-b", "f20",
giturl]
+ #call(cmd, cwd=tmp)
+ commands.clone(module, tmp, branch)
+
+ # import the source rpm into git and save filenames of sources
+ uploadfiles = commands.import_srpm(filepath)
+
+ # save the source files into lookaside cache
+ commands.upload(uploadfiles, replace=True)
+
+ # git push
+ message = "Import of {}".format(os.path.basename(filepath))
+ #call(["git", "commit", "-m", message],
cwd=repo_dir)
+ #call(["git", "push"], cwd=repo_dir)
+ try:
+ commands.commit(message)
+ commands.push()
+ except rpkgError:
+ pass
+ git_hash = commands.commithash
+ finally:
+ shutil.rmtree(tmp)
+ return git_hash
+
+
+class DistGitImporter():
+ def __init__(self):
+ self.config_reader = DistGitConfigReader()
+ self.opts = self.config_reader.read()
+
+ def run(self):
+ get_url = "{}/backend/uploading/".format(self.opts.frontend_base_url)
+ upload_url =
"{}/backend/upload-completed/".format(self.opts.frontend_base_url)
+ auth = ("user",self.opts.frontend_auth)
+ headers = {"content-type": "application/json"}
+
+ tmp = tempfile.mkdtemp()
+ try:
+ while(True):
+ # get the data
+ r = get(get_url)
+ try:
+ task = r.json()["builds"][0]
+
+ task_id = task["task_id"]
+ user = task["user"]
+ project = task["project"]
+ package = task["package"]
+ branch = task["branch"]
+ package_url = task["package_url"]
+ except:
+ time.sleep(10)
+ continue
+
+ # make sure repos exist
+ reponame = "{}/{}/{}".format(user, project, package)
+ call(["/usr/share/dist-git/git_package.sh", reponame])
+ call(["/usr/share/dist-git/git_branch.sh", branch, reponame])
+
+ # download the package
+ filepath = os.path.join(tmp, os.path.basename(package_url))
+ urllib.urlretrieve(package_url, filepath)
+
+ # import it and delete the srpm
+ git_hash = import_srpm(user, project, package, branch, filepath)
+ os.remove(filepath)
+
+ # send a response
+ data = {"task_id": task_id,
+ "repo_name": reponame,
+ "git_hash": git_hash}
+ post(upload_url, auth=auth, data=json.dumps(data), headers=headers)
+ finally:
+ shutil.rmtree(tmp)
+
+def main():
+ importer = DistGitImporter()
+ try:
+ importer.run()
+ except KeyboardInterrupt:
+ return
+
+if __name__ == "__main__":
+ main()
+
diff --git a/dist-git/dist-git/helpers.py b/dist-git/dist-git/helpers.py
new file mode 100644
index 0000000..91fbb89
--- /dev/null
+++ b/dist-git/dist-git/helpers.py
@@ -0,0 +1,61 @@
+from bunch import Bunch
+import os
+import ConfigParser
+
+def _get_conf(cp, section, option, default, mode=None):
+ """
+ To make returning items from config parser less irritating
+
+ :param mode: convert obtained value, possible modes:
+ - None (default): do nothing
+ - "bool" or "boolean"
+ - "int"
+ - "float"
+ """
+
+ if cp.has_section(section) and cp.has_option(section, option):
+ if mode is None:
+ return cp.get(section, option)
+ elif mode in ["bool", "boolean"]:
+ return cp.getboolean(section, option)
+ elif mode == "int":
+ return cp.getint(section, option)
+ elif mode == "float":
+ return cp.getfloat(section, option)
+ elif mode == "path":
+ path = cp.get(section, option)
+ if path.startswith("~"):
+ path = os.path.expanduser(path)
+ path = os.path.abspath(path)
+ path = os.path.normpath(path)
+
+ return path
+ return default
+
+
+class DistGitConfigReader(object):
+ def __init__(self, config_file=None):
+ self.config_file = config_file or "/etc/copr/copr-dist-git.conf"
+
+ def read(self):
+ try:
+ opts = self._read_unsafe()
+ return opts
+
+ except ConfigParser.Error as e:
+ raise CoprBackendError(
+ "Error parsing config file: {0}: {1}".format(
+ self.config_file, e))
+
+ def _read_unsafe(self):
+ cp = ConfigParser.ConfigParser()
+ cp.read(self.config_file)
+
+ opts = Bunch()
+
+ opts.frontend_base_url = _get_conf(
+ cp, "dist-git", "frontend_base_url",
"http://copr-fe")
+
+ opts.frontend_auth = _get_conf(
+ cp, "dist-git", "frontend_auth",
"PASSWORDHERE")
+ return opts
diff --git a/dist-git/dit_git_import.py b/dist-git/dit_git_import.py
deleted file mode 100644
index 01731b9..0000000
--- a/dist-git/dit_git_import.py
+++ /dev/null
@@ -1,91 +0,0 @@
-import os
-import types
-import shutil
-import tempfile
-from pyrpkg import Commands
-from pyrpkg.errors import rpkgError
-
-# Example usage:
-#
-# user = "asamalik"
-# project = "project-for-dist-git"
-# pkg_name = "devtoolset-3"
-# branch = "f20"
-# filepath = "/tmp/rh-php56-php-5.6.5-5.el7.src.rpm"
-#
-# git_hash = import_srpm(user, project, pkg_name, branch, filepath)
-
-
-def _my_upload(repo_dir, reponame, filename, filehash):
- """
- This is a replacement function for uploading sources.
- Rpkg uses upload.cgi for uploading which doesn't make sense
- on the local machine.
- """
- lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/"
- source = os.path.join(repo_dir, filename)
- destination = os.path.join(lookaside, reponame, filename, filehash, filename)
- if not os.path.exists(destination):
- os.makedirs(os.path.dirname(destination))
- shutil.copyfile(source, destination)
-
-def import_srpm(user, project, pkg, branch, filepath):
- """
- Imports a source rpm file into local dist git.
- Repository name is in the Copr Style: user/project/package
- filepath is a srpm file locally downloaded somewhere
- """
- # I need to use git via SSH because of gitolite as it manages
- # permissions with it's hook that relies on gitolite console
- # which is a default shell on SSH
- gitbaseurl = "ssh://frank@localhost/%(module)s"
- tmp = tempfile.mkdtemp()
- try:
- repo_dir = os.path.join(tmp, pkg)
-
- # use rpkg for importing the source rpm
- commands = Commands(path =repo_dir,
- lookaside ="",
- lookasidehash ="md5",
- lookaside_cgi ="",
- gitbaseurl =gitbaseurl,
- anongiturl ="",
- branchre ="",
- kojiconfig ="",
- build_client ="")
-
- # rpkg gets module_name as a basename of git url
- # we use module_name as "username/projectname/packagename"
- # basename is not working here - so I'm setting it manually
- module = "{}/{}/{}".format(user, project, pkg)
- commands.module_name = module
- # rpkg calls upload.cgi script on the dist git server
- # here, I just copy the source files manually with custom function
- # I also add one parameter "repo_dir" to that function with this hack
- commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir)
-
- # clone the pkg repository into tmp directory
- #module = "{}/{}/{}.git".format(user, project, pkg)
- #giturl = gitbaseurl % {'module': module}
- #cmd = ['git', 'clone', "-b", "f20",
giturl]
- #call(cmd, cwd=tmp)
- commands.clone(module, tmp, branch)
-
- # import the source rpm into git and save filenames of sources
- uploadfiles = commands.import_srpm(filepath)
-
- # save the source files into lookaside cache
- commands.upload(uploadfiles, replace=True)
-
- # git push
- message = "Import of {}".format(os.path.basename(filepath))
- #call(["git", "commit", "-m", message],
cwd=repo_dir)
- #call(["git", "push"], cwd=repo_dir)
- try:
- commands.commit(message)
- commands.push()
- except rpkgError:
- pass
- finally:
- shutil.rmtree(tmp)
- return commands.commithash