[PATCH] use createrepo_c instead of createrepo to make repos if it is available
by Dennis Gilmore
Signed-off-by: Dennis Gilmore <dennis(a)ausil.us>
---
builder/kojid | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/builder/kojid b/builder/kojid
index 3a92ec3..4e53fb5 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -4252,7 +4252,11 @@ class CreaterepoTask(BaseTaskHandler):
def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo):
koji.ensuredir(self.outdir)
- cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir]
+ if os.path.isfile('/usr/bin/createrepo_c'):
+ cmd = ['/usr/bin/createrepo_c']
+ else:
+ cmd = ['/usr/bin/createrepo']
+ cmd.extend(['-vd', '-o', self.outdir])
if pkglist is not None:
cmd.extend(['-i', pkglist])
if os.path.isfile(groupdata):
--
2.4.0
8 years, 1 month
[PATCH 1/2] Move config processing from CLI to koji.read_config().
by Daniel Mach
---
cli/koji | 85 ++------------------------------------------
koji/__init__.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 109 insertions(+), 82 deletions(-)
diff --git a/cli/koji b/cli/koji
index 47d9691..66f4d99 100755
--- a/cli/koji
+++ b/cli/koji
@@ -186,88 +186,9 @@ def get_options():
list_commands()
parser.error('Unknown command: %s' % args[0])
assert False
- # load local config
- defaults = {
- 'server' : 'http://localhost/kojihub',
- 'weburl' : 'http://localhost/koji',
- 'topurl' : None,
- 'pkgurl' : None,
- 'topdir' : '/mnt/koji',
- 'max_retries' : None,
- 'retry_interval': None,
- 'anon_retry' : None,
- 'offline_retry' : None,
- 'offline_retry_interval' : None,
- 'keepalive' : True,
- 'timeout' : None,
- 'use_fast_upload': False,
- 'poll_interval': 5,
- 'krbservice': 'host',
- 'cert': '~/.koji/client.crt',
- 'ca': '~/.koji/clientca.crt',
- 'serverca': '~/.koji/serverca.crt',
- 'authtype': None
- }
- #note: later config files override earlier ones
- configs = koji.config_directory_contents('/etc/koji.conf.d')
- if os.access('/etc/koji.conf', os.F_OK):
- configs.append('/etc/koji.conf')
- if options.configFile:
- fn = os.path.expanduser(options.configFile)
- if os.path.isdir(fn):
- contents = koji.config_directory_contents(fn)
- if not contents:
- parser.error("No config files found in directory: %s" % fn)
- configs.extend(contents)
- else:
- if not os.access(fn, os.F_OK):
- parser.error("No such file: %s" % fn)
- configs.append(fn)
- else:
- user_config_dir = os.path.expanduser("~/.koji/config.d")
- configs.extend(koji.config_directory_contents(user_config_dir))
- fn = os.path.expanduser("~/.koji/config")
- if os.access(fn, os.F_OK):
- configs.append(fn)
- got_conf = False
- for configFile in configs:
- f = open(configFile)
- config = ConfigParser.ConfigParser()
- config.readfp(f)
- f.close()
- if config.has_section(options.profile):
- got_conf = True
- for name, value in config.items(options.profile):
- #note the defaults dictionary also serves to indicate which
- #options *can* be set via the config file. Such options should
- #not have a default value set in the option parser.
- if defaults.has_key(name):
- if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'):
- defaults[name] = config.getboolean(options.profile, name)
- elif name in ('max_retries', 'retry_interval',
- 'offline_retry_interval', 'poll_interval', 'timeout'):
- try:
- defaults[name] = int(value)
- except ValueError:
- parser.error("value for %s config option must be a valid integer" % name)
- assert False
- else:
- defaults[name] = value
- if configs and not got_conf:
- warn("Warning: no configuration for profile name: %s" % options.profile)
- for name, value in defaults.iteritems():
- if getattr(options, name, None) is None:
- setattr(options, name, value)
- dir_opts = ('topdir', 'cert', 'ca', 'serverca')
- for name in dir_opts:
- # expand paths here, so we don't have to worry about it later
- value = os.path.expanduser(getattr(options, name))
- setattr(options, name, value)
-
- #honor topdir
- if options.topdir:
- koji.BASEDIR = options.topdir
- koji.pathinfo.topdir = options.topdir
+
+ defaults = koji.read_config(options.profile, user_config=options.configFile)
+ options._update_loose(defaults.__dict__)
#pkgurl is obsolete
if options.pkgurl:
diff --git a/koji/__init__.py b/koji/__init__.py
index f45ff70..9951a28 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -28,6 +28,7 @@ except ImportError:
sys.stderr.write("Warning: Could not install krbV module. Kerberos support will be disabled.\n")
sys.stderr.flush()
import base64
+import ConfigParser
import datetime
import errno
from fnmatch import fnmatch
@@ -35,6 +36,7 @@ import httplib
import logging
import logging.handlers
from koji.util import md5_constructor
+import optparse
import os
import os.path
import pwd
@@ -1457,6 +1459,110 @@ def config_directory_contents(dir_name):
return configs
+def _config_directory_contents(dir_name):
+ configs = []
+ try:
+ conf_dir_contents = os.listdir(dir_name)
+ except OSError, exception:
+ if exception.errno != errno.ENOENT:
+ raise
+ else:
+ for name in sorted(conf_dir_contents):
+ if not name.endswith('.conf'):
+ continue
+ config_full_name = os.path.join(dir_name, name)
+ configs.append(config_full_name)
+ return configs
+
+
+def read_config(profile_name, user_config=None):
+ config_defaults = {
+ 'server': 'http://localhost/kojihub',
+ 'weburl': 'http://localhost/koji',
+ 'topurl': None,
+ 'pkgurl': None,
+ 'topdir': '/mnt/koji',
+ 'max_retries': None,
+ 'retry_interval': None,
+ 'anon_retry': None,
+ 'offline_retry': None,
+ 'offline_retry_interval': None,
+ 'keepalive': True,
+ 'timeout': None,
+ 'use_fast_upload': False,
+ 'poll_interval': 5,
+ 'krbservice': 'host',
+ 'cert': '~/.koji/client.crt',
+ 'ca': '~/.koji/clientca.crt',
+ 'serverca': '~/.koji/serverca.crt',
+ 'authtype': None
+ }
+
+ int_options = ['max_retries', 'retry_interval', 'offline_retry_interval', 'poll_interval', 'timeout']
+ bool_options = ['anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload']
+ path_options = ['topdir', 'cert', 'ca', 'serverca']
+
+ result = config_defaults.copy()
+ for option in config_defaults:
+ if option in path_options:
+ result[option] = os.path.expanduser(result[option])
+
+ configs = []
+
+ # main config
+ configs.append("/etc/koji.conf")
+
+ # conf.d
+ configs.extend(_config_directory_contents("/etc/koji.conf.d"))
+
+ # user config
+ configs.append(os.path.expanduser("~/.koji/config"))
+
+ # user conf.d
+ configs.extend(_config_directory_contents(os.path.expanduser("~/.koji/conf.d")))
+
+ # TODO: read configs via xdg.BaseDirectory.load_config_path("koji")
+
+ # user config specified in runtime
+ if user_config is not None:
+ configs.append(user_config)
+
+ # read configs in particular order, use the last value found
+ for config_path in configs:
+ if not os.access(config_path, os.F_OK):
+ continue
+ config = ConfigParser.SafeConfigParser()
+ config.readfp(open(config_path, "r"))
+
+ if profile_name not in config.sections():
+ continue
+
+ # check for invalid options
+ invalid_options = []
+ for option in config.options(profile_name):
+ if option not in result:
+ invalid_options.append(option)
+
+ if invalid_options:
+ raise ValueError("Invalid options: %s" % ", ".join(invalid_options))
+
+ for option in config.options(profile_name):
+ if option in bool_options:
+ result[option] = config.getboolean(profile_name, option)
+ elif option in int_options:
+ result[option] = config.getint(profile_name, option)
+ else:
+ result[option] = config.get(profile_name, option)
+ if option in path_options:
+ result[option] = os.path.expanduser(result[option])
+
+ result["profile"] = profile_name
+
+ # convert dict to optparse Values
+ options = optparse.Values(result)
+ return options
+
+
class PathInfo(object):
# ASCII numbers and upper- and lower-case letter for use in tmpdir()
ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]
--
2.5.0
8 years, 1 month
Frequent, intermittent failures of buildSRPMFromSCM since koji-1.10
by John Florian
About half of my builds fail with ...
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/koji/daemon.py", line 1161, in runTask
response = (handler.run(),)
File "/usr/lib/python2.7/site-packages/koji/tasks.py", line 158, in run
return koji.util.call_with_argcheck(self.handler, self.params, self.opts)
File "/usr/lib/python2.7/site-packages/koji/util.py", line 154, in call_with_argcheck
return func(*args, **kwargs)
File "/usr/sbin/kojid", line 3817, in handler
broot.init()
File "/usr/sbin/kojid", line 460, in init
rv = self.mock(['--init'])
File "/usr/sbin/kojid", line 408, in mock
incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger)
File "/usr/lib/python2.7/site-packages/koji/daemon.py", line 48, in incremental_upload
fast_incremental_upload(session, fname, fd, path, retries, logger)
File "/usr/lib/python2.7/site-packages/koji/daemon.py", line 87, in fast_incremental_upload
result = session.rawUpload(contents, offset, path, fname, overwrite=True)
File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1577, in __call__
return self.__func(self.__name,args,opts)
File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1920, in _callMethod
return self._sendCall(handler, headers, request)
File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1831, in _sendCall
return self._sendOneCall(handler, headers, request)
File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1850, in _sendOneCall
cnx.send(request)
File "/usr/lib64/python2.7/httplib.py", line 820, in send
self.sock.sendall(data)
File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 111, in sendall
self.close()
File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 82, in close
self.shutdown()
File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 53, in shutdown
self.__dict__["conn"].shutdown()
Error: []
If I simply resubmit the same build request enough times the job will eventually succeed, but it's really annoying as is. This all started when I upgraded to koji-1.10. Is anyone else seeing this or did I perhaps screw something up with the upgrade? I filed a bug[0] a few days ago but have heard nothing there.
[0] https://bugzilla.redhat.com/show_bug.cgi?id=1266245
--
John Florian
8 years, 2 months
[PATCH 0/3] Added support for plugins on client
by Christos Triantafyllidis
This patch allows the usage of plugins on cli. Majority of the changes are just a re-ordering of existing code.
Cheers,
Christos
Christos Triantafyllidis (3):
Added support for plugins at client
Added plugin configuration in client conf file
Passing options variable to plugins
cli/koji | 91 +++++++++++++++++++++++++++++++++++++----------------------
cli/koji.conf | 8 ++++++
2 files changed, 65 insertions(+), 34 deletions(-)
--
2.4.3
8 years, 2 months
mock-1.2.13 is available
by Miroslav Suchý
I just released new mock release (mock-1.2.13). It is bugfix release,
but some bugfix may be interesting for you:
* Fedora 23 configs are reverted back to use yum again. To be on pair
with Koji
* Lot of fixes for --new-chroot option
* Mockchain can download SRPM from Dropbox
* DNF does not install weak dependencies by default
* When cleaning up chroots, mock now exclude mountpoints
* When you build using DNF (rawhide) on systems, which does not have DNF
(EL6, 7), mock will print warning, wait for confirmation, tell you how
to suppress this warning next time. Nevertheless this warning is not
fatal and Mock can continue using YUM.
* Previously package_state plugin always used YUM, now it use DNF when
chroot is configured to use DNF.
* When file `/usr/bin/yum-deprecated` is present on your machine, then
variable `config_opts['yum_command']` is set to this v
alue by default.
* Several others bugfixes
https://bodhi.fedoraproject.org/updates/FEDORA-2015-16053
Mirek Suchy
8 years, 2 months
[PATCH] PAM support for hub and BasicAuth for web
by Christos Triantafyllidis
Hello,
The following patch is adding support for PAM authentication for the
koji-hub and BasicAuth for the koji-web.
This is useful for our internal use case as it allows us to login without
the overhead of setting up either a CA or a kerberos realm for our users.
The configuration is backwards compatible and hopefully similar to the
other authntication methods.
To active PAM support on hub you define the option:
PAMService = koji
in hub.conf. The value will be the name of the PAM service. Note the call
to the PAM module is done via unpriviledged call thus the use of pam_unix
won't be possible.
Note that activating this option will have as result that username/password
combinations from the DB will no longer be checked (similarly to when
activating kerberos or SSL client auth).
The BasicAuth for koji-web requires 2 changes:
a) To enable WSGIPassAuthorization for /koji/login in httpd configuration.
That passes the authorization variable from the apache to the application.
b) Set the "BasicAuthRealm" option to the Basic Authentication Realm that
will be presented to the user to login.
Finally python-pam package has been added to the hub's dependencies.
Cheers,
Christos
Christos Triantafyllidis (1):
- Added PAM support for hub - Added BasicAuth support for web
hub/hub.conf | 4 +++-
hub/kojixmlrpc.py | 2 ++
koji.spec | 1 +
koji/auth.py | 33 +++++++++++++++++++++++++--------
koji/server.py | 2 ++
www/conf/kojiweb.conf | 5 +++++
www/conf/web.conf | 3 +++
www/kojiweb/index.py | 18 +++++++++++++++++-
www/kojiweb/wsgi_publisher.py | 9 +++++++--
9 files changed, 65 insertions(+), 12 deletions(-)
--
2.4.3
8 years, 2 months
[PATCH] add support for Image Factory generation of VMWare Fusion Vagrant boxes
by Ian McLeod
---
builder/kojid | 11 +++++++++--
cli/koji | 2 +-
2 files changed, 10 insertions(+), 3 deletions(-)
diff --git a/builder/kojid b/builder/kojid
index c0759a8..4ea93f2 100755
--- a/builder/kojid
+++ b/builder/kojid
@@ -3063,7 +3063,7 @@ class BaseImageTask(OzImageTask):
Some image formats require others to be processed first, which is why
we have to do this. raw files in particular may not be kept.
"""
- supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker', 'vagrant-virtualbox', 'vagrant-libvirt', 'vpc')
+ supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker', 'vagrant-virtualbox', 'vagrant-libvirt', 'vagrant-vmware-fusion', 'vpc')
for f in formats:
if f not in supported:
raise koji.ApplianceError('Invalid format: %s' % f)
@@ -3098,6 +3098,7 @@ class BaseImageTask(OzImageTask):
'vsphere-ova': self._buildOVA,
'vagrant-virtualbox': self._buildOVA,
'vagrant-libvirt': self._buildOVA,
+ 'vagrant-vmware-fusion': self._buildOVA,
'docker': self._buildDocker
}
# add a handler to the logger so that we capture ImageFactory's logging
@@ -3247,8 +3248,14 @@ class BaseImageTask(OzImageTask):
if format == 'vagrant-libvirt':
format = 'rhevm-ova'
img_opts['rhevm_ova_format'] = 'vagrant-libvirt'
+ if format == 'vagrant-vmware-fusion':
+ format = 'vsphere-ova'
+ img_opts['vsphere_ova_format'] = 'vagrant-vmware-fusion'
+ # The initial disk image transform for VMWare Fusion/Workstation requires a "standard" VMDK
+ # not the stream oriented format used for VirtualBox or regular VMWare OVAs
+ img_opts['vsphere_vmdk_format'] = 'standard'
targ = self._do_target_image(self.base_img.base_image.identifier,
- format.replace('-ova', ''))
+ format.replace('-ova', ''), img_opts=img_opts)
targ2 = self._do_target_image(targ.target_image.identifier, 'OVA',
img_opts=img_opts)
return {'image': targ2.target_image.data}
diff --git a/cli/koji b/cli/koji
index 47d9691..c74171c 100755
--- a/cli/koji
+++ b/cli/koji
@@ -5265,7 +5265,7 @@ def handle_image_build(options, session, args):
"""Create a disk image given an install tree"""
formats = ('vmdk', 'qcow', 'qcow2', 'vdi', 'vpc', 'rhevm-ova',
'vsphere-ova', 'vagrant-virtualbox', 'vagrant-libvirt',
- 'docker', 'raw-xz')
+ 'vagrant-vmware-fusion', 'docker', 'raw-xz')
usage = _("usage: %prog image-build [options] <name> <version> " +
"<target> <install-tree-url> <arch> [<arch>...]")
usage += _("\n %prog image-build --config FILE")
--
2.1.0
8 years, 2 months
koji and mergerepo_c
by Thomas
Hi Folks,
Quick question, would you consider to use mergerepo_c with option
--all [1] in koji ?
I am happy to work on a patch if it would be accepted. Maybe, we can
keep default behavior and enable with an option on the tag.
My use case is to be able to ship different release built against fix
external repo package that may not be latest version
(Buildrequires:pkg = 1.0.0 while external repo has already 1.1.0)
[1]:
--all
Include all packages with the same name and arch if version or
release is different. If used --method argument is ignored!
--
Thomas
8 years, 2 months
Koji and DNF status?
by Miroslav Suchý
I would like to ask what is current status of migrating Koji to use DNF for building?
I am mainly interested whether Koji will use DNF for building F23 packages, or you postpone it for F24?
Because in such case I would revert fedora-23-*.cfg default configs in Mock to use Yum so Mock is on pair with Koji.
--
Miroslav Suchy, RHCA
Red Hat, Senior Software Engineer, #brno, #devexp, #fedora-buildsys
8 years, 2 months
redhat-rpm-config and circular dependencies
by Colin Walters
Because Fedora is a self-hosted system, circular dependencies are a fact of life. Self-hosted compilers and the like will always exist.
But I think the circular BR nature of redhat-rpm-config and macro packages is unnecessary self-inflicted pain. Currently, I am trying to backport (into a downstream distribution) the introduction of go-srpm-macros into redhat-rpm-config:
http://pkgs.fedoraproject.org/cgit/redhat-rpm-config.git/commit/?id=ba49b...
Yet because redhat-rpm-config itself is pulled into the build root for go-srpm-macros, it introduces a circular build dependency.
I could imagine external ways out of this (try dropping the BR of rrc for go-srpm-macros externally), but it would seem to me to be a lot saner just to include the macros themselves in redhat-rpm-config.
Thoughts? (Is this the right list?)
8 years, 3 months