Re: [koji] Pull-Request #16 `Enable keepcache in koji dnf conf`
by Mike Bonnet
On 12/17/15 12:13 PM, pagure(a)pagure.io wrote:
>
> ausil opened a new pull-request against the project: `koji` that you are following:
> ``
> Enable keepcache in koji dnf conf
> ``
What's the cache expiration policy? Is there a size limit? I'm concerned
that this could cause builder disks to fill up with cached packages.
7 years, 8 months
koji-dojo (docker images)
by John Casey
Hi all,
I wanted to send a quick note to say that I've started work on a suite
of Docker images that can be used to stand up some very basic Koji
services. These are designed to support automated testing of
applications that need to integrate with Koji. They're still very new,
so I haven't pushed anything to the public Docker registry yet (though
that's the plan for the near future).
The Git repository is here:
https://github.com/jdcasey/koji-dojo
If you're interested, I've included a basic README that should help you
get started. If you want more detail about how the container support one
another, and how the database is setup, I'd encourage you to take a look
at the */docker-scripts/run.sh files.
I plan to use this image suite to enable continuous integration builds
for some software I'm writing, and to explore the way the koji client
and hub communicate with one another in a live environment.
Enjoy.
-john
7 years, 9 months
[PATCH] adjust default seach result order for large tables
by Mike McLean
Based on a user request. For some searches, ordering by id descending
(approximately "most recent") makes more sense.
---
www/kojiweb/index.py | 17 +++++++++++++++--
1 file changed, 15 insertions(+), 2 deletions(-)
diff --git a/www/kojiweb/index.py b/www/kojiweb/index.py
index 876d956..433d943 100644
--- a/www/kojiweb/index.py
+++ b/www/kojiweb/index.py
@@ -2178,8 +2178,20 @@ _infoURLs = {'package': 'packageinfo?packageID=%(id)i',
_VALID_SEARCH_CHARS = r"""a-zA-Z0-9"""
_VALID_SEARCH_SYMS = r""" @.,_/\()%+-*?|[]^$"""
_VALID_SEARCH_RE = re.compile('^[' + _VALID_SEARCH_CHARS + re.escape(_VALID_SEARCH_SYMS) + ']+$')
-
-def search(environ, start=None, order='name'):
+_DEFAULT_SEARCH_ORDER = {
+ # For searches against large tables, use '-id' to show most recent first
+ 'build' : '-id',
+ 'rpm' : '-id',
+ 'maven' : '-id',
+ 'win' : '-id',
+ # for other tables, ordering by name makes much more sense
+ 'tag' : 'name',
+ 'target' : 'name',
+ 'package' : 'name',
+ # any type not listed will default to 'name'
+}
+
+def search(environ, start=None, order=None):
values = _initValues(environ, 'Search', 'search')
server = _getServer(environ)
values['error'] = None
@@ -2211,6 +2223,7 @@ def search(environ, start=None, order='name'):
if not infoURL:
raise koji.GenericError, 'unknown search type: %s' % type
values['infoURL'] = infoURL
+ order = order or _DEFAULT_SEARCH_ORDER.get(type, 'name')
values['order'] = order
results = kojiweb.util.paginateMethod(server, values, 'search', args=(terms, type, match),
--
2.4.3
7 years, 9 months
[PATCH] fix using https url with non-ssl auth in python 2.7.9+
by Mike McLean
In python 2.7.9, a context option was added to httplib.HTTPSConnection and
changed its behavior to performs certificate and hostname checks by default.
While this is definitely an improvement, we were relying on the old behavior.
This change restores that (until we can switch to proper verification).
---
koji/__init__.py | 6 ++++++
koji/ssl/__init__.py | 7 +++++++
2 files changed, 13 insertions(+)
diff --git a/koji/__init__.py b/koji/__init__.py
index 2406a02..ebdd4b8 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -45,6 +45,7 @@ import shutil
import signal
import socket
import ssl.SSLCommon
+from ssl import ssl as pyssl
import struct
import tempfile
import time
@@ -1614,6 +1615,11 @@ class ClientSession(object):
default_port = 443
elif scheme == 'https':
cnxOpts = {}
+ if sys.version_info[:3] >= (2, 7, 9):
+ #ctx = pyssl.SSLContext(pyssl.PROTOCOL_SSLv23)
+ ctx = pyssl._create_unverified_context()
+ # TODO - we should default to verifying where possible
+ cnxOpts['context'] = ctx
cnxClass = httplib.HTTPSConnection
default_port = 443
elif scheme == 'http':
diff --git a/koji/ssl/__init__.py b/koji/ssl/__init__.py
index 180fed6..0be8717 100644
--- a/koji/ssl/__init__.py
+++ b/koji/ssl/__init__.py
@@ -1 +1,8 @@
# identify this as the ssl module
+
+# our own ssl submodule masks python's in the main lib, so we import this here
+try:
+ import ssl # python's ssl module
+except ImportError:
+ # ssl module added in 2.6
+ pass
--
2.4.3
7 years, 9 months
[PATCH] remove additional dead ssl code
by Mike McLean
---
koji/ssl/SSLCommon.py | 58 --------------
koji/ssl/XMLRPCServerProxy.py | 177 ------------------------------------------
2 files changed, 235 deletions(-)
delete mode 100644 koji/ssl/XMLRPCServerProxy.py
diff --git a/koji/ssl/SSLCommon.py b/koji/ssl/SSLCommon.py
index 5a9a5e4..56efc05 100644
--- a/koji/ssl/SSLCommon.py
+++ b/koji/ssl/SSLCommon.py
@@ -47,46 +47,6 @@ def CreateSSLContext(certs):
return ctx
-
-class PlgBaseServer(SocketServer.ThreadingTCPServer):
- allow_reuse_address = 1
-
- def __init__(self, server_addr, req_handler):
- self._quit = False
- self.allow_reuse_address = 1
- SocketServer.ThreadingTCPServer.__init__(self, server_addr, req_handler)
-
- def stop(self):
- self._quit = True
-
- def serve_forever(self):
- while not self._quit:
- self.handle_request()
- self.server_close()
-
-
-class PlgBaseSSLServer(PlgBaseServer):
- """ SSL-enabled variant """
-
- def __init__(self, server_address, req_handler, certs, timeout=None):
- self._timeout = timeout
- self.ssl_ctx = CreateSSLContext(certs)
-
- PlgBaseServer.__init__(self, server_address, req_handler)
-
- sock = socket.socket(self.address_family, self.socket_type)
- con = SSL.Connection(self.ssl_ctx, sock)
- self.socket = SSLConnection.SSLConnection(con)
- if sys.version_info[:3] >= (2, 3, 0):
- self.socket.settimeout(self._timeout)
- self.server_bind()
- self.server_activate()
-
- host, port = self.socket.getsockname()[:2]
- self.server_name = socket.getfqdn(host)
- self.server_port = port
-
-
class PlgHTTPSConnection(httplib.HTTPConnection):
"This class allows communication via SSL."
@@ -119,21 +79,3 @@ class PlgHTTPSConnection(httplib.HTTPConnection):
break
else:
raise socket.error, "failed to connect"
-
-
-
-class PlgHTTPS(httplib.HTTP):
- """Compatibility with 1.5 httplib interface
-
- Python 1.5.2 did not have an HTTPS class, but it defined an
- interface for sending http requests that is also useful for
- https.
- """
-
- _http_vsn = 11
- _http_vsn_str = 'HTTP/1.1'
-
- _connection_class = PlgHTTPSConnection
-
- def __init__(self, host='', port=None, ssl_context=None, strict=None, timeout=None):
- self._setup(self._connection_class(host, port, ssl_context, strict, timeout))
diff --git a/koji/ssl/XMLRPCServerProxy.py b/koji/ssl/XMLRPCServerProxy.py
deleted file mode 100644
index 78273c8..0000000
--- a/koji/ssl/XMLRPCServerProxy.py
+++ /dev/null
@@ -1,177 +0,0 @@
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Library General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
-#
-# Modified by Dan Williams <dcbw(a)redhat.com>
-# Further modified by Mike Bonnet <mikeb(a)redhat.com>
-
-import os, sys
-import SSLCommon
-import urllib
-import xmlrpclib
-
-__version__='0.12'
-
-class PlgSSL_Transport(xmlrpclib.Transport):
-
- user_agent = "pyOpenSSL_XMLRPC/%s - %s" % (__version__, xmlrpclib.Transport.user_agent)
-
- def __init__(self, ssl_context, timeout=None, use_datetime=0):
- if sys.version_info[:3] >= (2, 5, 0):
- xmlrpclib.Transport.__init__(self, use_datetime)
- self.ssl_ctx=ssl_context
- self._timeout = timeout
- self._https = None
-
- def make_connection(self, host):
- # Handle username and password.
- try:
- host, extra_headers, x509 = self.get_host_info(host)
- except AttributeError:
- # Yay for Python 2.2
- pass
- _host, _port = urllib.splitport(host)
- if hasattr(xmlrpclib.Transport, 'single_request'):
- cnx_class = SSLCommon.PlgHTTPSConnection
- else:
- cnx_class = SSLCommon.PlgHTTPS
- self._https = cnx_class(_host, (_port and int(_port) or 443), ssl_context=self.ssl_ctx, timeout=self._timeout)
- return self._https
-
- def close(self):
- if self._https:
- self._https.close()
- self._https = None
-
-
-class Plg_ClosableTransport(xmlrpclib.Transport):
- """Override make_connection so we can close it."""
- def __init__(self):
- self._http = None
-
- def make_connection(self, host):
- # create a HTTP connection object from a host descriptor
- import httplib
- host, extra_headers, x509 = self.get_host_info(host)
- self._http = httplib.HTTP(host)
- return self._http
-
- def close(self):
- if self._http:
- self._http.close()
- self._http = None
-
-
-class PlgXMLRPCServerProxy(xmlrpclib.ServerProxy):
- def __init__(self, uri, certs, timeout=None, verbose=0, allow_none=0):
- if certs and len(certs) > 0:
- self.ctx = SSLCommon.CreateSSLContext(certs)
- self._transport = PlgSSL_Transport(ssl_context=self.ctx, timeout=timeout)
- else:
- self._transport = Plg_ClosableTransport()
- xmlrpclib.ServerProxy.__init__(self, uri, transport=self._transport,
- verbose=verbose, allow_none=allow_none)
-
- def cancel(self):
- self._transport.close()
-
-
-###########################################################
-# Testing stuff
-###########################################################
-
-
-import threading
-import time
-import random
-import OpenSSL
-import socket
-
-client_start = False
-
-threadlist_lock = threading.Lock()
-threadlist = {}
-
-class TestClient(threading.Thread):
- def __init__(self, certs, num, tm):
- self.server = PlgXMLRPCServerProxy("https://127.0.0.1:8886", certs, timeout=20)
- self.num = i
- self.tm = tm
- threading.Thread.__init__(self)
-
- def run(self):
- while not client_start:
- time.sleep(0.05)
- i = 0
- while i < 5:
- reply = None
- try:
- reply = self.server.ping(self.num, i)
- except OpenSSL.SSL.Error, e:
- reply = "OpenSSL Error (%s)" % e
- except socket.timeout, e:
- reply = "Socket timeout (%s)" % e
- threadlist_lock.acquire()
- self.tm.inc()
- threadlist_lock.release()
- print "TRY(%d / %d): %s" % (self.num, i, reply)
- time.sleep(0.05)
- i = i + 1
- threadlist_lock.acquire()
- del threadlist[self]
- threadlist_lock.release()
-
-class TimeoutCounter:
- def __init__(self):
- self._timedout = 0
- self._lock = threading.Lock();
-
- def inc(self):
- self._lock.acquire()
- self._timedout = self._timedout + 1
- self._lock.release()
-
- def get(self):
- return self._timedout
-
-if __name__ == '__main__':
- if len(sys.argv) < 4:
- print "Usage: python XMLRPCServerProxy.py key_and_cert peer_ca_cert"
- sys.exit(1)
-
- certs = {}
- certs['key_and_cert'] = sys.argv[1]
- certs['peer_ca_cert'] = sys.argv[3]
-
- tm = TimeoutCounter()
- i = 100
- while i > 0:
- t = TestClient(certs, i, tm)
- threadlist[t] = None
- print "Created thread %d." % i
- t.start()
- i = i - 1
-
- time.sleep(3)
- print "Unleashing threads."
- client_start = True
- while True:
- try:
- time.sleep(0.25)
- threadlist_lock.acquire()
- if len(threadlist) == 0:
- break
- threadlist_lock.release()
- except KeyboardInterrupt:
- os._exit(0)
- print "All done. (%d timed out)" % tm.get()
--
2.4.3
7 years, 9 months
[PATCH] new archivetypes: shell scripts and batch files
by Mike McLean
---
docs/schema.sql | 2 ++
1 file changed, 2 insertions(+)
diff --git a/docs/schema.sql b/docs/schema.sql
index 1212a94..d8ce975 100644
--- a/docs/schema.sql
+++ b/docs/schema.sql
@@ -819,6 +819,8 @@ insert into archivetypes (name, description, extensions) values ('json', 'JSON d
insert into archivetypes (name, description, extensions) values ('key', 'Key file', 'key');
insert into archivetypes (name, description, extensions) values ('dot', 'DOT graph description', 'dot gv');
insert into archivetypes (name, description, extensions) values ('groovy', 'Groovy script file', 'groovy gvy');
+insert into archivetypes (name, description, extensions) values ('batch', 'Batch file', 'bat');
+insert into archivetypes (name, description, extensions) values ('shell', 'Shell script', 'sh');
-- Do we want to enforce a constraint that a build can only generate one
--
2.4.3
7 years, 9 months
[PATCH] new archivetypes: shell scripts and batch files
by Mike McLean
---
docs/schema.sql | 2 ++
koji/__init__.py | 14 +++++++++-----
2 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/docs/schema.sql b/docs/schema.sql
index 1212a94..d8ce975 100644
--- a/docs/schema.sql
+++ b/docs/schema.sql
@@ -819,6 +819,8 @@ insert into archivetypes (name, description, extensions) values ('json', 'JSON d
insert into archivetypes (name, description, extensions) values ('key', 'Key file', 'key');
insert into archivetypes (name, description, extensions) values ('dot', 'DOT graph description', 'dot gv');
insert into archivetypes (name, description, extensions) values ('groovy', 'Groovy script file', 'groovy gvy');
+insert into archivetypes (name, description, extensions) values ('batch', 'Batch file', 'bat');
+insert into archivetypes (name, description, extensions) values ('shell', 'Shell script', 'sh');
-- Do we want to enforce a constraint that a build can only generate one
diff --git a/koji/__init__.py b/koji/__init__.py
index 8e297dc..82dbd05 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -44,7 +44,8 @@ import rpm
import shutil
import signal
import socket
-import ssl.SSLCommon
+import koji.ssl.SSLCommon
+import ssl
import struct
import tempfile
import time
@@ -1608,12 +1609,15 @@ class ClientSession(object):
self._path = uri[2]
default_port = 80
if self.opts.get('certs'):
- ctx = ssl.SSLCommon.CreateSSLContext(self.opts['certs'])
+ ctx = koji.ssl.SSLCommon.CreateSSLContext(self.opts['certs'])
cnxOpts = {'ssl_context' : ctx}
- cnxClass = ssl.SSLCommon.PlgHTTPSConnection
+ cnxClass = koji.ssl.SSLCommon.PlgHTTPSConnection
default_port = 443
elif scheme == 'https':
cnxOpts = {}
+ if sys.version_info[:3] >= (2, 7, 9):
+ ctx = ssl.SSLContext()
+ cnxOpts['ssl_context'] = ctx
cnxClass = httplib.HTTPSConnection
default_port = 443
elif scheme == 'http':
@@ -1748,13 +1752,13 @@ class ClientSession(object):
certs['ca_cert'] = ca
certs['peer_ca_cert'] = serverca
- ctx = ssl.SSLCommon.CreateSSLContext(certs)
+ ctx = koji.ssl.SSLCommon.CreateSSLContext(certs)
self._cnxOpts = {'ssl_context' : ctx}
# 60 second timeout during login
old_timeout = self._cnxOpts.get('timeout')
self._cnxOpts['timeout'] = 60
try:
- self._cnxClass = ssl.SSLCommon.PlgHTTPSConnection
+ self._cnxClass = koji.ssl.SSLCommon.PlgHTTPSConnection
if self._port == 80 and not self.explicit_port:
self._port = 443
sinfo = self.callMethod('sslLogin', proxyuser)
--
2.4.3
7 years, 9 months
[PATCH] fall back to client side info for authtype
by Mike McLean
---
cli/koji | 2 +-
koji/__init__.py | 5 +++++
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/cli/koji b/cli/koji
index 42bce30..6df17c7 100755
--- a/cli/koji
+++ b/cli/koji
@@ -6687,7 +6687,7 @@ def handle_moshimoshi(options, session, args):
print "%s, %s!" % (random.choice(greetings), u["name"],)
print ""
print "You are using the hub at %s" % (session.baseurl,)
- authtype = u.get("authtype")
+ authtype = u.get('authtype', getattr(session, 'authtype', None))
if authtype == koji.AUTHTYPE_NORMAL:
print "Authenticated via password"
elif authtype == koji.AUTHTYPE_KERB:
diff --git a/koji/__init__.py b/koji/__init__.py
index a3d75d8..8e297dc 100644
--- a/koji/__init__.py
+++ b/koji/__init__.py
@@ -1594,6 +1594,7 @@ class ClientSession(object):
self.opts = opts
self._connection = None
self._setup_connection()
+ self.authtype = None
self.setSession(sinfo)
self.multicall = False
self._calls = []
@@ -1644,6 +1645,7 @@ class ClientSession(object):
self.callnum = None
# do we need to do anything else here?
self._setup_connection()
+ self.authtype = None
else:
self.logged_in = True
self.callnum = 0
@@ -1654,6 +1656,7 @@ class ClientSession(object):
if not sinfo:
return False
self.setSession(sinfo)
+ self.authtype = AUTHTYPE_NORMAL
return True
def subsession(self):
@@ -1724,6 +1727,7 @@ class ClientSession(object):
return False
self.setSession(sinfo)
+ self.authtype = AUTHTYPE_KERB
return True
def _serverPrincipal(self, cprinc):
@@ -1765,6 +1769,7 @@ class ClientSession(object):
self.opts['certs'] = certs
self.setSession(sinfo)
+ self.authtype = AUTHTYPE_SSL
return True
def logout(self):
--
2.1.0
7 years, 9 months
[PATCH] don't break listArchives argument ordering
by Mike McLean
---
hub/kojihub.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/hub/kojihub.py b/hub/kojihub.py
index f2b207b..f19ad6f 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -3630,8 +3630,8 @@ def get_image_build(buildInfo, strict=False):
raise koji.GenericError, 'no such image build: %s' % buildInfo
return result
-def list_archives(buildID=None, buildrootID=None, imageID=None, componentBuildrootID=None, hostID=None, type=None,
- filename=None, size=None, checksum=None, typeInfo=None, queryOpts=None):
+def list_archives(buildID=None, buildrootID=None, componentBuildrootID=None, hostID=None, type=None,
+ filename=None, size=None, checksum=None, typeInfo=None, queryOpts=None, imageID=None):
"""
Retrieve information about archives.
If buildID is not null it will restrict the list to archives built by the build with that ID.
--
2.1.0
7 years, 9 months
[PATCH] fix query opts in repo_references
by Mike McLean
---
hub/kojihub.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/hub/kojihub.py b/hub/kojihub.py
index acdea2b..f2b207b 100644
--- a/hub/kojihub.py
+++ b/hub/kojihub.py
@@ -2418,7 +2418,7 @@ def repo_references(repo_id):
values = {'repo_id': repo_id}
clauses = ['repo_id=%(repo_id)s', 'retire_event IS NULL']
query = QueryProcessor(columns=fields, aliases=aliases, tables=['standard_buildroot'],
- clauses=clauses, values=values, opts = {'asList':True})
+ clauses=clauses, values=values)
#check results for bad states
ret = []
for data in query.execute():
--
2.1.0
7 years, 10 months