Vinzenz Feenstra has uploaded a new change for review.
Change subject: Allow GZIP transport on XMLRPC
......................................................................
Allow GZIP transport on XMLRPC
Change-Id: Iebdaa04b17e2c1df1c1852ed536c5d6d8ec8d88b
Signed-off-by: Vinzenz Feenstra <vfeenstr(a)redhat.com>
---
M lib/vdsm/utils.py
M lib/vdsm/vdscli.py.in
2 files changed, 113 insertions(+), 2 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/01/25201/1
diff --git a/lib/vdsm/utils.py b/lib/vdsm/utils.py
index b682dec..81397ef 100644
--- a/lib/vdsm/utils.py
+++ b/lib/vdsm/utils.py
@@ -37,6 +37,7 @@
import fcntl
import functools
import glob
+import gzip
import io
import itertools
import logging
@@ -44,6 +45,7 @@
import os
import platform
import pwd
+import re
import select
import shutil
import signal
@@ -148,6 +150,28 @@
raise
+##
+# Encode a string using the gzip content encoding such as specified by the
+# Content-Encoding: gzip
+# in the HTTP header, as described in RFC 1952
+#
+# @param data the unencoded data
+# @return the encoded data
+
+def gzip_encode(data):
+ """data -> gzip encoded data
+
+ Encode data using the gzip content encoding as described in RFC 1952
+ """
+ f = StringIO()
+ gzf = gzip.GzipFile(mode="wb", fileobj=f, compresslevel=1)
+ gzf.write(data)
+ gzf.close()
+ encoded = f.getvalue()
+ f.close()
+ return encoded
+
+
class IPXMLRPCRequestHandler(SimpleXMLRPCRequestHandler):
if config.getboolean('vars', 'xmlrpc_http11'):
@@ -171,6 +195,26 @@
# the methods only on Python 2.6.
if sys.version_info[:2] == (2, 6):
+ def __init__(self, *args, **kwargs):
+ self.encode_threshold = 1400
+ SimpleXMLRPCRequestHandler.__init__(self, *args, **kwargs)
+
+ # a re to match a gzip Accept-Encoding
+ aepattern = re.compile(r"""
+ \s* ([^\s;]+) \s* #content-coding
+ (;\s* q \s*=\s* ([0-9\.]+))? #q
+ """, re.VERBOSE | re.IGNORECASE)
+
+ def accept_encodings(self):
+ r = {}
+ ae = self.headers.get("Accept-Encoding", "")
+ for e in ae.split(","):
+ match = self.aepattern.match(e)
+ if match:
+ v = match.group(3)
+ v = float(v) if v else 1.0
+ r[match.group(1)] = v
+ return r
def do_POST(self):
# Check that the path is legal
@@ -183,7 +227,7 @@
# We read this in chunks to avoid straining
# socket.read(); around the 10 or 15Mb mark, some platforms
# begin to have problems (bug #792570).
- max_chunk_size = 10*1024*1024
+ max_chunk_size = 10 * 1024 * 1024
size_remaining = int(self.headers["content-length"])
L = []
while size_remaining:
@@ -218,9 +262,20 @@
# got a valid XML RPC response
self.send_response(200)
self.send_header("Content-type", "text/xml")
+ pureLen = len(response)
+ if self.encode_threshold is not None:
+ if pureLen > self.encode_threshold:
+ q = self.accept_encodings().get("gzip", 0)
+ if q:
+ response = gzip_encode(response)
+ self.send_header("Content-encoding",
"gzip")
self.send_header("Content-length", str(len(response)))
self.end_headers()
self.wfile.write(response)
+ import datetime
+ import re
+ with open('/tmp/rpc-stats.log', 'a') as statslog:
+ statslog.write("%s - %s - %d - %d\n" %
(datetime.datetime.utcnow().isoformat(),
re.search(r'(?<=<methodName>)(.+)(?=<\/methodName)', data).group(0),
len(response), pureLen))
def report_404(self):
self.send_response(404)
diff --git a/lib/vdsm/vdscli.py.in b/lib/vdsm/vdscli.py.in
index 5fa7528..e9e5c26 100644
--- a/lib/vdsm/vdscli.py.in
+++ b/lib/vdsm/vdscli.py.in
@@ -19,10 +19,12 @@
# Refer to the README and COPYING files for full details of the license
#
+import gzip
import xmlrpclib
import subprocess
import os
import re
+import StringIO
import sys
from xml.parsers.expat import ExpatError
from . import SecureXMLRPCServer
@@ -34,7 +36,61 @@
d_port = '54321'
+if sys.version_info[:2] == (2, 6):
+ class GzipDecodedResponse(gzip.GzipFile if gzip else object):
+ """a file-like object to decode a response encoded with the gzip
+ method, as described in RFC 1952.
+ """
+ def __init__(self, response):
+ data = response.read()
+ with open("/tmp/client.dump", "wb") as f:
+ f.write(data)
+ self.stringio = StringIO.StringIO(data)
+ gzip.GzipFile.__init__(self, mode="rb", fileobj=self.stringio)
+
+ def close(self):
+ gzip.GzipFile.close(self)
+ self.stringio.close()
+
+ def wrap_request(transport):
+ self = transport
+
+ def wrapped_request(host, handler, request_body, verbose=0):
+ # issue XML-RPC request
+ h = self.make_connection(host)
+ if verbose:
+ h.set_debuglevel(1)
+
+ self.send_request(h, handler, request_body)
+ self.send_host(h, host)
+ self.send_user_agent(h)
+ h.putheader("Accept-Encoding", "gzip")
+ self.send_content(h, request_body)
+
+ errcode, errmsg, headers = h.getreply()
+
+ if errcode != 200:
+ raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg,
+ headers)
+
+ try:
+ sock = h._conn.sock
+ except AttributeError:
+ sock = None
+
+ self.verbose = verbose
+ #print headers.dict
+ if headers.get('Content-encoding', '') == 'gzip':
+ return self.parse_response(GzipDecodedResponse(h.getfile()))
+ else:
+ return self._parse_response(h.getfile(), sock)
+
+ transport.request = wrapped_request
+
+
def wrap_transport(transport):
+ if sys.version_info[:2] == (2, 6):
+ wrap_request(transport)
old_parse_response = transport.parse_response
def wrapped_parse_response(*args, **kwargs):
@@ -42,7 +98,7 @@
return old_parse_response(*args, **kwargs)
except ExpatError:
sys.stderr.write('Parsing error was thrown during parsing '
- 'response when provided: {}'.format(args[1]))
+ 'response when provided: {0}'.format(args))
raise
transport.parse_response = wrapped_parse_response
return transport
--
To view, visit
http://gerrit.ovirt.org/25201
To unsubscribe, visit
http://gerrit.ovirt.org/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iebdaa04b17e2c1df1c1852ed536c5d6d8ec8d88b
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Vinzenz Feenstra <vfeenstr(a)redhat.com>