[Beaker-devel] [RFC PATCH 3/4] harness: support beaker proxy protocol

Lucas Meneghel Rodrigues lmr at redhat.com
Tue Apr 2 23:57:39 UTC 2013


On 04/02/2013 06:18 PM, Don Zickus wrote:
> This patch contains the pieces to communicate with Beaker's new stable API.
>
> It is a thin layer that takes commands from the harness and translates them into
> the appropriate http GET, POST or PUT command.
>
> One thing I did to make debugging easier is to allow the ability to write to
> files locally (offline mode).  The harness doesn't care where the data is sent,
> so writing locally allow me to quickly test and address problems without complicating
> it with broken http requests.
>
> I also implemented chunk uploading to minimize memory usage.
>
> I spent most of my time testing the log uploading and task result reporting
> aspect of the API.  Everything seems to work great.  I expect the 'status'
> and 'watchdog' parts to just work.
>
> Mostly of interest to the beaker folks.  Autotest folks can review for style,
> syntax and entertainment.
>
> Signed-off-by: Don Zickus <dzickus at redhat.com>
> ---
>   client/bkr_proxy.py |  418 +++++++++++++++++++++++++++++++++++++++++++++++++++
>   1 files changed, 418 insertions(+), 0 deletions(-)
>   create mode 100644 client/bkr_proxy.py
>
> diff --git a/client/bkr_proxy.py b/client/bkr_proxy.py
> new file mode 100644
> index 0000000..03b2fdc
> --- /dev/null
> +++ b/client/bkr_proxy.py
> @@ -0,0 +1,418 @@
> +# bkr_proxy.py
> +#
> +# Copyright (C) 2011 Jan Stancek <jstancek at redhat.com>
> +#
> +# This program is free software; you can redistribute it and/or modify
> +# it under the terms of the GNU General Public License as published by
> +# the Free Software Foundation; either version 2 of the License, or
> +# (at your option) any later version.
> +#
> +# This program is distributed in the hope that it will be useful,
> +# but WITHOUT ANY WARRANTY; without even the implied warranty of
> +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> +# GNU General Public License for more details.
> +#
> +# You should have received a copy of the GNU General Public License
> +# along with this program; if not, write to the Free Software
> +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
> +
> +# Started by Jan Stancek <jstancek at redhat.com> 2011
> +"""
> +bkr_proxy - class used to talk to beaker
> +"""
> +__author__ = """Don Zickus 2013"""
> +
> +
> +import time
> +import os
> +import sys
> +import base64
> +import xmlrpclib
> +import random
> +import logging
> +import shutil
> +import httplib
> +import time
> +import re
> +import urllib, urllib2
> +from types import *
> +from autotest.client.shared import utils

There are a bunch of unused modules here. More comments below:

> +log = logging
> +
> +
> +AUTOTEST_CACHE_DIR = '/var/cache/autotest'
> +
> +class BkrProxyException(Exception):
> +    def __init__(self, text):
> +        Exception.__init__(self, text)
> +
> +'''Hard coded internal paths'''
> +def make_path_cmdlog(r):
> +    """
> +    Converts a recipe id into an internal path for logging purposes
> +
> +    @param r: recipe id
> +
> +    @return a path to the internal command log
> +    """
> +
> +    path = AUTOTEST_CACHE_DIR + '/recipes/' + r
> +    if not os.path.exists(path):
> +        os.makedirs(path)
> +    if not os.path.isdir(path):
> +        raise BkrProxyException("Path(%s) exists and is not a directory" % path)
> +    return path + '/cmd_log'
> +
> +def make_path_bkrcache(r):
> +    """
> +    Converts a recipe id into an internal path for cache'ing recipe
> +
> +    @param r: recipe id
> +
> +    @return a path to the internal recipe cache file
> +    """
> +
> +    return AUTOTEST_CACHE_DIR + '/recipes/' + r + '/beaker_recipe.cache'
> +
> +'''End Hard coded internal paths'''
> +
> +"""
> +Hard coded paths as described in the Beaker Server API
> +http://beaker-project.org/dev/proposals/harness-api.html
> +"""
> +
> +def make_path_recipe(r):
> +    """
> +    Converts a recipe id into a beaker path
> +
> +    @param r: recipe id
> +
> +    @return a beaker path to the recipe id
> +    """
> +
> +    return '/recipes/' + r
> +
> +def make_path_watchdog(r):
> +    """
> +    Converts a recipe id into a beaker path for the watchdog
> +
> +    @param r: recipe id
> +
> +    @return a beaker path of the recipe's watchdog file
> +    """
> +
> +    return '/recipes/' + r + '/watchdog'
> +
> +def make_path_status(r, t=None):
> +    """
> +    Converts id into a beaker path to status file
> +
> +    Given a recipe id and/or a task id, translate them into
> +    the proper beaker path to the status file.  Recipe only, returns
> +    the path to the recipe's status, whereas including a task returns
> +    the path to the task's status.
> +
> +    @param r: recipe id
> +    @param t: task id
> +
> +    @return a beaker path of the recipe's/task's status file
> +    """
> +
> +    rpath = '/recipes/' + r
> +    tpath = t and '/tasks/' + t or ''
> +
> +    return rpath + tpath + '/status'
> +
> +def make_path_result(r, t):
> +    """
> +    Converts task id into a beaker path to result file
> +
> +    Given a recipe id and a task id, translate them into
> +    the proper beaker path to the result file.
> +
> +    @param r: recipe id
> +    @param t: task id
> +
> +    @return a beaker path of the task's result file
> +    """
> +
> +    rpath = '/recipes/' + r
> +    tpath = '/tasks/' + t
> +
> +    return rpath + tpath + '/results/'
> +
> +def make_path_log(r, t=None, i=None):
> +    """
> +    Converts id into a beaker path to log file
> +
> +    Given a recipe id, a task id, and/or result id, translate
> +    them into the proper beaker path to the log file.  Depending
> +    on which log file is needed, provide the appropriate params.
> +    Note the dependency, a result id needs a task id and recipe id,
> +    while a task id needs a recipe id.
> +
> +    @param r: recipe id
> +    @param t: task id
> +    @param i: result id
> +
> +    @return a beaker path of the task's result file
> +    """
> +
> +    rpath = '/recipes/' + r
> +    tpath = t and '/tasks/' + t or ''
> +    ipath = i and '/results/' + i or ''
> +
> +    return rpath + tpath + ipath + '/logs'
> +
> +'''End Hard coded paths'''
> +
> +def copy_remote(data, dest, use_put=None):
> +    """
> +    Copy data to a remote server using http calls POST or PUT
> +
> +    Using http POST and PUT methods, copy data over http.  To use
> +    PUT method, provide a dictionary of values to be populated in
> +    the Content-Range and Content-Length headers.  Otherwise default
> +    is to use POST method.
> +
> +    Traps on HTTPError 500 and 400
> +
> +    @param data: encoded data string to copy remotely
> +    @param dest: remote server URL
> +    @param use_put: dictionary of items if using PUT method
> +
> +    @return html header info for post processing
> +    """
> +
> +    ret = None
> +    req = urllib2.Request(dest, data=data)
> +    if use_put:
> +        req.add_header('Content-Type', 'application/octet-stream')
> +        end = use_put['start'] + use_put['size'] - 1
> +        req.add_header('Content-Range', 'bytes %s-%s/%s' % (use_put['start'],
> +                        end, use_put['total']))
> +        req.add_header('Content-Length', '%s' % use_put['size'])
> +        req.get_method = lambda: 'PUT'
> +
> +    try:
> +        res = utils.urlopen(req)
> +        ret = res.info()
> +        res.close()
> +    except urllib2.HTTPError as e:

^ This construct was introduced in py 2.5 and will break things when 
running on py 2.4. As we strive to keep compatibility with it, you've 
got to replace this with a comma:

     except urllib2.HTTPError, e:

I did fix all of the little problems I found and send them to your 
email, so you can apply them and re-send the patchset.


More information about the Beaker-devel mailing list