Add support to imgfac for building images for multiple target and
pushing images to multiple providers.
New 'image' and 'build' IWHD objects are introduced as outlined here:
https://fedorahosted.org/pipermail/aeolus-devel/2011-May/001667.html
The build cmdline now looks like:
$> imgfac --template <template> --target <target> [--target
<target> ...]
The push cmdline is now:
$> imgfac --image <image-UUID> --credentials <credentials-XML> --provider
<provider> [--provider <provider> ...]
One item that requires further thought is what to do about determining
the target image appropriate for a given provider. Currently, we have a
fairly lame heuristic, but perhaps we should allow conductor to supply
this.
---
Documentation/man/imgfac.1 | 34 +++--
README.markdown | 20 ++--
imagefactory.conf | 2 +
imagefactory/ApplicationConfiguration.py | 20 ++-
imagefactory/BuildDispatcher.py | 211 ++++++++++++++++++++++++++++--
imagefactory/ImageWarehouse.py | 41 ++++++
imagefactory/builders/BaseBuilder.py | 6 +-
imagefactory/builders/FedoraBuilder.py | 14 +-
imagefactory/builders/IBuilder.py | 2 +-
imagefactory/builders/MockBuilder.py | 4 +-
imagefactory/builders/RHELBuilder.py | 2 +-
imgfac.py | 16 +--
tests/testBuildDispatcher.py | 2 +-
tests/testImageWarehouse.py | 24 ++++
14 files changed, 334 insertions(+), 64 deletions(-)
diff --git a/Documentation/man/imgfac.1 b/Documentation/man/imgfac.1
index 137fbd9..d434fa0 100644
--- a/Documentation/man/imgfac.1
+++ b/Documentation/man/imgfac.1
@@ -10,9 +10,13 @@ Currently supported private cloud services: RHEV-M, KVM.
.SH SYNOPSIS
.B imgfac [-h] [--version] [-v] [--debug] [--foreground] [--config CONFIG]
.B [imgdir IMGDIR] [--timeout TIMEOUT] [--qmf] [--qpidd QPIDD]
-.B [--template TEMPLATE] [--target TARGET] [--image IMAGE]
-.B [--provider PROVIDER] [--credentials CREDENTIALS]
-.B [--warehouse WAREHOUSE] [--target_bucket TARGET_BUCKET]
+.B [--image IMAGE]
+.B [--template TEMPLATE] [--target TARGET] [--target TARGET]...
+.B [--credentials CREDENTIALS] [--provider PROVIDER] [--provider PROVIDER]...
+.B [--warehouse WAREHOUSE]
+.B [--image_bucket IMAGE_BUCKET]
+.B [--build_bucket BUILD_BUCKET]
+.B [--target_bucket TARGET_BUCKET]
.B [--template_bucket TEMPLATE_BUCKET]
.B [--icicle_bucket ICICLE_BUCKET]
.B [--provider_bucket PROVIDER_BUCKET]
@@ -37,6 +41,9 @@ Image factory configuration file to use. (default:
/etc/imagefactory.conf)
.B --imgdir IMGDIR
Build image files in location specified. After the image is finished it will be saved in
this location. (default: /tmp)
+.B --image IMAGE
+The UUID of the image to build or push. This is a UUID of an object in IWHD's image
bucket.
+
.B --timeout TIMEOUT
Set the timeout period for image building in seconds. (default: 3600)
@@ -59,27 +66,24 @@ URL of qpidd to connect to. (default: localhost)
Build specified system and exit.
.B --template TEMPLATE
-Template to build from. Template can be passed as URL,UUID or XML. Please reffer to the
+Template to build from. Template can be passed as URL, PATH, UUID or XML. Please refer to
the
.BR XML
.BR EXAMPLES
section at the bottom.
.B --target TARGET
-Cloud service to target. This parameter is intended to specify the location as a target
for cloud providers that use regions, eg. US-West is a region of Amazon EC2.
-
+The list of cloud services to target. Examples include ec2, rhev-m, vmware, rackspace,
condorcloud and gogrid.
-.SS Image instantiation:
- Instantiate an image and exit.
+.SS Image pushing:
+ Push an image and exit.
-.B --image IMAGE
-Image UUID to instantiate. The UUID is stored in Image Warehouse.
.B --provider PROVIDER
-Cloud service provider upon which to instantiate the image, eg. rackspace
+The list of cloud service providers to push the emacs. Examples include ec2-us-east-1,
ec2-us-west-1, rackspace.
.B --credentials CREDENTIALS
-Cloud provider credentials. Credentials must be passed as XML file. Please reffer to the
+Cloud provider credentials. Credentials must be passed as XML file. Please refer to the
.BR XML
.BR EXAMPLES
section at the bottom.
@@ -90,6 +94,12 @@ section at the bottom.
.B --warehouse WAREHOUSE
URL of the warehouse location to store images. (default:
http://localhost:9090/)
+.B --image_bucket IMAGE_BUCKET
+Name of warehouse bucket to look in for images. (default: None)
+
+.B --build_bucket BUILD_BUCKET
+Name of warehouse bucket to look in for builds. (default: None)
+
.B --target_bucket TARGET_BUCKET
Name of warehouse bucket to look in for target images. (default: None)
diff --git a/README.markdown b/README.markdown
index 7f777e1..e7a469f 100644
--- a/README.markdown
+++ b/README.markdown
@@ -2,9 +2,9 @@
usage: imgfac [-h] [--version] [-v] [--debug] [--foreground] [--config CONFIG]
[imgdir IMGDIR] [--warehouse WAREHOUSE] [--timeout TIMEOUT]
- [--qmf] [--broker BROKER] [--template TEMPLATE]
- [--target TARGET] [--image IMAGE] [--provider PROVIDER]
- [--credentials CREDENTIALS]
+ [--qmf] [--broker BROKER] [--image IMAGE]
+ [--template TEMPLATE] [--target TARGET]
+ [--provider PROVIDER] [--credentials CREDENTIALS]
System image creation tool...
@@ -17,6 +17,7 @@
(default: False)
--config CONFIG Configuration file to use. (default:
/etc/imagefactory.conf)
+ --image IMAGE The UUID of the image to build or push
--imgdir IMGDIR Build image files in location specified. (default:
/tmp)
--warehouse WAREHOUSE
@@ -34,16 +35,15 @@
Build specified system and exit.
--template TEMPLATE Template XML file to build from.
- --target TARGET Cloud service to target
+ --target TARGET List of cloud services to target
- Image instantiation:
- Instantiate an image and exit.
+ Image pushing:
+ Push an image and exit.
- --image IMAGE Image to instantiate
- --provider PROVIDER Cloud service provider upon which to instantiate the
- image
+ --provider PROVIDER List of cloud service providers to which the image
+ should be pushed
--credentials CREDENTIALS
- Cloud provider credentials
+ Cloud provider credentials XML
## Dependencies: ##
diff --git a/imagefactory.conf b/imagefactory.conf
index b98de71..454ea89 100644
--- a/imagefactory.conf
+++ b/imagefactory.conf
@@ -1,5 +1,7 @@
{
"warehouse": "http://localhost:9090/",
+ "image_bucket": "images",
+ "build_bucket": "builds",
"target_bucket": "target_images",
"template_bucket": "templates",
"icicle_bucket": "icicles",
diff --git a/imagefactory/ApplicationConfiguration.py
b/imagefactory/ApplicationConfiguration.py
index 57b668a..9de2b78 100644
--- a/imagefactory/ApplicationConfiguration.py
+++ b/imagefactory/ApplicationConfiguration.py
@@ -53,30 +53,36 @@ class ApplicationConfiguration(object):
and does not connect to a qpidd."""
qmf_description = """Provide a QMFv2 agent interface. (see
https://cwiki.apache.org/qpid/qmfv2-project-page.html for more
information)"""
cli_build_description = """Build specified system and
exit."""
- cli_push_description = """Instantiate an image and
exit."""
+ cli_push_description = """Push an image and
exit."""
warehouse_description = """Options for specifying Image Warehouse
(
http://aeolusproject.org/imagewarehouse.html) base URL and bucket
names."""
argparser = argparse.ArgumentParser(description=main_description,
prog='imgfac')
argparser.add_argument('--version', action='version',
version='%(prog)s 0.1', help='Version info')
argparser.add_argument('-v', '--verbose',
action='store_true', default=False, help='Set verbose logging.')
argparser.add_argument('--debug', action='store_true',
default=False, help='Set really verbose logging for debugging.')
+ argparser.add_argument('--image', help='UUID of iwhd image object to
rebuild or push')
argparser.add_argument('--foreground', action='store_true',
default=False, help='Stay in the foreground and avoid launching a daemon. (default:
%(default)s)')
argparser.add_argument('--config',
default='/etc/imagefactory.conf', help='Configuration file to use. (default:
%(default)s)')
argparser.add_argument('--imgdir', default='/tmp',
help='Build image files in location specified. (default: %(default)s)')
argparser.add_argument('--timeout', type=int, default=3600, help='Set
the timeout period for image building in seconds. (default: %(default)s)')
argparser.add_argument('--tmpdir', default='/tmp', help='Use
the specified location for temporary files. (default: %(default)s)')
+
group_qmf = argparser.add_argument_group(title='QMF agent',
description=qmf_description)
group_qmf.add_argument('--qmf', action='store_true',
default=False, help='Turn on QMF agent interface. (default: %(default)s)')
group_qmf.add_argument('--qpidd', default='localhost',
help='URL of qpidd to connect to. (default: %(default)s)')
+
group_build = argparser.add_argument_group(title='Image building',
description=cli_build_description)
group_build.add_argument('--template', help='Template XML file to
build from.')
- group_build.add_argument('--target', help='Cloud service to
target')
- group_push = argparser.add_argument_group(title='Image instantiation',
description=cli_push_description)
- group_push.add_argument('--image', help='Image to instantiate')
- group_push.add_argument('--provider', help='Cloud service provider
upon which to instantiate the image')
- group_push.add_argument('--credentials', help='Cloud provider
credentials')
+ group_build.add_argument('--target', action='append',
help='Cloud services to target (e.g. ec2, rhev-m, rackspace, condorcloud, etc.)')
+
+ group_push = argparser.add_argument_group(title='Image pushing',
description=cli_push_description)
+ group_push.add_argument('--provider', action='append',
help='Cloud service providers to push the image (e.g. ec2-us-east-1, rackspace,
etc.)')
+ group_push.add_argument('--credentials', help='Cloud provider
credentials XML (i.e. <provider_credentials/> document)')
+
group_warehouse = argparser.add_argument_group(title='Image Warehouse',
description=warehouse_description)
group_warehouse.add_argument('--warehouse',
default='http://localhost:9090/', help='URL of the warehouse location to store
images. (default: %(default)s)')
+ group_warehouse.add_argument('--image_bucket', help='Name of
warehouse bucket to look in images. (default: %(default)s)')
+ group_warehouse.add_argument('--build_bucket', help='Name of
warehouse bucket to look in builds. (default: %(default)s)')
group_warehouse.add_argument('--target_bucket', help='Name of
warehouse bucket to look in for target images. (default: %(default)s)')
group_warehouse.add_argument('--template_bucket', help='Name of
warehouse bucket to look in for templates. (default: %(default)s)')
group_warehouse.add_argument('--icicle_bucket', help='Name of
warehouse bucket to look in for icicles. (default: %(default)s)')
@@ -93,7 +99,7 @@ class ApplicationConfiguration(object):
elif(sys.argv[0].endswith("imgfac.py")):
return self.argparser.parse_args()
elif(sys.argv[0].endswith("unittest")):
- return self.argparser.parse_args('--target_bucket unittests_target_images
--template_bucket unittests_templates --icicle_bucket unittests_icicles --provider_bucket
unittests_provider_images'.split())
+ return self.argparser.parse_args('--image_bucket unittests_images
--build_bucket unittests_builds --target_bucket unittests_target_images --template_bucket
unittests_templates --icicle_bucket unittests_icicles --provider_bucket
unittests_provider_images'.split())
else:
return self.argparser.parse_args([])
diff --git a/imagefactory/BuildDispatcher.py b/imagefactory/BuildDispatcher.py
index fe9743a..564b220 100644
--- a/imagefactory/BuildDispatcher.py
+++ b/imagefactory/BuildDispatcher.py
@@ -18,38 +18,53 @@
import sys
import libxml2
import logging
+import os.path
import props
from threading import Thread, Lock
from imagefactory.builders import *
+from imagefactory.ApplicationConfiguration import ApplicationConfiguration
+from imagefactory.ImageFactoryException import ImageFactoryException
+from imagefactory.ImageWarehouse import ImageWarehouse
from imagefactory.Template import Template
class BuildDispatcher(object):
template = props.prop("_template", "The template property.")
target = props.prop("_target", "The target property.")
+ image_id = props.prop("_image_id", "The UUID of the image.")
+ build_id = props.prop("_build_id", "The UUID of the build.")
+ provider = props.prop("_provider", "The provider being pushed
to.")
status = props.prop("_status", "The status property.")
percent_complete = props.prop("_percent_complete", "The
percent_complete property.")
new_image_id = props.prop("_new_image_id" "The image property.")
- def __init__(self, template, target):
+ def __init__(self, template, target, image_id = '', build_id = ''):
super(BuildDispatcher, self).__init__()
self.log = logging.getLogger('%s.%s' % (__name__,
self.__class__.__name__))
- self.template = template
+ self.template = template if isinstance(template, Template) else
Template(template)
self.target = target
+ self.image_id = image_id
+ self.build_id = build_id
+ self.provider = None
self.status = "New"
self.percent_complete = 0
+ self._watcher = None
self._builder = self._get_builder()
self._builder.delegate = self
self.new_image_id = self._builder.new_image_id
- def build_image(self):
- self._start_builder_thread("build_image")
+ def build_image(self, watcher=None):
+ self._watcher = watcher
+ kwargs = dict(build_id=self.build_id)
+ self._start_builder_thread("build_image", arg_dict=kwargs)
- def push_image(self, target_image_id, provider, credentials):
+ def push_image(self, target_image_id, provider, credentials, watcher=None):
+ self._watcher = watcher
+ self.provider = provider
kwargs = dict(target_image_id=target_image_id, provider=provider,
credentials=credentials)
self._start_builder_thread("push_image", arg_dict=kwargs)
@@ -58,6 +73,9 @@ class BuildDispatcher(object):
def builder_did_update_status(self, builder, old_status, new_status):
self.status = new_status
+ if self.status == "COMPLETED" and self._watcher:
+ self._watcher.completed()
+ self._watcher = None
def builder_did_update_percentage(self, builder, original_percentage,
new_percentage):
self.percent_complete = new_percentage
@@ -66,13 +84,9 @@ class BuildDispatcher(object):
pass
def _get_builder(self):
- template_object = Template(self.template)
-
builder_class = MockBuilder.MockBuilder
if (self.target != "mock"): # If target is mock always run mock builder
regardless of template
- parsed_doc = libxml2.parseDoc(template_object.xml)
- node = parsed_doc.xpathEval('/template/os/name')
- os_name = node[0].content
+ os_name = self.__class__._xml_node(self.template.xml,
'/template/os/name')
class_name = "%sBuilder" % (os_name, )
try:
module_name = "imagefactory.builders.%s" % (class_name, )
@@ -81,10 +95,185 @@ class BuildDispatcher(object):
except AttributeError, e:
self.log.exception("CAUGHT EXCEPTION: %s \n Could not find builder
class for %s, returning MockBuilder!", e, os_name)
- return builder_class(template_object, self.target)
+ return builder_class(self.template, self.target)
def _start_builder_thread(self, method_name, arg_dict={}):
thread_name = "%s.%s()" % (self.new_image_id, method_name)
# using args to pass the method we want to call on the target object.
builder_thread = Thread(target = self._builder, name=thread_name,
args=(method_name), kwargs=arg_dict)
builder_thread.start()
+
+ @classmethod
+ def build_image_for_targets(cls, image_id, build_id, template, targets):
+ warehouse = cls._get_warehouse()
+
+ template = Template(template)
+
+ image_id = cls._ensure_image(warehouse, image_id, template)
+ build_id = cls._ensure_build(warehouse, image_id, build_id)
+
+ watcher = BuildWatcher(image_id, build_id, len(targets), warehouse)
+
+ dispatchers = []
+ for target in targets:
+ dispatcher = BuildDispatcher(template, target, image_id, build_id)
+ dispatcher.build_image(watcher)
+ dispatchers.append(dispatcher)
+
+ return dispatchers
+
+ @classmethod
+ def push_image_to_providers(cls, image_id, build_id, providers, credentials):
+ warehouse = cls._get_warehouse()
+
+ if not build_id:
+ build_id = cls._latest_unpushed(warehouse, image_id)
+
+ watcher = PushWatcher(image_id, build_id, len(providers), warehouse)
+
+ dispatchers = []
+ for provider in providers:
+ target = cls._map_provider_to_target(provider)
+
+ target_image_id = cls._target_image_for_build_and_target(warehouse, build_id,
target)
+
+ template = cls._template_for_target_image_id(warehouse, target_image_id)
+
+ dispatcher = BuildDispatcher(template, target, image_id, build_id)
+ dispatcher.push_image(target_image_id, provider, credentials, watcher)
+ dispatchers.append(dispatcher)
+
+ return dispatchers
+
+ @classmethod
+ def _get_warehouse(cls):
+ return
ImageWarehouse(ApplicationConfiguration().configuration['warehouse'])
+
+ @classmethod
+ def _xml_node(cls, xml, xpath):
+ nodes = libxml2.parseDoc(xml).xpathEval(xpath)
+ if not nodes:
+ return None
+ return nodes[0].content
+
+ @classmethod
+ def _ensure_image(cls, warehouse, image_id, template):
+ if image_id:
+ return image_id
+
+ name = cls._xml_node(template.xml, '/template/name')
+ if name:
+ image_xml = '<image><name>%s</name></image>'
% name
+ else:
+ image_xml = '</image>'
+
+ return warehouse.store_image(None, image_xml)
+
+ @classmethod
+ def _ensure_build(cls, warehouse, image_id, build_id):
+ if build_id:
+ return build_id
+ return warehouse.store_build(None, dict(image = image_id))
+
+ @classmethod
+ def _latest_unpushed(cls, warehouse, image_id):
+ return warehouse.metadata_for_id_of_type(['latest_unpushed'], image_id,
'image')['latest_unpushed']
+
+ @classmethod
+ def _target_image_for_build_and_target(cls, warehouse, build_id, target):
+ return warehouse.query("target_image", "$build == \"%s\"
&& $target == \"%s\"" % (build_id, target))[0]
+
+ @classmethod
+ def _template_for_target_image_id(cls, warehouse, target_image_id):
+ return warehouse.metadata_for_id_of_type(['template'], target_image_id,
'target_image')['template']
+
+ @classmethod
+ def _is_rhevm_provider(cls, provider):
+ rhevm_json = '/etc/rhevm.json'
+ if not os.path.exists(rhevm_json):
+ return False
+
+ rhevm_sites = {}
+ f = open(rhevm_json, 'r')
+ try:
+ rhevm_sites = json.loads(f.read())
+ finally:
+ f.close()
+
+ return provider in rhevm_sites
+
+ # FIXME: this is a hack; conductor is the only one who really
+ # knows this mapping, so perhaps it should provide it?
+ # e.g. pass a provider => target dict into push_image
+ # rather than just a list of providers. Perhaps just use
+ # this heuristic for the command line?
+ #
+ # provider semantics, per target:
+ # - ec2: region, one of ec2-us-east-1, ec2-us-west-1, ec2-ap-southeast-1,
ec2-ap-northeast-1, ec2-eu-west-1
+ # - condorcloud: ignored
+ # - rhev-m: a key in /etc/rhevm.json and passed to op=register&site=provider
+ # - mock: any provider with 'mock' prefix
+ # - rackspace: provider is rackspace
+ #
+ @classmethod
+ def _map_provider_to_target(cls, provider):
+ if provider.startswith('ec2-'):
+ return 'ec2'
+ elif provider == 'rackspace':
+ return 'rackspace'
+ elif cls._is_rhevm_provider(provider):
+ return 'rhev-m'
+ elif provider.startswith('mock'):
+ return 'mock'
+ else:
+ return 'condorcloud' # condorcloud ignores provider
+
+class Watcher(object):
+ def __init__(self, image_id, build_id, remaining, warehouse):
+ self.remaining = remaining
+ self.warehouse = warehouse
+ self.image_id = image_id
+ self.build_id = build_id
+
+ def completed(self):
+ self.remaining -= 1
+ if self.remaining == 0:
+ self.all_completed()
+
+ def all_completed(self):
+ pass
+
+ def _image_attr(self, attr):
+ return self.warehouse.metadata_for_id_of_type([attr], self.image_id,
'image')[attr]
+
+ def _set_image_attr(self, attr, value):
+ self.warehouse.set_metadata_for_id_of_type({attr : value}, self.image_id,
'image')
+
+ def _latest_build(self):
+ return self._image_attr('latest_build')
+
+ def _set_latest_build(self, build_id):
+ self._set_image_attr('latest_build', build_id)
+
+ def _latest_unpushed(self):
+ return self._image_attr('latest_unpushed')
+
+ def _set_latest_unpushed(self, build_id):
+ self._set_image_attr('latest_unpushed', build_id)
+
+ def _set_build_parent(self, parent_id):
+ self.warehouse.set_metadata_for_id_of_type({'parent' : parent_id},
self.build_id, 'build')
+
+class BuildWatcher(Watcher):
+ def all_completed(self):
+ parent_id = self._latest_unpushed()
+ if not parent_id:
+ parent_id = self._latest_build()
+ self._set_latest_unpushed(self.build_id)
+ if parent_id:
+ self._set_build_parent(parent_id)
+
+class PushWatcher(Watcher):
+ def all_completed(self):
+ self._set_latest_build(self.build_id)
+ self._set_latest_unpushed(None)
diff --git a/imagefactory/ImageWarehouse.py b/imagefactory/ImageWarehouse.py
index a5e74e5..973f7c4 100644
--- a/imagefactory/ImageWarehouse.py
+++ b/imagefactory/ImageWarehouse.py
@@ -24,6 +24,7 @@ import httplib2
import urllib
import uuid
import os
+import libxml2
import props
from imagefactory.ApplicationConfiguration import ApplicationConfiguration
@@ -31,6 +32,8 @@ from imagefactory.ApplicationConfiguration import
ApplicationConfiguration
class ImageWarehouse(object):
url = props.prop("_url", "The url property.")
+ image_bucket = props.prop("_image_bucket", "The image_bucket
property.")
+ build_bucket = props.prop("_build_bucket", "The build_bucket
property.")
target_image_bucket = props.prop("_target_image_bucket", "The
target_image_bucket property.")
template_bucket = props.prop("_template_bucket", "The template_bucket
property.")
icicle_bucket = props.prop("_icicle_bucket", "The icicle_bucket
property.")
@@ -51,6 +54,8 @@ class ImageWarehouse(object):
url = ApplicationConfiguration().configuration['warehouse']
self.log.debug("Property (url) not specified. Pulling from application
configuration: %s" % (url, ))
+ self.image_bucket =
ApplicationConfiguration().configuration['image_bucket']
+ self.build_bucket =
ApplicationConfiguration().configuration['build_bucket']
self.target_image_bucket =
ApplicationConfiguration().configuration['target_bucket']
self.template_bucket =
ApplicationConfiguration().configuration['template_bucket']
self.icicle_bucket =
ApplicationConfiguration().configuration['icicle_bucket']
@@ -99,6 +104,14 @@ class ImageWarehouse(object):
return object_url
+ def query(self, object_type, expression):
+ object_url = self.__url_for_id_of_type("_query", object_type,
create=False)
+ self.log.debug("Querying (%s) with expression (%s)" % (object_url,
expression))
+ xml = self._http_post(object_url, expression,
'application/x-www-form-urlencoded')
+ if not xml:
+ return []
+ return map(lambda n: n.content,
libxml2.parseDoc(xml).xpathEval("/objects/object/key"))
+
def post_on_object_with_id_of_type(self, object_id, object_type, post_data):
object_url = self.__url_for_id_of_type(object_id, object_type, create=False)
return self._http_post(object_url, urllib.urlencode(post_data),
'application/x-www-form-urlencoded')
@@ -152,6 +165,34 @@ class ImageWarehouse(object):
metadata.update( { item : self._http_get("%s/%s" % (object_url,
item)) } )
return metadata
+ def store_image(self, image_id, image_xml, metadata=None):
+ if(not image_id):
+ image_id = str(uuid.uuid4())
+ object_url = self.__url_for_id_of_type(image_id, object_type="image")
+
+ self._http_put(object_url, image_xml)
+
+ meta_data = dict(uuid=str(image_id), object_type="image")
+ if(metadata):
+ meta_data.update(metadata)
+ self.set_metadata_for_object_at_url(meta_data, object_url)
+
+ return image_id
+
+ def store_build(self, build_id, metadata=None):
+ if(not build_id):
+ build_id = str(uuid.uuid4())
+ object_url = self.__url_for_id_of_type(build_id, object_type="build")
+
+ self._http_put(object_url)
+
+ meta_data = dict(uuid=str(build_id), object_type="build")
+ if(metadata):
+ meta_data.update(metadata)
+ self.set_metadata_for_object_at_url(meta_data, object_url)
+
+ return build_id
+
def store_target_image(self, target_image_id, image_file_path, metadata=None):
object_url = self.__url_for_id_of_type(target_image_id,
"target_image")
try:
diff --git a/imagefactory/builders/BaseBuilder.py b/imagefactory/builders/BaseBuilder.py
index 32499a7..03f6bf3 100644
--- a/imagefactory/builders/BaseBuilder.py
+++ b/imagefactory/builders/BaseBuilder.py
@@ -125,7 +125,7 @@ class BaseBuilder(object):
getattr(self, str().join(args))(**kwargs)
# Image actions
- def build_image(self):
+ def build_image(self, build_id=None):
"""Build the image file. This method is implemented by subclasses
of BaseBuilder to handle
OS specific build mechanics."""
raise NotImplementedError
@@ -135,10 +135,12 @@ class BaseBuilder(object):
OS specific build mechanics."""
raise NotImplementedError
- def store_image(self, target_parameters=None):
+ def store_image(self, build_id, target_parameters=None):
template_id = self.warehouse.store_template(self.template.xml,
self.template.identifier)
icicle_id = self.warehouse.store_icicle(self.output_descriptor)
metadata = dict(template=template_id, target=self.target, icicle=icicle_id,
target_parameters=target_parameters)
+ if build_id:
+ metadata['build'] = build_id
self.warehouse.store_target_image(self.new_image_id, self.image,
metadata=metadata)
def push_image(self, target_image_id, provider, credentials):
diff --git a/imagefactory/builders/FedoraBuilder.py
b/imagefactory/builders/FedoraBuilder.py
index aa3b637..86760fd 100644
--- a/imagefactory/builders/FedoraBuilder.py
+++ b/imagefactory/builders/FedoraBuilder.py
@@ -104,14 +104,14 @@ class FedoraBuilder(BaseBuilder):
# We don't really care about the name so just force uniqueness
self.guest.name = self.guest.name + "-" + self.new_image_id
- def build_image(self):
+ def build_image(self, build_id=None):
try:
if self.target in self.upload_clouds or (self.target == "ec2" and
self.app_config["ec2_build_style"] == "upload"):
self.init_guest("local")
- self.build_upload()
+ self.build_upload(build_id)
elif self.target in self.nonul_clouds or (self.target == "ec2" and
self.app_config["ec2_build_style"] == "snapshot"):
# No actual need to have a guest object here so don't bother
- self.build_snapshot()
+ self.build_snapshot(build_id)
else:
raise ImageFactoryException("Invalid build target (%s) passed to
build_image()" % (self.target))
except:
@@ -121,7 +121,7 @@ class FedoraBuilder(BaseBuilder):
self.status="FAILED"
raise
- def build_snapshot(self):
+ def build_snapshot(self, build_id):
# All we need do here is store the relevant bits in the Warehouse
self.log.debug("Building Linux for non-upload cloud (%s)" %
(self.target))
self.image = "%s/placeholder-linux-image-%s" %
(self.app_config['imgdir'], self.new_image_id)
@@ -129,13 +129,13 @@ class FedoraBuilder(BaseBuilder):
image_file.write("Placeholder for non upload cloud Linux image")
image_file.close()
self.log.debug("Storing placeholder object for non upload cloud
image")
- self.store_image()
+ self.store_image(build_id)
self.percent_complete = 100
self.status = "COMPLETED"
self.log.debug("Completed placeholder warehouse object for linux non-upload
image...")
sleep(5)
- def build_upload(self):
+ def build_upload(self, build_id):
self.log.debug("build_upload() called on FedoraBuilder...")
self.log.debug("Building for target %s with warehouse config %s" %
(self.target, self.app_config['warehouse']))
self.status="BUILDING"
@@ -202,7 +202,7 @@ class FedoraBuilder(BaseBuilder):
else:
target_parameters="No target parameters for cloud type %s" %
(self.target)
- self.store_image(target_parameters)
+ self.store_image(build_id, target_parameters)
self.log.debug("Image warehouse storage complete")
self.percent_complete=100
self.status="COMPLETED"
diff --git a/imagefactory/builders/IBuilder.py b/imagefactory/builders/IBuilder.py
index 15a75d8..7a2f1f6 100644
--- a/imagefactory/builders/IBuilder.py
+++ b/imagefactory/builders/IBuilder.py
@@ -60,7 +60,7 @@ class IBuilder(Interface):
output_descriptor = Attribute("""ICICLE document describing what was
actually installed in the image.""")
delegate = Attribute("""Object conforming to the BuilderDelegate
interface.""")
# methods
- def build_image():
+ def build_image(build_id=None):
"""Tell the builder to start building the
image."""
def abort():
diff --git a/imagefactory/builders/MockBuilder.py b/imagefactory/builders/MockBuilder.py
index 05851a1..5b79154 100644
--- a/imagefactory/builders/MockBuilder.py
+++ b/imagefactory/builders/MockBuilder.py
@@ -40,7 +40,7 @@ class MockBuilder(BaseBuilder):
self.app_config = ApplicationConfiguration().configuration
# Image actions
- def build_image(self):
+ def build_image(self, build_id=None):
if(self.template.xml == "<template>FAIL</template>"):
self.log.debug("build_image() failed for MockBuilder...")
failing_thread = FailureThread(target=self, kwargs=dict(message="Testing
failure conditions via mock target builder..."))
@@ -83,7 +83,7 @@ class MockBuilder(BaseBuilder):
self.status = "COMPLETED"
self.log.debug("Completed mock image build...")
- self.store_image()
+ self.store_image(build_id)
def push_image(self, target_image_id, provider, credentials):
self.status = "INITIALIZING"
diff --git a/imagefactory/builders/RHELBuilder.py b/imagefactory/builders/RHELBuilder.py
index 3f8d1be..f3d278c 100644
--- a/imagefactory/builders/RHELBuilder.py
+++ b/imagefactory/builders/RHELBuilder.py
@@ -30,7 +30,7 @@ class RHELBuilder(BaseBuilder):
super(RHELBuilder, self).__init__(template, target)
# Image actions
- def build_image(self):
+ def build_image(self, build_id=None):
pass
def abort(self):
diff --git a/imgfac.py b/imgfac.py
index 3b24067..30e1fea 100755
--- a/imgfac.py
+++ b/imgfac.py
@@ -156,9 +156,9 @@ class Application(object):
self.setup_logging()
if (self.app_config['template'] and
self.app_config['target']):
- self.dispatcher = BuildDispatcher(self.app_config['target'],
self.app_config['template'])
- self.dispatcher.build_image()
- print("Image created with id: %s" % (self.dispatcher.image_id,
))
+ dispatchers =
BuildDispatcher.build_image_for_targets(self.app_config['image'], None,
self.app_config['template'], self.app_config['target'])
+ for dispatcher in dispatchers:
+ print("Building build %s of image %s to target %s" %
(dispatcher.build_id, dispatcher.image_id, dispatcher.target))
elif (self.app_config['image'] and
self.app_config['provider'] and self.app_config['credentials']):
credentials = self.app_config['credentials']
@@ -172,13 +172,9 @@ class Application(object):
print("Unexpected content or formatting of
credentials...")
sys.exit(1)
- warehouse = ImageWarehouse(self.app_config['warehouse'])
- metadata = warehouse.metadata_for_id_of_type(('template',
'target'), self.app_config['image'], object_type='image')
-
- self.dispatcher = BuildDispatcher(metadata['target'],
metadata['template'])
- self.dispatcher.push_image(self.app_config['image'],
self.app_config['provider'], credentials)
- print("Image instance created with id: %s" %
(self.dispatcher.image_id, ))
-
+ dispatchers =
BuildDispatcher.push_image_to_providers(self.app_config['image'], None,
self.app_config['provider'], credentials)
+ for dispatcher in dispatchers:
+ print("Pushing build %s of image %s to provider %s" %
(dispatcher.build_id, dispatcher.image_id, dispatcher.provider))
if __name__ == "__main__":
application = Application()
diff --git a/tests/testBuildDispatcher.py b/tests/testBuildDispatcher.py
index 411d51d..226da12 100644
--- a/tests/testBuildDispatcher.py
+++ b/tests/testBuildDispatcher.py
@@ -34,7 +34,7 @@ class testBuildDispatcher(unittest.TestCase):
template_xml =
"<template><name>f14jeos</name><os><name>Fedora</name></os></template>"
dispatcher = BuildDispatcher.BuildDispatcher(template_xml, "mock")
self.assertIsInstance(dispatcher._builder, MockBuilder.MockBuilder)
- self.assertEqual(dispatcher.template, template_xml)
+ self.assertEqual(dispatcher.template.xml, template_xml)
self.assertEqual(dispatcher.target, "mock")
if __name__ == '__main__':
diff --git a/tests/testImageWarehouse.py b/tests/testImageWarehouse.py
index a63d3a8..ebe0bb8 100644
--- a/tests/testImageWarehouse.py
+++ b/tests/testImageWarehouse.py
@@ -98,6 +98,30 @@ class testImageWarehouse(unittest.TestCase):
self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id))
self.assertTrue(self.warehouse.remove_icicle_with_id(icicle_id2))
+ def testImageAndBuildMethods(self):
+ """Tests CRUD operations on images, builds and metadata on those
objects..."""
+ image_xml = '<image/>'
+
+ image_id = self.warehouse.store_image(None, image_xml, self.metadata)
+
+ self.assertIsNotNone(image_id)
+
+ image_body, metadata = self.warehouse.object_with_id_of_type(image_id,
'image', self.metadata.keys())
+
+ self.assertEqual(image_xml, image_body)
+ self.assertEqual(self.metadata, metadata)
+
+ build_id = self.warehouse.store_build(None, self.metadata)
+
+ build_body, metadata = self.warehouse.object_with_id_of_type(build_id,
'build', self.metadata.keys())
+
+ self.assertEqual('', build_body)
+ self.assertEqual(self.metadata, metadata)
+
+ ids = self.warehouse.query('build', '$object_type ==
"build" && $key1 == "value1"')
+
+ self.assertIn(build_id, ids)
+
def testBucketCreation(self):
#
self.assert_(self.warehouse.create_bucket_at_url("%s/unittests-create_bucket/%s"
% (self.warehouse.url, str(uuid.uuid4()))))
self.warehouse.create_bucket_at_url("%s/unittests-create_bucket" %
(self.warehouse.url, ))
--
1.7.4.4