Hi, this is first version of warehouse integration. build and push methods for image building are now mocked up, there is not real image_builder_service call in their body.
Requires patches: delayed_job, provider type, Image Warehouse API library for conductor.
From: Jan Provaznik jprovazn@redhat.com
--- src/app/models/provider_image.rb | 4 ++- .../20110221125633_add_uuid_to_providerimage.rb | 9 +++++++ src/spec/models/provider_image_spec.rb | 24 ++++++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletions(-) create mode 100644 src/db/migrate/20110221125633_add_uuid_to_providerimage.rb create mode 100644 src/spec/models/provider_image_spec.rb
diff --git a/src/app/models/provider_image.rb b/src/app/models/provider_image.rb index 861daea..0ac23c2 100644 --- a/src/app/models/provider_image.rb +++ b/src/app/models/provider_image.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 20110207110131 +# Schema version: 20110221125633 # # Table name: provider_images # @@ -9,6 +9,7 @@ # provider_image_key :string(255) # uploaded :boolean # registered :boolean +# uuid :string(255) #
class ProviderImage < ActiveRecord::Base @@ -17,4 +18,5 @@ class ProviderImage < ActiveRecord::Base
validates_presence_of :provider_id validates_presence_of :image_id + validates_uniqueness_of :uuid, :allow_nil => true end diff --git a/src/db/migrate/20110221125633_add_uuid_to_providerimage.rb b/src/db/migrate/20110221125633_add_uuid_to_providerimage.rb new file mode 100644 index 0000000..6964437 --- /dev/null +++ b/src/db/migrate/20110221125633_add_uuid_to_providerimage.rb @@ -0,0 +1,9 @@ +class AddUuidToProviderimage < ActiveRecord::Migration + def self.up + add_column :provider_images, :uuid, :string + end + + def self.down + remove_column :provider_images, :uuid + end +end diff --git a/src/spec/models/provider_image_spec.rb b/src/spec/models/provider_image_spec.rb new file mode 100644 index 0000000..419d976 --- /dev/null +++ b/src/spec/models/provider_image_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' + +describe ProviderImage do + it "should have a provider_id" do + i = Factory.build(:mock_provider_image) + i.should be_valid + i.provider = nil + i.should_not be_valid + end + + it "should have an image_id" do + i = Factory.build(:mock_provider_image) + i.should be_valid + i.image = nil + i.should_not be_valid + end + + it "should have a unique uuid" do + i = Factory(:mock_provider_image, :uuid => '1') + i.should be_valid + i = Factory.build(:mock_provider_image, :uuid => '1') + i.should_not be_valid + end +end
From: Jan Provaznik jprovazn@redhat.com
--- aeolus-conductor.spec.in | 1 + src/app/models/image_warehouse_object.rb | 2 +- src/config/environment.rb | 1 + 3 files changed, 3 insertions(+), 1 deletions(-)
diff --git a/aeolus-conductor.spec.in b/aeolus-conductor.spec.in index 7af22c1..fc34a68 100644 --- a/aeolus-conductor.spec.in +++ b/aeolus-conductor.spec.in @@ -34,6 +34,7 @@ Requires: rubygem(imagebuilder-console) Requires: rubygem(rack-restful_submit) Requires: rubygem(sunspot_rails) Requires: rubygem(delayed_job) +Requires: rubygem(uuidtools) Requires: postgresql Requires: postgresql-server Requires: ruby-postgres diff --git a/src/app/models/image_warehouse_object.rb b/src/app/models/image_warehouse_object.rb index 48dd04f..df01ea3 100644 --- a/src/app/models/image_warehouse_object.rb +++ b/src/app/models/image_warehouse_object.rb @@ -36,7 +36,7 @@ module ImageWarehouseObject
def generate_uuid # TODO: generate real uuid here, e.g. with some ruby uuid generator - self.uuid ||= "#{self.name}-#{Time.now.to_f.to_s}" + self.uuid ||= UUIDTools::UUID.timestamp_create.to_s end
end diff --git a/src/config/environment.rb b/src/config/environment.rb index cedcb32..2e4b254 100644 --- a/src/config/environment.rb +++ b/src/config/environment.rb @@ -55,6 +55,7 @@ Rails::Initializer.run do |config| config.gem 'rack-restful_submit', :version => '1.1.2' config.gem 'sunspot_rails', :lib => 'sunspot/rails' config.gem 'delayed_job', :version => '~>2.0.4' + config.gem 'uuidtools'
config.middleware.swap Rack::MethodOverride, 'Rack::RestfulSubmit'
From: Jan Provaznik jprovazn@redhat.com
Icicle represnts output from image build (or image push) in xml format. --- src/app/models/icicle.rb | 17 +++++++++++++++++ src/app/models/provider_image.rb | 12 +++++++++++- src/db/migrate/20110222153957_create_icicles.rb | 15 +++++++++++++++ src/spec/factories/icicle.rb | 3 +++ src/spec/factories/provider_image.rb | 1 + src/spec/models/icicle_spec.rb | 17 +++++++++++++++++ src/spec/models/provider_image_spec.rb | 10 ++++++++++ 7 files changed, 74 insertions(+), 1 deletions(-) create mode 100644 src/app/models/icicle.rb create mode 100644 src/db/migrate/20110222153957_create_icicles.rb create mode 100644 src/spec/factories/icicle.rb create mode 100644 src/spec/models/icicle_spec.rb
diff --git a/src/app/models/icicle.rb b/src/app/models/icicle.rb new file mode 100644 index 0000000..14037f9 --- /dev/null +++ b/src/app/models/icicle.rb @@ -0,0 +1,17 @@ +# == Schema Information +# Schema version: 20110223132404 +# +# Table name: icicles +# +# id :integer not null, primary key +# uuid :string(255) +# xml :text +# created_at :datetime +# updated_at :datetime +# + +class Icicle < ActiveRecord::Base + validates_presence_of :uuid + validates_uniqueness_of :uuid + has_many :provider_images +end diff --git a/src/app/models/provider_image.rb b/src/app/models/provider_image.rb index 0ac23c2..c2d9e2e 100644 --- a/src/app/models/provider_image.rb +++ b/src/app/models/provider_image.rb @@ -1,5 +1,5 @@ # == Schema Information -# Schema version: 20110221125633 +# Schema version: 20110223132404 # # Table name: provider_images # @@ -10,13 +10,23 @@ # uploaded :boolean # registered :boolean # uuid :string(255) +# icicle_id :integer #
class ProviderImage < ActiveRecord::Base belongs_to :provider belongs_to :image + belongs_to :icicle
validates_presence_of :provider_id validates_presence_of :image_id validates_uniqueness_of :uuid, :allow_nil => true + + after_destroy :destroy_orphaned_icicles + + def destroy_orphaned_icicles + if self.icicle and self.icicle.provider_images.empty? + self.icicle.destroy + end + end end diff --git a/src/db/migrate/20110222153957_create_icicles.rb b/src/db/migrate/20110222153957_create_icicles.rb new file mode 100644 index 0000000..a48dec8 --- /dev/null +++ b/src/db/migrate/20110222153957_create_icicles.rb @@ -0,0 +1,15 @@ +class CreateIcicles < ActiveRecord::Migration + def self.up + create_table :icicles do |t| + t.string :uuid + t.text :xml + t.timestamps + end + add_column :provider_images, :icicle_id, :integer + end + + def self.down + drop_table :icicles + remove_column :provider_images, :icicle_id + end +end diff --git a/src/spec/factories/icicle.rb b/src/spec/factories/icicle.rb new file mode 100644 index 0000000..3961561 --- /dev/null +++ b/src/spec/factories/icicle.rb @@ -0,0 +1,3 @@ +Factory.define :icicle do |i| + i.sequence(:uuid) { |n| "icicle#{n}" } +end diff --git a/src/spec/factories/provider_image.rb b/src/spec/factories/provider_image.rb index 55c2281..2dbe453 100644 --- a/src/spec/factories/provider_image.rb +++ b/src/spec/factories/provider_image.rb @@ -4,6 +4,7 @@ Factory.define :provider_image do |ri| ri.sequence(:provider_image_key) { |n| "provider_image_key#(n)" } ri.uploaded true ri.registered true + ri.association :icicle end
Factory.define :mock_provider_image, :parent => :provider_image do |i| diff --git a/src/spec/models/icicle_spec.rb b/src/spec/models/icicle_spec.rb new file mode 100644 index 0000000..d4a1e86 --- /dev/null +++ b/src/spec/models/icicle_spec.rb @@ -0,0 +1,17 @@ +require 'spec_helper' + +describe Icicle do + it "should not be valid if uuid is not set" do + i = Factory.build(:icicle, :uuid => nil) + i.valid?.should be_false + i.uuid = '' + i.valid?.should be_false + end + + it "should have a unique uuid" do + old = Factory(:icicle) + old.should be_valid + new = Factory.build(:icicle, :uuid => old.uuid) + new.should_not be_valid + end +end diff --git a/src/spec/models/provider_image_spec.rb b/src/spec/models/provider_image_spec.rb index 419d976..4ad2a50 100644 --- a/src/spec/models/provider_image_spec.rb +++ b/src/spec/models/provider_image_spec.rb @@ -21,4 +21,14 @@ describe ProviderImage do i = Factory.build(:mock_provider_image, :uuid => '1') i.should_not be_valid end + + it "should delete orphaned icicles" do + icicle = Factory(:icicle) + i1 = Factory(:mock_provider_image, :icicle => icicle) + i2 = Factory(:mock_provider_image, :icicle => icicle) + i1.destroy + Icicle.find_by_id(icicle.id).should_not be_nil + i2.destroy + Icicle.find_by_id(icicle.id).should be_nil + end end
From: Jan Provaznik jprovazn@redhat.com
--- src/app/models/image_warehouse_object.rb | 38 +++++++++++++++++++++++++---- src/app/models/template.rb | 13 ++++++++- src/config/image_warehouse.yml | 2 +- src/spec/factories/template.rb | 6 ++++ src/spec/models/image_spec.rb | 5 ---- 5 files changed, 50 insertions(+), 14 deletions(-)
diff --git a/src/app/models/image_warehouse_object.rb b/src/app/models/image_warehouse_object.rb index df01ea3..7717ccd 100644 --- a/src/app/models/image_warehouse_object.rb +++ b/src/app/models/image_warehouse_object.rb @@ -16,6 +16,9 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, # MA 02110-1301, USA. A copy of the GNU General Public License is # also available at http://www.gnu.org/copyleft/gpl.html. + +require 'warehouse_client' + module ImageWarehouseObject
WAREHOUSE_CONFIG = YAML.load_file("#{RAILS_ROOT}/config/image_warehouse.yml") @@ -24,13 +27,36 @@ module ImageWarehouseObject @xml ||= ImageDescriptorXML.new(self[:xml].to_s) end
+ # this should be overriden in a model if we want to save additional attrs + # for the model + def warehouse_attrs + {:uuid => self.uuid, :object_type => self.class.class_name} + end + + # this should be overriden in a model if we want to save body + def warehouse_body + nil + end + + # TODO: it would be nice not call upload, when save was invoked by warehouse + # sync script def upload - self.uri = File.join(WAREHOUSE_CONFIG['baseurl'], "#{uuid}") - response = Typhoeus::Request.put(self.uri, :body => xml.to_xml, :timeout => 30000) - if response.code == 200 - update_attribute(:uploaded, true) - else - raise "failed to upload template (code #{response.code}): #{response.body}" + whouse = Warehouse::Client.new(WAREHOUSE_CONFIG['baseurl']) + # TODO: for now there is no way how it check if bucket exists in warehouse + # so we try to create bucket everytime, if bucket exists, warehouse returns + # 500 Internal server error + whouse.create_bucket(warehouse_bucket) rescue true + # TODO: we delete existing object if it exists + whouse.bucket(warehouse_bucket).object(self.uuid).delete! rescue true + whouse.bucket(warehouse_bucket).create_object(self.uuid, warehouse_body, warehouse_attrs) + end + + def delete_in_warehouse + whouse = Warehouse::Client.new(WAREHOUSE_CONFIG['baseurl']) + begin + whouse.bucket(warehouse_bucket).object(self.uuid).delete! + rescue + logger.error "failed to delete #{self.uuid} in warehouse: #{$!}" end end
diff --git a/src/app/models/template.rb b/src/app/models/template.rb index 7ef0449..b84409c 100644 --- a/src/app/models/template.rb +++ b/src/app/models/template.rb @@ -26,6 +26,7 @@ require 'typhoeus' class Template < ActiveRecord::Base include PermissionedObject include ImageWarehouseObject + searchable do text :name, :as => :code_substring text :platform, :as => :code_substring @@ -38,8 +39,8 @@ class Template < ActiveRecord::Base has_many :instances has_and_belongs_to_many :assemblies before_validation :generate_uuid - before_save :update_xml - before_destroy :no_instances? + before_save [:update_xml, :upload] + before_destroy [:no_instances?, :delete_in_warehouse]
has_many :permissions, :as => :permission_object, :dependent => :destroy, :include => [:role], @@ -137,4 +138,12 @@ class Template < ActiveRecord::Base xml.clear_groups groups.to_a.each {|group| xml.add_group(group)} end + + def warehouse_body + self.xml.to_xml + end + + def warehouse_bucket + 'templates' + end end diff --git a/src/config/image_warehouse.yml b/src/config/image_warehouse.yml index cc7545c..ede2baa 100644 --- a/src/config/image_warehouse.yml +++ b/src/config/image_warehouse.yml @@ -1,3 +1,3 @@ -baseurl: http://localhost:9090/templates/ +baseurl: http://localhost:9090 # to set up the bucket, run: # curl -X PUT http://localhost:9090/templates diff --git a/src/spec/factories/template.rb b/src/spec/factories/template.rb index 0305147..1bb6fa6 100644 --- a/src/spec/factories/template.rb +++ b/src/spec/factories/template.rb @@ -1,4 +1,10 @@ Factory.define :template do |i| i.sequence(:name) { |n| "template#{n}" } i.platform 'fedora' + i.after_build do |tpl| + if tpl.respond_to?(:stub!) + tpl.stub!(:upload).and_return(true) + tpl.stub!(:delete_in_warehouse).and_return(true) + end + end end diff --git a/src/spec/models/image_spec.rb b/src/spec/models/image_spec.rb index a5c3534..d364919 100644 --- a/src/spec/models/image_spec.rb +++ b/src/spec/models/image_spec.rb @@ -59,11 +59,6 @@ describe Image do account.stub!(:connect).and_return(client) account.stub!(:valid_credentials?).and_return(true) account.save! - Template.stub!(:new).and_return do |*args| - tpl = Template.proxied_by_rspec__new(*args) - tpl.should_receive(:upload).and_return(true) - tpl - end
lambda do lambda do
From: Jan Provaznik jprovazn@redhat.com
--- .../controllers/image_factory/builds_controller.rb | 5 +- src/app/models/image.rb | 75 ++++++++++++++----- src/app/models/image_warehouse_object.rb | 9 ++- src/app/models/provider.rb | 8 -- src/app/models/provider_image.rb | 29 ++++++++ src/features/step_definitions/template_steps.rb | 2 +- src/spec/controllers/builds_controller_spec.rb | 8 +-- src/spec/factories/image.rb | 1 + src/spec/models/image_spec.rb | 10 +-- 9 files changed, 100 insertions(+), 47 deletions(-)
diff --git a/src/app/controllers/image_factory/builds_controller.rb b/src/app/controllers/image_factory/builds_controller.rb index ac0a3e2..035bb38 100644 --- a/src/app/controllers/image_factory/builds_controller.rb +++ b/src/app/controllers/image_factory/builds_controller.rb @@ -23,17 +23,18 @@ class ImageFactory::BuildsController < ApplicationController return end
- @tpl.upload unless @tpl.uploaded errors = {} warnings = [] params[:targets].each do |target_id| begin target = ProviderType.find(target_id) - Image.build(@tpl, target) + Image.create_and_build!(@tpl, target) rescue ImageExistsError warnings << $!.message rescue errors[target ? target.name : target_id] = $!.message + logger.error $!.message + logger.error $!.backtrace.join("\n ") end end flash[:warning] = 'Warning: ' + warnings.join unless warnings.empty? diff --git a/src/app/models/image.rb b/src/app/models/image.rb index cfa15e4..55e8f6e 100644 --- a/src/app/models/image.rb +++ b/src/app/models/image.rb @@ -40,6 +40,7 @@ class ImageExistsError < Exception;end
class Image < ActiveRecord::Base include SearchFilter + include ImageWarehouseObject
before_save :generate_uuid
@@ -57,6 +58,8 @@ class Image < ActiveRecord::Base validates_presence_of :template_id validates_presence_of :provider_type_id
+ validates_uniqueness_of :template_id, :scope => :provider_type_id + SEARCHABLE_COLUMNS = %w(name)
STATE_QUEUED = 'queued' @@ -73,31 +76,50 @@ class Image < ActiveRecord::Base self.uuid ||= "image-#{self.template_id}-#{Time.now.to_f.to_s}" end
- def self.build(template, target) - # FIXME: This will need to be enhanced to handle multiple - # providers of same type, only one is supported right now - if Image.find_by_template_id_and_provider_type_id(template.id, target.id) - raise ImageExistsError, "An attempted build of this template for the target '#{target.name}' already exists" + # TODO: for now when build is finished we call upload automatically for all providers + def after_update + if self.status_changed? and self.status == STATE_COMPLETE + # TODO: use after_commit callback in rails 3 - it's better to have it outside + # update transaction + begin + invoke_sync + upload_to_all_providers_with_account + rescue => e + logger.error e.message + logger.error e.backtrace.join("\n ") + end end + end
- unless provider = Provider.find_by_target_with_account(target.id) - raise "There is no provider for '#{target.name}' type with valid account." + def build + # TODO: this is just stubbed upload call, + # when new image_builder_service is done, replace + # with real request to image_builder_service + begin + unless self.provider_type.build_supported + raise "Build is not supported on images with provider type #{self.provider_type.name}" + end + # put build request to image_builder_service here + self.status = STATE_COMPLETE + rescue + # TODO - associate with event model and create event from exception + self.status = STATE_FAILED end + save! + end
- img = nil - Image.transaction do - img = Image.create!( - :name => "#{template.name}/#{target.codename}", - :provider_type_id => target.id, - :template_id => template.id, - :status => Image::STATE_QUEUED - ) - ProviderImage.create!( - :image_id => img.id, - :provider_id => provider - ) + def self.create_and_build!(template, provider_type) + if Image.find_by_template_id_and_provider_type_id(template.id, provider_type.id) + raise ImageExistsError, "An attempted build of this template for the target '#{provider_type.name}' already exists" end - return img + img = Image.create!( + :name => "#{template.name}/#{provider_type.codename}", + :provider_type_id => provider_type.id, + :template_id => template.id, + :status => Image::STATE_QUEUED + ) + img.delay.build + img end
def self.single_import(providername, username, password, image_id) @@ -164,6 +186,19 @@ class Image < ActiveRecord::Base
private
+ def upload_to_all_providers_with_account + provider_type.providers.each do |p| + # upload only to providers with account + unless p.provider_accounts.empty? + ProviderImage.create!( + :uuid => UUIDTools::UUID.timestamp_create.to_s, + :image => self, + :provider => p + ).delay.upload + end + end + end + def self.get_account(providername, username, password) unless provider = Provider.find_by_name(providername) raise "There is not provider with name '#{providername}'" diff --git a/src/app/models/image_warehouse_object.rb b/src/app/models/image_warehouse_object.rb index 7717ccd..40a7c3b 100644 --- a/src/app/models/image_warehouse_object.rb +++ b/src/app/models/image_warehouse_object.rb @@ -27,13 +27,12 @@ module ImageWarehouseObject @xml ||= ImageDescriptorXML.new(self[:xml].to_s) end
- # this should be overriden in a model if we want to save additional attrs - # for the model + # this should be overriden in a model if model wants to save additional attrs def warehouse_attrs {:uuid => self.uuid, :object_type => self.class.class_name} end
- # this should be overriden in a model if we want to save body + # this should be overriden in a model if model wants to save a body def warehouse_body nil end @@ -65,4 +64,8 @@ module ImageWarehouseObject self.uuid ||= UUIDTools::UUID.timestamp_create.to_s end
+ def invoke_sync + # TODO: invoke warehouse_sync here (after switching to eventmachine) + # pass self.uuid to sync only changed object + end end diff --git a/src/app/models/provider.rb b/src/app/models/provider.rb index b933fdb..d7d77ea 100644 --- a/src/app/models/provider.rb +++ b/src/app/models/provider.rb @@ -129,14 +129,6 @@ class Provider < ActiveRecord::Base cloud_accounts.collect {|account| account.pools}.flatten.uniq end
- # returns first provider of provider_type which has at least one cloud account - def self.find_by_target_with_account(provider_type) - Provider.all(:conditions => {:provider_type_id => provider_type}).each do |p| - return p unless p.provider_accounts.empty? - end - nil - end - # TODO: implement or remove - this is meant to contain a hash of # supported provider_types to use in populating form, though if we # infer that field, we don't need this. diff --git a/src/app/models/provider_image.rb b/src/app/models/provider_image.rb index c2d9e2e..7d54a37 100644 --- a/src/app/models/provider_image.rb +++ b/src/app/models/provider_image.rb @@ -14,6 +14,8 @@ #
class ProviderImage < ActiveRecord::Base + include ImageWarehouseObject + belongs_to :provider belongs_to :image belongs_to :icicle @@ -21,9 +23,36 @@ class ProviderImage < ActiveRecord::Base validates_presence_of :provider_id validates_presence_of :image_id validates_uniqueness_of :uuid, :allow_nil => true + validates_uniqueness_of :image_id, :scope => :provider_id
after_destroy :destroy_orphaned_icicles
+ def upload + # TODO: this is just stubbed upload call, + # when new image_builder_service is done, replace + # with real request to image_builder_service + begin + # put upload request to image_builder_service here + self.uploaded = true + self.registered = true + rescue + self.uploaded = false + self.registered = false + end + save! + end + + def after_update + if self.uploaded_changed? and self.uploaded == true + begin + invoke_sync + rescue => e + logger.error e.message + logger.error e.backtrace.join("\n ") + end + end + end + def destroy_orphaned_icicles if self.icicle and self.icicle.provider_images.empty? self.icicle.destroy diff --git a/src/features/step_definitions/template_steps.rb b/src/features/step_definitions/template_steps.rb index 3491eb4..79d2e96 100644 --- a/src/features/step_definitions/template_steps.rb +++ b/src/features/step_definitions/template_steps.rb @@ -98,7 +98,7 @@ Given /^there is Amazon AWS provider account$/ do end
Given /^there is Amazon AWS build for this template$/ do - Image.build(@template, ProviderType.find_by_codename("ec2")) + Image.create_and_build!(@template, ProviderType.find_by_codename("ec2")) end
Given /^there is an imported template$/ do diff --git a/src/spec/controllers/builds_controller_spec.rb b/src/spec/controllers/builds_controller_spec.rb index b0f1885..1bcfa5c 100644 --- a/src/spec/controllers/builds_controller_spec.rb +++ b/src/spec/controllers/builds_controller_spec.rb @@ -24,14 +24,8 @@ describe ImageFactory::BuildsController do
it "should create a new Image" do lambda do - post :create, :template_id => @template.id, :targets => ProviderType.find_by_codename("mock").id + post :create, :template_id => @template.id, :targets => [ProviderType.find_by_codename("mock").id] end.should change(Image, :count).by(1) end - - it "should create a new ProviderImage" do - lambda do - post :create, :template_id => @template.id, :targets => ProviderType.find_by_codename("mock").id - end.should change(ProviderImage, :count).by(1) - end end end diff --git a/src/spec/factories/image.rb b/src/spec/factories/image.rb index c8b34e3..d794dbd 100644 --- a/src/spec/factories/image.rb +++ b/src/spec/factories/image.rb @@ -6,4 +6,5 @@ Factory.define :image do |i| i.status 'queued' i.provider_type_id { ProviderType.find_by_codename("ec2").id } i.association(:template) + i.after_build {|img| img.stub!(:build).and_return(true) if img.respond_to?(:stub!)} end diff --git a/src/spec/models/image_spec.rb b/src/spec/models/image_spec.rb index d364919..9635f20 100644 --- a/src/spec/models/image_spec.rb +++ b/src/spec/models/image_spec.rb @@ -35,10 +35,9 @@ describe Image do i.should be_valid end
- it "should not build image if image already exists for specified target" do - old = Factory.build(:image) - old.save! - lambda {Image.build(old.template, old.provider_type)}.should raise_error(ImageExistsError) + it "should not create and build image if image already exists for specified target" do + old = Factory(:image) + lambda {Image.create_and_build!(old.template, old.provider_type)}.should raise_error(ImageExistsError) end
it "should build image if there is provider and cloud account for specified target" do @@ -47,9 +46,8 @@ describe Image do acc.save! tpl = Factory.build(:template) tpl.save! - img = Image.build(tpl, ProviderType.find_by_codename("mock")) + img = Image.create_and_build!(tpl, ProviderType.find_by_codename("mock")) Image.find(img).should == img - ProviderImage.find_by_image_id(img.id).should_not be_nil end
it "should import image" do
From: Jan Provaznik jprovazn@redhat.com
periodically pulls data from warehouse and updates conductor's db --- src/app/models/template.rb | 8 ++ src/warehouse_sync/server | 88 ++++++++++++++++++++ src/warehouse_sync/warehouse_sync.rb | 150 ++++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 0 deletions(-) create mode 100755 src/warehouse_sync/server create mode 100644 src/warehouse_sync/warehouse_sync.rb
diff --git a/src/app/models/template.rb b/src/app/models/template.rb index b84409c..21efc6f 100644 --- a/src/app/models/template.rb +++ b/src/app/models/template.rb @@ -73,6 +73,14 @@ class Template < ActiveRecord::Base write_attribute(:xml, xml.to_xml) end
+ def update_from_xml + self.name = xml.name + self.summary = xml.description + self.platform = xml.platform + self.platform_version = xml.platform_version + self.architecture = xml.architecture + end + def providers # TODO: rewrite cleanly ProviderImage.all( diff --git a/src/warehouse_sync/server b/src/warehouse_sync/server new file mode 100755 index 0000000..ea53c8f --- /dev/null +++ b/src/warehouse_sync/server @@ -0,0 +1,88 @@ +#!/usr/bin/ruby + +$: << File.dirname(__FILE__) + +require 'rubygems' +require 'optparse' +require 'warehouse_sync' + +DELAY = 1 +WAREHOUSE_CONFIG = YAML.load_file(File.join(File.dirname(__FILE__), "../config/image_warehouse.yml")) + +# show timestamp in log file +class Logger + def format_message(severity, timestamp, progname, msg) + time = timestamp.getutc.strftime("%Y-%m-%d %H:%M:%S") + "\n#{time}:#{severity.to_s.upcase}: #{msg}" + end +end + +help = false +daemon = true +log_dir = "/var/log/aeolus-conductor" +pid_dir = "/var/run/aeolus-conductor" + +optparse = OptionParser.new do |opts| + + opts.banner = <<BANNER +Usage: +warehouse_sync [options] + +Options: +BANNER + opts.on( '-f', '--pid-file PATH', "Use PATH to the warehouse_sync pid directory (defaults to #{pid_dir})") do |newpath| + pid_dir = newpath + end + opts.on( '-h', '--help', '') { help = true } + opts.on( '-l', '--log PATH', "Use PATH to the warehouse_sync log directory (defaults to #{log_dir}). Use '-' for stdout") do |newpath| + log_dir = newpath + end + opts.on( '-n', '--nodaemon', 'Do not daemonize (useful in combination with -l for debugging)') { daemon = false } +end + +begin + optparse.parse! +rescue OptionParser::InvalidOption => e + puts "Invalid option #{e.args}" + puts + puts optparse + exit(1) +end + +if help + puts optparse + exit(0) +end + +if log_dir == '-' + LOG_FILE = STDOUT +else + LOG_FILE = "#{log_dir}/warehouse_sync.log" +end + +logger = Logger.new(LOG_FILE) +logger.level = Logger::DEBUG +logger.datetime_format = "%Y-%m-%d %H:%M:%S" +logger.info "Warehouse_sync starting up" + +def create_pid_file(file) + FileUtils.mkdir_p File.dirname(file) + open(file, "w") {|f| f.write(Process.pid) } + File.chmod(0644, file) +end + +# daemonize +if daemon + # note that this requires 'active_support', which we get for free from dutils + Process.daemon +end + +begin + #create_pid_file("#{pid_dir}/warehouse_sync.pid") + WarehouseSync.new(:uri => WAREHOUSE_CONFIG['baseurl'], :logger => logger, :delay => DELAY * 60).run +rescue Exception => e + logger.error "#{e.backtrace.shift}: #{e.message}" + e.backtrace.each do |step| + logger.error "\tfrom #{step}" + end +end diff --git a/src/warehouse_sync/warehouse_sync.rb b/src/warehouse_sync/warehouse_sync.rb new file mode 100644 index 0000000..b9fdbe9 --- /dev/null +++ b/src/warehouse_sync/warehouse_sync.rb @@ -0,0 +1,150 @@ +$: << File.join(File.dirname(__FILE__), "../dutils") + +require 'rubygems' +require 'dutils' +require 'warehouse_client' + +class WarehouseSync + def initialize(opts) + @uri = opts[:uri] + @delay = opts[:delay] || 10*60 + @logger = opts[:logger] + @whouse = Warehouse::Client.new(@uri) + end + + def run + while true + begin + pull_icicles + pull_templates + pull_images + pull_provider_images + rescue => e + @logger.error e.message + @logger.error "backtrace:\n" + e.backtrace.join("\n ") + ensure + @logger.debug "sleep #{@delay}" + sleep @delay + end + end + end + + def pull_icicles + @logger.debug "*** Getting icicles" + @whouse.bucket('icicles').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid]) + icicle = Icicle.find_by_uuid(attrs[:uuid]) || Icicle.new(:uuid => attrs[:uuid]) + icicle.xml = obj.body + icicle.uuid = attrs[:uuid] + update_changes(icicle) + end + end + end + + def pull_templates + @logger.debug "*** Getting templates" + @whouse.bucket('templates').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid]) + #unless tpl = Template.find_by_uuid(attrs[:uuid]) + # raise "Template with uuid #{attrs[:uuid]} not found" + #end + tpl = Template.find_by_uuid(attrs[:uuid]) || Template.new(:uuid => attrs[:uuid]) + tpl.xml = obj.body + tpl.update_from_xml + update_changes(tpl) + end + end + end + + def pull_images + @logger.debug "*** Getting images" + @whouse.bucket('images').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid, :target, :template]) + #img = Image.find_by_uuid(attrs[:uuid]) || Image.new(:uuid => attrs[:uuid]) + unless img = Image.find_by_uuid(attrs[:uuid]) + raise "image with uuid #{attrs[:uuid]} not found" + end + unless attrs[:target] + raise "target uuid is not set" + end + unless ptype = ProviderType.find_by_codename(attrs[:target]) + raise "provider type #{attrs[:target]} not found" + end + unless attrs[:template] + raise "template uuid is not set" + end + unless tpl = Template.find_by_uuid(attrs[:template]) + raise "Template with uuid #{attrs[:template]} not found" + end + img.provider_type_id = ptype.id + img.template_id = tpl.id + update_changes(img) + end + end + end + + def pull_provider_images + @logger.debug "*** Getting provider images" + @whouse.bucket('provider_images').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid, :image, :icicle]) + # we don't allow create non-existing ProviderImage in conductor because + # we don't know provider_id (provider attribute contains only url or + # string which is not unique in conductor) + unless pimg = ProviderImage.find_by_uuid(attrs[:uuid]) + raise "provider image with uuid #{attrs[:uuid]} not found" + end + unless attrs[:image] + raise "image uuid is not set" + end + unless img = Image.find_by_uuid(attrs[:image]) + raise "image with uuid #{attrs[:image]} not found" + end + unless attrs[:icicle] + raise "icicle uuid is not set" + end + unless icicle = Icicle.find_by_uuid(attrs[:icicle]) + raise "icicle with uuid #{attrs[:icicle]} not found" + end + pimg.image_id = img.id + pimg.icicle_id = icicle.id + # TODO: set target_identifier too? + update_changes(img) + end + end + end + + private + + def safely_process(obj) + begin + yield obj + rescue => e + @logger.error "Error while processing #{obj.key} (skipping): #{e.message}" + @logger.debug e.backtrace.join("\n ") + end + end + + def update_changes(obj) + if obj.new_record? + @logger.debug "#{obj.class.class_name} #{obj.uuid} is not in DB, saving" + obj.save! + elsif obj.changed? + @logger.debug "#{obj.class.class_name} #{obj.uuid} has been changed:" + log_changes(obj) + obj.save! + else + @logger.debug "#{obj.class.class_name} #{obj.uuid} is without changes" + end + end + + def log_changes(obj) + obj.changed.each do |attr| + @logger.debug "old #{attr}: #{obj.send(attr + '_was')}" + @logger.debug "new #{attr}: #{obj[attr]}" + end + end +end
aeolus-devel@lists.fedorahosted.org