From: Jan Provaznik <jprovazn(a)redhat.com>
periodically pulls data from warehouse and updates conductor's db
---
src/app/models/template.rb | 8 ++
src/warehouse_sync/server | 88 ++++++++++++++++++++
src/warehouse_sync/warehouse_sync.rb | 150 ++++++++++++++++++++++++++++++++++
3 files changed, 246 insertions(+), 0 deletions(-)
create mode 100755 src/warehouse_sync/server
create mode 100644 src/warehouse_sync/warehouse_sync.rb
diff --git a/src/app/models/template.rb b/src/app/models/template.rb
index b84409c..21efc6f 100644
--- a/src/app/models/template.rb
+++ b/src/app/models/template.rb
@@ -73,6 +73,14 @@ class Template < ActiveRecord::Base
write_attribute(:xml, xml.to_xml)
end
+ def update_from_xml
+ self.name = xml.name
+ self.summary = xml.description
+ self.platform = xml.platform
+ self.platform_version = xml.platform_version
+ self.architecture = xml.architecture
+ end
+
def providers
# TODO: rewrite cleanly
ProviderImage.all(
diff --git a/src/warehouse_sync/server b/src/warehouse_sync/server
new file mode 100755
index 0000000..ea53c8f
--- /dev/null
+++ b/src/warehouse_sync/server
@@ -0,0 +1,88 @@
+#!/usr/bin/ruby
+
+$: << File.dirname(__FILE__)
+
+require 'rubygems'
+require 'optparse'
+require 'warehouse_sync'
+
+DELAY = 1
+WAREHOUSE_CONFIG = YAML.load_file(File.join(File.dirname(__FILE__),
"../config/image_warehouse.yml"))
+
+# show timestamp in log file
+class Logger
+ def format_message(severity, timestamp, progname, msg)
+ time = timestamp.getutc.strftime("%Y-%m-%d %H:%M:%S")
+ "\n#{time}:#{severity.to_s.upcase}: #{msg}"
+ end
+end
+
+help = false
+daemon = true
+log_dir = "/var/log/aeolus-conductor"
+pid_dir = "/var/run/aeolus-conductor"
+
+optparse = OptionParser.new do |opts|
+
+ opts.banner = <<BANNER
+Usage:
+warehouse_sync [options]
+
+Options:
+BANNER
+ opts.on( '-f', '--pid-file PATH', "Use PATH to the warehouse_sync
pid directory (defaults to #{pid_dir})") do |newpath|
+ pid_dir = newpath
+ end
+ opts.on( '-h', '--help', '') { help = true }
+ opts.on( '-l', '--log PATH', "Use PATH to the warehouse_sync log
directory (defaults to #{log_dir}). Use '-' for stdout") do |newpath|
+ log_dir = newpath
+ end
+ opts.on( '-n', '--nodaemon', 'Do not daemonize (useful in
combination with -l for debugging)') { daemon = false }
+end
+
+begin
+ optparse.parse!
+rescue OptionParser::InvalidOption => e
+ puts "Invalid option #{e.args}"
+ puts
+ puts optparse
+ exit(1)
+end
+
+if help
+ puts optparse
+ exit(0)
+end
+
+if log_dir == '-'
+ LOG_FILE = STDOUT
+else
+ LOG_FILE = "#{log_dir}/warehouse_sync.log"
+end
+
+logger = Logger.new(LOG_FILE)
+logger.level = Logger::DEBUG
+logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+logger.info "Warehouse_sync starting up"
+
+def create_pid_file(file)
+ FileUtils.mkdir_p File.dirname(file)
+ open(file, "w") {|f| f.write(Process.pid) }
+ File.chmod(0644, file)
+end
+
+# daemonize
+if daemon
+ # note that this requires 'active_support', which we get for free from dutils
+ Process.daemon
+end
+
+begin
+ #create_pid_file("#{pid_dir}/warehouse_sync.pid")
+ WarehouseSync.new(:uri => WAREHOUSE_CONFIG['baseurl'], :logger => logger,
:delay => DELAY * 60).run
+rescue Exception => e
+ logger.error "#{e.backtrace.shift}: #{e.message}"
+ e.backtrace.each do |step|
+ logger.error "\tfrom #{step}"
+ end
+end
diff --git a/src/warehouse_sync/warehouse_sync.rb b/src/warehouse_sync/warehouse_sync.rb
new file mode 100644
index 0000000..b9fdbe9
--- /dev/null
+++ b/src/warehouse_sync/warehouse_sync.rb
@@ -0,0 +1,150 @@
+$: << File.join(File.dirname(__FILE__), "../dutils")
+
+require 'rubygems'
+require 'dutils'
+require 'warehouse_client'
+
+class WarehouseSync
+ def initialize(opts)
+ @uri = opts[:uri]
+ @delay = opts[:delay] || 10*60
+ @logger = opts[:logger]
+ @whouse = Warehouse::Client.new(@uri)
+ end
+
+ def run
+ while true
+ begin
+ pull_icicles
+ pull_templates
+ pull_images
+ pull_provider_images
+ rescue => e
+ @logger.error e.message
+ @logger.error "backtrace:\n" + e.backtrace.join("\n ")
+ ensure
+ @logger.debug "sleep #{@delay}"
+ sleep @delay
+ end
+ end
+ end
+
+ def pull_icicles
+ @logger.debug "*** Getting icicles"
+ @whouse.bucket('icicles').objects.each do |bucket_obj|
+ safely_process(bucket_obj) do |obj|
+ attrs = obj.attrs([:uuid])
+ icicle = Icicle.find_by_uuid(attrs[:uuid]) || Icicle.new(:uuid =>
attrs[:uuid])
+ icicle.xml = obj.body
+ icicle.uuid = attrs[:uuid]
+ update_changes(icicle)
+ end
+ end
+ end
+
+ def pull_templates
+ @logger.debug "*** Getting templates"
+ @whouse.bucket('templates').objects.each do |bucket_obj|
+ safely_process(bucket_obj) do |obj|
+ attrs = obj.attrs([:uuid])
+ #unless tpl = Template.find_by_uuid(attrs[:uuid])
+ # raise "Template with uuid #{attrs[:uuid]} not found"
+ #end
+ tpl = Template.find_by_uuid(attrs[:uuid]) || Template.new(:uuid =>
attrs[:uuid])
+ tpl.xml = obj.body
+ tpl.update_from_xml
+ update_changes(tpl)
+ end
+ end
+ end
+
+ def pull_images
+ @logger.debug "*** Getting images"
+ @whouse.bucket('images').objects.each do |bucket_obj|
+ safely_process(bucket_obj) do |obj|
+ attrs = obj.attrs([:uuid, :target, :template])
+ #img = Image.find_by_uuid(attrs[:uuid]) || Image.new(:uuid => attrs[:uuid])
+ unless img = Image.find_by_uuid(attrs[:uuid])
+ raise "image with uuid #{attrs[:uuid]} not found"
+ end
+ unless attrs[:target]
+ raise "target uuid is not set"
+ end
+ unless ptype = ProviderType.find_by_codename(attrs[:target])
+ raise "provider type #{attrs[:target]} not found"
+ end
+ unless attrs[:template]
+ raise "template uuid is not set"
+ end
+ unless tpl = Template.find_by_uuid(attrs[:template])
+ raise "Template with uuid #{attrs[:template]} not found"
+ end
+ img.provider_type_id = ptype.id
+ img.template_id = tpl.id
+ update_changes(img)
+ end
+ end
+ end
+
+ def pull_provider_images
+ @logger.debug "*** Getting provider images"
+ @whouse.bucket('provider_images').objects.each do |bucket_obj|
+ safely_process(bucket_obj) do |obj|
+ attrs = obj.attrs([:uuid, :image, :icicle])
+ # we don't allow create non-existing ProviderImage in conductor because
+ # we don't know provider_id (provider attribute contains only url or
+ # string which is not unique in conductor)
+ unless pimg = ProviderImage.find_by_uuid(attrs[:uuid])
+ raise "provider image with uuid #{attrs[:uuid]} not found"
+ end
+ unless attrs[:image]
+ raise "image uuid is not set"
+ end
+ unless img = Image.find_by_uuid(attrs[:image])
+ raise "image with uuid #{attrs[:image]} not found"
+ end
+ unless attrs[:icicle]
+ raise "icicle uuid is not set"
+ end
+ unless icicle = Icicle.find_by_uuid(attrs[:icicle])
+ raise "icicle with uuid #{attrs[:icicle]} not found"
+ end
+ pimg.image_id = img.id
+ pimg.icicle_id = icicle.id
+ # TODO: set target_identifier too?
+ update_changes(img)
+ end
+ end
+ end
+
+ private
+
+ def safely_process(obj)
+ begin
+ yield obj
+ rescue => e
+ @logger.error "Error while processing #{obj.key} (skipping):
#{e.message}"
+ @logger.debug e.backtrace.join("\n ")
+ end
+ end
+
+ def update_changes(obj)
+ if obj.new_record?
+ @logger.debug "#{obj.class.class_name} #{obj.uuid} is not in DB, saving"
+ obj.save!
+ elsif obj.changed?
+ @logger.debug "#{obj.class.class_name} #{obj.uuid} has been changed:"
+ log_changes(obj)
+ obj.save!
+ else
+ @logger.debug "#{obj.class.class_name} #{obj.uuid} is without changes"
+ end
+ end
+
+ def log_changes(obj)
+ obj.changed.each do |attr|
+ @logger.debug "old #{attr}: #{obj.send(attr + '_was')}"
+ @logger.debug "new #{attr}: #{obj[attr]}"
+ end
+ end
+end
--
1.7.4