From: Jan Provaznik jprovazn@redhat.com
periodically pulls data from warehouse and updates conductor's db --- src/app/models/template.rb | 8 ++ src/warehouse_sync/server | 88 ++++++++++++++++++++ src/warehouse_sync/warehouse_sync.rb | 150 ++++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 0 deletions(-) create mode 100755 src/warehouse_sync/server create mode 100644 src/warehouse_sync/warehouse_sync.rb
diff --git a/src/app/models/template.rb b/src/app/models/template.rb index b84409c..21efc6f 100644 --- a/src/app/models/template.rb +++ b/src/app/models/template.rb @@ -73,6 +73,14 @@ class Template < ActiveRecord::Base write_attribute(:xml, xml.to_xml) end
+ def update_from_xml + self.name = xml.name + self.summary = xml.description + self.platform = xml.platform + self.platform_version = xml.platform_version + self.architecture = xml.architecture + end + def providers # TODO: rewrite cleanly ProviderImage.all( diff --git a/src/warehouse_sync/server b/src/warehouse_sync/server new file mode 100755 index 0000000..ea53c8f --- /dev/null +++ b/src/warehouse_sync/server @@ -0,0 +1,88 @@ +#!/usr/bin/ruby + +$: << File.dirname(__FILE__) + +require 'rubygems' +require 'optparse' +require 'warehouse_sync' + +DELAY = 1 +WAREHOUSE_CONFIG = YAML.load_file(File.join(File.dirname(__FILE__), "../config/image_warehouse.yml")) + +# show timestamp in log file +class Logger + def format_message(severity, timestamp, progname, msg) + time = timestamp.getutc.strftime("%Y-%m-%d %H:%M:%S") + "\n#{time}:#{severity.to_s.upcase}: #{msg}" + end +end + +help = false +daemon = true +log_dir = "/var/log/aeolus-conductor" +pid_dir = "/var/run/aeolus-conductor" + +optparse = OptionParser.new do |opts| + + opts.banner = <<BANNER +Usage: +warehouse_sync [options] + +Options: +BANNER + opts.on( '-f', '--pid-file PATH', "Use PATH to the warehouse_sync pid directory (defaults to #{pid_dir})") do |newpath| + pid_dir = newpath + end + opts.on( '-h', '--help', '') { help = true } + opts.on( '-l', '--log PATH', "Use PATH to the warehouse_sync log directory (defaults to #{log_dir}). Use '-' for stdout") do |newpath| + log_dir = newpath + end + opts.on( '-n', '--nodaemon', 'Do not daemonize (useful in combination with -l for debugging)') { daemon = false } +end + +begin + optparse.parse! +rescue OptionParser::InvalidOption => e + puts "Invalid option #{e.args}" + puts + puts optparse + exit(1) +end + +if help + puts optparse + exit(0) +end + +if log_dir == '-' + LOG_FILE = STDOUT +else + LOG_FILE = "#{log_dir}/warehouse_sync.log" +end + +logger = Logger.new(LOG_FILE) +logger.level = Logger::DEBUG +logger.datetime_format = "%Y-%m-%d %H:%M:%S" +logger.info "Warehouse_sync starting up" + +def create_pid_file(file) + FileUtils.mkdir_p File.dirname(file) + open(file, "w") {|f| f.write(Process.pid) } + File.chmod(0644, file) +end + +# daemonize +if daemon + # note that this requires 'active_support', which we get for free from dutils + Process.daemon +end + +begin + #create_pid_file("#{pid_dir}/warehouse_sync.pid") + WarehouseSync.new(:uri => WAREHOUSE_CONFIG['baseurl'], :logger => logger, :delay => DELAY * 60).run +rescue Exception => e + logger.error "#{e.backtrace.shift}: #{e.message}" + e.backtrace.each do |step| + logger.error "\tfrom #{step}" + end +end diff --git a/src/warehouse_sync/warehouse_sync.rb b/src/warehouse_sync/warehouse_sync.rb new file mode 100644 index 0000000..b9fdbe9 --- /dev/null +++ b/src/warehouse_sync/warehouse_sync.rb @@ -0,0 +1,150 @@ +$: << File.join(File.dirname(__FILE__), "../dutils") + +require 'rubygems' +require 'dutils' +require 'warehouse_client' + +class WarehouseSync + def initialize(opts) + @uri = opts[:uri] + @delay = opts[:delay] || 10*60 + @logger = opts[:logger] + @whouse = Warehouse::Client.new(@uri) + end + + def run + while true + begin + pull_icicles + pull_templates + pull_images + pull_provider_images + rescue => e + @logger.error e.message + @logger.error "backtrace:\n" + e.backtrace.join("\n ") + ensure + @logger.debug "sleep #{@delay}" + sleep @delay + end + end + end + + def pull_icicles + @logger.debug "*** Getting icicles" + @whouse.bucket('icicles').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid]) + icicle = Icicle.find_by_uuid(attrs[:uuid]) || Icicle.new(:uuid => attrs[:uuid]) + icicle.xml = obj.body + icicle.uuid = attrs[:uuid] + update_changes(icicle) + end + end + end + + def pull_templates + @logger.debug "*** Getting templates" + @whouse.bucket('templates').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid]) + #unless tpl = Template.find_by_uuid(attrs[:uuid]) + # raise "Template with uuid #{attrs[:uuid]} not found" + #end + tpl = Template.find_by_uuid(attrs[:uuid]) || Template.new(:uuid => attrs[:uuid]) + tpl.xml = obj.body + tpl.update_from_xml + update_changes(tpl) + end + end + end + + def pull_images + @logger.debug "*** Getting images" + @whouse.bucket('images').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid, :target, :template]) + #img = Image.find_by_uuid(attrs[:uuid]) || Image.new(:uuid => attrs[:uuid]) + unless img = Image.find_by_uuid(attrs[:uuid]) + raise "image with uuid #{attrs[:uuid]} not found" + end + unless attrs[:target] + raise "target uuid is not set" + end + unless ptype = ProviderType.find_by_codename(attrs[:target]) + raise "provider type #{attrs[:target]} not found" + end + unless attrs[:template] + raise "template uuid is not set" + end + unless tpl = Template.find_by_uuid(attrs[:template]) + raise "Template with uuid #{attrs[:template]} not found" + end + img.provider_type_id = ptype.id + img.template_id = tpl.id + update_changes(img) + end + end + end + + def pull_provider_images + @logger.debug "*** Getting provider images" + @whouse.bucket('provider_images').objects.each do |bucket_obj| + safely_process(bucket_obj) do |obj| + attrs = obj.attrs([:uuid, :image, :icicle]) + # we don't allow create non-existing ProviderImage in conductor because + # we don't know provider_id (provider attribute contains only url or + # string which is not unique in conductor) + unless pimg = ProviderImage.find_by_uuid(attrs[:uuid]) + raise "provider image with uuid #{attrs[:uuid]} not found" + end + unless attrs[:image] + raise "image uuid is not set" + end + unless img = Image.find_by_uuid(attrs[:image]) + raise "image with uuid #{attrs[:image]} not found" + end + unless attrs[:icicle] + raise "icicle uuid is not set" + end + unless icicle = Icicle.find_by_uuid(attrs[:icicle]) + raise "icicle with uuid #{attrs[:icicle]} not found" + end + pimg.image_id = img.id + pimg.icicle_id = icicle.id + # TODO: set target_identifier too? + update_changes(img) + end + end + end + + private + + def safely_process(obj) + begin + yield obj + rescue => e + @logger.error "Error while processing #{obj.key} (skipping): #{e.message}" + @logger.debug e.backtrace.join("\n ") + end + end + + def update_changes(obj) + if obj.new_record? + @logger.debug "#{obj.class.class_name} #{obj.uuid} is not in DB, saving" + obj.save! + elsif obj.changed? + @logger.debug "#{obj.class.class_name} #{obj.uuid} has been changed:" + log_changes(obj) + obj.save! + else + @logger.debug "#{obj.class.class_name} #{obj.uuid} is without changes" + end + end + + def log_changes(obj) + obj.changed.each do |attr| + @logger.debug "old #{attr}: #{obj.send(attr + '_was')}" + @logger.debug "new #{attr}: #{obj[attr]}" + end + end +end