From: Martyn Taylor <mtaylor(a)redhat.com>
---
src/app/util/condormatic.rb | 198 ++++++++++++++++++++++-------------
src/condor_refreshd/condor_refreshd | 77 ++++---------
2 files changed, 150 insertions(+), 125 deletions(-)
diff --git a/src/app/util/condormatic.rb b/src/app/util/condormatic.rb
index 03484ff..5fe3863 100644
--- a/src/app/util/condormatic.rb
+++ b/src/app/util/condormatic.rb
@@ -20,6 +20,8 @@
require 'nokogiri'
require 'socket'
+REFRESHD_LOG_FILE = "/var/log/aeolus-conductor/condor_refreshd.log"
+
def condormatic_instance_create(task)
begin
@@ -35,26 +37,26 @@ def condormatic_instance_create(task)
# the ability to get the exit value of the command in ruby 1.8.
pipe = IO.popen("condor_submit 2>&1", "w+")
pipe.puts "universe = grid\n"
- Rails.logger.error "universe = grid\n"
+ logger.error "universe = grid\n"
pipe.puts "executable = #{job_name}\n"
- Rails.logger.error "executable = #{job_name}\n"
+ logger.error "executable = #{job_name}\n"
pipe.puts "grid_resource = deltacloud $$(provider_url)\n"
- Rails.logger.error "grid_resource = deltacloud $$(provider_url)\n"
+ logger.error "grid_resource = deltacloud $$(provider_url)\n"
pipe.puts "DeltacloudUsername = $$(username)\n"
- Rails.logger.error "DeltacloudUsername = $$(username)\n"
+ logger.error "DeltacloudUsername = $$(username)\n"
pipe.puts "DeltacloudPassword = $$(password)\n"
- Rails.logger.error "DeltacloudPassword = $$(password)\n"
+ logger.error "DeltacloudPassword = $$(password)\n"
pipe.puts "DeltacloudImageId = $$(image_key)\n"
- Rails.logger.error "DeltacloudImageId = $$(image_key)\n"
+ logger.error "DeltacloudImageId = $$(image_key)\n"
pipe.puts "DeltacloudHardwareProfile = $$(hardware_profile_key)\n"
- Rails.logger.error "DeltacloudHardwareProfile =
$$(hardware_profile_key)\n"
+ logger.error "DeltacloudHardwareProfile = $$(hardware_profile_key)\n"
pipe.puts "DeltacloudKeyname = $$(keypair)\n"
- Rails.logger.error "DeltacloudKeyname = $$(keypair)\n"
+ logger.error "DeltacloudKeyname = $$(keypair)\n"
if realm != nil
pipe.puts "DeltacloudRealmId = $$(realm_key)\n"
- Rails.logger.error "DeltacloudRealmId = $$(realm_key)\n"
+ logger.error "DeltacloudRealmId = $$(realm_key)\n"
end
requirements = "requirements = front_end_hardware_profile_id ==
\"#{instance.hardware_profile.id}\" && image ==
\"#{instance.template.id}\""
@@ -66,23 +68,23 @@ def condormatic_instance_create(task)
requirements += "\n"
pipe.puts requirements
- Rails.logger.error requirements
+ logger.error requirements
pipe.puts "notification = never\n"
- Rails.logger.error "notification = never\n"
+ logger.error "notification = never\n"
pipe.puts "queue\n"
- Rails.logger.error "queue\n"
+ logger.error "queue\n"
pipe.close_write
out = pipe.read
pipe.close
- Rails.logger.error "$? (return value?) is #{$?}"
+ logger.error "$? (return value?) is #{$?}"
raise ("Error calling condor_submit: #{out}") if $? != 0
rescue Exception => ex
task.state = Task::STATE_FAILED
- Rails.logger.error ex.message
- Rails.logger.error ex.backtrace
+ logger.error ex.message
+ logger.error ex.backtrace
else
# FIXME: We're kinda lying here.. we don't know the state for the task but I
don't think that matters so much
# as we are just going to use the 'task' table as a kind of audit log.
@@ -126,87 +128,51 @@ end
def condormatic_instance_stop(task)
instance = task.instance_of?(InstanceTask) ? task.instance : task
- Rails.logger.info("calling condor_rm -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
+ logger.info("calling condor_rm -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
pipe = IO.popen("condor_rm -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
out = pipe.read
pipe.close
- Rails.logger.info("condor_rm return status is #{$?}")
- Rails.logger.error("Error calling condor_rm (exit code #{$?}) on job:
#{out}") if $? != 0
+ logger.info("condor_rm return status is #{$?}")
+ logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if
$? != 0
end
def condormatic_instance_reset_error(instance)
condormatic_instance_stop(instance)
- Rails.logger.info("calling condor_rm -forcex -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
+ logger.info("calling condor_rm -forcex -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
pipe = IO.popen("condor_rm -forcex -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
out = pipe.read
pipe.close
- Rails.logger.info("condor_rm return status is #{$?}")
- Rails.logger.error("Error calling condor_rm (exit code #{$?}) on job:
#{out}") if $? != 0
+ logger.info("condor_rm return status is #{$?}")
+ logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if
$? != 0
end
def condormatic_instance_destroy(task)
instance = task.instance
- Rails.logger.info("calling condor_rm -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
+ logger.info("calling condor_rm -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
pipe = IO.popen("condor_rm -constraint 'Cmd ==
\"#{instance.condor_job_id}\"' 2>&1")
out = pipe.read
pipe.close
- Rails.logger.info("condor_rm return status is #{$?}")
- Rails.logger.error("Error calling condor_rm (exit code #{$?}) on job:
#{out}") if $? != 0
+ logger.info("condor_rm return status is #{$?}")
+ logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if
$? != 0
end
def condormatic_classads_sync
- Rails.logger.info "Starting condormatic_classads_sync..."
- index = 0
- providers = Provider.find(:all)
-
-
- # we first need to invalidate old ADS
- # FIXME: this invalidates *all* ads, including those not really
- # related to deltacloud. It would be best if we could find a
- # way to restrict the classads that we actually invalidate,
- # but it seems that the invalidate only matches against Name,
- # Machine, and SlotID. Unfortunately classads don't seem to
- # have a regex match, so we can't regex match on Name. My
- # attempts with Machine also failed, for unknown reasons.
- #
- # The other way we could go about invalidating the ones
- # we care about is with something like:
- #
- # condor_status -startd -f "%s\n" Name
- #
- # which will show only the names of the startd classads.
- # Then we could do the regex matching in ruby, and iterate
- # through the provider_combination_* ones. This is a bit
- # racy, though, so I'm more inclined to just invalidate
- # everything at present.
- Rails.logger.info "Starting classad invalidate..."
- begin
- pipe = IO.popen("condor_advertise INVALIDATE_STARTD_ADS 2>&1",
"w+")
- pipe.puts 'MyType="Query"'
- pipe.puts 'TargetType="Machine"'
- pipe.close_write
- out = pipe.read
- pipe.close
+ logger = Logger.new(REFRESHD_LOG_FILE)
+ logger.level = Logger::DEBUG
- Rails.logger.info "Did invalidate, output is #{out}"
+ invalidate_provider_classads()
- if $? != 0
- Rails.logger.error "Unable to invalidate classads: #{out}"
- raise "Unable to invalidate classads, classad sync failed"
- end
- rescue Errno::EPIPE
- # if we failed to run condor_advertise, then in all likelihood condor isn't
- # installed or isn't running. In that case, there can't be any old
- # classads, so we just go on
- end
+ logger.info "Starting condormatic_classads_sync..."
+ index = 0
+ providers = Provider.find(:all)
- Rails.logger.info "Syncing classads.."
+ logger.info "Syncing classads.."
ads = []
front_end_hardware_profiles = HardwareProfile.find(:all, :conditions =>
["provider_id IS NULL"])
@@ -270,21 +236,109 @@ def condormatic_classads_sync
pipe.puts "provider_account_id=\"#{account.id}\""
pipe.puts "keypair=\"#{account.instance_key.name}\""
rescue Exception => ex
- Rails.logger.error "Error writing provider classad to condor."
- Rails.logger.error ex.message
- Rails.logger.error ex.backtrace
+ logger.error "Error writing provider classad to condor."
+ logger.error ex.message
+ logger.error ex.backtrace
end
pipe.close_write
out = pipe.read
pipe.close
- Rails.logger.error "Unable to submit condor classad: #{out}" if $? != 0
+ logger.error "Unable to submit condor classad: #{out}" if $? != 0
index += 1
end
}
- Rails.logger.info "done"
+ logger.info "done"
+end
+
+def invalidate_provider_classads()
+ # we first need to invalidate old ADS
+ # FIXME: this invalidates *all* ads, including those not really
+ # related to deltacloud. It would be best if we could find a
+ # way to restrict the classads that we actually invalidate,
+ # but it seems that the invalidate only matches against Name,
+ # Machine, and SlotID. Unfortunately classads don't seem to
+ # have a regex match, so we can't regex match on Name. My
+ # attempts with Machine also failed, for unknown reasons.
+ #
+ # The other way we could go about invalidating the ones
+ # we care about is with something like:
+ #
+ # condor_status -startd -f "%s\n" Name
+ #
+ # which will show only the names of the startd classads.
+ # Then we could do the regex matching in ruby, and iterate
+ # through the provider_combination_* ones. This is a bit
+ # racy, though, so I'm more inclined to just invalidate
+ # everything at present.
+
+ logger = Logger.new(REFRESHD_LOG_FILE)
+ logger.level = Logger::DEBUG
+ begin
+ # Get list of class ad names
+ pipe = IO.popen('condor_status -startd -f "%s\n" Name',
"w+")
+ pipe.close_write
+ out = pipe.read
+ pipe.close
+
+ out.split("\n").each do |name|
+ if(name.include?("provider_combination"))
+ pipe = IO.popen("/usr/sbin/condor_advertise INVALIDATE_STARTD_ADS
2>&1", "w+")
+ pipe.puts 'MyType="Query"'
+ pipe.puts 'TargetType="Machine"'
+ pipe.puts 'Name="' + name + '"'
+ pipe.close_write
+ out = pipe.read
+ pipe.close
+ end
+ end
+
+ logger.info "Did invalidate, output is #{out}"
+
+ if $? != 0
+ logger.error "Unable to invalidate classads: #{out}"
+ raise "Unable to invalidate classads, classad sync failed #{out}"
+ end
+ rescue Errno::EPIPE
+ # if we failed to run condor_advertise, then in all likelihood condor isn't
+ # installed or isn't running. In that case, there can't be any old
+ # classads, so we just go on
+ end
+end
+
+def condormatic_queue_classad_sync(timeout)
+ logger = Logger.new(REFRESHD_LOG_FILE)
+ logger.level = Logger::DEBUG
+ begin
+ pipe = IO.popen("condor_submit 2>&1", "w+")
+ pipe.puts "Executable =
/usr/share/aeolus-conductor/condor_refreshd/condor_refreshd\n"
+ logger.error "Executable =
/usr/share/aeolus-conductor/condor_refreshd/condor_refreshd\n"
+ pipe.puts "Requirements = true\n"
+ logger.error "Requirements = true\n"
+ #pipe.puts "Arguments = RAILS_ENV=production\n"
+ #logger.error "Arguments = -d " + timeout.to_s + "
RAILS_ENV=production\n"
+ pipe.puts "environment = RAILS_ENV=production\n"
+ logger.error "environment = RAILS_ENV=production\n"
+ pipe.puts "Log = log\n"
+ pipe.puts "Output = output\n"
+ pipe.puts "Error = error\n"
+ pipe.puts "Queue\n"
+ logger.error "Queue\n"
+ pipe.close_write
+ out = pipe.read
+ pipe.close
+
+ logger.error "$? (return value?) is #{$?}"
+ raise ("Error calling condor_submit: #{out}") if $? != 0
+
+ rescue Exception => ex
+ puts "Failed To add Condor Job"
+ puts ex
+ logger.error ex.message
+ logger.error ex.backtrace
+ end
end
def kick_condor
diff --git a/src/condor_refreshd/condor_refreshd b/src/condor_refreshd/condor_refreshd
index 273902f..162fe3f 100755
--- a/src/condor_refreshd/condor_refreshd
+++ b/src/condor_refreshd/condor_refreshd
@@ -26,33 +26,23 @@ require 'optparse'
require 'socket'
require 'dutils'
-port = 7890
help = false
-daemon = true
-refreshd_log_dir = "/var/log/aeolus-conductor"
-refreshd_pid_dir = "/var/run/aeolus-conductor"
-timeout = 15
+delay = 0
+refreshd_log_dir = "/var/log/aeolus-conductor/"
+refresh_log = "condor_refreshd.log"
optparse = OptionParser.new do |opts|
-
opts.banner = <<BANNER
Usage: condor_refreshd [options]
Options:
BANNER
- opts.on( '-f', '--pid-file PATH', "Use PATH to the condor_refreshd
pid directory (defaults to #{refreshd_pid_dir})") do |newpath|
- refreshd_pid_dir = newpath
- end
opts.on( '-h', '--help', '') { help = true }
opts.on( '-l', '--log PATH', "Use PATH to the condor_refreshd log
directory (defaults to #{refreshd_log_dir}). Use '-' for stdout") do
|newpath|
refreshd_log_dir = newpath
end
- opts.on( '-n', '--nodaemon', 'Do not daemonize (useful in
combination with -l for debugging)') { daemon = false }
- opts.on( '-p', '--port PORT', 'Use PORT (default: 7890)') do
|newport|
- port = newport
- end
- opts.on( '-t', '--timeout TIMEOUT', 'Automatically refresh classads
after TIMEOUT minutes') do |newtime|
- timeout = newtime
+ opts.on( '-d', '--delay SECONDS', 'do not run for SECONDS before
refreshing classads.') do |newdelay|
+ delay = newdelay
end
end
@@ -76,49 +66,30 @@ if refreshd_log_dir == '-'
else
REFRESHD_LOG_FILE = "#{refreshd_log_dir}/condor_refreshd.log"
end
-logger = Logger.new(REFRESHD_LOG_FILE)
-logger.level = Logger::DEBUG
-logger.info "Starting condor_refreshd..."
-socket = Socket.new(Socket::AF_INET, Socket::SOCK_DGRAM, 0)
-in_addr = Socket.pack_sockaddr_in(port, '127.0.0.1')
-socket.bind(in_addr)
+logger = Logger.new(REFRESHD_LOG_FILE)
-# daemonize
-if daemon
- # note that this requires 'active_support', which we get for free from dutils
- Process.daemon
-end
+logger.level = Logger::DEBUG
begin
- REFRESHD_PID_FILE = "#{refreshd_pid_dir}/condor_refreshd.pid"
- FileUtils.mkdir_p File.dirname(REFRESHD_PID_FILE)
- open(REFRESHD_PID_FILE, "w") {|f| f.write(Process.pid) }
- File.chmod(0644, REFRESHD_PID_FILE)
-
- while true
- begin
- condormatic_classads_sync
- results = select([socket], nil, nil, timeout * 60)
- # if results was nil, we timed out waiting, and we need to do a sync
- # but not read from the socket
- if not results.nil? and results[0][0] == socket
- socket.recvfrom(1024)
- logger.info "Doing classad sync after getting data from the socket"
- else
- logger.info "Doing classad sync after timeout"
- end
- rescue => e
- logger.error "#{e.backtrace.shift}: #{e.message}"
- e.backtrace.each do |step|
- logger.error "\tfrom #{step}"
- end
+ # Ideally get the start time here.
+ #sleep(delay)
+ #sleep(20)
+ begin
+ logger.info("----------------------------------------------")
+ logger.info(Time.now.to_s + ": " + "Executing ClassAd Sync")
+ condormatic_classads_sync
+ rescue => e
+ logger.error "#{e.backtrace.shift}: #{e.message}"
+ e.backtrace.each do |step|
+ logger.error "\tfrom #{step}"
end
end
-rescue => e
- logger.error "#{e.backtrace.shift}: #{e.message}"
- e.backtrace.each do |step|
- logger.error "\tfrom #{step}"
- end
+ # Get the end time, calculate elapsed time, subtract it from 55 (5 sec buffer), and use
that as the arg to the timeout.
+ condormatic_queue_classad_sync(30)
+
+ logger.info(Time.now.to_s + ": Refresh Completed")
+ logger.info("----------------------------------------------")
+ exit(0)
end
--
1.7.4