[PATCH aeolus] aeolus-connector rpm spec file and init script
by Angus Thomas
From: Angus Thomas <athomas(a)redhat.com>
Whitespace fixed
---
.../image_factory_connector/aeolus-connector.init | 76 ++++++++++++++++++++
.../rubygem-image_factory_connector.spec | 61 ++++++++++++++++
2 files changed, 137 insertions(+), 0 deletions(-)
create mode 100755 services/image_factory/image_factory_connector/aeolus-connector.init
create mode 100644 services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec
diff --git a/services/image_factory/image_factory_connector/aeolus-connector.init b/services/image_factory/image_factory_connector/aeolus-connector.init
new file mode 100755
index 0000000..624922c
--- /dev/null
+++ b/services/image_factory/image_factory_connector/aeolus-connector.init
@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# aeolus-connector startup script for aeolus-connector
+#
+# chkconfig: - 2345 97 03
+# description: aeolus-connector is a microapp to talk to
+# Aeolus Image Factory QMF console
+
+RETVAL=0
+prog="aeolus-connector"
+exec=/usr/bin/image_factory_connector
+lockfile=/var/lock/subsys/aeolus-connector
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+port=2003
+logfile=/var/log/aeolus-conductor/aeolus-connector.log
+[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog
+
+start() {
+ if [ -e $lockfile ] ; then
+ echo "$prog subsystem locked"
+ exit 1
+ fi
+ if [ $UID -ne 0 ] ; then
+ echo "User has insufficient privilege."
+ exit 2
+ fi
+ [ -x $exec ] || exit 3
+ echo -n $"Starting $prog: "
+ daemon $exec -p $port start >> $logfile 2>&1 &
+ retval=$?
+ echo
+ [ $retval -eq 0 ] && touch $lockfile
+}
+
+stop() {
+ if [ $UID -ne 0 ] ; then
+ echo "User has insufficient privilege."
+ exit 4
+ fi
+ echo -n $"Stopping $prog: "
+ killall image_factory_connector
+ retval=$?
+ echo
+ [ $retval -eq 0 ] && rm -f $lockfile
+}
+
+restart() {
+ stop
+ start
+}
+
+rh_status() {
+ status image_factory_connector
+}
+
+case "$1" in
+ start)
+ $1
+ ;;
+ stop)
+ $1
+ ;;
+ restart)
+ $1
+ ;;
+ status)
+ rh_status
+ ;;
+ *)
+ echo $"Usage: $0 {start|stop|status|restart}"
+ exit 2
+esac
+exit $?
diff --git a/services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec b/services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec
new file mode 100644
index 0000000..c3a4a78
--- /dev/null
+++ b/services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec
@@ -0,0 +1,61 @@
+# Generated from image_factory_connector-0.0.1.gem by gem2rpm -*- rpm-spec -*-
+%define ruby_sitelib %(ruby -rrbconfig -e "puts Config::CONFIG['sitelibdir']")
+%define gemdir %(ruby -rubygems -e 'puts Gem::dir' 2>/dev/null)
+%define gemname image_factory_connector
+%define geminstdir %{gemdir}/gems/%{gemname}-%{version}
+
+Summary: Sinatra microapp to talk to Aeolus Image Factory QMF console
+Name: rubygem-%{gemname}
+Version: 0.0.1
+Release: 1%{?dist}
+Group: Development/Languages
+License: GPLv2+ or Ruby
+URL: http://aeolusproject.org
+Source0: %{gemname}-%{version}.gem
+Source1: aeolus-connector.init
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
+Requires: rubygems
+Requires: rubygem(image_factory_console) >= 0.2.0
+Requires: rubygem(builder) >= 2.1.2
+Requires: rubygem(sinatra) >= 1.0
+Requires: rubygem(rspec) >= 1.3.0
+BuildRequires: rubygems
+BuildArch: noarch
+Provides: rubygem(%{gemname}) = %{version}
+
+%description
+REST api and callback mechanism to wrap qmf for the Aeolus project
+
+
+%prep
+
+%build
+
+%install
+rm -rf %{buildroot}
+mkdir -p %{buildroot}%{gemdir}
+gem install --local --install-dir %{buildroot}%{gemdir} \
+ --force --rdoc %{SOURCE0}
+mkdir -p %{buildroot}/%{_bindir}
+mv %{buildroot}%{gemdir}/bin/* %{buildroot}/%{_bindir}
+rmdir %{buildroot}%{gemdir}/bin
+find %{buildroot}%{geminstdir}/bin -type f | xargs chmod a+x
+mkdir -p %{buildroot}/etc/init.d
+cp %{SOURCE1} %{buildroot}/etc/init.d/aeolus-connector
+
+%clean
+rm -rf %{buildroot}
+
+%files
+%defattr(-, root, root, -)
+%{_bindir}/image_factory_connector
+%{gemdir}/gems/%{gemname}-%{version}/
+%doc %{gemdir}/doc/%{gemname}-%{version}
+%{gemdir}/cache/%{gemname}-%{version}.gem
+%{gemdir}/specifications/%{gemname}-%{version}.gemspec
+%attr(755,root,root) /etc/init.d/aeolus-connector
+
+
+%changelog
+* Mon Mar 14 2011 Angus Thomas <athomas(a)redhat.com> - 0.0.1-1
+- Initial package
--
1.7.4
13 years, 2 months
[PATCH aeolus] Condor Refresh
by Martyn Taylor
From: Martyn Taylor <mtaylor(a)redhat.com>
---
src/app/util/condormatic.rb | 198 ++++++++++++++++++++++-------------
src/condor_refreshd/condor_refreshd | 77 ++++---------
2 files changed, 150 insertions(+), 125 deletions(-)
diff --git a/src/app/util/condormatic.rb b/src/app/util/condormatic.rb
index 03484ff..5fe3863 100644
--- a/src/app/util/condormatic.rb
+++ b/src/app/util/condormatic.rb
@@ -20,6 +20,8 @@
require 'nokogiri'
require 'socket'
+REFRESHD_LOG_FILE = "/var/log/aeolus-conductor/condor_refreshd.log"
+
def condormatic_instance_create(task)
begin
@@ -35,26 +37,26 @@ def condormatic_instance_create(task)
# the ability to get the exit value of the command in ruby 1.8.
pipe = IO.popen("condor_submit 2>&1", "w+")
pipe.puts "universe = grid\n"
- Rails.logger.error "universe = grid\n"
+ logger.error "universe = grid\n"
pipe.puts "executable = #{job_name}\n"
- Rails.logger.error "executable = #{job_name}\n"
+ logger.error "executable = #{job_name}\n"
pipe.puts "grid_resource = deltacloud $$(provider_url)\n"
- Rails.logger.error "grid_resource = deltacloud $$(provider_url)\n"
+ logger.error "grid_resource = deltacloud $$(provider_url)\n"
pipe.puts "DeltacloudUsername = $$(username)\n"
- Rails.logger.error "DeltacloudUsername = $$(username)\n"
+ logger.error "DeltacloudUsername = $$(username)\n"
pipe.puts "DeltacloudPassword = $$(password)\n"
- Rails.logger.error "DeltacloudPassword = $$(password)\n"
+ logger.error "DeltacloudPassword = $$(password)\n"
pipe.puts "DeltacloudImageId = $$(image_key)\n"
- Rails.logger.error "DeltacloudImageId = $$(image_key)\n"
+ logger.error "DeltacloudImageId = $$(image_key)\n"
pipe.puts "DeltacloudHardwareProfile = $$(hardware_profile_key)\n"
- Rails.logger.error "DeltacloudHardwareProfile = $$(hardware_profile_key)\n"
+ logger.error "DeltacloudHardwareProfile = $$(hardware_profile_key)\n"
pipe.puts "DeltacloudKeyname = $$(keypair)\n"
- Rails.logger.error "DeltacloudKeyname = $$(keypair)\n"
+ logger.error "DeltacloudKeyname = $$(keypair)\n"
if realm != nil
pipe.puts "DeltacloudRealmId = $$(realm_key)\n"
- Rails.logger.error "DeltacloudRealmId = $$(realm_key)\n"
+ logger.error "DeltacloudRealmId = $$(realm_key)\n"
end
requirements = "requirements = front_end_hardware_profile_id == \"#{instance.hardware_profile.id}\" && image == \"#{instance.template.id}\""
@@ -66,23 +68,23 @@ def condormatic_instance_create(task)
requirements += "\n"
pipe.puts requirements
- Rails.logger.error requirements
+ logger.error requirements
pipe.puts "notification = never\n"
- Rails.logger.error "notification = never\n"
+ logger.error "notification = never\n"
pipe.puts "queue\n"
- Rails.logger.error "queue\n"
+ logger.error "queue\n"
pipe.close_write
out = pipe.read
pipe.close
- Rails.logger.error "$? (return value?) is #{$?}"
+ logger.error "$? (return value?) is #{$?}"
raise ("Error calling condor_submit: #{out}") if $? != 0
rescue Exception => ex
task.state = Task::STATE_FAILED
- Rails.logger.error ex.message
- Rails.logger.error ex.backtrace
+ logger.error ex.message
+ logger.error ex.backtrace
else
# FIXME: We're kinda lying here.. we don't know the state for the task but I don't think that matters so much
# as we are just going to use the 'task' table as a kind of audit log.
@@ -126,87 +128,51 @@ end
def condormatic_instance_stop(task)
instance = task.instance_of?(InstanceTask) ? task.instance : task
- Rails.logger.info("calling condor_rm -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
+ logger.info("calling condor_rm -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
pipe = IO.popen("condor_rm -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
out = pipe.read
pipe.close
- Rails.logger.info("condor_rm return status is #{$?}")
- Rails.logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if $? != 0
+ logger.info("condor_rm return status is #{$?}")
+ logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if $? != 0
end
def condormatic_instance_reset_error(instance)
condormatic_instance_stop(instance)
- Rails.logger.info("calling condor_rm -forcex -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
+ logger.info("calling condor_rm -forcex -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
pipe = IO.popen("condor_rm -forcex -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
out = pipe.read
pipe.close
- Rails.logger.info("condor_rm return status is #{$?}")
- Rails.logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if $? != 0
+ logger.info("condor_rm return status is #{$?}")
+ logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if $? != 0
end
def condormatic_instance_destroy(task)
instance = task.instance
- Rails.logger.info("calling condor_rm -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
+ logger.info("calling condor_rm -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
pipe = IO.popen("condor_rm -constraint 'Cmd == \"#{instance.condor_job_id}\"' 2>&1")
out = pipe.read
pipe.close
- Rails.logger.info("condor_rm return status is #{$?}")
- Rails.logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if $? != 0
+ logger.info("condor_rm return status is #{$?}")
+ logger.error("Error calling condor_rm (exit code #{$?}) on job: #{out}") if $? != 0
end
def condormatic_classads_sync
- Rails.logger.info "Starting condormatic_classads_sync..."
- index = 0
- providers = Provider.find(:all)
-
-
- # we first need to invalidate old ADS
- # FIXME: this invalidates *all* ads, including those not really
- # related to deltacloud. It would be best if we could find a
- # way to restrict the classads that we actually invalidate,
- # but it seems that the invalidate only matches against Name,
- # Machine, and SlotID. Unfortunately classads don't seem to
- # have a regex match, so we can't regex match on Name. My
- # attempts with Machine also failed, for unknown reasons.
- #
- # The other way we could go about invalidating the ones
- # we care about is with something like:
- #
- # condor_status -startd -f "%s\n" Name
- #
- # which will show only the names of the startd classads.
- # Then we could do the regex matching in ruby, and iterate
- # through the provider_combination_* ones. This is a bit
- # racy, though, so I'm more inclined to just invalidate
- # everything at present.
- Rails.logger.info "Starting classad invalidate..."
- begin
- pipe = IO.popen("condor_advertise INVALIDATE_STARTD_ADS 2>&1", "w+")
- pipe.puts 'MyType="Query"'
- pipe.puts 'TargetType="Machine"'
- pipe.close_write
- out = pipe.read
- pipe.close
+ logger = Logger.new(REFRESHD_LOG_FILE)
+ logger.level = Logger::DEBUG
- Rails.logger.info "Did invalidate, output is #{out}"
+ invalidate_provider_classads()
- if $? != 0
- Rails.logger.error "Unable to invalidate classads: #{out}"
- raise "Unable to invalidate classads, classad sync failed"
- end
- rescue Errno::EPIPE
- # if we failed to run condor_advertise, then in all likelihood condor isn't
- # installed or isn't running. In that case, there can't be any old
- # classads, so we just go on
- end
+ logger.info "Starting condormatic_classads_sync..."
+ index = 0
+ providers = Provider.find(:all)
- Rails.logger.info "Syncing classads.."
+ logger.info "Syncing classads.."
ads = []
front_end_hardware_profiles = HardwareProfile.find(:all, :conditions => ["provider_id IS NULL"])
@@ -270,21 +236,109 @@ def condormatic_classads_sync
pipe.puts "provider_account_id=\"#{account.id}\""
pipe.puts "keypair=\"#{account.instance_key.name}\""
rescue Exception => ex
- Rails.logger.error "Error writing provider classad to condor."
- Rails.logger.error ex.message
- Rails.logger.error ex.backtrace
+ logger.error "Error writing provider classad to condor."
+ logger.error ex.message
+ logger.error ex.backtrace
end
pipe.close_write
out = pipe.read
pipe.close
- Rails.logger.error "Unable to submit condor classad: #{out}" if $? != 0
+ logger.error "Unable to submit condor classad: #{out}" if $? != 0
index += 1
end
}
- Rails.logger.info "done"
+ logger.info "done"
+end
+
+def invalidate_provider_classads()
+ # we first need to invalidate old ADS
+ # FIXME: this invalidates *all* ads, including those not really
+ # related to deltacloud. It would be best if we could find a
+ # way to restrict the classads that we actually invalidate,
+ # but it seems that the invalidate only matches against Name,
+ # Machine, and SlotID. Unfortunately classads don't seem to
+ # have a regex match, so we can't regex match on Name. My
+ # attempts with Machine also failed, for unknown reasons.
+ #
+ # The other way we could go about invalidating the ones
+ # we care about is with something like:
+ #
+ # condor_status -startd -f "%s\n" Name
+ #
+ # which will show only the names of the startd classads.
+ # Then we could do the regex matching in ruby, and iterate
+ # through the provider_combination_* ones. This is a bit
+ # racy, though, so I'm more inclined to just invalidate
+ # everything at present.
+
+ logger = Logger.new(REFRESHD_LOG_FILE)
+ logger.level = Logger::DEBUG
+ begin
+ # Get list of class ad names
+ pipe = IO.popen('condor_status -startd -f "%s\n" Name', "w+")
+ pipe.close_write
+ out = pipe.read
+ pipe.close
+
+ out.split("\n").each do |name|
+ if(name.include?("provider_combination"))
+ pipe = IO.popen("/usr/sbin/condor_advertise INVALIDATE_STARTD_ADS 2>&1", "w+")
+ pipe.puts 'MyType="Query"'
+ pipe.puts 'TargetType="Machine"'
+ pipe.puts 'Name="' + name + '"'
+ pipe.close_write
+ out = pipe.read
+ pipe.close
+ end
+ end
+
+ logger.info "Did invalidate, output is #{out}"
+
+ if $? != 0
+ logger.error "Unable to invalidate classads: #{out}"
+ raise "Unable to invalidate classads, classad sync failed #{out}"
+ end
+ rescue Errno::EPIPE
+ # if we failed to run condor_advertise, then in all likelihood condor isn't
+ # installed or isn't running. In that case, there can't be any old
+ # classads, so we just go on
+ end
+end
+
+def condormatic_queue_classad_sync(timeout)
+ logger = Logger.new(REFRESHD_LOG_FILE)
+ logger.level = Logger::DEBUG
+ begin
+ pipe = IO.popen("condor_submit 2>&1", "w+")
+ pipe.puts "Executable = /usr/share/aeolus-conductor/condor_refreshd/condor_refreshd\n"
+ logger.error "Executable = /usr/share/aeolus-conductor/condor_refreshd/condor_refreshd\n"
+ pipe.puts "Requirements = true\n"
+ logger.error "Requirements = true\n"
+ #pipe.puts "Arguments = RAILS_ENV=production\n"
+ #logger.error "Arguments = -d " + timeout.to_s + " RAILS_ENV=production\n"
+ pipe.puts "environment = RAILS_ENV=production\n"
+ logger.error "environment = RAILS_ENV=production\n"
+ pipe.puts "Log = log\n"
+ pipe.puts "Output = output\n"
+ pipe.puts "Error = error\n"
+ pipe.puts "Queue\n"
+ logger.error "Queue\n"
+ pipe.close_write
+ out = pipe.read
+ pipe.close
+
+ logger.error "$? (return value?) is #{$?}"
+ raise ("Error calling condor_submit: #{out}") if $? != 0
+
+ rescue Exception => ex
+ puts "Failed To add Condor Job"
+ puts ex
+ logger.error ex.message
+ logger.error ex.backtrace
+ end
end
def kick_condor
diff --git a/src/condor_refreshd/condor_refreshd b/src/condor_refreshd/condor_refreshd
index 273902f..162fe3f 100755
--- a/src/condor_refreshd/condor_refreshd
+++ b/src/condor_refreshd/condor_refreshd
@@ -26,33 +26,23 @@ require 'optparse'
require 'socket'
require 'dutils'
-port = 7890
help = false
-daemon = true
-refreshd_log_dir = "/var/log/aeolus-conductor"
-refreshd_pid_dir = "/var/run/aeolus-conductor"
-timeout = 15
+delay = 0
+refreshd_log_dir = "/var/log/aeolus-conductor/"
+refresh_log = "condor_refreshd.log"
optparse = OptionParser.new do |opts|
-
opts.banner = <<BANNER
Usage: condor_refreshd [options]
Options:
BANNER
- opts.on( '-f', '--pid-file PATH', "Use PATH to the condor_refreshd pid directory (defaults to #{refreshd_pid_dir})") do |newpath|
- refreshd_pid_dir = newpath
- end
opts.on( '-h', '--help', '') { help = true }
opts.on( '-l', '--log PATH', "Use PATH to the condor_refreshd log directory (defaults to #{refreshd_log_dir}). Use '-' for stdout") do |newpath|
refreshd_log_dir = newpath
end
- opts.on( '-n', '--nodaemon', 'Do not daemonize (useful in combination with -l for debugging)') { daemon = false }
- opts.on( '-p', '--port PORT', 'Use PORT (default: 7890)') do |newport|
- port = newport
- end
- opts.on( '-t', '--timeout TIMEOUT', 'Automatically refresh classads after TIMEOUT minutes') do |newtime|
- timeout = newtime
+ opts.on( '-d', '--delay SECONDS', 'do not run for SECONDS before refreshing classads.') do |newdelay|
+ delay = newdelay
end
end
@@ -76,49 +66,30 @@ if refreshd_log_dir == '-'
else
REFRESHD_LOG_FILE = "#{refreshd_log_dir}/condor_refreshd.log"
end
-logger = Logger.new(REFRESHD_LOG_FILE)
-logger.level = Logger::DEBUG
-logger.info "Starting condor_refreshd..."
-socket = Socket.new(Socket::AF_INET, Socket::SOCK_DGRAM, 0)
-in_addr = Socket.pack_sockaddr_in(port, '127.0.0.1')
-socket.bind(in_addr)
+logger = Logger.new(REFRESHD_LOG_FILE)
-# daemonize
-if daemon
- # note that this requires 'active_support', which we get for free from dutils
- Process.daemon
-end
+logger.level = Logger::DEBUG
begin
- REFRESHD_PID_FILE = "#{refreshd_pid_dir}/condor_refreshd.pid"
- FileUtils.mkdir_p File.dirname(REFRESHD_PID_FILE)
- open(REFRESHD_PID_FILE, "w") {|f| f.write(Process.pid) }
- File.chmod(0644, REFRESHD_PID_FILE)
-
- while true
- begin
- condormatic_classads_sync
- results = select([socket], nil, nil, timeout * 60)
- # if results was nil, we timed out waiting, and we need to do a sync
- # but not read from the socket
- if not results.nil? and results[0][0] == socket
- socket.recvfrom(1024)
- logger.info "Doing classad sync after getting data from the socket"
- else
- logger.info "Doing classad sync after timeout"
- end
- rescue => e
- logger.error "#{e.backtrace.shift}: #{e.message}"
- e.backtrace.each do |step|
- logger.error "\tfrom #{step}"
- end
+ # Ideally get the start time here.
+ #sleep(delay)
+ #sleep(20)
+ begin
+ logger.info("----------------------------------------------")
+ logger.info(Time.now.to_s + ": " + "Executing ClassAd Sync")
+ condormatic_classads_sync
+ rescue => e
+ logger.error "#{e.backtrace.shift}: #{e.message}"
+ e.backtrace.each do |step|
+ logger.error "\tfrom #{step}"
end
end
-rescue => e
- logger.error "#{e.backtrace.shift}: #{e.message}"
- e.backtrace.each do |step|
- logger.error "\tfrom #{step}"
- end
+ # Get the end time, calculate elapsed time, subtract it from 55 (5 sec buffer), and use that as the arg to the timeout.
+ condormatic_queue_classad_sync(30)
+
+ logger.info(Time.now.to_s + ": Refresh Completed")
+ logger.info("----------------------------------------------")
+ exit(0)
end
--
1.7.4
13 years, 2 months
[PATCH aeolus 1/2] Added Overrides to Condor Job
by Martyn Taylor
From: Martyn Taylor <mtaylor(a)redhat.com>
---
src/app/util/condormatic.rb | 7 ++++++-
1 files changed, 6 insertions(+), 1 deletions(-)
diff --git a/src/app/util/condormatic.rb b/src/app/util/condormatic.rb
index 03484ff..cf038dd 100644
--- a/src/app/util/condormatic.rb
+++ b/src/app/util/condormatic.rb
@@ -49,6 +49,12 @@ def condormatic_instance_create(task)
Rails.logger.error "DeltacloudImageId = $$(image_key)\n"
pipe.puts "DeltacloudHardwareProfile = $$(hardware_profile_key)\n"
Rails.logger.error "DeltacloudHardwareProfile = $$(hardware_profile_key)\n"
+ pipe.puts "DeltacloudHardwareProfileMemory = $$(hardware_profile_memory)\n"
+ Rails.logger.error "DeltacloudHardwareProfileMemory = $$(hardware_profile_memory)\n"
+ pipe.puts "DeltacloudHardwareProfileCPU = $$(hardware_profile_cpu)\n"
+ Rails.logger.error "DeltacloudHardwareProfileCPU = $$(hardware_profile_cpu)\n"
+ pipe.puts "DeltacloudHardwareProfileStorage = $$(hardware_profile_storage)\n"
+ Rails.logger.error "DeltacloudHardwareProfileStorage = $$(hardware_profile_storage)\n"
pipe.puts "DeltacloudKeyname = $$(keypair)\n"
Rails.logger.error "DeltacloudKeyname = $$(keypair)\n"
@@ -262,7 +268,6 @@ def condormatic_classads_sync
pipe.puts "hardware_profile_memory=\"#{overrides[:memory]}\""
pipe.puts "hardware_profile_storage=\"#{overrides[:storage]}\""
pipe.puts "hardware_profile_cpu=\"#{overrides[:cpu]}\""
- pipe.puts "hardware_profile_architecuture=\"#{overrides[:architecture]}\""
pipe.puts "realm_key=\"#{realm ? realm.external_key : ''}\""
pipe.puts "provider_url=\"#{account.provider.url}\""
pipe.puts "username=\"#{account.username}\""
--
1.7.4
13 years, 2 months
Multiple platform support for templates
by Jan Provazník
I've decided to send this patch even if I haven't done full e2e test (as it
depends on new imagefactory which is not in yet and I didn't achieve to setup
it today). This patch is quite simple, it only enables more than one repository
for image building. All worked for me on conductor side, though I'm not sure if
it really works with new imagefactory.
requirements:
new imagefactory
richard su's patches:
"[PATCH conductor 1/2] delayed_jobs to build and push with new image factory console"
"[PATCH conductor 2/2] template xml change to conform to new format"
'rake dc:prepare_repos' should be run
13 years, 2 months
[PATCH aeolus] aeolus-connector rpm spec file and init script
by Angus Thomas
From: Angus Thomas <athomas(a)redhat.com>
Spec file packages the image_factory_connector gem and the init script.
---
.../image_factory_connector/aeolus-connector.init | 77 ++++++++++++++++++++
.../rubygem-image_factory_connector.spec | 61 ++++++++++++++++
2 files changed, 138 insertions(+), 0 deletions(-)
create mode 100755 services/image_factory/image_factory_connector/aeolus-connector.init
create mode 100644 services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec
diff --git a/services/image_factory/image_factory_connector/aeolus-connector.init b/services/image_factory/image_factory_connector/aeolus-connector.init
new file mode 100755
index 0000000..e4237fd
--- /dev/null
+++ b/services/image_factory/image_factory_connector/aeolus-connector.init
@@ -0,0 +1,77 @@
+#!/bin/bash
+#
+# aeolus-connector startup script for aeolus-connector
+#
+# chkconfig: - 2345 97 03
+# description: aeolus-connector is a microapp to talk to
+# Aeolus Image Factory QMF console
+
+RETVAL=0
+prog="aeolus-connector"
+exec=/usr/bin/image_factory_connector
+lockfile=/var/lock/subsys/aeolus-connector
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+port=2003
+logfile=/var/log/aeolus-conductor/aeolus-connector.log
+[ -e /etc/sysconfig/$prog ] && . /etc/sysconfig/$prog
+
+start() {
+ if [ -e $lockfile ] ; then
+ echo "$prog subsystem locked"
+ exit 1
+ fi
+ if [ $UID -ne 0 ] ; then
+ echo "User has insufficient privilege."
+ exit 2
+ fi
+ [ -x $exec ] || exit 3
+ echo -n $"Starting $prog: "
+ daemon $exec -p $port start >> $logfile 2>&1 &
+ retval=$?
+ echo
+ [ $retval -eq 0 ] && touch $lockfile
+}
+
+stop() {
+ if [ $UID -ne 0 ] ; then
+ echo "User has insufficient privilege."
+ exit 4
+ fi
+ echo -n $"Stopping $prog: "
+ killall image_factory_connector
+ retval=$?
+ echo
+ [ $retval -eq 0 ] && rm -f $lockfile
+}
+
+restart() {
+ stop
+ start
+}
+
+rh_status() {
+ status image_factory_connector
+}
+
+case "$1" in
+ start)
+ $1
+ ;;
+ stop)
+ $1
+ ;;
+ restart)
+ $1
+ ;;
+ status)
+ rh_status
+ ;;
+ *)
+ echo $"Usage: $0 {start|stop|status|restart}"
+ exit 2
+esac
+exit $?
+
diff --git a/services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec b/services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec
new file mode 100644
index 0000000..c3a4a78
--- /dev/null
+++ b/services/image_factory/image_factory_connector/rubygem-image_factory_connector.spec
@@ -0,0 +1,61 @@
+# Generated from image_factory_connector-0.0.1.gem by gem2rpm -*- rpm-spec -*-
+%define ruby_sitelib %(ruby -rrbconfig -e "puts Config::CONFIG['sitelibdir']")
+%define gemdir %(ruby -rubygems -e 'puts Gem::dir' 2>/dev/null)
+%define gemname image_factory_connector
+%define geminstdir %{gemdir}/gems/%{gemname}-%{version}
+
+Summary: Sinatra microapp to talk to Aeolus Image Factory QMF console
+Name: rubygem-%{gemname}
+Version: 0.0.1
+Release: 1%{?dist}
+Group: Development/Languages
+License: GPLv2+ or Ruby
+URL: http://aeolusproject.org
+Source0: %{gemname}-%{version}.gem
+Source1: aeolus-connector.init
+BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n)
+Requires: rubygems
+Requires: rubygem(image_factory_console) >= 0.2.0
+Requires: rubygem(builder) >= 2.1.2
+Requires: rubygem(sinatra) >= 1.0
+Requires: rubygem(rspec) >= 1.3.0
+BuildRequires: rubygems
+BuildArch: noarch
+Provides: rubygem(%{gemname}) = %{version}
+
+%description
+REST api and callback mechanism to wrap qmf for the Aeolus project
+
+
+%prep
+
+%build
+
+%install
+rm -rf %{buildroot}
+mkdir -p %{buildroot}%{gemdir}
+gem install --local --install-dir %{buildroot}%{gemdir} \
+ --force --rdoc %{SOURCE0}
+mkdir -p %{buildroot}/%{_bindir}
+mv %{buildroot}%{gemdir}/bin/* %{buildroot}/%{_bindir}
+rmdir %{buildroot}%{gemdir}/bin
+find %{buildroot}%{geminstdir}/bin -type f | xargs chmod a+x
+mkdir -p %{buildroot}/etc/init.d
+cp %{SOURCE1} %{buildroot}/etc/init.d/aeolus-connector
+
+%clean
+rm -rf %{buildroot}
+
+%files
+%defattr(-, root, root, -)
+%{_bindir}/image_factory_connector
+%{gemdir}/gems/%{gemname}-%{version}/
+%doc %{gemdir}/doc/%{gemname}-%{version}
+%{gemdir}/cache/%{gemname}-%{version}.gem
+%{gemdir}/specifications/%{gemname}-%{version}.gemspec
+%attr(755,root,root) /etc/init.d/aeolus-connector
+
+
+%changelog
+* Mon Mar 14 2011 Angus Thomas <athomas(a)redhat.com> - 0.0.1-1
+- Initial package
--
1.7.4
13 years, 2 months
[PATCH conductor 2/2] template xml change to conform to new format
by Richard Su
new format: https://github.com/clalancette/oz/blob/master/examples/f14x86_64.tdl
---
src/app/models/template.rb | 13 +++++-
src/app/util/image_descriptor_xml.rb | 74 +++++++++++++++++++++------------
2 files changed, 57 insertions(+), 30 deletions(-)
diff --git a/src/app/models/template.rb b/src/app/models/template.rb
index b84409c..f4dc905 100644
--- a/src/app/models/template.rb
+++ b/src/app/models/template.rb
@@ -67,9 +67,11 @@ class Template < ActiveRecord::Base
def update_xml
xml.name = self.name
xml.description = self.summary
- xml.platform = self.platform
- xml.platform_version = self.platform_version
- xml.architecture = self.architecture
+ platform = {}
+ platform['name'] = self.platform.capitalize
+ platform['version'] = self.platform_version
+ platform['arch'] = self.architecture
+ xml.platform = platform
write_attribute(:xml, xml.to_xml)
end
@@ -146,4 +148,9 @@ class Template < ActiveRecord::Base
def warehouse_bucket
'templates'
end
+
+ def warehouse_url
+ baseurl = YAML.load_file("#{RAILS_ROOT}/config/image_warehouse.yml")['baseurl'] + "/templates/"
+ self.uuid ? baseurl + self.uuid : baseurl
+ end
end
diff --git a/src/app/util/image_descriptor_xml.rb b/src/app/util/image_descriptor_xml.rb
index 70f2e34..3a4bbae 100644
--- a/src/app/util/image_descriptor_xml.rb
+++ b/src/app/util/image_descriptor_xml.rb
@@ -28,9 +28,9 @@ class ImageDescriptorXML
@doc = Nokogiri::XML(xmlstr)
# create at least root node if it doesn't exist
unless @doc.root
- @doc.root = Nokogiri::XML::Node.new('image', @doc)
+ @doc.root = Nokogiri::XML::Node.new('template', @doc)
end
- @root = @doc.root.at_xpath('/image')
+ @root = @doc.root.at_xpath('/template')
end
def name=(str)
@@ -42,45 +42,65 @@ class ImageDescriptorXML
return get_node_text('name')
end
+ def description=(str)
+ node = get_or_create_node('description')
+ node.content = str
+ end
+
+ def description
+ return get_node_text('description')
+ end
+
def platform
- node = @root.at_xpath('/image/os')
+ node = @root.at_xpath('/template/os')
return node ? node['name'] : nil
end
- def platform=(str)
+ def platform=(platform_hash)
# FIXME: we remove all repos because we don't know which one is for
# platform
# update: we don't add platform repo, image builder chooses right one from OS
# name, but we add all other repos
#recreate_repo_nodes
- get_or_create_node('os')['name'] = str
+ platform_node = get_or_create_node('os')
+ platform_node.xpath('.//*').remove
+ platform_hash.each do |key, value|
+ snode = Nokogiri::XML::Node.new(key, @doc)
+ platform_node << snode
+ snode.content = value
+ end
+ install_node = Nokogiri::XML::Node.new('install', @doc)
+ install_node.set_attribute('type', 'url')
+ platform_node << install_node
+ url_node = Nokogiri::XML::Node.new('url', @doc)
+ # TODO: change when more than one os is supported by conductor
+ # url_node.content = "http://download.fedoraproject.org/pub/fedora/linux/releases/13/Fedora/x86..."
+ url_node.content = YAML.load_file("#{RAILS_ROOT}/config/image_descriptor_package_repositories.yml")['baseurl']
+ install_node << url_node
end
def platform_version
- node = @root.at_xpath('/image/os')
- return node ? node['version'] : nil
- end
-
- def platform_version=(str)
- get_or_create_node('os')['version'] = str
+ node = @root.at_xpath('/image/os/version')
+ return node ? node.content : nil
end
def architecture
- node = @root.at_xpath('/image/os')
- return node ? node['architecture'] : nil
+ node = @root.at_xpath('/template/os/arch')
+ return node ? node.content : nil
end
def architecture=(str)
get_or_create_node('os')['architecture'] = str
end
- def description=(str)
- node = get_or_create_node('description')
- node.content = str
- end
-
- def description
- return get_node_text('description')
+ def services=(services)
+ service_node = get_or_create_node('services')
+ service_node.xpath('.//service').remove
+ platform_hash.each do |key, value|
+ snode = Nokogiri::XML::Node.new(key, @doc)
+ platform_node << snode
+ snode.content = value
+ end
end
def services=(services)
@@ -97,7 +117,7 @@ class ImageDescriptorXML
def services
unless @services
@services = []
- @root.xpath('/image/services/service').each do |s|
+ @root.xpath('/template/services/service').each do |s|
services << s.text
end
end
@@ -109,13 +129,13 @@ class ImageDescriptorXML
end
def packages
- @root.xpath('/image/packages/package').map do |s|
+ @root.xpath('/template/packages/package').map do |s|
s.at_xpath('.//name').text
end
end
def groups
- @root.xpath('/image/groups/group').map do |s|
+ @root.xpath('/template/groups/group').map do |s|
s.at_xpath('.//name').text
end
end
@@ -138,7 +158,7 @@ class ImageDescriptorXML
end
def remove_package(package)
- @root.xpath('/image/packages/package').each do |s|
+ @root.xpath('/template/packages/package').each do |s|
if name = s.at_xpath('.//name') and name.text.to_s == package
s.remove
end
@@ -146,11 +166,11 @@ class ImageDescriptorXML
end
def clear_packages
- @root.xpath('/image/packages').each { |s| s.remove }
+ @root.xpath('/template/packages').each { |s| s.remove }
end
def clear_groups
- @root.xpath('/image/groups').each { |s| s.remove }
+ @root.xpath('/template/groups').each { |s| s.remove }
end
private
@@ -180,7 +200,7 @@ class ImageDescriptorXML
end
def get_node_text(path)
- node = @root.at_xpath('/image/' + path)
+ node = @root.at_xpath('/template/' + path)
return node ? node.text : nil
end
--
1.7.4
13 years, 2 months
[PATCH conductor 1/2] delayed_jobs to build and push with new image factory console
by Richard Su
The jobs submits requests to the console. The console communicates
with the new factory. The responses are in xml and indicates the uuid.
---
src/app/models/build_job.rb | 62 ++++++++++++++++++++++++++++++++
src/app/models/image.rb | 18 ++-------
src/app/models/provider_image.rb | 6 ---
src/app/models/push_job.rb | 65 ++++++++++++++++++++++++++++++++++
src/config/image_factory_console.yml | 2 +
src/spec/factories/build_job.rb | 7 ++++
src/spec/factories/push_job.rb | 7 ++++
src/spec/models/build_job_spec.rb | 9 +++++
src/spec/models/push_job_spec.rb | 9 +++++
src/spec/models/template_spec.rb | 6 +++
10 files changed, 171 insertions(+), 20 deletions(-)
create mode 100644 src/app/models/build_job.rb
create mode 100644 src/app/models/push_job.rb
create mode 100644 src/config/image_factory_console.yml
create mode 100644 src/spec/factories/build_job.rb
create mode 100644 src/spec/factories/push_job.rb
create mode 100644 src/spec/models/build_job_spec.rb
create mode 100644 src/spec/models/push_job_spec.rb
diff --git a/src/app/models/build_job.rb b/src/app/models/build_job.rb
new file mode 100644
index 0000000..336aa11
--- /dev/null
+++ b/src/app/models/build_job.rb
@@ -0,0 +1,62 @@
+#
+# Copyright (C) 2011 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301, USA. A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+
+require 'typhoeus'
+require 'nokogiri'
+
+class BuildJob < Struct.new(:image_id, :hydra)
+
+ def perform
+ @hydra = hydra
+ Rails.logger.info "--- start ---"
+ Rails.logger.info "BuildJob for image_id: #{image_id}"
+ begin
+ if (@hydra == nil)
+ @hydra = Typhoeus::Hydra.new
+ end
+ image = Image.find(image_id)
+ request = Typhoeus::Request.new(YAML.load_file("#{RAILS_ROOT}/config/image_factory_console.yml")['buildurl'],
+ :method => :post,
+ :timeout => 60*1000, # in milliseconds
+ :params => {
+ :template => image.template.warehouse_url,
+ :target => image.provider_type.codename})
+ request.on_complete do |response|
+ if response.success?
+ Rails.logger.info "success "
+ xml = Nokogiri::XML(response.body)
+ @return_uuid = xml.xpath('//image/uuid').first.text
+ image.uuid = @return_uuid
+ image.save
+ Rails.logger.info "uuid: #{return_uuid}"
+ else
+ return_uuid = nil
+ Rails.logger.error "failure"
+ Rails.logger.error response.body
+ end
+ end
+ @hydra.queue(request)
+ @hydra.run
+ rescue Exception => e
+ Rails.logger.error "Exception: "
+ Rails.logger.error e
+ end
+ Rails.logger.info "--- done ---"
+ @return_uuid
+ end
+end
diff --git a/src/app/models/image.rb b/src/app/models/image.rb
index 2109fe6..3f9653c 100644
--- a/src/app/models/image.rb
+++ b/src/app/models/image.rb
@@ -91,15 +91,6 @@ class Image < ActiveRecord::Base
end
end
- def build
- # TODO: this is just stubbed upload call,
- # when new image_builder_service is done, replace
- # with real request to image_builder_service
- unless self.provider_type.build_supported
- raise "Build is not supported on images with provider type #{self.provider_type.name}"
- end
- end
-
def self.create_and_build!(template, provider_type)
if Image.find_by_template_id_and_provider_type_id(template.id, provider_type.id)
raise ImageExistsError, "An attempted build of this template for the target '#{provider_type.name}' already exists"
@@ -114,10 +105,9 @@ class Image < ActiveRecord::Base
:template_id => template.id,
:status => Image::STATE_QUEUED
)
- img.delay.build
+ Delayed::Job.enqueue(BuildJob.new(img.id))
img
end
-
def self.single_import(providername, username, password, image_id)
account = Image.get_account(providername, username, password)
Image.import(account, image_id)
@@ -185,11 +175,11 @@ class Image < ActiveRecord::Base
provider_type.providers.each do |p|
# upload only to providers with account
unless p.provider_accounts.empty?
- ProviderImage.create!(
- :uuid => UUIDTools::UUID.timestamp_create.to_s,
+ pimg = ProviderImage.create!(
:image => self,
:provider => p
- ).delay.push
+ )
+ Delayed::Job.enqueue(PushJob.new(pimg.id))
end
end
end
diff --git a/src/app/models/provider_image.rb b/src/app/models/provider_image.rb
index ea2209b..81d7be1 100644
--- a/src/app/models/provider_image.rb
+++ b/src/app/models/provider_image.rb
@@ -31,12 +31,6 @@ class ProviderImage < ActiveRecord::Base
ACTIVE_STATES = [ STATE_QUEUED ]
INACTIVE_STATES = [STATE_COMPLETE, STATE_FAILED, STATE_CANCELED]
- def push
- # TODO: this is just stubbed upload call,
- # when new image_builder_service is done, replace
- # with real request to image_builder_service
- end
-
def after_update
if self.status_changed? and self.status == STATE_COMPLETE
begin
diff --git a/src/app/models/push_job.rb b/src/app/models/push_job.rb
new file mode 100644
index 0000000..6dce8aa
--- /dev/null
+++ b/src/app/models/push_job.rb
@@ -0,0 +1,65 @@
+#
+# Copyright (C) 2011 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
+# MA 02110-1301, USA. A copy of the GNU General Public License is
+# also available at http://www.gnu.org/copyleft/gpl.html.
+
+require 'typhoeus'
+require 'nokogiri'
+
+class PushJob < Struct.new(:provider_image_id, :hydra)
+ def perform
+ Rails.logger.info "--- start ---"
+ Rails.logger.info "PushJob for provider_image_id: #{provider_image_id}"
+ begin
+ @hydra = hydra
+ if (@hydra == nil)
+ @hydra = Typhoeus::Hydra.new
+ end
+ provider_image = ProviderImage.find(provider_image_id)
+ # TODO: what if a provider has multiple accounts
+ # for now pick first account
+ provider_account = provider_image.provider.provider_accounts.first
+ request = Typhoeus::Request.new(YAML.load_file("#{RAILS_ROOT}/config/image_factory_console.yml")['pushurl'],
+ :method => :post,
+ :timeout => 60*1000, # in milliseconds
+ :params => {
+ :image_id => provider_image.image.uuid,
+ :provider => provider_image.provider.name,
+ :credentials => provider_account.build_credentials
+ })
+ request.on_complete do |response|
+ if response.success?
+ Rails.logger.info "success "
+ xml = Nokogiri::XML(response.body)
+ @return_uuid = xml.xpath('//image/uuid').first.text
+ provider_image.uuid = @return_uuid
+ provider_image.save
+ Rails.logger.info "uuid: #{return_uuid}"
+ else
+ Rails.logger.error "failure"
+ Rails.logger.error response.body
+ end
+ end
+ @hydra.queue(request)
+ @hydra.run
+ rescue Exception => e
+ Rails.logger.error "Exception: "
+ Rails.logger.error e
+ end
+ Rails.logger.info "--- done ---"
+ @return_uuid
+ end
+end
diff --git a/src/config/image_factory_console.yml b/src/config/image_factory_console.yml
new file mode 100644
index 0000000..b5d66d6
--- /dev/null
+++ b/src/config/image_factory_console.yml
@@ -0,0 +1,2 @@
+buildurl: http://localhost:2003/build
+pushurl: http://localhost:2003/push
diff --git a/src/spec/factories/build_job.rb b/src/spec/factories/build_job.rb
new file mode 100644
index 0000000..b9faa9c
--- /dev/null
+++ b/src/spec/factories/build_job.rb
@@ -0,0 +1,7 @@
+Factory.define :build_job do |p|
+ hydra = Typhoeus::Hydra.new
+ response = Typhoeus::Response.new(:code => 200, :headers => "", :body => "<?xml version=\"1.0\" encoding=\"UTF-8\"><image><uuid>build-job-stub-uuid</uuid></image>", :time => 0.1)
+ hydra.stub(:post, YAML.load_file("#{RAILS_ROOT}/config/image_factory_console.yml")['buildurl']).and_return(response)
+ p.image_id { Factory.create(:image).id }
+ p.hydra hydra
+end
diff --git a/src/spec/factories/push_job.rb b/src/spec/factories/push_job.rb
new file mode 100644
index 0000000..b3d82f0
--- /dev/null
+++ b/src/spec/factories/push_job.rb
@@ -0,0 +1,7 @@
+Factory.define :push_job do |p|
+ hydra = Typhoeus::Hydra.new
+ response = Typhoeus::Response.new(:code => 200, :headers => "", :body => "<?xml version=\"1.0\" encoding=\"UTF-8\"><image><uuid>push-job-stub-uuid</uuid></image>", :time => 0.1)
+ hydra.stub(:post, YAML.load_file("#{RAILS_ROOT}/config/image_factory_console.yml")['pushurl']).and_return(response)
+ p.provider_image_id { Factory.create(:provider_image, :provider => Factory.create(:mock_provider_account).provider).id }
+ p.hydra hydra
+end
diff --git a/src/spec/models/build_job_spec.rb b/src/spec/models/build_job_spec.rb
new file mode 100644
index 0000000..0c4ecf8
--- /dev/null
+++ b/src/spec/models/build_job_spec.rb
@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe BuildJob do
+ it "should match request and parse stubbed xml" do
+ job = Factory.build(:build_job)
+ uuid = job.perform
+ uuid.should == "build-job-stub-uuid"
+ end
+end
diff --git a/src/spec/models/push_job_spec.rb b/src/spec/models/push_job_spec.rb
new file mode 100644
index 0000000..48348e7
--- /dev/null
+++ b/src/spec/models/push_job_spec.rb
@@ -0,0 +1,9 @@
+require 'spec_helper'
+
+describe PushJob do
+ it "should match request and parse stubbed xml" do
+ job = Factory.build(:push_job)
+ uuid = job.perform
+ uuid.should == "push-job-stub-uuid"
+ end
+end
diff --git a/src/spec/models/template_spec.rb b/src/spec/models/template_spec.rb
index 2249bf3..21df655 100644
--- a/src/spec/models/template_spec.rb
+++ b/src/spec/models/template_spec.rb
@@ -37,4 +37,10 @@ describe Template do
tpl2 = Template.find(tpl)
tpl2.packages.should == ['test']
end
+
+ it "should have warehouse url" do
+ t = Factory.build(:template)
+ t.uuid = "uuid"
+ t.warehouse_url.should == YAML.load_file("#{RAILS_ROOT}/config/image_warehouse.yml")['baseurl'] + "/templates/" + t.uuid
+ end
end
--
1.7.4
13 years, 2 months
delayed_jobs and template xml patches, v1
by Richard Su
To do end-to-end, these patches requires you to have the
image_factory_connector and the new image factory running. See
conductor/services/image_factory/README on how to setup.
Note: there is an install timeout issue in the new image factory that is
being investigated. Only the build job has been tested thus far.
13 years, 2 months
[PATCH] task #649: remove image builder service bits from conductor
by Mo Morsi
to be replaced w/ aeolus-connector
---
aeolus-conductor.spec.in | 12 +-
conf/conductor-image_builder_service | 78 --------
src/image_builder_service/image_builder_service | 234 -----------------------
3 files changed, 6 insertions(+), 318 deletions(-)
delete mode 100755 conf/conductor-image_builder_service
delete mode 100755 src/image_builder_service/image_builder_service
diff --git a/aeolus-conductor.spec.in b/aeolus-conductor.spec.in
index 6e44e8b..e020e05 100644
--- a/aeolus-conductor.spec.in
+++ b/aeolus-conductor.spec.in
@@ -126,8 +126,8 @@ mv %{buildroot}/%{app_root}/doc %{buildroot}/%{app_root}/test %{buildroot}/%{doc
%{__cp} conf/aeolus-conductor %{buildroot}%{_initrddir}
%{__cp} conf/conductor-dbomatic %{buildroot}%{_initrddir}
%{__cp} conf/conductor-condor_refreshd %{buildroot}%{_initrddir}
-%{__cp} conf/conductor-image_builder_service %{buildroot}%{_initrddir}
%{__cp} conf/conductor-delayed_job %{buildroot}%{_initrddir}
+%{__cp} conf/aeolus-connector %{buildroot}%{_initrddir}
%{__cp} conf/aeolus-conductor-httpd.conf %{buildroot}%{_sysconfdir}/httpd/conf.d/aeolus-conductor.conf
%{__cp} conf/aeolus-conductor.logrotate %{buildroot}%{_sysconfdir}/logrotate.d/aeolus-conductor
%{__cp} conf/aeolus-conductor.sysconf %{buildroot}%{_sysconfdir}/sysconfig/aeolus-conductor
@@ -157,7 +157,7 @@ touch %{buildroot}%{_localstatedir}/log/%{name}/rails.log
touch %{buildroot}%{_localstatedir}/log/%{name}/dbomatic.log
touch %{buildroot}%{_localstatedir}/run/%{name}/event_log_position
touch %{buildroot}%{_localstatedir}/log/%{name}/condor_refreshd.log
-touch %{buildroot}%{_localstatedir}/log/%{name}/image_builder_service.log
+touch %{buildroot}%{_localstatedir}/log/%{name}/connector.log
[ -e %{buildroot}%{app_root}/log ] || %{__mkdir} -p %{buildroot}%{app_root}/log
touch %{buildroot}%{app_root}/log/delayed_job.log
@@ -193,8 +193,8 @@ getent passwd aeolus >/dev/null || \
/sbin/chkconfig --add aeolus-conductor
/sbin/chkconfig --add conductor-dbomatic
/sbin/chkconfig --add conductor-condor_refreshd
-/sbin/chkconfig --add conductor-image_builder_service
/sbin/chkconfig --add conductor-delayed_job
+/sbin/chkconfig --add aeolus-connector
%preun daemons
if [ $1 = 0 ]; then
@@ -204,10 +204,10 @@ if [ $1 = 0 ]; then
/sbin/chkconfig --del conductor-dbomatic
/sbin/service conductor-condor_refreshd stop > /dev/null 2>&1
/sbin/chkconfig --del conductor-condor_refreshd
-/sbin/service conductor-image_builder_service stop > /dev/null 2>&1
-/sbin/chkconfig --del conductor-image_builder_service
/sbin/service conductor-delayed_job stop > /dev/null 2>&1
/sbin/chkconfig --del conductor-delayed_job
+/sbin/service aeolus-connector stop > /dev/null 2>&1
+/sbin/chkconfig --del aeolus-connector
fi
%files
@@ -227,8 +227,8 @@ fi
%{_initrddir}/aeolus-conductor
%{_initrddir}/conductor-dbomatic
%{_initrddir}/conductor-condor_refreshd
-%{_initrddir}/conductor-image_builder_service
%{_initrddir}/conductor-delayed_job
+%{_initrddir}/aeolus-connector
%config(noreplace) %{_sysconfdir}/logrotate.d/%{name}
%config(noreplace) %{_sysconfdir}/sysconfig/aeolus-conductor
%config(noreplace) %{_sysconfdir}/sysconfig/conductor-rails
diff --git a/conf/conductor-image_builder_service b/conf/conductor-image_builder_service
deleted file mode 100755
index 947b0ac..0000000
--- a/conf/conductor-image_builder_service
+++ /dev/null
@@ -1,78 +0,0 @@
-#!/bin/bash
-#
-#
-# conductor-image_builder_service startup script for conductor-image_builder_service
-#
-# chkconfig: - 93 07
-# description: conductor-image_builder_service builds cloud images
-# from the Aeolus Conductor db
-
-[ -r /etc/sysconfig/conductor-rails ] && . /etc/sysconfig/conductor-rails
-
-[ -r /etc/sysconfig/aeolus-conductor ] && . /etc/sysconfig/aeolus-conductor
-
-CONDUCTOR_DIR="${CONDUCTOR_DIR:-/usr/share/aeolus-conductor}"
-BUILDER_LOG="${BUILDER_LOG:-/var/log/aeolus-conductor/image_builder_service.log}"
-BUILDER_PID="${BUILDER_PID:-/var/run/aeolus-conductor/image_builder_service.pid}"
-BUILDER_LOCKFILE="${BUILDER_LOCKFILE:-/var/lock/subsys/conductor-image_builder_service}"
-AEOLUS_USER="${AEOLUS_USER:-aeolus}"
-
-BUILDER_PATH=$CONDUCTOR_DIR/image_builder_service
-BUILDER_PROG=image_builder_service
-
-. /etc/init.d/functions
-
-start() {
- echo -n "Starting conductor-image_builder_service: "
-
- daemon --user=$AEOLUS_USER $BUILDER_PATH/$BUILDER_PROG
- RETVAL=$?
- if [ $RETVAL -eq 0 ] && touch $BUILDER_LOCKFILE ; then
- echo_success
- echo
- else
- echo_failure
- echo
- fi
-}
-
-stop() {
- echo -n "Shutting down conductor-image_builder_service: "
- killproc -p $BUILDER_PID $BUILDER_PROG
- RETVAL=$?
- if [ $RETVAL -eq 0 ] && rm -f $BUILDER_LOCKFILE ; then
- echo_success
- echo
- else
- echo_failure
- echo
- fi
-}
-
-case "$1" in
- start)
- start
- ;;
- stop)
- stop
- ;;
- restart)
- stop
- start
- ;;
- reload)
- ;;
- force-reload)
- restart
- ;;
- status)
- status -p $BUILDER_PID $BUILDER_PROG
- RETVAL=$?
- ;;
- *)
- echo "Usage: conductor-image_builder_service {start|stop|restart|status}"
- exit 1
- ;;
-esac
-
-exit $RETVAL
diff --git a/src/image_builder_service/image_builder_service b/src/image_builder_service/image_builder_service
deleted file mode 100755
index 19ea709..0000000
--- a/src/image_builder_service/image_builder_service
+++ /dev/null
@@ -1,234 +0,0 @@
-#!/usr/bin/env ruby
-#
-# Copyright (C) 2010 Red Hat, Inc.
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
-# MA 02110-1301, USA. A copy of the GNU General Public License is
-# also available at http://www.gnu.org/copyleft/gpl.html.
-
-$: << File.join(File.dirname(__FILE__), "../dutils")
-require 'dutils'
-require "image_builder_console"
-require 'logger'
-require 'optparse'
-
-help = false
-daemon = true
-builder_pid_dir = "/var/run/aeolus-conductor"
-builder_log_dir = "/var/log/aeolus-conductor"
-timeout = 8
-
-optparse = OptionParser.new do |opts|
-
- opts.banner = <<BANNER
-Usage:
-image_builder_service [options]
-
-Options:
-BANNER
- opts.on( '-f', '--pid-file PATH', "Use PATH to the image_builder_service pid directory (defaults to #{builder_pid_dir})") do |newpath|
- builder_pid_dir = newpath
- end
- opts.on( '-h', '--help', '') { help = true }
- opts.on( '-l', '--log PATH', "Use PATH to the image_builder_service log directory (defaults to #{builder_log_dir}). Use '-' for stdout") do |newpath|
- builder_log_dir = newpath
- end
- opts.on( '-n', '--nodaemon', 'Do not daemonize (useful in combination with -l for debugging)') { daemon = false }
- opts.on( '-t', '--timeout SEC', 'Sleep for SEC between each poll for a new job (default is #{timeout})') do |newtime|
- timeout = newtime
- end
-end
-
-begin
- optparse.parse!
-rescue OptionParser::InvalidOption => e
- puts "Invalid option #{e.args}"
- puts
- puts optparse
- exit(1)
-end
-
-if help
- puts optparse
- exit(0)
-end
-
-# setup the logger
-if builder_log_dir == '-'
- BUILDER_LOG_FILE = STDOUT
-else
- BUILDER_LOG_FILE = "#{builder_log_dir}/image_builder_service.log"
-end
-logger = Logger.new(BUILDER_LOG_FILE)
-logger.level = Logger::DEBUG
-
-# daemonize
-if daemon
- # note that this requires 'active_support', which we get for free from dutils
- Process.daemon
-end
-
-#TODO: Make this whole thing less fragile
-class ImageBuilderService
- def initialize(logger, timeout)
- @log = logger
- @timeout = timeout.to_i
- @console ||= ImageBuilderConsole.new(:logger=>@log)
- @activebuilds ||= []
- @log.info "Image Builder Service initialized..."
- poll
- end
-
- def check_for_queued
- queue = Image.find(:all, :conditions => {:status => Image::STATE_QUEUED})
- cur_builds = Image.find(:all, :conditions => {:status => Image::ACTIVE_STATES})
- if queue.size > 0 || cur_builds.size > 0
- @log.debug "========================================"
- @log.debug "Queued Builds: " + queue.size.to_s
- @log.debug "All Incomplete Builds: " + cur_builds.size.to_s
- @log.debug "========================================"
- end
- find_orphaned(cur_builds) if cur_builds.size > queue.size
- queue.each {|t|
- build(t)
- }
- end
-
- def build(image)
- #targets.each do |t|
- @log.debug "========================================"
- @log.debug "target: " + image.provider_type.codename + ", status: " + image.status
- @log.debug "========================================"
- # FIXME: this should be contained elsewhere (probably Image model) so we
- # can keep logic out of here. Also, this currently only handles one
- # account, we will need to be able to specify at some point.
- creds = nil
- image.provider_type.providers.each do |p|
- unless p.provider_accounts.empty?
- creds = p.provider_accounts.first.build_credentials
- break
- end
- end
- #TODO: switch this back to uri once ActiveBuild retrieves it properly
- ab = @console.build_image(image.template.xml.to_xml, image.provider_type.codename, image.uuid, creds)
- if ab
- update_build_list(ab, image)
- image.build_id = ab.object_id.to_s
- image.save!
- @log.debug "========================================"
- @log.debug "Build id saved as: " + image.build_id
- @log.debug "========================================"
- end
- #end
- end
-
- def check_build_num
- @activebuilds.size
- end
-
- def find_orphaned(cur_builds)
- cur_builds.delete_if do |b|
- break if @activebuilds.size == 0
- @activebuilds.each do |ab|
- b.build_id.eql?(ab[:build_id])
- end
- end
- if cur_builds.size > 0
- @log.debug "========================================"
- @log.debug "There appear to be " + cur_builds.size.to_s + " untracked builds."
- @log.debug "Attempting to get status updates...."
- @log.debug "========================================"
- found = []
- cur_builds.each do |t|
- found << {:ab => @console.find_build(t.build_id),
- :target => t }
- end
- @log.debug "========================================"
- @log.debug "Retrieved " + found.size.to_s + " builds to update."
- @log.debug "========================================"
- found.each do |f|
- update_build_list(f[:ab], f[:target])
- end
- end
- end
-
- def update_agg(obj,new_status)
- @log.debug "========================================"
- @log.debug "Getting ar object to update using " + obj[:build].target.inspect + " and " + obj[:ar_id].inspect + " ..."
- @log.debug "========================================"
- image = Image.find(:first, :conditions => { :provider_type_id => ProviderType.find_by_codename(obj[:build].target.to_s).id,
- :template_id => obj[:ar_id].to_i })
- @log.debug "========================================"
- @log.debug "Updating with status: " + new_status
- @log.debug "========================================"
- image.status = new_status
- image.save!
- if new_status == 'complete'
- ri = image.provider_images.first
- ri.provider_image_key = obj[:build].finished_image
- ri.status = ProviderImage::STATE_COMPLETE
- ri.save!
-
- # FIXME: Might be cleaner to do this in an active record observer instead.
- @log.debug "========================================"
- @log.debug "calling condor classad sync."
- @log.debug "========================================"
- kick_condor
- end
- @log.debug "========================================"
- @log.debug "database updated!"
- @log.debug "========================================"
- end
-
- private
- def poll()
- loop do
- check_for_queued
- @activebuilds.delete_if do |ab|
- cur_status = @console.check_status(ab[:build])
- update_agg(ab, cur_status) unless cur_status.eql?(ab[:status])
- @log.debug "========================================"
- @log.debug "Status for target #{ab[:build].target}: #{cur_status}"
- @log.debug "Builds in array: " + check_build_num.to_s
- @log.debug "========================================"
- Image::INACTIVE_STATES.include?(cur_status)
- end
- sleep @timeout
-
- end
- end
-
- def update_build_list(ab, target)
- @activebuilds <<
- { :ar_id => target.template.id,
- :build => ab,
- :status => target.status,
- :build_id => ab.object_id.to_s
- }
- end
-end
-
-begin
- BUILDER_PID_FILE = "#{builder_pid_dir}/image_builder_service.pid"
- FileUtils.mkdir_p File.dirname(BUILDER_PID_FILE)
- open(BUILDER_PID_FILE, "w") {|f| f.write(Process.pid) }
- File.chmod(0644, BUILDER_PID_FILE)
-
- ImageBuilderService.new(logger, timeout)
-rescue => e
- logger.error "#{e.backtrace.shift}: #{e.message}"
- e.backtrace.each do |step|
- logger.error "\tfrom #{step}"
- end
-end
--
1.7.2.3
13 years, 2 months