[PATCH conductor 4/8] Added deployment launch retry
jprovazn at redhat.com
jprovazn at redhat.com
Thu Jun 14 13:36:31 UTC 2012
From: Jan Provaznik <jprovazn at redhat.com>
If a deployment launch fails and the deployment is rolled back and
there are some other instance matches which can be used for launch,
we try to re-launch the deployment with another instance match.
https://www.aeolusproject.org/redmine/issues/3370
---
src/app/models/deployment.rb | 39 ++++++++++++++++++++++++++++++++++-
src/app/models/instance.rb | 14 ++++++++++++
src/spec/models/deployment_spec.rb | 33 +++++++++++++++++++++++++++++-
3 files changed, 83 insertions(+), 3 deletions(-)
diff --git a/src/app/models/deployment.rb b/src/app/models/deployment.rb
index 792a6ac..ac5820f 100644
--- a/src/app/models/deployment.rb
+++ b/src/app/models/deployment.rb
@@ -82,6 +82,7 @@ class Deployment < ActiveRecord::Base
before_create :set_pool_family
before_create :set_new_state
after_save :log_state_change
+ after_save :handle_completed_rollback
USER_MUTABLE_ATTRS = ['name']
@@ -230,11 +231,20 @@ class Deployment < ActiveRecord::Base
end
def launch!(user)
+ self.reload unless self.new_record?
self.state = STATE_PENDING
save!
all_inst_match, account, errs = find_match_with_common_account
- unless all_inst_match
+ if all_inst_match
+ self.events << Event.create(
+ :source => self,
+ :event_time => DateTime.now,
+ :status_code => 'deployment_launch_match',
+ :summary => "Attempting to launch this deployment on provider account "\
+ "#{account.name}"
+ )
+ else
raise I18n.t('deployments.errors.match_not_found',
:errors => errs.join(", "))
end
@@ -259,7 +269,9 @@ class Deployment < ActiveRecord::Base
# occurs (for example DC API is not running, or config server is not
# running), then CREATE_FAILED is set in instance.launch!
instances.each do |instance|
+ instance.reset_attrs unless instance.state == Instance::STATE_NEW
match = all_inst_match.find{|m| m.instance.id == instance.id}
+ instance.instance_matches << match
begin
instance.launch!(match,
user,
@@ -547,7 +559,8 @@ class Deployment < ActiveRecord::Base
all_matches = instances.map do |instance|
m, e = instance.matches
errors = e.map {|e| "#{instance.name}: #{e}"}
- m
+ # filter matches we used in previous retries
+ m.select {|inst_match| !instance.includes_instance_match?(inst_match)}
end
pool.pool_family.provider_accounts_by_priority.each do |account|
@@ -715,4 +728,26 @@ class Deployment < ActiveRecord::Base
)
end
end
+
+ def handle_completed_rollback
+ if self.state_changed? and self.state == STATE_ROLLBACK_COMPLETE
+ begin
+ if self.events.where(
+ :status_code => 'deployment_launch_match').count > 9
+ raise "There was too many launch retries, aborting"
+ end
+ launch!(self.owner)
+ rescue
+ self.events << Event.create(
+ :source => self,
+ :event_time => DateTime.now,
+ :status_code => 'deployment_launch_failed',
+ :summary => "Failed to launch deployment: #{$!.message}"
+ )
+ update_attribute(:state, STATE_FAILED)
+ logger.error $!.message
+ logger.error $!.backtrace.join("\n ")
+ end
+ end
+ end
end
diff --git a/src/app/models/instance.rb b/src/app/models/instance.rb
index 88631c1..4dd8589 100644
--- a/src/app/models/instance.rb
+++ b/src/app/models/instance.rb
@@ -387,6 +387,10 @@ class Instance < ActiveRecord::Base
[matched, errors]
end
+ def includes_instance_match?(match)
+ instance_matches.any?{|m| m.equals?(match)}
+ end
+
def launch!(match, user, config_server, config)
# create a taskomatic task
task = InstanceTask.create!({:user => user,
@@ -498,6 +502,16 @@ class Instance < ActiveRecord::Base
end
end
+ def reset_attrs
+ # TODO: is it OK to upload params to config server multiple times?
+ # do we now keep instance config on config server by default?
+ update_attributes(:state => Instance::STATE_NEW,
+ :provider_account => nil,
+ :public_addresses => nil,
+ :private_addresses => nil)
+ instance_key.destroy if instance_key
+ end
+
private
def self.apply_search_filter(search)
diff --git a/src/spec/models/deployment_spec.rb b/src/spec/models/deployment_spec.rb
index 1f4ea9e..cf4cde0 100644
--- a/src/spec/models/deployment_spec.rb
+++ b/src/spec/models/deployment_spec.rb
@@ -455,7 +455,7 @@ describe Deployment do
end
end
- context "in rollback_in_progress state" do
+ context "in rollback_in_progress" do
before :each do
@deployment.state = Deployment::STATE_ROLLBACK_IN_PROGRESS
@deployment.save!
@@ -468,6 +468,37 @@ describe Deployment do
@deployment.instances << @inst1
@deployment.instances << @inst2
end
+
+ it "should relaunch the deployment if rollback is finished" do
+ Deployment.any_instance.should_receive(:launch!)
+ @inst2.update_attribute(:state, Instance::STATE_STOPPED)
+ end
+
+ describe "on relaunch" do
+ before :each do
+ account = Factory.create(:mock_provider_account)
+ match1 = FactoryGirl.build(:instance_match,
+ :provider_account => account,
+ :instance_id => @inst1.id)
+ match2 = FactoryGirl.build(:instance_match,
+ :provider_account => account,
+ :instance_id => @inst2.id)
+ Instance.any_instance.stub(:launch!).and_return(true)
+ Deployment.any_instance.stub(:find_match_with_common_account).
+ and_return([[match1, match2], account, []])
+ @inst2.update_attribute(:state, Instance::STATE_STOPPED)
+ end
+
+ it "should set pending state for the deployment" do
+ @deployment.reload
+ @deployment.state.should == Deployment::STATE_PENDING
+ end
+
+ it "should save instance match for each instance" do
+ @inst1.instance_matches.count.should > 0
+ @inst2.instance_matches.count.should > 0
+ end
+ end
end
context "in incomplete state" do
--
1.7.7.6
More information about the aeolus-devel
mailing list