[PATCH conductor 4/6] Added deployment launch retry

jprovazn at redhat.com jprovazn at redhat.com
Tue Jun 5 13:43:41 UTC 2012


From: Jan Provaznik <jprovazn at redhat.com>

If a deployment launch fails and the deployment is rolled back and
there are some other instance matches which can be used for launch,
we try to re-launch the deployment with another instance match.

https://www.aeolusproject.org/redmine/issues/3370
---
 src/app/models/deployment.rb       |   39 ++++++++++++++++++++++++++++++++++-
 src/app/models/instance.rb         |   13 ++++++++++++
 src/spec/models/deployment_spec.rb |   33 +++++++++++++++++++++++++++++-
 3 files changed, 82 insertions(+), 3 deletions(-)

diff --git a/src/app/models/deployment.rb b/src/app/models/deployment.rb
index 55e557a..c3a4a48 100644
--- a/src/app/models/deployment.rb
+++ b/src/app/models/deployment.rb
@@ -82,6 +82,7 @@ class Deployment < ActiveRecord::Base
   before_create :set_pool_family
   before_create :set_new_state
   after_save :log_state_change
+  after_save :handle_completed_rollback
 
   USER_MUTABLE_ATTRS = ['name']
 
@@ -230,11 +231,20 @@ class Deployment < ActiveRecord::Base
   end
 
   def launch!(user)
+    self.reload unless self.new_record?
     self.state = STATE_PENDING
     save!
     all_inst_match, account, errs = find_match_with_common_account
 
-    unless all_inst_match
+    if all_inst_match
+      self.events << Event.create(
+        :source => self,
+        :event_time => DateTime.now,
+        :status_code => 'deployment_launch_match',
+        :summary => "Attempting to launch this deployment on provider account "\
+                    "#{account.name}"
+      )
+    else
       raise I18n.t('deployments.errors.match_not_found',
                    :errors => errs.join(", "))
     end
@@ -259,7 +269,9 @@ class Deployment < ActiveRecord::Base
     # occurs (for example DC API is not running, or config server is not
     # running), then CREATE_FAILED is set in instance.launch!
     instances.each do |instance|
+      instance.reset_attrs unless instance.state == Instance::STATE_NEW
       match = all_inst_match.find{|m| m.instance.id == instance.id}
+      instance.instance_matches << match
       begin
         instance.launch!(match,
                          user,
@@ -547,7 +559,8 @@ class Deployment < ActiveRecord::Base
     all_matches = instances.map do |instance|
       m, e = instance.matches
       errors = e.map {|e| "#{instance.name}: #{e}"}
-      m
+      # filter matches we used in previous retries
+      m.select {|inst_match| !instance.includes_instance_match?(inst_match)}
     end
 
     pool.pool_family.provider_accounts_by_priority.each do |account|
@@ -711,4 +724,26 @@ class Deployment < ActiveRecord::Base
       )
     end
   end
+
+  def handle_completed_rollback
+    if self.state_changed? and self.state == STATE_ROLLBACK_COMPLETE
+      begin
+        if self.events.where(
+            :status_code => 'deployment_launch_match').count > 9
+          raise "There was too many launch retries, aborting"
+        end
+        launch!(self.owner)
+      rescue
+        self.events << Event.create(
+          :source => self,
+          :event_time => DateTime.now,
+          :status_code => 'deployment_launch_failed',
+          :summary => "Failed to launch deployment: #{$!.message}"
+        )
+        update_attribute(:state, STATE_FAILED)
+        logger.error $!.message
+        logger.error $!.backtrace.join("\n    ")
+      end
+    end
+  end
 end
diff --git a/src/app/models/instance.rb b/src/app/models/instance.rb
index 270cdd5..6a256ae 100644
--- a/src/app/models/instance.rb
+++ b/src/app/models/instance.rb
@@ -386,6 +386,10 @@ class Instance < ActiveRecord::Base
     [matched, errors]
   end
 
+  def includes_instance_match?(match)
+    instance_matches.any?{|m| m.equals?(match)}
+  end
+
   def launch!(match, user, config_server, config)
     # create a taskomatic task
     task = InstanceTask.create!({:user        => user,
@@ -495,6 +499,15 @@ class Instance < ActiveRecord::Base
     end
   end
 
+  def reset_attrs
+    # TODO: is it OK to upload params to config server multiple times?
+    # do we now keep instance config on config server by default?
+    update_attributes(:state => Instance::STATE_NEW,
+                           :public_addresses => nil,
+                           :private_addresses => nil)
+    instance_key.destroy if instance_key
+  end
+
   private
 
   def self.apply_search_filter(search)
diff --git a/src/spec/models/deployment_spec.rb b/src/spec/models/deployment_spec.rb
index bd18bff..77db72a 100644
--- a/src/spec/models/deployment_spec.rb
+++ b/src/spec/models/deployment_spec.rb
@@ -449,7 +449,7 @@ describe Deployment do
       end
     end
 
-    context "in rollback_in_progress state" do
+    context "in rollback_in_progress" do
       before :each do
         @deployment.state = Deployment::STATE_ROLLBACK_IN_PROGRESS
         @deployment.save!
@@ -462,6 +462,37 @@ describe Deployment do
         @deployment.instances << @inst1
         @deployment.instances << @inst2
       end
+
+      it "should relaunch the deployment if rollback is finished" do
+        Deployment.any_instance.should_receive(:launch!)
+        @inst2.update_attribute(:state, Instance::STATE_STOPPED)
+      end
+
+      describe "on relaunch" do
+        before :each do
+          account = Factory.create(:mock_provider_account)
+          match1 = FactoryGirl.build(:instance_match,
+                                     :provider_account => account,
+                                     :instance_id => @inst1.id)
+          match2 = FactoryGirl.build(:instance_match,
+                                     :provider_account => account,
+                                     :instance_id => @inst2.id)
+          Instance.any_instance.stub(:launch!).and_return(true)
+          Deployment.any_instance.stub(:find_match_with_common_account).
+            and_return([[match1, match2], account, []])
+          @inst2.update_attribute(:state, Instance::STATE_STOPPED)
+        end
+
+        it "should set pending state for the deployment" do
+          @deployment.reload
+          @deployment.state.should == Deployment::STATE_PENDING
+        end
+
+        it "should save instance match for each instance" do
+          @inst1.instance_matches.count.should > 0
+          @inst2.instance_matches.count.should > 0
+        end
+      end
     end
 
     context "in incomplete state" do
-- 
1.7.7.6




More information about the aeolus-devel mailing list