[PATCH conductor, rev1] aeolus-upgrade enhances upgrade process

Steve Linabery slinaber at redhat.com
Wed Aug 15 21:18:36 UTC 2012


Brought in and refactored katello-upgrade[1] to more closely follow their
system of handling upgrades.

This script and associate files provide us with:
1) better logging of upgrade process (previously limited to watching stdout)
2) history of what upgrade scripts have been applied
3) directory where we can drop incremental upgrade scripts for
   more modular upgrade control.
4) requires user to stop services manually or use --auto-stop option, thus
   avoiding unintentional data loss in currently running services

[1] https://github.com/Katello/katello/blob/master/puppet/bin/katello-upgrade
---
 aeolus-conductor.spec.in                           |    4 +-
 src/script/aeolus-upgrade                          |  477 ++++++++++++++++++++
 src/script/upgrade                                 |    7 -
 .../default/0001_migrate_conductor_db.sh           |   27 ++
 .../default/0002_run_aeolus-configure.sh           |   13 +
 src/upgrade-scripts/upgrade/0001_data_defn.sh      |   28 ++
 src/upgrade-scripts/upgrade/0002_example_script.py |   17 +
 7 files changed, 565 insertions(+), 8 deletions(-)
 create mode 100755 src/script/aeolus-upgrade
 delete mode 100755 src/script/upgrade
 create mode 100755 src/upgrade-scripts/default/0001_migrate_conductor_db.sh
 create mode 100755 src/upgrade-scripts/default/0002_run_aeolus-configure.sh
 create mode 100755 src/upgrade-scripts/upgrade/0001_data_defn.sh
 create mode 100755 src/upgrade-scripts/upgrade/0002_example_script.py

diff --git a/aeolus-conductor.spec.in b/aeolus-conductor.spec.in
index 0eb5c4f..1c62014 100644
--- a/aeolus-conductor.spec.in
+++ b/aeolus-conductor.spec.in
@@ -353,6 +353,7 @@ touch %{buildroot}%{app_root}/log/delayed_job.log
 
 # copy script files over
 %{__cp} -r src/script %{buildroot}%{app_root}
+%{__cp} -r src/upgrade-scripts %{buildroot}%{app_root}
 
 %{__mkdir} -p %{buildroot}%{_sysconfdir}/%{name}
 
@@ -457,7 +458,8 @@ fi
 %{app_root}/vendor/provider_selection
 %config %{_sysconfdir}/%{name}
 %doc AUTHORS COPYING
-%{app_root}/script/upgrade
+%{app_root}/script/aeolus-upgrade
+%{app_root}/upgrade-scripts
 
 %files daemons
 %{_initrddir}/aeolus-conductor
diff --git a/src/script/aeolus-upgrade b/src/script/aeolus-upgrade
new file mode 100755
index 0000000..df89c31
--- /dev/null
+++ b/src/script/aeolus-upgrade
@@ -0,0 +1,477 @@
+#!/usr/bin/ruby
+
+require 'optparse'
+
+
+AEOLUS_UPGRADE_DIR = ENV['AEOLUS_UPGRADE_DIR'] || '/usr/share/aeolus-conductor/upgrade-scripts/'
+HISTORY_FILE_PATH = ENV['AEOLUS_UPGRADE_HISTORY'] || '/var/lib/aeolus-conductor/upgrade-history'
+DEFAULT_LOG_FILE = '/var/log/aeolus-conductor/aeolus_upgrade.log'
+DEFAULT_SCRIPTS_DIR = 'default/'
+UPGRADE_SCRIPTS_DIR = 'upgrade/'
+
+# error codes for exit_with function
+ERROR_CODES = {
+  :success => 0,
+  :general_error => 1,
+  :option_parser_error => 2,
+  :not_root => 3,
+  :io_error => 4,
+  :unknown => 127,
+}
+
+BACKEND_SERVICES = ["mongod",
+                    "iwhd",
+                    "postgresql",
+                    "httpd",
+                    "deltacloud-core",
+                    "libvirtd",
+                    "aeolus-conductor",
+                    "conductor-dbomatic",
+                    "imagefactory",
+                    "ntpd"]
+STDOUT_NAME = "stdout"
+
+# Terminate script with error code from ERROR_CODES hash
+def exit_with(code = :unknown)
+  code = ERROR_CODES[code.to_sym] || ERROR_CODES[:unknown]
+  exit code
+end
+
+# Indent each line of text with spacing
+def indent_text(text, spacing=" ")
+  text.gsub(/^/, spacing)
+end
+
+# Prints message and waits for Y/n answer.
+# Returns true if the answer is 'Y', false if it is 'n'.
+def confirm(message)
+  yes = 'y'
+  no = 'n'
+  begin
+    print message + " (y/n): "
+    answer = gets.strip
+  end while !(answer == yes or answer == no)
+  return answer == yes
+end
+
+
+# Stores history of executed scripts
+class UpgradeHistory
+
+  def initialize(path_to_history_file)
+    @history_file = path_to_history_file
+  end
+
+  # Returns last inserted executed script
+  def get_last_script
+    last_line = nil
+    f = open_history_file 'r'
+    f.each_line do |line|
+        last_line = line
+    end
+    f.close
+    last_line
+  end
+
+  # Ddds a script to history
+  def add_script(script)
+    f = open_history_file 'a'
+    f.puts File.basename(script)
+    f.close
+  end
+
+  private
+
+  def ensure_file(path)
+    if not File.exists? path
+      f = File.new(path, "w")
+      f.close
+    end
+  end
+
+  def open_history_file(mode)
+    ensure_file @history_file
+    File.open(@history_file, mode)
+  rescue => e
+    $stderr.puts e.message
+    exit_with :io_error
+  end
+
+end
+
+
+# Dummy history with /dev/null-like behaviour
+class DevNullHistory
+
+  def get_last_script
+    nil
+  end
+
+  def add_script(script)
+  end
+end
+
+
+# Queue of upgrade scripts.
+class UpgradeQueue
+
+  # Load scripts form a directory
+  # Takes path to a directory with upgrade scripts, an upgrade history instance
+  # and an instance of script parser.
+  # The queue finds all scripts applicable to current deployment that have not been
+  # executed yet.
+  def load(script_dir, history, script_parser)
+    @history = history
+    @script_dir = script_dir
+    @script_parser = script_parser
+    @scripts = nil
+  end
+
+  # Returns all scripts in the queue that have not been executed yet
+  def get_scripts
+    return [] if not @history or not @script_dir
+
+    @scripts ||= upgrade_scripts(@script_dir, @history.get_last_script)
+    @scripts
+  end
+
+  # Process all scripts in the queue. Execute a block for each of them
+  def process &block
+    while not self.get_scripts.empty?
+      yield self.get_scripts.first, @history
+      self.get_scripts.shift
+    end
+  end
+
+  def empty?
+    self.get_scripts.empty?
+  end
+
+  def length
+    self.get_scripts.length
+  end
+
+  protected
+
+  def upgrade_scripts(dir_path, starting_from = nil)
+    script_infos = find_upgrade_scripts(dir_path, starting_from).map do |path|
+      @script_parser.get_info(path)
+    end
+
+    return filter_scripts_for("aeolus-conductor", script_infos)
+  end
+
+  def filter_scripts_for(deployment, script_infos)
+    script_infos.delete_if do |script|
+      not script[:apply].include? deployment
+    end
+  end
+
+  def find_upgrade_scripts(path, starting_from = nil)
+    scripts = Dir.glob(path+'*').select{|f| File.executable?(f)}.sort
+    if not starting_from.nil?
+      scripts = scripts.delete_if {|script| (script <= path+starting_from)}
+    end
+    return scripts
+  end
+
+end
+
+# Parses upgrade scripts headers
+# The files can contain:
+# '#name: NAME' - on a single line, pretty name of the script
+# '#description: DESC' - can be multiline, longer description of the script
+class ScriptParser
+
+  # Returns info available from the file header
+  def get_info(script_path)
+    @info = {}
+    parse_script_file script_path
+    @info[:name] ||= ""
+    @info[:apply] ||= []
+    @info[:filename] = File.basename(script_path)
+    @info[:path] = script_path
+    @info
+  end
+
+  private
+
+  def parse_script_file(script_path)
+    f = File.open(script_path, 'r')
+    f.each_line do |line|
+      process_line line
+    end
+    f.close
+  end
+
+  def process_line(line)
+    @in_desc ||= false
+
+    if line =~ /#\s*name:\s*(.*)\s*$/
+      @info[:name] = $1
+      @in_desc = false
+
+    elsif line =~ /#\s*apply:\s*(.*)$/
+      @info[:apply] = $1.strip.split(/\s/)
+      @in_desc = false
+
+    elsif /#\s*description:\s*(.*)\s*$/.match line
+      @info[:description] = $1
+      @in_desc = true
+
+    elsif ( line =~ /^\s*#.*/ ) and @in_desc
+      @info[:description] ||= ""
+      @info[:description] += "\n" + line.gsub(/\s*#/, "").strip
+      @info[:description] = @info[:description].strip
+
+    else
+      @in_desc = false
+
+    end
+  end
+
+end
+
+
+# The upgrade process itself.
+# It creates two upgrade queues:
+#  - one for default scripts that are executed everytime
+#  - one for one-time scripts that get executed only once
+# Tries to execute the scripts one by one. If any of them fails, the process
+# is stopped.
+# Prints a result status at the end.
+class UpgradeProcess
+
+  LINE_LEN = 80
+
+  def run(options)
+    @options = options
+
+    # check if backend services are stopped
+    ensure_services_are_off
+
+    fake_history    = DevNullHistory.new
+    upgrade_history = UpgradeHistory.new(HISTORY_FILE_PATH)
+    @default_queue = UpgradeQueue.new
+    @default_queue.load(AEOLUS_UPGRADE_DIR+DEFAULT_SCRIPTS_DIR, fake_history, ScriptParser.new) if not @options[:skip_default]
+    @upgrade_queue = UpgradeQueue.new
+    @upgrade_queue.load(AEOLUS_UPGRADE_DIR+UPGRADE_SCRIPTS_DIR, upgrade_history, ScriptParser.new)
+
+    print_header
+
+    @step_count = @upgrade_queue.length
+    @step_count += @default_queue.length
+
+    if @step_count == 0
+      print_nothing_to_do
+      exit_with :success
+    end
+
+    process_queue(@default_queue)
+    process_queue(@upgrade_queue) if @default_queue.empty?
+
+    print_result
+
+    if finished_step_count < @step_count
+      exit_with :general_error
+    else
+      exit_with :success
+    end
+  end
+
+  protected
+
+  def ensure_services_are_off
+    stopped = true
+    BACKEND_SERVICES.each do |service|
+      if !is_stopped? service
+        puts "Service '%s' can not be running while aeolus-upgrade is in progress" % service
+        stopped = false
+      end
+    end
+    if !stopped
+      print_line
+      if @options[:auto_stop]
+        stop_services
+      else
+        puts "This script can be run in auto-stop mode with the -a / --auto-stop option to"
+        puts "disable services automatically."
+        puts "You may always use `service aeolus-services stop` to stop all the services"
+        puts ""
+        exit_service_stop
+      end
+    end
+  end
+
+  def stop_services
+    service = 'aeolus-services'
+    puts "We will stop all aeolus services using #{service} stop."
+    if confirm("PROCEED?")
+      if !stop(service)
+        puts "There was an issue stopping your services. Please resolve this manually"
+        puts "and try again"
+        exit_with :general_error
+      end
+    else
+      exit_service_stop
+    end
+  end
+
+  def exit_service_stop
+    puts "Exiting. Please stop your services and try again"
+    exit_with :general_error
+  end
+
+  def print_line(char="=", repeats=LINE_LEN)
+    puts char*repeats
+  end
+
+  def print_nothing_to_do
+    print_line
+    puts "Nothing to do"
+  end
+
+  def print_header
+    print_line
+    puts " Aeolus-conductor upgrade"
+  end
+
+  def print_step_info info
+    print_line
+    puts
+    puts "%s/%s: %s (%s)" % [@current_step, @step_count, info[:name], info[:filename]]
+    puts indent_text(info[:description]) if info[:description]
+    puts
+  end
+
+  def print_result
+    print_line
+    puts "Upgrade successful" if finished_step_count == @step_count
+    puts "Finished %i of %i upgrade steps" % [finished_step_count, @step_count]
+  end
+
+  def finished_step_count
+    @step_count - @upgrade_queue.length - @default_queue.length
+  end
+
+  def process_queue(queue)
+    @current_step ||= 1
+    queue.process do |script_info, history|
+      print_step_info(script_info)
+
+      if not @options[:dry_run]
+        break if not @options[:assume_yes] and not confirm("Do you want to proceed?")
+        break if not process_script(script_info)
+
+        history.add_script script_info[:path]
+      end
+
+      @current_step+=1
+    end
+  end
+
+  def process_script(script_info)
+    if execute_script(script_info[:path])
+      puts
+      puts script_info[:name]+" OK."
+      puts
+      return true
+    else
+      puts
+      puts script_info[:name]+" FAILED."
+      puts
+      return false
+    end
+  end
+
+  def execute_script script
+    command = script
+    command += ' &>>'+ log_file if log_file != STDOUT_NAME
+
+    log_start(script) if log_file != STDOUT_NAME
+    result = system(command)
+    log_result(result) if log_file != STDOUT_NAME
+
+    return result
+  end
+
+  def log_start script
+    system('printf "\n[$(date)] '+ script +'\n" >> '+ log_file)
+  end
+
+  def log_result success
+    system('printf "SUCCEEDED\n" >> '+ log_file) if success
+    system('printf "FAILED\n" >> '+ log_file) if not success
+  end
+
+  def log_file
+    return @options[:log_file] || DEFAULT_LOG_FILE
+  end
+
+end
+
+# test if given service is stopped
+def is_stopped? (service)
+  return !system('/sbin/service %s status 2>/dev/null >/dev/null' % service)
+end
+
+# system appears to return TRUE in some cases even when the
+# system call fails. non catastrophic errors, such as the service
+# already being off
+def stop(service)
+  return system('/sbin/service %s stop' % service)
+end
+
+# Parse and return script options
+def parse_options
+  opts = {}
+  opts[:assume_yes] = false
+
+  begin
+    option_parser = OptionParser.new
+    option_parser.banner = "Usage: #{$0} [options]"
+
+    option_parser.on_tail.on('-y', '--assumeyes', 'Assume yes on confirmations') do
+      opts[:assume_yes] = true
+    end
+
+    option_parser.on_tail.on('-a', '--autostop', 'Automatically stop services') do
+      opts[:auto_stop] = true
+    end
+
+    option_parser.on_tail.on('-d', '--dry-run', 'Prints the upgrade steps without modifying anything') do
+      opts[:dry_run] = true
+    end
+
+    option_parser.on_tail.on('-s', '--skip-default', 'Skips the default upgrade steps') do
+      opts[:skip_default] = true
+    end
+
+    option_parser.on_tail.on('--log=LOG_FILE', 'Log file, can also be set to stdout') do |value|
+      opts[:log_file] = value
+    end
+
+    option_parser.on_tail.on('-h', '--help', 'Show this short summary') do
+      puts option_parser
+      exit_with :success
+    end
+
+    option_parser.parse!
+  rescue => e
+    $stderr.puts e.message
+    $stderr.puts option_parser
+    exit_with :option_parser_error
+  end
+  opts
+end
+
+
+# check if running as root
+unless Process.uid == 0
+  $stderr.puts "You must run aeolus-upgrade as root"
+  exit_with :not_root
+end
+
+# start the upgrade process
+upgrade = UpgradeProcess.new
+upgrade.run(parse_options)
diff --git a/src/script/upgrade b/src/script/upgrade
deleted file mode 100755
index e3f658e..0000000
--- a/src/script/upgrade
+++ /dev/null
@@ -1,7 +0,0 @@
-#!/bin/sh
-
-cd /usr/share/aeolus-conductor
-aeolus-services stop
-service postgresql start
-RAILS_ENV=production rake dc:upgrade
-aeolus-services start
diff --git a/src/upgrade-scripts/default/0001_migrate_conductor_db.sh b/src/upgrade-scripts/default/0001_migrate_conductor_db.sh
new file mode 100755
index 0000000..e5c0d14
--- /dev/null
+++ b/src/upgrade-scripts/default/0001_migrate_conductor_db.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+
+#name: Migrate aeolus-conductor rails database
+#apply: aeolus-conductor
+#description: Executes rake db:migrate for the aeolus-conductor database
+
+# default configuration values (should be the same as in our sysv init script)
+if [ -f /etc/sysconfig/aeolus-conductor ]; then . /etc/sysconfig/aeolus-conductor; fi
+AEOLUS_HOME=${AEOLUS_HOME:-/usr/share/aeolus-conductor}
+AEOLUS_ENV=${AEOLUS_ENV:-production}
+
+SERVICES="postgresql"
+for SERVICE in $SERVICES; do
+    service $SERVICE start
+done
+
+
+pushd $AEOLUS_HOME >/dev/null
+RAILS_ENV=$AEOLUS_ENV rake db:migrate --trace 2>&1
+ret_code=$?
+popd >/dev/null
+
+for SERVICE in $SERVICES; do
+    service $SERVICE stop
+done
+
+exit $ret_code
diff --git a/src/upgrade-scripts/default/0002_run_aeolus-configure.sh b/src/upgrade-scripts/default/0002_run_aeolus-configure.sh
new file mode 100755
index 0000000..e84f1b5
--- /dev/null
+++ b/src/upgrade-scripts/default/0002_run_aeolus-configure.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+#name: Execute aeolus-configure
+#apply: aeolus-conductor
+#description: Executes aeolus-configure. Note: this restarts aeolus services. Skip this step if you want to manually run aeolus-configure later.
+
+# default configuration values (should be the same as in our sysv init script)
+if [ -f /etc/sysconfig/aeolus-conductor ]; then . /etc/sysconfig/aeolus-conductor; fi
+
+aeolus-configure 2>&1
+ret_code=$?
+
+exit $ret_code
diff --git a/src/upgrade-scripts/upgrade/0001_data_defn.sh b/src/upgrade-scripts/upgrade/0001_data_defn.sh
new file mode 100755
index 0000000..df1a144
--- /dev/null
+++ b/src/upgrade-scripts/upgrade/0001_data_defn.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+#name: Update aeolus-conductor Data Definition
+#apply: aeolus-conductor
+#description: Adds a role definition to the existing seeded data definition. noop if role is already defined.
+
+# default configuration values (should be the same as in our sysv init script)
+if [ -f /etc/sysconfig/aeolus-conductor ]; then . /etc/sysconfig/aeolus-conductor; fi
+AEOLUS_HOME=${AEOLUS_HOME:-/usr/share/aeolus-conductor}
+AEOLUS_ENV=${AEOLUS_ENV:-production}
+
+SERVICES="postgresql"
+for SERVICE in $SERVICES; do
+    service $SERVICE start
+done
+
+
+pushd $AEOLUS_HOME >/dev/null
+#if data defn has been previously added, this is a noop
+RAILS_ENV=$AEOLUS_ENV rake dc:upgrade --trace 2>&1
+ret_code=$?
+popd >/dev/null
+
+for SERVICE in $SERVICES; do
+    service $SERVICE stop
+done
+
+exit $ret_code
diff --git a/src/upgrade-scripts/upgrade/0002_example_script.py b/src/upgrade-scripts/upgrade/0002_example_script.py
new file mode 100755
index 0000000..25b15ba
--- /dev/null
+++ b/src/upgrade-scripts/upgrade/0002_example_script.py
@@ -0,0 +1,17 @@
+#!/usr/bin/python
+
+#name: Example script 2
+#apply: katello
+#description: Empty python script
+
+import sys
+
+# TODO
+print "Test script output"
+print "Test script output 2"
+sys.stdout.flush()
+
+print >> sys.stderr, "Test script error output"
+sys.stderr.flush()
+
+exit(0)
\ No newline at end of file
-- 
1.7.7.6




More information about the aeolus-devel mailing list