This patch adds a DataService Class that allows the collection of statistics in order to produce QoS and Quota Stats
From: martyntaylor mtaylor@redhat.com
--- src/app/util/data_service.rb | 112 +++++++++++++++++++++++++ src/spec/util/data_service_spec.rb | 159 ++++++++++++++++++++++++++++++++++++ 2 files changed, 271 insertions(+), 0 deletions(-) create mode 100644 src/app/util/data_service.rb create mode 100644 src/spec/util/data_service_spec.rb
diff --git a/src/app/util/data_service.rb b/src/app/util/data_service.rb new file mode 100644 index 0000000..c48d24a --- /dev/null +++ b/src/app/util/data_service.rb @@ -0,0 +1,112 @@ +class DataService + + QoSDataPoint = Struct.new(:time, :average, :max, :min) + QuotaUsagePoint = Struct.new(:used, :max) + TotalQuotaUsagePoint = Struct.new(:name, :no_instances) + + # This will return array of data points between start and end, if there is a data point where the interval start + interval end + # is greater than the end time, it will be ignored + # Example: + # start = 12.30, end = 12.32, interval = 45secs + # Intervals: 12.30.00 - 12.30.45, 12.30.45 - 12.31.30 will be returned. Interval 12.31.30 - 12.32.15 will not + def self.qos_task_submission_stats(start_time, end_time, interval_length, parent, action) + + instances = [] + + if parent.class == Provider + cloud_accounts = CloudAccount.find(:all, :conditions => {:provider_id => provider.id}) + cloud_accounts.each do |cloud_account| + instances.concat(instances) + end + elsif parent.class == Pool || parent.class == CloudAccount + instances = parent.instances + else + return nil + end + + return calculate_qos_task_submission_stats(start_time, end_time, interval_length, instances, action) + end + + def self.tasks_submissions_mean_max_min(time, tasks) + + first_pass = true + + total_time = nil + maximum_time = nil + minimum_time = nil + + tasks.each do |task| + + if(first_pass == true) + total_time = task.submission_time + maximum_time = task.submission_time + minimum_time = task.submission_time + first_pass = false + else + total_time += task.submission_time + + if task.submission_time > maximum_time + maximum_time = task.submission_time + end + + if task.submission_time< minimum_time + minimum_time = task.submission_time + end + end + + end + average_time = total_time / tasks.length + + return QoSDataPoint.new(time, average_time, maximum_time, minimum_time) + end + + # Returns the Used and Maximum Number of Instances in Quota + def self.quota_utilisation(parent) + quota = parent.quota + if quota + return QuotaUsagePoint.new(quota.total_instances, quota.maximum_total_instances) + end + end + + def self.total_quota_utilisation(provider) + data_points = [] + free_instances = 0 + + cloud_accounts = CloudAccount.find(:all, :conditions => {:provider_id => provider.id}) + cloud_accounts.each do |cloud_account| + quota = cloud_account.quota + if quota + data_points << TotalQuotaUsagePoint.new(cloud_account.username, quota.total_instances) + free_instances += (quota.maximum_total_instances - quota.total_instances) + end + end + data_points << TotalQuotaUsagePoint.new("free", free_instances) + return data_points + end + + private + def self.calculate_qos_task_submission_stats(start_time, end_time, interval_length, instances, action) + + data = [] + until start_time > (end_time - interval_length) do + interval_time = start_time + interval_length + + tasks = Task.find(:all, :conditions => { :time_submitted => start_time..interval_time, + :time_started => start_time..Time.now, + :failure_code => nil, + :action => action, + :task_target_id => instances + }) + if tasks.length > 0 + data << tasks_submissions_mean_max_min(start_time, tasks) + else + data << QoSDataPoint.new(start_time, 0, 0, 0) + end + + start_time = interval_time + end + + return data + end + +end diff --git a/src/spec/util/data_service_spec.rb b/src/spec/util/data_service_spec.rb new file mode 100644 index 0000000..68dad3e --- /dev/null +++ b/src/spec/util/data_service_spec.rb @@ -0,0 +1,159 @@ +require 'spec_helper' +#TODO this should be dealt with upstream, possibly in spec_helper +require 'app/util/data_service' + +describe DataService do + + it "should calculate the quota usage for a provider with a numbner of cloud accounts" do + client = mock('DeltaCloud', :null_object => true) + provider = Factory.build(:mock_provider) + provider.stub!(:connect).and_return(client) + provider.save! + + data = [[25, 10], [40, 20], [20, 20]] + free = 0 + for i in 0..2 + cloud_account = Factory.build(:cloud_account, :provider => provider, :username => "username" + i.to_s) + cloud_account.stub!(:valid_credentials?).and_return(true) + cloud_account.save! + + quota = Factory(:quota, :maximum_total_instances => data[i][0], :total_instances => data[i][1]) + cloud_account.quota_id = quota.id + cloud_account.save! + + free += (data[i][0] - data[i][1]) + end + + data_points = DataService.total_quota_utilisation(provider) + data_points[0].should == DataService::TotalQuotaUsagePoint.new("username0", data[0][1]) + data_points[1].should == DataService::TotalQuotaUsagePoint.new("username1", data[1][1]) + data_points[2].should == DataService::TotalQuotaUsagePoint.new("username2", data[2][1]) + data_points[3].should == DataService::TotalQuotaUsagePoint.new("free", free) + + end + + it "should calculate the total number of instances and maximum numbder of instances of a cloud account" do + client = mock('DeltaCloud', :null_object => true) + provider = Factory.build(:mock_provider) + provider.stub!(:connect).and_return(client) + provider.save! + + cloud_account = Factory.build(:cloud_account, :provider => provider) + cloud_account.stub!(:valid_credentials?).and_return(true) + cloud_account.save! + + quota = Factory(:quota, :maximum_total_instances => 50, :total_instances => 20) + cloud_account.quota_id = quota.id + + data_point = DataService.quota_utilisation(cloud_account) + data_point.should == DataService::QuotaUsagePoint.new(20, 50) + end + + it "should calculate the average, max and min task submission times" do + tasks = [] + instance = Factory :instance + + for i in 1..10 do + time = Time.now + task = Task.new(:instance => instance, :type => "InstanceTask", :state => Task::STATE_PENDING, :failure_code => nil) + task.time_submitted = time + time += i + task.time_started = time + task.save + tasks << task + end + + data_point = DataService.tasks_submissions_mean_max_min(Time.now, tasks) + + data_point.average.should == 5.5 + data_point.min.should == 1 + data_point.max.should == 10 + end + + it "should create data points for the average, max and min task submission times between two times at given intervals" do + pool = Factory :pool + instance = Factory(:instance, :pool_id => pool.id) + + expected_averages = [ 20, 40, 60, 80, 100] + no_intervals = expected_averages.length + interval_length = 30 + + end_time = Time.now.getutc + start_time = end_time - (interval_length * no_intervals) + + generate_tasks(start_time, interval_length, instance, expected_averages) + + data_points = DataService.qos_task_submission_stats(start_time, end_time, interval_length, pool, InstanceTask::ACTION_CREATE) + + for i in 0...data_points.length + average_time = expected_averages[i] + dp = data_points[i] + + dp.average.should == average_time + # The multiplications could be set as static numbers but are left as calculations for easier understanding of code + dp.max.should == (average_time / 10) * 2 * 9 + dp.min.should == (average_time / 10) * 2 + end + end + + it "should create data points for mean, max and min task submission times at given intervals for a provider with multiple accounts" do + pool = Factory :pool + + expected_averages = [] + expected_averages[0] = [ 20, 40, 60, 80, 100] + expected_averages[1] = [ 40, 60, 80, 100, 120] + expected_averages[2] = [ 60, 80, 100, 120, 140] + + no_intervals = expected_averages.length + interval_length = 30 + end_time = Time.now.getutc + start_time = end_time - (interval_length * no_intervals) + + client = mock('DeltaCloud', :null_object => true) + provider = Factory.build(:mock_provider) + provider.stub!(:connect).and_return(client) + provider.save! + + cloud_accounts = [] + expected_averages.each do |expected_average| + cloud_account = Factory.build(:cloud_account, :provider => provider, :username => "username" + expected_average[0].to_s) + cloud_account.stub!(:valid_credentials?).and_return(true) + cloud_account.save! + + instance = Factory(:instance, :cloud_account_id => cloud_account.id, :pool_id => pool.id) + generate_tasks(start_time, interval_length, instance, expected_average) + end + + data_points = DataService.qos_task_submission_stats(start_time, end_time, interval_length, pool, InstanceTask::ACTION_CREATE) + + for i in 0...data_points.length + dp = data_points[i] + dp.average.should == expected_averages[1][i] + dp.max.should == (expected_averages[2][i] / 10) * 2 * 9 + dp.min.should == (expected_averages[0][i] / 10) * 2 + end + end + + def generate_tasks(start_time, interval_length, instance, expected_averages) + interval_time = start_time + expected_averages.each do |avg| + submission_time = interval_time + (interval_length / 2) + for i in 1..9 do + started_time = submission_time + ((avg / 10) * 2) * i + + task = InstanceTask.new(:instance => instance, + :type => "InstanceTask", + :state => Task::STATE_QUEUED, + :failure_code => nil, + :action => InstanceTask::ACTION_CREATE, + :task_target_id => instance.id) + task.created_at = submission_time + task.time_submitted = submission_time + task.time_started = started_time + task.save! + end + interval_time += interval_length + end + end + +end
deltacloud-devel@lists.fedorahosted.org