diff --git a/bin/perfcp b/bin/perfcp new file mode 100755 index 0000000..8358624 --- /dev/null +++ b/bin/perfcp @@ -0,0 +1,185 @@ +#!/usr/bin/env ruby + +# Copyright (c) 2013 Altiscale, inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License + +# This script tests read and write performance of a file system +# It copies files from a source folder into a sink folder and calculates +# average times with standard deviation + +require 'logger' +require 'optparse' +require 'ostruct' +require 'terminal-table' +require 'descriptive_statistics' +require 'fileutils' + +LOG_LEVELS = %w( + debug + info + warn + error + fatal +) + +logger = Logger.new(STDOUT) +log_map = Hash[LOG_LEVELS.map.with_index.to_a] +settings = OpenStruct.new +options = OptionParser.new do |opts| + settings.log_level = 'info' + settings.times = 1 + + opts.on('-s', + '--source SOURCE', + 'Source folder containing files to be copied.') do |sources| + settings.sources = sources + end + + opts.on('-d', + '--dest DESTINATION', + 'Destination folder.') do |dest| + settings.dest = dest + end + + opts.on('-t', + '--times NUM_TIMES', + Integer, + 'Times to run script, default 1.') do |times| + settings.times = times + end + + opts.on('-i', + '--init INIT_SCRIPT', + 'Optional init script(runs each time).') do |init| + settings.init = init + end + + opts.on('-l', + '--log-level LEVEL', + "Log level: #{LOG_LEVELS.join(', ')}") do |log_level| + settings.log_level = log_level + end + + opts.on('-h', + '--help', + 'Show this help.') do + puts options.to_s + exit + end +end + +# Parse options and make sure we have our required ones. +begin + options.parse! + mandatory = [:sources, :dest] + missing = [] + mandatory.each do |setting| + if !settings.marshal_dump.has_key?(setting) + missing << setting.to_s.gsub('_', '-') + end + end + if !missing.empty? + puts "Missing required options: #{missing.join(', ')}" + puts options.to_s + exit 1 + end + unless LOG_LEVELS.include?(settings.log_level) + puts "Invalid log_level (#{settings.log_level}). \ + Valid values: #{LOG_LEVELS.join(', ')}" + exit 1 + end +rescue OptionParser::InvalidOption, OptionParser::MissingArgument + puts $ERROR_INFO.to_s + puts options.to_s + exit 1 +end + + +logger.level = log_map[settings.log_level] +logger.debug "Starting perfcp with sources in #{settings.sources}" + +#The heading row of the table we are going to print +headings = ["File", "Size"] + +#This table will store the file names in the 1st coloumn, file sizes in the 2nd +#column and times to copy those files in subsequent coloumns. Last 2 coloumns +#will be used for mean and standard deviation +table = Array.new + +# Get the list of files and sizes (and make it static) +i = 0 +Dir.foreach(settings.sources) do |item| + next if item == '.' or item == '..' + table.push(Array.new) + file = File.new(settings.sources + "/" + item) + table[i].push(item) + table[i].push(file.size()) + i += 1; +end + +#Create the destination directory if it doesn't exist already +if !Dir.exists? settings.dest + Dir.mkdir(settings.dest) + logger.debug "Destination directory #{settings.dest} not found. Creating it" +end + +# For each of the times +for i in 1..settings.times.to_i + logger.debug "Run: #{i}" + headings.push("Run #{i}") + + # Execute init script + if settings.init then + logger.debug("Running init script: " + settings.init) + system(settings.init) + end + + #Delete directory if it exists already + if Dir.exists? "#{settings.dest}/Run#{i}" + FileUtils.rm_rf("#{settings.dest}/Run#{i}") + logger.debug "Directory #{settings.dest}/Run#{i} existed already. Deleting it" + end + + #Create a new directory in the destination for this run + command = "mkdir #{settings.dest}/Run#{i}" + logger.debug "Making directory. Command: #{command}" + system command + abort("Couldn't create a directory for Run #{i}") if $? != 0 + + # Do the actual copies. For each file + table.each do |row| + command = "cp #{settings.sources}/#{row[0]} #{settings.dest}/Run#{i}" + logger.debug "Copying file #{row[0]}. Command: #{command}" + + #Push the time taken to do the copy into the row + start = Time.now + system command + row.push(Time.now - start) + abort("Couldn't copy file #{row[0]} during Run #{i}") if $? != 0 + end +end + +#Calculate the mean and standard deviations of the times. +#The times are in the 2nd coloumn onwards +headings.push("Average", "Std. Dev.") +table.each do |row| + times = row[2..row.size]; + row.push( times.mean ) + row.push( times.standard_deviation ) +end + +terminal_table = Terminal::Table.new :headings => headings, :rows => table + +puts "These were the times: " +puts terminal_table \ No newline at end of file diff --git a/lib/perf_framework/version.rb b/lib/perf_framework/version.rb index a3ec9cf..30489bb 100644 --- a/lib/perf_framework/version.rb +++ b/lib/perf_framework/version.rb @@ -14,5 +14,5 @@ # Version module PerfFramework - VERSION = '0.0.3' + VERSION = '0.0.4' end diff --git a/perf_framework.gemspec b/perf_framework.gemspec index 4ffd516..49f3f4b 100644 --- a/perf_framework.gemspec +++ b/perf_framework.gemspec @@ -26,4 +26,6 @@ Gem::Specification.new do |spec| spec.add_runtime_dependency "net-ssh" spec.add_runtime_dependency "net-scp" spec.add_runtime_dependency "aws-sdk" + spec.add_runtime_dependency "terminal-table" + spec.add_runtime_dependency "descriptive_statistics" end