Aws dp 402 perf nfs fuse

bin/perfcp

@@ -0,0 +1,185 @@
+#!/usr/bin/env ruby
+
+# Copyright (c) 2013 Altiscale, inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License
+
+# This script tests read and write performance of a file system
+# It copies files from a source folder into a sink folder and calculates
+# average times with standard deviation
+
+require 'logger'
+require 'optparse'
+require 'ostruct'
+require 'terminal-table'
+require 'descriptive_statistics'
+require 'fileutils'
+
+LOG_LEVELS = %w(
+  debug
+  info
+  warn
+  error
+  fatal
+)
+
+logger = Logger.new(STDOUT)
+log_map = Hash[LOG_LEVELS.map.with_index.to_a]
+settings = OpenStruct.new
+options = OptionParser.new do |opts|
+  settings.log_level = 'info'
+  settings.times = 1
+
+  opts.on('-s',
+          '--source SOURCE',
+          'Source folder containing files to be copied.') do |sources|
+    settings.sources = sources
+  end
+
+  opts.on('-d',
+          '--dest DESTINATION',
+          'Destination folder.') do |dest|
+    settings.dest = dest
+  end
+
+  opts.on('-t',
+          '--times NUM_TIMES',
+          Integer,
+          'Times to run script, default 1.') do |times|
+    settings.times = times
+  end
+
+  opts.on('-i',
+          '--init INIT_SCRIPT',
+          'Optional init script(runs each time).') do |init|
+    settings.init = init
+  end
+
+  opts.on('-l',
+          '--log-level LEVEL',
+          "Log level: #{LOG_LEVELS.join(', ')}") do |log_level|
+    settings.log_level = log_level
+  end
+
+  opts.on('-h',
+          '--help',
+          'Show this help.') do
+    puts options.to_s
+    exit
+  end
+end
+
+# Parse options and make sure we have our required ones.
+begin
+  options.parse!
+  mandatory = [:sources, :dest]
+  missing = []
+  mandatory.each do |setting|
+    if !settings.marshal_dump.has_key?(setting)
+      missing << setting.to_s.gsub('_', '-')
+    end
+  end
+  if !missing.empty?
+    puts "Missing required options: #{missing.join(', ')}"
+    puts options.to_s
+    exit 1
+  end
+  unless LOG_LEVELS.include?(settings.log_level)
+    puts "Invalid log_level (#{settings.log_level}). \
+    Valid values: #{LOG_LEVELS.join(', ')}"
+    exit 1
+  end
+rescue OptionParser::InvalidOption, OptionParser::MissingArgument
+  puts $ERROR_INFO.to_s
+  puts options.to_s
+  exit 1
+end
+
+
+logger.level = log_map[settings.log_level]
+logger.debug "Starting perfcp with sources in #{settings.sources}"
+
+#The heading row of the table we are going to print
+headings = ["File", "Size"]
+
+#This table will store the file names in the 1st coloumn, file sizes in the 2nd
+#column and times to copy those files in subsequent coloumns. Last 2 coloumns 
+#will be used for mean and standard deviation
+table = Array.new
+
+# Get the list of files and sizes (and make it static)
+i = 0
+Dir.foreach(settings.sources) do |item|
+  next if item == '.' or item == '..'
+  table.push(Array.new)
+  file = File.new(settings.sources + "/" + item)
+  table[i].push(item)
+  table[i].push(file.size())
+  i += 1;
+end
+
+#Create the destination directory if it doesn't exist already
+if !Dir.exists? settings.dest
+  Dir.mkdir(settings.dest)
+  logger.debug "Destination directory #{settings.dest} not found. Creating it"
+end
+
+# For each of the times 
+for i in 1..settings.times.to_i
+  logger.debug "Run: #{i}"
+  headings.push("Run #{i}")
+
+  # Execute init script
+  if settings.init then
+    logger.debug("Running init script: " + settings.init)
+    system(settings.init)
+  end
+
+  #Delete directory if it exists already
+  if Dir.exists? "#{settings.dest}/Run#{i}"
+    FileUtils.rm_rf("#{settings.dest}/Run#{i}")
+    logger.debug "Directory #{settings.dest}/Run#{i} existed already. Deleting it"
+  end
+
+  #Create a new directory in the destination for this run
+  command = "mkdir #{settings.dest}/Run#{i}"
+  logger.debug "Making directory. Command: #{command}"
+  system command
+  abort("Couldn't create a directory for Run #{i}") if $? != 0
+
+  # Do the actual copies. For each file
+  table.each do |row|
+    command = "cp #{settings.sources}/#{row[0]} #{settings.dest}/Run#{i}"
+    logger.debug "Copying file #{row[0]}. Command: #{command}"
+
+    #Push the time taken to do the copy into the row
+    start = Time.now
+    system command
+    row.push(Time.now - start)
+    abort("Couldn't copy file #{row[0]} during Run #{i}") if $? != 0
+  end
+end
+
+#Calculate the mean and standard deviations of the times.
+#The times are in the 2nd coloumn onwards
+headings.push("Average", "Std. Dev.")
+table.each do |row|
+  times = row[2..row.size];
+  row.push( times.mean )
+  row.push( times.standard_deviation )
+end
+
+terminal_table = Terminal::Table.new :headings => headings, :rows => table
+
+puts "These were the times: "
+puts terminal_table

lib/perf_framework/version.rb

@@ -14,5 +14,5 @@
 
 # Version
 module PerfFramework
-  VERSION = '0.0.3'
+  VERSION = '0.0.4'
 end

perf_framework.gemspec

@@ -26,4 +26,6 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "net-ssh"
   spec.add_runtime_dependency "net-scp"
   spec.add_runtime_dependency "aws-sdk"
+  spec.add_runtime_dependency "terminal-table"
+  spec.add_runtime_dependency "descriptive_statistics"
 end