From 9729be86fb0749a30ec116cf8ebda57cd809461d Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Wed, 5 Jun 2013 09:22:40 -0500 Subject: [PATCH 01/25] Add non-200 responses line to printout --- beeswithmachineguns/bees.py | 6 ++++++ 1 file changed, 6 insertions(+) mode change 100644 => 100755 beeswithmachineguns/bees.py diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py old mode 100644 new mode 100755 index 8141f6b..ac6f78e --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -246,11 +246,13 @@ def _attack(params): requests_per_second_search = re.search('Requests\ per\ second:\s+([0-9.]+)\ \[#\/sec\]\ \(mean\)', ab_results) failed_requests = re.search('Failed\ requests:\s+([0-9.]+)', ab_results) complete_requests_search = re.search('Complete\ requests:\s+([0-9]+)', ab_results) + non_200_responses_search = re.search('Non-2xx responses:\s+([0-9]+)', ab_results) response['ms_per_request'] = float(ms_per_request_search.group(1)) response['requests_per_second'] = float(requests_per_second_search.group(1)) response['failed_requests'] = float(failed_requests.group(1)) response['complete_requests'] = float(complete_requests_search.group(1)) + response['non_200_responses'] = float(non_200_responses_search.group(1)) stdin, stdout, stderr = client.exec_command('cat %(csv_filename)s' % params) response['request_time_cdf'] = [] @@ -302,6 +304,10 @@ def _print_results(results, params, csv_filename): complete_results = [r['failed_requests'] for r in complete_bees] total_failed_requests = sum(complete_results) print ' Failed requests:\t\t%i' % total_failed_requests + + non_200_results = [r['non_200_responses'] for r in complete_bees] + total_non_200_results = sum(non_200_results) + print ' Non-200 Responses:\t\t%i' % total_non_200_results complete_results = [r['requests_per_second'] for r in complete_bees] mean_requests = sum(complete_results) From 1a2675fc71c1b553157fbe18c79da18aee61da04 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Wed, 5 Jun 2013 11:04:42 -0500 Subject: [PATCH 02/25] Up the verbosity level of AB --- beeswithmachineguns/bees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index ac6f78e..2fd16dc 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -231,7 +231,7 @@ def _attack(params): options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params params['options'] = options - benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params + benchmark_command = 'ab -v 2 -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params stdin, stdout, stderr = client.exec_command(benchmark_command) response = {} From db784d8dedb56f7cb4560624cdf914340662d9e3 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Wed, 5 Jun 2013 11:10:14 -0500 Subject: [PATCH 03/25] Fix regex for non-200 reponses --- beeswithmachineguns/bees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 2fd16dc..30bf9e9 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -246,7 +246,7 @@ def _attack(params): requests_per_second_search = re.search('Requests\ per\ second:\s+([0-9.]+)\ \[#\/sec\]\ \(mean\)', ab_results) failed_requests = re.search('Failed\ requests:\s+([0-9.]+)', ab_results) complete_requests_search = re.search('Complete\ requests:\s+([0-9]+)', ab_results) - non_200_responses_search = re.search('Non-2xx responses:\s+([0-9]+)', ab_results) + non_200_responses_search = re.search('Non-2xx\ responses:\s+([0-9]+)', ab_results) response['ms_per_request'] = float(ms_per_request_search.group(1)) response['requests_per_second'] = float(requests_per_second_search.group(1)) From d9f597225620320966a93c238ebf889c0fc850c9 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Wed, 5 Jun 2013 11:13:09 -0500 Subject: [PATCH 04/25] Escape char in regex --- beeswithmachineguns/bees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 30bf9e9..6790b5f 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -246,7 +246,7 @@ def _attack(params): requests_per_second_search = re.search('Requests\ per\ second:\s+([0-9.]+)\ \[#\/sec\]\ \(mean\)', ab_results) failed_requests = re.search('Failed\ requests:\s+([0-9.]+)', ab_results) complete_requests_search = re.search('Complete\ requests:\s+([0-9]+)', ab_results) - non_200_responses_search = re.search('Non-2xx\ responses:\s+([0-9]+)', ab_results) + non_200_responses_search = re.search('Non\-2xx\ responses:\s+([0-9]+)', ab_results) response['ms_per_request'] = float(ms_per_request_search.group(1)) response['requests_per_second'] = float(requests_per_second_search.group(1)) From 13b53897dba1e6631ff3b22cfa0c48dbac8660d2 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Wed, 5 Jun 2013 11:21:31 -0500 Subject: [PATCH 05/25] We may not always have non-200 responses --- beeswithmachineguns/bees.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 6790b5f..746190b 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -252,7 +252,11 @@ def _attack(params): response['requests_per_second'] = float(requests_per_second_search.group(1)) response['failed_requests'] = float(failed_requests.group(1)) response['complete_requests'] = float(complete_requests_search.group(1)) - response['non_200_responses'] = float(non_200_responses_search.group(1)) + + if non_200_responses_search is None: + response['non_200_responses'] = 0 + else + response['non_200_responses'] = float(non_200_responses_search.group(1)) stdin, stdout, stderr = client.exec_command('cat %(csv_filename)s' % params) response['request_time_cdf'] = [] From 0b20dbdb4c6536e3e53b01c373d9652015120708 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Wed, 5 Jun 2013 11:23:34 -0500 Subject: [PATCH 06/25] I'm so bad syntactically --- beeswithmachineguns/bees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 746190b..705e25b 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -255,7 +255,7 @@ def _attack(params): if non_200_responses_search is None: response['non_200_responses'] = 0 - else + else: response['non_200_responses'] = float(non_200_responses_search.group(1)) stdin, stdout, stderr = client.exec_command('cat %(csv_filename)s' % params) From 4df99173de8afe1590655cb036101fd3b8f78cc8 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Thu, 6 Jun 2013 14:46:41 -0500 Subject: [PATCH 07/25] Don't use verbose output --- beeswithmachineguns/bees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 705e25b..50df538 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -231,7 +231,7 @@ def _attack(params): options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params params['options'] = options - benchmark_command = 'ab -v 2 -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params + benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params stdin, stdout, stderr = client.exec_command(benchmark_command) response = {} From edfffb0c27daa8de4bf25c47175a84c71a0adcc8 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Thu, 6 Jun 2013 15:01:35 -0500 Subject: [PATCH 08/25] Show more detailed failed requests data --- beeswithmachineguns/bees.py | 45 +++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 50df538..b4b978b 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -247,12 +247,41 @@ def _attack(params): failed_requests = re.search('Failed\ requests:\s+([0-9.]+)', ab_results) complete_requests_search = re.search('Complete\ requests:\s+([0-9]+)', ab_results) non_200_responses_search = re.search('Non\-2xx\ responses:\s+([0-9]+)', ab_results) + + """ + If there are failed requests, get the breakdown + (Connect: 0, Receive: 0, Length: 338, Exceptions: 0) + """ + failed_connect_search = re.search('\s+\(Connect:\s+([0-9.]+)', ab_results) + failed_receive_search = re.search('\s+\(.+Receive:\s+([0-9.]+)', ab_results) + failed_length_search = re.search('\s+\(.+Length:\s+([0-9.]+)', ab_results) + failed_exceptions_search = re.search('\s+\(.+Exceptions:\s+([0-9.]+)', ab_results) response['ms_per_request'] = float(ms_per_request_search.group(1)) response['requests_per_second'] = float(requests_per_second_search.group(1)) response['failed_requests'] = float(failed_requests.group(1)) response['complete_requests'] = float(complete_requests_search.group(1)) + if failed_connect_search is None: + response['failed_connect'] = 0 + else: + response['failed_connect'] = float(failed_connect_search.group(1)) + + if failed_receive_search is None: + response['failed_receive'] = 0 + else: + response['failed_receive'] = float(failed_receive_search.group(1)) + + if failed_length_search is None: + response['failed_length'] = 0 + else: + response['failed_length'] = float(failed_length_search.group(1)) + + if failed_exceptions_search is None: + response['failed_exceptions_connect'] = 0 + else: + response['failed_exceptions_connect'] = float(failed_exceptions_search.group(1)) + if non_200_responses_search is None: response['non_200_responses'] = 0 else: @@ -309,6 +338,22 @@ def _print_results(results, params, csv_filename): total_failed_requests = sum(complete_results) print ' Failed requests:\t\t%i' % total_failed_requests + complete_results = [r['failed_connect'] for r in complete_bees] + total_failed_connect_requests = sum(complete_results) + print ' Connect:\t\t%i' % total_failed_connect_requests + + complete_results = [r['failed_receive'] for r in complete_bees] + total_failed_receive_requests = sum(complete_results) + print ' Receive:\t\t%i' % total_failed_receive_requests + + complete_results = [r['failed_length'] for r in complete_bees] + total_failed_length_requests = sum(complete_results) + print ' Length:\t\t%i' % total_failed_length_requests + + complete_results = [r['failed_exceptions_connect'] for r in complete_bees] + total_failed_exception_requests = sum(complete_results) + print ' Exception:\t\t%i' % total_failed_exception_requests + non_200_results = [r['non_200_responses'] for r in complete_bees] total_non_200_results = sum(non_200_results) print ' Non-200 Responses:\t\t%i' % total_non_200_results From 4302b90e637445e9f6aeb4de03f059ba70a33496 Mon Sep 17 00:00:00 2001 From: Matt Kornatz Date: Thu, 6 Jun 2013 15:02:14 -0500 Subject: [PATCH 09/25] Fix variable name --- beeswithmachineguns/bees.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index b4b978b..0dbd423 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -278,9 +278,9 @@ def _attack(params): response['failed_length'] = float(failed_length_search.group(1)) if failed_exceptions_search is None: - response['failed_exceptions_connect'] = 0 + response['failed_exceptions'] = 0 else: - response['failed_exceptions_connect'] = float(failed_exceptions_search.group(1)) + response['failed_exceptions'] = float(failed_exceptions_search.group(1)) if non_200_responses_search is None: response['non_200_responses'] = 0 @@ -350,7 +350,7 @@ def _print_results(results, params, csv_filename): total_failed_length_requests = sum(complete_results) print ' Length:\t\t%i' % total_failed_length_requests - complete_results = [r['failed_exceptions_connect'] for r in complete_bees] + complete_results = [r['failed_exceptions'] for r in complete_bees] total_failed_exception_requests = sum(complete_results) print ' Exception:\t\t%i' % total_failed_exception_requests From 149870d93a19f39628763f3347e45f4893d6b694 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Sun, 30 Jun 2013 15:35:54 +0300 Subject: [PATCH 10/25] switched from using deprecated optparse to using argparse --- beeswithmachineguns/main.py | 111 +++++++++++------------------------- 1 file changed, 33 insertions(+), 78 deletions(-) diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py index 1b84154..221b2cb 100644 --- a/beeswithmachineguns/main.py +++ b/beeswithmachineguns/main.py @@ -27,106 +27,62 @@ import bees from urlparse import urlparse -from optparse import OptionParser, OptionGroup +from argparse import ArgumentParser def parse_options(): """ Handle the command line arguments for spinning up bees """ - parser = OptionParser(usage=""" -bees COMMAND [options] + parser = ArgumentParser(description=""" + Bees with Machine Guns. + A utility for arming (creating) many bees (small EC2 instances) to attack + (load test) targets (web applications). + """) -Bees with Machine Guns + subparsers = parser.add_subparsers(title='commands', dest='command') + up_cmd = subparsers.add_parser("up", help='Start a batch of load testing servers.', description= + """Start a batch of load testing servers. + In order to spin up new servers you will need to specify at least the -k command, which is the name of the EC2 keypair to use for creating and connecting to the new servers. The bees will expect to find a .pem file with this name in ~/.ssh/.""") -A utility for arming (creating) many bees (small EC2 instances) to attack -(load test) targets (web applications). + # Required + up_cmd.add_argument('-k', '--key', metavar="KEY", dest='key', required=True, help="The ssh key pair name to use to connect to the new servers.") -commands: - up Start a batch of load testing servers. - attack Begin the attack on a specific url. - down Shutdown and deactivate the load testing servers. - report Report the status of the load testing servers. - """) + up_cmd.add_argument('-s', '--servers', metavar="SERVERS", dest='servers', type=int, default=5, help="The number of servers to start (default: 5).") + up_cmd.add_argument('-g', '--group', metavar="GROUP", dest='group', default='default', help="The security group(s) to run the instances under (default: default).") + up_cmd.add_argument('-z', '--zone', metavar="ZONE", dest='zone', default='us-east-1d', help="The availability zone to start the instances in (default: us-east-1d).") + up_cmd.add_argument('-i', '--instance', metavar="INSTANCE", dest='instance', default='ami-ff17fb96', help="The instance-id to use for each server from (default: ami-ff17fb96).") + up_cmd.add_argument('-t', '--type', metavar="TYPE", dest='type', default='t1.micro', help="The instance-type to use for each server (default: t1.micro).") + up_cmd.add_argument('-l', '--login', metavar="LOGIN", dest='login', default='newsapps', help="The ssh username name to use to connect to the new servers (default: newsapps).") + up_cmd.add_argument('-v', '--subnet', metavar="SUBNET", dest='subnet', default=None, help="The vpc subnet id in which the instances should be launched. (default: None).") - up_group = OptionGroup(parser, "up", - """In order to spin up new servers you will need to specify at least the -k command, which is the name of the EC2 keypair to use for creating and connecting to the new servers. The bees will expect to find a .pem file with this name in ~/.ssh/.""") + attack_cmd = subparsers.add_parser("attack", help='Begin the attack on a specific url.', description= + """Begin the attack on a specific url. + Beginning an attack requires only that you specify the -u option with the URL you wish to target.""") # Required - up_group.add_option('-k', '--key', metavar="KEY", nargs=1, - action='store', dest='key', type='string', - help="The ssh key pair name to use to connect to the new servers.") - - up_group.add_option('-s', '--servers', metavar="SERVERS", nargs=1, - action='store', dest='servers', type='int', default=5, - help="The number of servers to start (default: 5).") - up_group.add_option('-g', '--group', metavar="GROUP", nargs=1, - action='store', dest='group', type='string', default='default', - help="The security group(s) to run the instances under (default: default).") - up_group.add_option('-z', '--zone', metavar="ZONE", nargs=1, - action='store', dest='zone', type='string', default='us-east-1d', - help="The availability zone to start the instances in (default: us-east-1d).") - up_group.add_option('-i', '--instance', metavar="INSTANCE", nargs=1, - action='store', dest='instance', type='string', default='ami-ff17fb96', - help="The instance-id to use for each server from (default: ami-ff17fb96).") - up_group.add_option('-t', '--type', metavar="TYPE", nargs=1, - action='store', dest='type', type='string', default='t1.micro', - help="The instance-type to use for each server (default: t1.micro).") - up_group.add_option('-l', '--login', metavar="LOGIN", nargs=1, - action='store', dest='login', type='string', default='newsapps', - help="The ssh username name to use to connect to the new servers (default: newsapps).") - up_group.add_option('-v', '--subnet', metavar="SUBNET", nargs=1, - action='store', dest='subnet', type='string', default=None, - help="The vpc subnet id in which the instances should be launched. (default: None).") - - parser.add_option_group(up_group) - - attack_group = OptionGroup(parser, "attack", - """Beginning an attack requires only that you specify the -u option with the URL you wish to target.""") + attack_cmd.add_argument('-u', '--url', metavar="URL", dest='url', required=True, help="URL of the target to attack.") - # Required - attack_group.add_option('-u', '--url', metavar="URL", nargs=1, - action='store', dest='url', type='string', - help="URL of the target to attack.") - attack_group.add_option('-p', '--post-file', metavar="POST_FILE", nargs=1, - action='store', dest='post_file', type='string', default=False, - help="The POST file to deliver with the bee's payload.") - attack_group.add_option('-m', '--mime-type', metavar="MIME_TYPE", nargs=1, - action='store', dest='mime_type', type='string', default='text/plain', - help="The MIME type to send with the request.") - attack_group.add_option('-n', '--number', metavar="NUMBER", nargs=1, - action='store', dest='number', type='int', default=1000, - help="The number of total connections to make to the target (default: 1000).") - attack_group.add_option('-c', '--concurrent', metavar="CONCURRENT", nargs=1, - action='store', dest='concurrent', type='int', default=100, - help="The number of concurrent connections to make to the target (default: 100).") - attack_group.add_option('-H', '--headers', metavar="HEADERS", nargs=1, - action='store', dest='headers', type='string', default='', + attack_cmd.add_argument('-p', '--post-file', metavar="POST_FILE", dest='post_file', default=False, help="The POST file to deliver with the bee's payload.") + attack_cmd.add_argument('-m', '--mime-type', metavar="MIME_TYPE", dest='mime_type', default='text/plain', help="The MIME type to send with the request.") + attack_cmd.add_argument('-n', '--number', metavar="NUMBER", dest='number', type=int, default=1000, help="The number of total connections to make to the target (default: 1000).") + attack_cmd.add_argument('-c', '--concurrent', metavar="CONCURRENT", dest='concurrent', type=int, default=100, help="The number of concurrent connections to make to the target (default: 100).") + attack_cmd.add_argument('-H', '--headers', metavar="HEADERS", dest='headers', default='', help="HTTP headers to send to the target to attack. Multiple headers should be separated by semi-colons, e.g header1:value1;header2:value2") - attack_group.add_option('-e', '--csv', metavar="FILENAME", nargs=1, - action='store', dest='csv_filename', type='string', default='', - help="Store the distribution of results in a csv file for all completed bees (default: '').") - - parser.add_option_group(attack_group) + attack_cmd.add_argument('-e', '--csv', metavar="FILENAME", dest='csv_filename', default='', help="Store the distribution of results in a csv file for all completed bees (default: '').") - (options, args) = parser.parse_args() + down_cmd = subparsers.add_parser("down", help='Shutdown and deactivate the load testing servers.', description='Shutdown and deactivate the load testing servers.') + report_cmd = subparsers.add_parser("report", help='Report the status of the load testing servers.', description='Report the status of the load testing servers.') - if len(args) <= 0: - parser.error('Please enter a command.') + options = parser.parse_args() - command = args[0] + command = options.command if command == 'up': - if not options.key: - parser.error('To spin up new instances you need to specify a key-pair name with -k') - if options.group == 'default': print 'New bees will use the "default" EC2 security group. Please note that port 22 (SSH) is not normally open on this group. You will need to use to the EC2 tools to open it before you will be able to attack.' bees.up(options.servers, options.group, options.zone, options.instance, options.type, options.login, options.key, options.subnet) elif command == 'attack': - if not options.url: - parser.error('To run an attack you need to specify a url with -u') - parsed = urlparse(options.url) if not parsed.scheme: parsed = urlparse("http://" + options.url) @@ -148,6 +104,5 @@ def parse_options(): elif command == 'report': bees.report() - def main(): parse_options() From b4569c0c08f79840208e25d0badf33b4e5197d83 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Sun, 30 Jun 2013 16:28:09 +0300 Subject: [PATCH 11/25] fixed wrong indentation which caused security groups not to work --- beeswithmachineguns/bees.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 0dbd423..561838e 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -87,10 +87,9 @@ def _get_security_group_ids(connection, security_group_names, subnet): if subnet == None: if group.vpc_id == None: ids.append(group.id) - elif group.vpc_id != None: - ids.append(group.id) - - return ids + elif group.vpc_id != None: + ids.append(group.id) + return ids # Methods From efab950720dddad4e4cfc47ec75cd55ed8e57009 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Sun, 30 Jun 2013 18:39:38 +0300 Subject: [PATCH 12/25] added the option to run test by timelimit instead of number of requests --- beeswithmachineguns/bees.py | 30 +++++++++++++++++++----------- beeswithmachineguns/main.py | 4 +++- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 561838e..e95b2c7 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -230,7 +230,10 @@ def _attack(params): options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params params['options'] = options - benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params + if params['timelimit'] > 0: + benchmark_command = 'ab -r -t %(timelimit)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params + else: + benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params stdin, stdout, stderr = client.exec_command(benchmark_command) response = {} @@ -409,7 +412,7 @@ def _print_results(results, params, csv_filename): row.append(r['request_time_cdf'][i]["Time in ms"]) writer.writerow(row) -def attack(url, n, c, **options): +def attack(url, n, c, t, **options): """ Test the root url of this site. """ @@ -442,20 +445,24 @@ def attack(url, n, c, **options): instance_count = len(instances) - if n < instance_count * 2: - print 'bees: error: the total number of requests must be at least %d (2x num. instances)' % (instance_count * 2) - return if c < instance_count: print 'bees: error: the number of concurrent requests must be at least %d (num. instances)' % instance_count return - if n < c: - print 'bees: error: the number of concurrent requests (%d) must be at most the same as number of requests (%d)' % (c, n) - return - - requests_per_instance = int(float(n) / instance_count) connections_per_instance = int(float(c) / instance_count) + if t > 0: + print 'Each of %i bees will fire for %s seconds, %s at a time.' % (instance_count, t, connections_per_instance) + requests_per_instance = 50000; + else: + if n < instance_count * 2: + print 'bees: error: the total number of requests must be at least %d (2x num. instances)' % (instance_count * 2) + return + if n < c: + print 'bees: error: the number of concurrent requests (%d) must be at most the same as number of requests (%d)' % (c, n) + return + + requests_per_instance = int(float(n) / instance_count) - print 'Each of %i bees will fire %s rounds, %s at a time.' % (instance_count, requests_per_instance, connections_per_instance) + print 'Each of %i bees will fire %s rounds, %s at a time.' % (instance_count, requests_per_instance, connections_per_instance) params = [] @@ -467,6 +474,7 @@ def attack(url, n, c, **options): 'url': url, 'concurrent_requests': connections_per_instance, 'num_requests': requests_per_instance, + 'timelimit': t, 'username': username, 'key_name': key_name, 'headers': headers, diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py index 221b2cb..2aecc7e 100644 --- a/beeswithmachineguns/main.py +++ b/beeswithmachineguns/main.py @@ -69,6 +69,8 @@ def parse_options(): attack_cmd.add_argument('-H', '--headers', metavar="HEADERS", dest='headers', default='', help="HTTP headers to send to the target to attack. Multiple headers should be separated by semi-colons, e.g header1:value1;header2:value2") attack_cmd.add_argument('-e', '--csv', metavar="FILENAME", dest='csv_filename', default='', help="Store the distribution of results in a csv file for all completed bees (default: '').") + attack_cmd.add_argument('-t', '--timelimit', metavar="TIMELIMIT", dest='timelimit', type=int, default=0, + help="Maximum number of seconds to spend for benchmarking. This implies a -n 50000 internally. Use this to benchmark the server within a fixed total amount of time (default: no limit).") down_cmd = subparsers.add_parser("down", help='Shutdown and deactivate the load testing servers.', description='Shutdown and deactivate the load testing servers.') report_cmd = subparsers.add_parser("report", help='Report the status of the load testing servers.', description='Report the status of the load testing servers.') @@ -97,7 +99,7 @@ def parse_options(): csv_filename=options.csv_filename, ) - bees.attack(options.url, options.number, options.concurrent, **additional_options) + bees.attack(options.url, options.number, options.concurrent, options.timelimit, **additional_options) elif command == 'down': bees.down() From 66d7a7593a6be9a9bcfa0a27cd491dd6d8f80c51 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Sun, 7 Jul 2013 11:51:12 +0300 Subject: [PATCH 13/25] added percentage of failures to the output and "Mission Assessment" consideration changed the failed request details and non-200 to be more concise and similar to ab results --- beeswithmachineguns/bees.py | 43 +++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index e95b2c7..a0f50eb 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -338,27 +338,24 @@ def _print_results(results, params, csv_filename): complete_results = [r['failed_requests'] for r in complete_bees] total_failed_requests = sum(complete_results) - print ' Failed requests:\t\t%i' % total_failed_requests - - complete_results = [r['failed_connect'] for r in complete_bees] - total_failed_connect_requests = sum(complete_results) - print ' Connect:\t\t%i' % total_failed_connect_requests - - complete_results = [r['failed_receive'] for r in complete_bees] - total_failed_receive_requests = sum(complete_results) - print ' Receive:\t\t%i' % total_failed_receive_requests - - complete_results = [r['failed_length'] for r in complete_bees] - total_failed_length_requests = sum(complete_results) - print ' Length:\t\t%i' % total_failed_length_requests - - complete_results = [r['failed_exceptions'] for r in complete_bees] - total_failed_exception_requests = sum(complete_results) - print ' Exception:\t\t%i' % total_failed_exception_requests - + total_failed_percent = total_failed_requests/total_complete_requests*100 + print ' Failed requests:\t\t%i (%.2f%%)' % (total_failed_requests, total_failed_percent) + + if total_failed_requests > 0: + complete_results = [r['failed_connect'] for r in complete_bees] + total_failed_connect_requests = sum(complete_results) + complete_results = [r['failed_receive'] for r in complete_bees] + total_failed_receive_requests = sum(complete_results) + complete_results = [r['failed_length'] for r in complete_bees] + total_failed_length_requests = sum(complete_results) + complete_results = [r['failed_exceptions'] for r in complete_bees] + total_failed_exception_requests = sum(complete_results) + print ' (Connect: %i, Receive: %i, Length: %i, Exception: %i)' % (total_failed_connect_requests, total_failed_receive_requests, total_failed_length_requests, total_failed_exception_requests) + non_200_results = [r['non_200_responses'] for r in complete_bees] total_non_200_results = sum(non_200_results) - print ' Non-200 Responses:\t\t%i' % total_non_200_results + if total_non_200_results > 0: + print ' Non-200 Responses:\t\t%i' % total_non_200_results complete_results = [r['requests_per_second'] for r in complete_bees] mean_requests = sum(complete_results) @@ -388,13 +385,13 @@ def _print_results(results, params, csv_filename): print ' 50%% responses faster than:\t%f [ms]' % request_time_cdf[49] print ' 90%% responses faster than:\t%f [ms]' % request_time_cdf[89] - if mean_response < 500: + if mean_response < 500 and total_failed_percent < 0.1: print 'Mission Assessment: Target crushed bee offensive.' - elif mean_response < 1000: + elif mean_response < 1000 and total_failed_percent < 1: print 'Mission Assessment: Target successfully fended off the swarm.' - elif mean_response < 1500: + elif mean_response < 1500 and total_failed_percent < 5: print 'Mission Assessment: Target wounded, but operational.' - elif mean_response < 2000: + elif mean_response < 2000 and total_failed_percent < 10: print 'Mission Assessment: Target severely compromised.' else: print 'Mission Assessment: Swarm annihilated target.' From 8b172fec925c19f380ce58415a4397b8ab732f5b Mon Sep 17 00:00:00 2001 From: Ephraim Date: Mon, 8 Jul 2013 12:02:43 +0300 Subject: [PATCH 14/25] added the -g (gnuplot) option --- beeswithmachineguns/bees.py | 39 +++++++++++++++++++++++++++++++++++-- beeswithmachineguns/main.py | 2 ++ 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index a0f50eb..734aed5 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -224,6 +224,15 @@ def _attack(params): print 'Bee %i lost sight of the target (connection timed out creating csv_filename).' % params['i'] return None + if params['gnuplot_filename']: + stdin, stdout, stderr = client.exec_command('tempfile -s .tsv') + params['tsv_filename'] = stdout.read().strip() + if params['tsv_filename']: + options += ' -g %(tsv_filename)s' % params + else: + print 'Bee %i lost sight of the target (connection timed out creating tsv_filename).' % params['i'] + return None + if params['post_file']: pem_file_path=_get_pem_path(params['key_name']) os.system("scp -q -o 'StrictHostKeyChecking=no' -i %s %s %s@%s:/tmp/honeycomb" % (pem_file_path, params['post_file'], params['username'], params['instance_name'])) @@ -298,6 +307,17 @@ def _attack(params): print 'Bee %i lost sight of the target (connection timed out reading csv).' % params['i'] return None + if params['gnuplot_filename']: + stdin, stdout, stderr = client.exec_command('cat %(tsv_filename)s' % params) + dr = csv.DictReader(stdout, delimiter='\t') + response['gnuplot_fields'] = dr.fieldnames + response['gnuplot'] = [] + for row in dr: + response['gnuplot'].append(row) + if not response['gnuplot']: + print 'Bee %i lost sight of the target (connection timed out reading tsv).' % params['i'] + return None + print 'Bee %i is out of ammo.' % params['i'] client.close() @@ -306,7 +326,7 @@ def _attack(params): except socket.error, e: return e -def _print_results(results, params, csv_filename): +def _print_results(results, params, csv_filename, gnuplot_filename): """ Print summarized load-testing results. """ @@ -408,6 +428,13 @@ def _print_results(results, params, csv_filename): for r in results: row.append(r['request_time_cdf'][i]["Time in ms"]) writer.writerow(row) + + if gnuplot_filename: + with open(gnuplot_filename, 'w') as stream: + writer = csv.DictWriter(stream, delimiter='\t', fieldnames=results[0]['gnuplot_fields']) + writer.writeheader() + for r in results: + writer.writerows(r['gnuplot']) def attack(url, n, c, t, **options): """ @@ -416,6 +443,7 @@ def attack(url, n, c, t, **options): username, key_name, zone, instance_ids = _read_server_list() headers = options.get('headers', '') csv_filename = options.get("csv_filename", '') + gnuplot_filename = options.get("gnuplot_filename", '') if csv_filename: try: @@ -423,6 +451,12 @@ def attack(url, n, c, t, **options): except IOError, e: raise IOError("Specified csv_filename='%s' is not writable. Check permissions or specify a different filename and try again." % csv_filename) + if gnuplot_filename: + try: + stream = open(gnuplot_filename, 'w') + except IOError, e: + raise IOError("Specified gnuplot_filename='%s' is not writable. Check permissions or specify a different filename and try again." % gnuplot_filename) + if not instance_ids: print 'No bees are ready to attack.' return @@ -477,6 +511,7 @@ def attack(url, n, c, t, **options): 'headers': headers, 'post_file': options.get('post_file'), 'mime_type': options.get('mime_type', ''), + 'gnuplot_filename': gnuplot_filename, }) print 'Stinging URL so it will be cached for the attack.' @@ -496,6 +531,6 @@ def attack(url, n, c, t, **options): print 'Offensive complete.' - _print_results(results, params, csv_filename) + _print_results(results, params, csv_filename, gnuplot_filename) print 'The swarm is awaiting new orders.' diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py index 2aecc7e..1070d5b 100644 --- a/beeswithmachineguns/main.py +++ b/beeswithmachineguns/main.py @@ -69,6 +69,7 @@ def parse_options(): attack_cmd.add_argument('-H', '--headers', metavar="HEADERS", dest='headers', default='', help="HTTP headers to send to the target to attack. Multiple headers should be separated by semi-colons, e.g header1:value1;header2:value2") attack_cmd.add_argument('-e', '--csv', metavar="FILENAME", dest='csv_filename', default='', help="Store the distribution of results in a csv file for all completed bees (default: '').") + attack_cmd.add_argument('-g', '--gnuplot', metavar="FILENAME", dest='gnuplot_filename', default='', help="Write all measured values out as a 'gnuplot' or TSV (Tab separate values) file (default: '').") attack_cmd.add_argument('-t', '--timelimit', metavar="TIMELIMIT", dest='timelimit', type=int, default=0, help="Maximum number of seconds to spend for benchmarking. This implies a -n 50000 internally. Use this to benchmark the server within a fixed total amount of time (default: no limit).") @@ -97,6 +98,7 @@ def parse_options(): post_file=options.post_file, mime_type=options.mime_type, csv_filename=options.csv_filename, + gnuplot_filename=options.gnuplot_filename, ) bees.attack(options.url, options.number, options.concurrent, options.timelimit, **additional_options) From 6fa30cd38fd32274fc2411875e99c432f777e8d4 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Mon, 8 Jul 2013 12:09:56 +0300 Subject: [PATCH 15/25] refactored some var names to make thing more legible --- beeswithmachineguns/bees.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index 734aed5..af4c1c4 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -352,24 +352,24 @@ def _print_results(results, params, csv_filename, gnuplot_filename): print ' No bees completed the mission. Apparently your bees are peace-loving hippies.' return - complete_results = [r['complete_requests'] for r in complete_bees] - total_complete_requests = sum(complete_results) + complete_requests = [r['complete_requests'] for r in complete_bees] + total_complete_requests = sum(complete_requests) print ' Complete requests:\t\t%i' % total_complete_requests - complete_results = [r['failed_requests'] for r in complete_bees] - total_failed_requests = sum(complete_results) - total_failed_percent = total_failed_requests/total_complete_requests*100 - print ' Failed requests:\t\t%i (%.2f%%)' % (total_failed_requests, total_failed_percent) + failed_requests = [r['failed_requests'] for r in complete_bees] + total_failed_requests = sum(failed_requests) + total_failed_percent = total_failed_requests/total_complete_requests + print ' Failed requests:\t\t{:,} ({:.2%})'.format(int(total_failed_requests), total_failed_percent) if total_failed_requests > 0: - complete_results = [r['failed_connect'] for r in complete_bees] - total_failed_connect_requests = sum(complete_results) - complete_results = [r['failed_receive'] for r in complete_bees] - total_failed_receive_requests = sum(complete_results) - complete_results = [r['failed_length'] for r in complete_bees] - total_failed_length_requests = sum(complete_results) - complete_results = [r['failed_exceptions'] for r in complete_bees] - total_failed_exception_requests = sum(complete_results) + failed_connect_requests = [r['failed_connect'] for r in complete_bees] + total_failed_connect_requests = sum(failed_connect_requests) + failed_receive_requests = [r['failed_receive'] for r in complete_bees] + total_failed_receive_requests = sum(failed_receive_requests) + failed_length_requests = [r['failed_length'] for r in complete_bees] + total_failed_length_requests = sum(failed_length_requests) + failed_exceptions_requests = [r['failed_exceptions'] for r in complete_bees] + total_failed_exception_requests = sum(failed_exceptions_requests) print ' (Connect: %i, Receive: %i, Length: %i, Exception: %i)' % (total_failed_connect_requests, total_failed_receive_requests, total_failed_length_requests, total_failed_exception_requests) non_200_results = [r['non_200_responses'] for r in complete_bees] @@ -377,12 +377,12 @@ def _print_results(results, params, csv_filename, gnuplot_filename): if total_non_200_results > 0: print ' Non-200 Responses:\t\t%i' % total_non_200_results - complete_results = [r['requests_per_second'] for r in complete_bees] - mean_requests = sum(complete_results) + requests_per_second = [r['requests_per_second'] for r in complete_bees] + mean_requests = sum(requests_per_second) print ' Requests per second:\t%f [#/sec]' % mean_requests - complete_results = [r['ms_per_request'] for r in complete_bees] - mean_response = sum(complete_results) / num_complete_bees + ms_per_request = [r['ms_per_request'] for r in complete_bees] + mean_response = sum(ms_per_request) / num_complete_bees print ' Time per request:\t\t%f [ms] (mean of bees)' % mean_response # Recalculate the global cdf based on the csv files collected from From 089de17af789feafc96503002be030c29404d8a9 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Mon, 8 Jul 2013 12:18:12 +0300 Subject: [PATCH 16/25] added the option to consider non-200 responses as failures added a "Successfull Requests per Second" value --- beeswithmachineguns/bees.py | 20 +++++++++++++++++--- beeswithmachineguns/main.py | 2 ++ 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index af4c1c4..c832c3a 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -257,6 +257,7 @@ def _attack(params): requests_per_second_search = re.search('Requests\ per\ second:\s+([0-9.]+)\ \[#\/sec\]\ \(mean\)', ab_results) failed_requests = re.search('Failed\ requests:\s+([0-9.]+)', ab_results) complete_requests_search = re.search('Complete\ requests:\s+([0-9]+)', ab_results) + time_taken_search = re.search('Time\ taken\ for\ tests:\s+([0-9]+)', ab_results) non_200_responses_search = re.search('Non\-2xx\ responses:\s+([0-9]+)', ab_results) """ @@ -272,6 +273,7 @@ def _attack(params): response['requests_per_second'] = float(requests_per_second_search.group(1)) response['failed_requests'] = float(failed_requests.group(1)) response['complete_requests'] = float(complete_requests_search.group(1)) + response['time_taken'] = float(time_taken_search.group(1)) if failed_connect_search is None: response['failed_connect'] = 0 @@ -326,7 +328,7 @@ def _attack(params): except socket.error, e: return e -def _print_results(results, params, csv_filename, gnuplot_filename): +def _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_failure): """ Print summarized load-testing results. """ @@ -356,7 +358,11 @@ def _print_results(results, params, csv_filename, gnuplot_filename): total_complete_requests = sum(complete_requests) print ' Complete requests:\t\t%i' % total_complete_requests - failed_requests = [r['failed_requests'] for r in complete_bees] + if non_200_is_failure: + failed_requests = [r['failed_requests']+r['non_200_responses'] for r in complete_bees] + else: + failed_requests = [r['failed_requests'] for r in complete_bees] + total_failed_requests = sum(failed_requests) total_failed_percent = total_failed_requests/total_complete_requests print ' Failed requests:\t\t{:,} ({:.2%})'.format(int(total_failed_requests), total_failed_percent) @@ -381,6 +387,13 @@ def _print_results(results, params, csv_filename, gnuplot_filename): mean_requests = sum(requests_per_second) print ' Requests per second:\t%f [#/sec]' % mean_requests + if non_200_is_failure: + successful_requests_per_second = [(r['complete_requests']-r['failed_requests']-r['non_200_responses'])/r['time_taken'] for r in complete_bees] + else: + successful_requests_per_second = [(r['complete_requests']-r['failed_requests'])/r['time_taken'] for r in complete_bees] + successful_mean_requests = sum(successful_requests_per_second) + print ' Successful Requests per second:\t%f [#/sec]' % successful_mean_requests + ms_per_request = [r['ms_per_request'] for r in complete_bees] mean_response = sum(ms_per_request) / num_complete_bees print ' Time per request:\t\t%f [ms] (mean of bees)' % mean_response @@ -444,6 +457,7 @@ def attack(url, n, c, t, **options): headers = options.get('headers', '') csv_filename = options.get("csv_filename", '') gnuplot_filename = options.get("gnuplot_filename", '') + non_200_is_failure = options.get("non_200_is_failure", False) if csv_filename: try: @@ -531,6 +545,6 @@ def attack(url, n, c, t, **options): print 'Offensive complete.' - _print_results(results, params, csv_filename, gnuplot_filename) + _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_failure) print 'The swarm is awaiting new orders.' diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py index 1070d5b..dae4179 100644 --- a/beeswithmachineguns/main.py +++ b/beeswithmachineguns/main.py @@ -72,6 +72,7 @@ def parse_options(): attack_cmd.add_argument('-g', '--gnuplot', metavar="FILENAME", dest='gnuplot_filename', default='', help="Write all measured values out as a 'gnuplot' or TSV (Tab separate values) file (default: '').") attack_cmd.add_argument('-t', '--timelimit', metavar="TIMELIMIT", dest='timelimit', type=int, default=0, help="Maximum number of seconds to spend for benchmarking. This implies a -n 50000 internally. Use this to benchmark the server within a fixed total amount of time (default: no limit).") + attack_cmd.add_argument('--non-200-is-failure', dest='non_200_is_failure', action='store_true', default=False, help="Treat non-200 responses as failures (treated as success by default).") down_cmd = subparsers.add_parser("down", help='Shutdown and deactivate the load testing servers.', description='Shutdown and deactivate the load testing servers.') report_cmd = subparsers.add_parser("report", help='Report the status of the load testing servers.', description='Report the status of the load testing servers.') @@ -99,6 +100,7 @@ def parse_options(): mime_type=options.mime_type, csv_filename=options.csv_filename, gnuplot_filename=options.gnuplot_filename, + non_200_is_failure=options.non_200_is_failure, ) bees.attack(options.url, options.number, options.concurrent, options.timelimit, **additional_options) From e9bb054df2acb248eb823ae870297904edcab495 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Sun, 14 Jul 2013 15:20:55 +0300 Subject: [PATCH 17/25] upped the number of requests when using the -n option so as not to be limited by 50k --- beeswithmachineguns/bees.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index c832c3a..eadf900 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -240,7 +240,7 @@ def _attack(params): params['options'] = options if params['timelimit'] > 0: - benchmark_command = 'ab -r -t %(timelimit)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params + benchmark_command = 'ab -r -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params else: benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params stdin, stdout, stderr = client.exec_command(benchmark_command) From 0a5d932aa9a8f410886c8108f7159822bd179983 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Sun, 14 Jul 2013 15:49:51 +0300 Subject: [PATCH 18/25] fixed display of non-200 when --non-200-is-failure option is used fixed mission assessment percentages --- beeswithmachineguns/bees.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py index eadf900..3fed30e 100755 --- a/beeswithmachineguns/bees.py +++ b/beeswithmachineguns/bees.py @@ -367,6 +367,9 @@ def _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_f total_failed_percent = total_failed_requests/total_complete_requests print ' Failed requests:\t\t{:,} ({:.2%})'.format(int(total_failed_requests), total_failed_percent) + non_200_results = [r['non_200_responses'] for r in complete_bees] + total_non_200_results = sum(non_200_results) + if total_failed_requests > 0: failed_connect_requests = [r['failed_connect'] for r in complete_bees] total_failed_connect_requests = sum(failed_connect_requests) @@ -376,11 +379,14 @@ def _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_f total_failed_length_requests = sum(failed_length_requests) failed_exceptions_requests = [r['failed_exceptions'] for r in complete_bees] total_failed_exception_requests = sum(failed_exceptions_requests) - print ' (Connect: %i, Receive: %i, Length: %i, Exception: %i)' % (total_failed_connect_requests, total_failed_receive_requests, total_failed_length_requests, total_failed_exception_requests) + if non_200_is_failure: + print ' (Connect: %i, Receive: %i, Length: %i, Exception: %i, Non-200: %i)' % \ + (total_failed_connect_requests, total_failed_receive_requests, total_failed_length_requests, total_failed_exception_requests, total_non_200_results) + else: + print ' (Connect: %i, Receive: %i, Length: %i, Exception: %i)' % \ + (total_failed_connect_requests, total_failed_receive_requests, total_failed_length_requests, total_failed_exception_requests) - non_200_results = [r['non_200_responses'] for r in complete_bees] - total_non_200_results = sum(non_200_results) - if total_non_200_results > 0: + if (not non_200_is_failure) and total_non_200_results > 0: print ' Non-200 Responses:\t\t%i' % total_non_200_results requests_per_second = [r['requests_per_second'] for r in complete_bees] @@ -418,13 +424,13 @@ def _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_f print ' 50%% responses faster than:\t%f [ms]' % request_time_cdf[49] print ' 90%% responses faster than:\t%f [ms]' % request_time_cdf[89] - if mean_response < 500 and total_failed_percent < 0.1: + if mean_response < 500 and total_failed_percent < 0.001: print 'Mission Assessment: Target crushed bee offensive.' - elif mean_response < 1000 and total_failed_percent < 1: + elif mean_response < 1000 and total_failed_percent < 0.01: print 'Mission Assessment: Target successfully fended off the swarm.' - elif mean_response < 1500 and total_failed_percent < 5: + elif mean_response < 1500 and total_failed_percent < 0.05: print 'Mission Assessment: Target wounded, but operational.' - elif mean_response < 2000 and total_failed_percent < 10: + elif mean_response < 2000 and total_failed_percent < 0.10: print 'Mission Assessment: Target severely compromised.' else: print 'Mission Assessment: Swarm annihilated target.' From 7e1664a5854f75e350591930ef8eed210d9e1c71 Mon Sep 17 00:00:00 2001 From: Ephraim Date: Tue, 16 Jul 2013 18:45:39 +0300 Subject: [PATCH 19/25] added the option to collect statistics from a set of tests --- README.textile | 23 +++++++++++++++ beeswithmachineguns/bees.py | 49 +++++++++++++++++++++++++++++-- beeswithmachineguns/main.py | 5 ++++ examples/LoadTest.gpi | 58 +++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 5 files changed, 134 insertions(+), 2 deletions(-) create mode 100644 examples/LoadTest.gpi diff --git a/README.textile b/README.textile index 8749c01..f461318 100644 --- a/README.textile +++ b/README.textile @@ -9,6 +9,7 @@ h2. Dependencies * Python 2.6 * boto * paramiko +* csvkit h2. Installation for users @@ -58,6 +59,28 @@ It then uses those 4 servers to send 10,000 requests, 250 at a time, to attack O Lastly, it spins down the 4 servers. *Please remember to do this*--we aren't responsible for your EC2 bills. +h2. Advanced Usage + +
+bees up -s 5 -g public -k frakkingtoasters -z us-west-1a -i ami-aabbccdd -l ubuntu
+for i in `seq 200 200 1000`
+do
+   echo "---- $i -----"
+   bees attack -t 900 -c $i -p query.dat -u http://www.ournewwebbyhotness.com/ --stats-file 15_Min_200_step.csv --non-200-is-failure --testname $i
+done
+bees down
+gnuplot -e "filename='15_Min_200_step'" examples/LoadTest.gpi
+
+ +This spins up 5 servers in the us-west-1a AZ from the specified AMI in security group 'public' using the EC2 keypair 'frakkingtoasters', whose private key is expected to reside at ~/.ssh/frakkingtoasters.pem. + +It then runs a series of 15 minute tests (which in this case are a post of some query.dat file) with an increasing number of concurrent users, all the while collecting all the resulting statistics in the 15_Min_200_step.csv file. Note that in this test non 200 responses are considered as errors. + +Next, the bees are spun down. + +Finally, a graph is created from the csv using gnuplot (an example gnuplot script can be found in the examples dir). + + For complete options type: 
diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py
index 3fed30e..2480e61 100755
--- a/beeswithmachineguns/bees.py
+++ b/beeswithmachineguns/bees.py
@@ -25,6 +25,8 @@
 """
 
 from multiprocessing import Pool
+from subprocess import check_output
+from collections import OrderedDict
 import os
 import re
 import socket
@@ -328,7 +330,7 @@ def _attack(params):
     except socket.error, e:
         return e
 
-def _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_failure):
+def _print_results(results, params, csv_filename, gnuplot_filename, stats_filename, existing_stats_file, testname, non_200_is_failure):
     """
     Print summarized load-testing results.
     """
@@ -454,6 +456,37 @@ def _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_f
             writer.writeheader()
             for r in results:
                 writer.writerows(r['gnuplot'])
+
+    if stats_filename:
+        csvstat_results = check_output(["csvstat", "-tc", "ttime", gnuplot_filename])
+        min_search = re.search('\sMin:\s+([0-9]+)', csvstat_results)
+        max_search = re.search('\sMax:\s+([0-9]+)', csvstat_results)
+        mean_search = re.search('\sMean:\s+([0-9.]+)', csvstat_results)
+        median_search = re.search('\sMedian:\s+([0-9.]+)', csvstat_results)
+        stdev_search = re.search('\sStandard\ Deviation:\s+([0-9.]+)', csvstat_results)
+
+        stats = OrderedDict()
+        stats['Name'] = testname
+        stats['Total'] = int(total_complete_requests)
+        stats['Success'] = int(total_complete_requests-total_failed_requests)
+        stats['% Success'] = stats['Success']/total_complete_requests
+        stats['Error'] = int(total_failed_requests)
+        stats['% Error'] = total_failed_percent
+        stats['TotalPerSecond'] = mean_requests
+        stats['SuccessPerSecond'] = successful_mean_requests
+        stats['Min'] = int(min_search.group(1))
+        stats['Max'] = int(max_search.group(1))
+        stats['Mean'] = float(mean_search.group(1))
+        stats['Median'] = float(median_search.group(1))
+        stats['StdDev'] = float(stdev_search.group(1))
+        for i in range(5, 100, 5):
+            stats['P' + str(i)] = request_time_cdf[i]
+
+        with open(stats_filename, 'a') as stream:
+            writer = csv.DictWriter(stream, fieldnames=stats)
+            if not existing_stats_file:
+                writer.writeheader()
+            writer.writerow(stats)
     
 def attack(url, n, c, t, **options):
     """
@@ -463,6 +496,9 @@ def attack(url, n, c, t, **options):
     headers = options.get('headers', '')
     csv_filename = options.get("csv_filename", '')
     gnuplot_filename = options.get("gnuplot_filename", '')
+    stats_filename = options.get("stats_filename", '')
+    existing_stats_file = False
+    testname = options.get("testname", '')
     non_200_is_failure = options.get("non_200_is_failure", False)
 
     if csv_filename:
@@ -471,6 +507,15 @@ def attack(url, n, c, t, **options):
         except IOError, e:
             raise IOError("Specified csv_filename='%s' is not writable. Check permissions or specify a different filename and try again." % csv_filename)
     
+    if stats_filename:
+        existing_stats_file = os.path.isfile(stats_filename)
+        try:
+            stream = open(stats_filename, 'a')
+        except IOError, e:
+            raise IOError("Specified stats_filename='%s' is not writable. Check permissions or specify a different filename and try again." % stats_filename)
+        if not gnuplot_filename:
+            gnuplot_filename = os.path.splitext(stats_filename)[0] + "." + testname + ".tsv"
+
     if gnuplot_filename:
         try:
             stream = open(gnuplot_filename, 'w')
@@ -551,6 +596,6 @@ def attack(url, n, c, t, **options):
 
     print 'Offensive complete.'
 
-    _print_results(results, params, csv_filename, gnuplot_filename, non_200_is_failure)
+    _print_results(results, params, csv_filename, gnuplot_filename, stats_filename, existing_stats_file, testname, non_200_is_failure)
 
     print 'The swarm is awaiting new orders.'
diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py
index dae4179..8ed02d2 100644
--- a/beeswithmachineguns/main.py
+++ b/beeswithmachineguns/main.py
@@ -72,6 +72,9 @@ def parse_options():
     attack_cmd.add_argument('-g', '--gnuplot', metavar="FILENAME", dest='gnuplot_filename', default='', help="Write all measured values out as a 'gnuplot' or TSV (Tab separate values) file (default: '').")
     attack_cmd.add_argument('-t', '--timelimit', metavar="TIMELIMIT", dest='timelimit', type=int, default=0,
                         help="Maximum number of seconds to spend for benchmarking. This implies a -n 50000 internally. Use this to benchmark the server within a fixed total amount of time (default: no limit).")
+    attack_cmd.add_argument('--stats-file', metavar="FILENAME", dest='stats_filename', default='',
+                        help="Store detailed graph ready stats across multiple tests in a csv file. Will create gnuplot files even if the -g/--gnuplot wasn't specified (default: '').")
+    attack_cmd.add_argument('--testname', metavar="NAME", dest='testname', default='unnamed', help="Name of current test. To be used in conjunction with --stats-file (default: 'unnamed').")
     attack_cmd.add_argument('--non-200-is-failure', dest='non_200_is_failure', action='store_true', default=False, help="Treat non-200 responses as failures (treated as success by default).")
 
     down_cmd = subparsers.add_parser("down", help='Shutdown and deactivate the load testing servers.', description='Shutdown and deactivate the load testing servers.')
@@ -100,6 +103,8 @@ def parse_options():
             mime_type=options.mime_type,
             csv_filename=options.csv_filename,
             gnuplot_filename=options.gnuplot_filename,
+            stats_filename=options.stats_filename,
+            testname=options.testname,
             non_200_is_failure=options.non_200_is_failure,
         )
 
diff --git a/examples/LoadTest.gpi b/examples/LoadTest.gpi
new file mode 100644
index 0000000..9c356b8
--- /dev/null
+++ b/examples/LoadTest.gpi
@@ -0,0 +1,58 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename=''" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 10
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename
+
+# Draw gridlines oriented on the y axis
+set grid y
+# Label the x-axis
+set xlabel 'Concurrent Users'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second and % Errors"
+set ytics nomirror 
+set y2tics 
+set ylabel 'Requests/Second' 
+set format y2 "%g %%"
+
+# Plot the data
+plot filename.'.csv' using 1:7 with lines lt 5 lw 3 axes x1y1, \
+             ''      using 1:8 with lines lt 2 lw 3 axes x1y1, \
+             ''      using 1:($6*100) with lines lt 1 lw 3 axes x1y2
+unset y2tics 
+unset y2label
+
+#
+# second graph
+#
+set title "Response Time"
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:15:9:32:31 with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+          ''         using 1:12:12:12:12 with candlesticks lt -1 notitle,\
+          ''         using 1:11 with lines lt -1 lw 3
diff --git a/requirements.txt b/requirements.txt
index 89e631f..933edec 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,2 +1,3 @@
 boto==2.8.0
 paramiko==1.10.1
+csvkit==0.5.0

From efa89cf3ddb604501322451e40b97c8aece313d7 Mon Sep 17 00:00:00 2001
From: Ephraim 
Date: Wed, 17 Jul 2013 19:09:21 +0300
Subject: [PATCH 20/25] switched to using mktemp instead of the deprecated
 tempfile changed to copying the tsv files using sftp to fix memory problems
 with large files

---
 beeswithmachineguns/bees.py | 32 +++++++++++++++-----------------
 1 file changed, 15 insertions(+), 17 deletions(-)

diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py
index 2480e61..dc10ded 100755
--- a/beeswithmachineguns/bees.py
+++ b/beeswithmachineguns/bees.py
@@ -25,8 +25,9 @@
 """
 
 from multiprocessing import Pool
-from subprocess import check_output
+from subprocess import check_output, call
 from collections import OrderedDict
+from tempfile import NamedTemporaryFile
 import os
 import re
 import socket
@@ -218,7 +219,7 @@ def _attack(params):
             for h in params['headers'].split(';'):
                 options += ' -H "%s"' % h
 
-        stdin, stdout, stderr = client.exec_command('tempfile -s .csv')
+        stdin, stdout, stderr = client.exec_command('mktemp --suffix=.csv')
         params['csv_filename'] = stdout.read().strip()
         if params['csv_filename']:
             options += ' -e %(csv_filename)s' % params
@@ -227,7 +228,7 @@ def _attack(params):
             return None
             
         if params['gnuplot_filename']:
-            stdin, stdout, stderr = client.exec_command('tempfile -s .tsv')
+            stdin, stdout, stderr = client.exec_command('mktemp --suffix=.tsv')
             params['tsv_filename'] = stdout.read().strip()
             if params['tsv_filename']:
                 options += ' -g %(tsv_filename)s' % params
@@ -312,15 +313,11 @@ def _attack(params):
             return None
 
         if params['gnuplot_filename']:
-            stdin, stdout, stderr = client.exec_command('cat %(tsv_filename)s' % params)
-            dr = csv.DictReader(stdout, delimiter='\t')
-            response['gnuplot_fields'] = dr.fieldnames
-            response['gnuplot'] = []
-            for row in dr:
-                response['gnuplot'].append(row)
-            if not response['gnuplot']:
-                print 'Bee %i lost sight of the target (connection timed out reading tsv).' % params['i']
-                return None
+            f = NamedTemporaryFile(suffix=".tsv", delete=False)
+            response['tsv_filename'] = f.name
+            f.close()
+            sftp = client.open_sftp()
+            sftp.get(params['tsv_filename'], response['tsv_filename'])
 
         print 'Bee %i is out of ammo.' % params['i']
 
@@ -451,11 +448,12 @@ def _print_results(results, params, csv_filename, gnuplot_filename, stats_filena
                 writer.writerow(row)
 
     if gnuplot_filename:
-        with open(gnuplot_filename, 'w') as stream:
-            writer = csv.DictWriter(stream, delimiter='\t', fieldnames=results[0]['gnuplot_fields'])
-            writer.writeheader()
-            for r in results:
-                writer.writerows(r['gnuplot'])
+        files = [r['tsv_filename'] for r in results]
+        # using csvkit utils to join the tsv files from all of the bees, adding a column to show whic bee produced each line. using sort because of performance problems with csvsort.
+        command = "csvstack -t -n bee -g " + ",".join(["%(i)s" % p for p in complete_bees_params]) + " " + " ".join(files) + " | csvcut -c 2-7,1 | sort -nk 5 -t ',' | sed 's/,/\t/g' > " + gnuplot_filename
+        call(command, shell=True)
+        # removing temp files
+        call(["rm"] + files)
 
     if stats_filename:
         csvstat_results = check_output(["csvstat", "-tc", "ttime", gnuplot_filename])

From 446a6c2621800060fc18fe08627db57780687860 Mon Sep 17 00:00:00 2001
From: Ephraim 
Date: Wed, 17 Jul 2013 19:12:43 +0300
Subject: [PATCH 21/25] changed requirements to be less restrictive

---
 requirements.txt | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index 933edec..cd711f9 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,3 @@
-boto==2.8.0
-paramiko==1.10.1
-csvkit==0.5.0
+boto>=2.8.0
+paramiko>=1.10.1
+csvkit>=0.5.0

From 692dc928e9aee40ab40bc8b279e2a93e2ddb2357 Mon Sep 17 00:00:00 2001
From: Ephraim 
Date: Tue, 23 Jul 2013 11:41:33 +0300
Subject: [PATCH 22/25] added compression for copying of gnuplot files fixed a
 crash when losing contact with a bee added some verbosity to steps which may
 take a while

---
 beeswithmachineguns/bees.py | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py
index dc10ded..0761985 100755
--- a/beeswithmachineguns/bees.py
+++ b/beeswithmachineguns/bees.py
@@ -207,10 +207,17 @@ def _attack(params):
     try:
         client = paramiko.SSHClient()
         client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+
+        if params['gnuplot_filename']:
+            use_compression = True
+        else:
+            use_compression = False
+
         client.connect(
             params['instance_name'],
             username=params['username'],
-            key_filename=_get_pem_path(params['key_name']))
+            key_filename=_get_pem_path(params['key_name']),
+            compress=use_compression)
 
         print 'Bee %i is firing her machine gun. Bang bang!' % params['i']
 
@@ -303,6 +310,8 @@ def _attack(params):
         else:
             response['non_200_responses'] = float(non_200_responses_search.group(1))
 
+        print 'Bee %i is out of ammo. She is collecting her pollen and flying back to the hive. This may take a while if she has a heavy load and/or the hive is far away...' % params['i']
+
         stdin, stdout, stderr = client.exec_command('cat %(csv_filename)s' % params)
         response['request_time_cdf'] = []
         for row in csv.DictReader(stdout):
@@ -319,8 +328,6 @@ def _attack(params):
             sftp = client.open_sftp()
             sftp.get(params['tsv_filename'], response['tsv_filename'])
 
-        print 'Bee %i is out of ammo.' % params['i']
-
         client.close()
 
         return response
@@ -448,7 +455,8 @@ def _print_results(results, params, csv_filename, gnuplot_filename, stats_filena
                 writer.writerow(row)
 
     if gnuplot_filename:
-        files = [r['tsv_filename'] for r in results]
+        print 'Joining gnuplot files from all bees.'
+        files = [r['tsv_filename'] for r in results if r is not None]
         # using csvkit utils to join the tsv files from all of the bees, adding a column to show whic bee produced each line. using sort because of performance problems with csvsort.
         command = "csvstack -t -n bee -g " + ",".join(["%(i)s" % p for p in complete_bees_params]) + " " + " ".join(files) + " | csvcut -c 2-7,1 | sort -nk 5 -t ',' | sed 's/,/\t/g' > " + gnuplot_filename
         call(command, shell=True)
@@ -456,6 +464,7 @@ def _print_results(results, params, csv_filename, gnuplot_filename, stats_filena
         call(["rm"] + files)
 
     if stats_filename:
+        print 'Calculating statistics.'
         csvstat_results = check_output(["csvstat", "-tc", "ttime", gnuplot_filename])
         min_search = re.search('\sMin:\s+([0-9]+)', csvstat_results)
         max_search = re.search('\sMax:\s+([0-9]+)', csvstat_results)

From f7503281fb95dcc204f971eb3d6f9b6a733704a6 Mon Sep 17 00:00:00 2001
From: Ephraim 
Date: Sun, 11 Aug 2013 10:42:21 +0000
Subject: [PATCH 23/25] Added search for security group in vpcs when not
 finding one with the default search to allow for AWS's default vpc

---
 beeswithmachineguns/bees.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py
index 0761985..14d0202 100755
--- a/beeswithmachineguns/bees.py
+++ b/beeswithmachineguns/bees.py
@@ -82,7 +82,7 @@ def _get_security_group_ids(connection, security_group_names, subnet):
     ids = []
     # Since we cannot get security groups in a vpc by name, we get all security groups and parse them by name later
     security_groups = connection.get_all_security_groups()
-	
+
     # Parse the name of each security group and add the id of any match to the group list
     for group in security_groups:
         for name in security_group_names:
@@ -92,6 +92,14 @@ def _get_security_group_ids(connection, security_group_names, subnet):
                         ids.append(group.id)
                 elif group.vpc_id != None:
                     ids.append(group.id)
+    if not ids:
+        print "Couldn't find security group, probably because you have a default vpc, looking for vpc security groups"
+        for group in security_groups:
+            for name in security_group_names:
+                if group.name == name:
+                    ids.append(group.id)
+    if not ids:
+        print "Warning: Couldn't find security group, using default!!!"
     return ids
 
 # Methods

From 1b9eeed89b4ecf82fdeeb6854e7d32cb12df6837 Mon Sep 17 00:00:00 2001
From: Ephraim 
Date: Sun, 18 Aug 2013 14:20:37 +0300
Subject: [PATCH 24/25] added the option to test multiple URLs in the same test

---
 beeswithmachineguns/bees.py | 18 ++++++++++++------
 beeswithmachineguns/main.py | 15 ++++++++-------
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py
index 14d0202..640f719 100755
--- a/beeswithmachineguns/bees.py
+++ b/beeswithmachineguns/bees.py
@@ -227,7 +227,7 @@ def _attack(params):
             key_filename=_get_pem_path(params['key_name']),
             compress=use_compression)
 
-        print 'Bee %i is firing her machine gun. Bang bang!' % params['i']
+        print 'Bee %i is firing her machine gun at (%s). Bang bang!' % (params['i'], params['url'])
 
         options = ''
         if params['headers'] is not '':
@@ -503,7 +503,7 @@ def _print_results(results, params, csv_filename, gnuplot_filename, stats_filena
                 writer.writeheader()
             writer.writerow(stats)
     
-def attack(url, n, c, t, **options):
+def attack(urls, n, c, t, **options):
     """
     Test the root url of this site.
     """
@@ -560,6 +560,11 @@ def attack(url, n, c, t, **options):
         print 'bees: error: the number of concurrent requests must be at least %d (num. instances)' % instance_count
         return
     connections_per_instance = int(float(c) / instance_count)
+    if instance_count < len(urls):
+        print "bees: error: the number of urls (%d) can't exceed the number of bees (%d)" % (len(urls), instance_count)
+        return
+    if instance_count % len(urls):
+       print "bees: warning: the load will not be evenly distributed between the urls because they can't be evenly divided between the bees [(%d bees) mod (%d urls) != 0]" % (instance_count, len(urls))
     if t > 0:
         print 'Each of %i bees will fire for %s seconds, %s at a time.' % (instance_count, t, connections_per_instance)
         requests_per_instance = 50000;
@@ -582,7 +587,7 @@ def attack(url, n, c, t, **options):
             'i': i,
             'instance_id': instance.id,
             'instance_name': instance.public_dns_name,
-            'url': url,
+            'url': urls[i % len(urls)],
             'concurrent_requests': connections_per_instance,
             'num_requests': requests_per_instance,
             'timelimit': t,
@@ -594,14 +599,15 @@ def attack(url, n, c, t, **options):
             'gnuplot_filename': gnuplot_filename,
         })
 
-    print 'Stinging URL so it will be cached for the attack.'
+    print 'Stinging URLs so they will be cached for the attack.'
 
     # Ping url so it will be cached for testing
     dict_headers = {}
     if headers is not '':
         dict_headers = headers = dict(h.split(':') for h in headers.split(';'))
-    request = urllib2.Request(url, headers=dict_headers)
-    urllib2.urlopen(request).read()
+    for url in urls:
+        request = urllib2.Request(url, headers=dict_headers)
+        urllib2.urlopen(request).read()
 
     print 'Organizing the swarm.'
 
diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py
index 8ed02d2..264ba88 100644
--- a/beeswithmachineguns/main.py
+++ b/beeswithmachineguns/main.py
@@ -60,7 +60,7 @@ def parse_options():
         Beginning an attack requires only that you specify the -u option with the URL you wish to target.""")
 
     # Required
-    attack_cmd.add_argument('-u', '--url', metavar="URL", dest='url', required=True, help="URL of the target to attack.")
+    attack_cmd.add_argument('-u', '--url', metavar="URL", dest='urls', action='append', required=True, help="URL(s) of the target to attack.")
 
     attack_cmd.add_argument('-p', '--post-file',  metavar="POST_FILE", dest='post_file', default=False, help="The POST file to deliver with the bee's payload.")
     attack_cmd.add_argument('-m', '--mime-type',  metavar="MIME_TYPE", dest='mime_type', default='text/plain', help="The MIME type to send with the request.")
@@ -90,12 +90,13 @@ def parse_options():
  
         bees.up(options.servers, options.group, options.zone, options.instance, options.type, options.login, options.key, options.subnet)
     elif command == 'attack':
-        parsed = urlparse(options.url)
-        if not parsed.scheme:
-            parsed = urlparse("http://" + options.url)
+        for url in options.urls:
+            parsed = urlparse(url)
+            if not parsed.scheme:
+                parsed = urlparse("http://" + url)
 
-        if not parsed.path:
-            parser.error('It appears your URL lacks a trailing slash, this will disorient the bees. Please try again with a trailing slash.')
+            if not parsed.path:
+                parser.error('It appears your URL lacks a trailing slash, this will disorient the bees. Please try again with a trailing slash.')
 
         additional_options = dict(
             headers=options.headers,
@@ -108,7 +109,7 @@ def parse_options():
             non_200_is_failure=options.non_200_is_failure,
         )
 
-        bees.attack(options.url, options.number, options.concurrent, options.timelimit, **additional_options)
+        bees.attack(options.urls, options.number, options.concurrent, options.timelimit, **additional_options)
 
     elif command == 'down':
         bees.down()

From bb2a3292c6c91b886c2cc50d87cbad2b3d335292 Mon Sep 17 00:00:00 2001
From: Ephraim Ofir 
Date: Sun, 25 Oct 2015 17:27:25 +0200
Subject: [PATCH 25/25] Added the option to use multiple post files in the same
 test. Not using keepalive in order to better represent distributed load.
 Using newer ab version (2.4) options for ignoring length errors and shorter
 socket timeout. Improved example gnuplot script and added some new ones.

---
 beeswithmachineguns/bees.py | 75 +++++++++++++++++++++++++++----------
 beeswithmachineguns/main.py |  4 +-
 examples/LoadTest.gpi       | 32 +++++++++++-----
 examples/LoadTestIter.gpi   | 71 +++++++++++++++++++++++++++++++++++
 examples/LoadTestNames.gpi  | 71 +++++++++++++++++++++++++++++++++++
 5 files changed, 222 insertions(+), 31 deletions(-)
 create mode 100644 examples/LoadTestIter.gpi
 create mode 100644 examples/LoadTestNames.gpi

diff --git a/beeswithmachineguns/bees.py b/beeswithmachineguns/bees.py
index 640f719..18698f9 100755
--- a/beeswithmachineguns/bees.py
+++ b/beeswithmachineguns/bees.py
@@ -25,9 +25,10 @@
 """
 
 from multiprocessing import Pool
-from subprocess import check_output, call
+from subprocess import check_output, call, CalledProcessError
 from collections import OrderedDict
 from tempfile import NamedTemporaryFile
+#from uuid import uuid4
 import os
 import re
 import socket
@@ -227,7 +228,7 @@ def _attack(params):
             key_filename=_get_pem_path(params['key_name']),
             compress=use_compression)
 
-        print 'Bee %i is firing her machine gun at (%s). Bang bang!' % (params['i'], params['url'])
+        print 'Bee %i is firing her machine gun (post file: %s) at (%s). Bang bang!' % (params['i'], params['post_file'], params['url'])
 
         options = ''
         if params['headers'] is not '':
@@ -254,22 +255,32 @@ def _attack(params):
         if params['post_file']:
             pem_file_path=_get_pem_path(params['key_name'])
             os.system("scp -q -o 'StrictHostKeyChecking=no' -i %s %s %s@%s:/tmp/honeycomb" % (pem_file_path, params['post_file'], params['username'], params['instance_name']))
-            options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params
+            options += ' -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params
+            #random_command = "sed -i 's/RANDOM/%s/' /tmp/honeycomb && cat /tmp/honeycomb" % str(uuid4())
+            #stdin, stdout, stderr = client.exec_command(random_command)
+            #print 'posting file: %s' % stdout.read()
+            #options += ' -k -T "%(mime_type)s; charset=UTF-8" -p /tmp/honeycomb' % params
 
         params['options'] = options
         if params['timelimit'] > 0:
-            benchmark_command = 'ab -r -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+            benchmark_command = 'ab -l -r -s 3 -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+            #benchmark_command = './ab -l 1000 -r -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+            #benchmark_command = 'ab -r -t %(timelimit)s -n 5000000 -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
         else:
-            benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+            benchmark_command = './ab -l -r -s 3 -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+            #benchmark_command = './ab -l 1000 -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
+            #benchmark_command = 'ab -r -n %(num_requests)s -c %(concurrent_requests)s -C "sessionid=NotARealSessionID" %(options)s "%(url)s"' % params
         stdin, stdout, stderr = client.exec_command(benchmark_command)
 
         response = {}
 
         ab_results = stdout.read()
+        ab_error = stderr.read()
         ms_per_request_search = re.search('Time\ per\ request:\s+([0-9.]+)\ \[ms\]\ \(mean\)', ab_results)
 
         if not ms_per_request_search:
-            print 'Bee %i lost sight of the target (connection timed out running ab).' % params['i']
+            #print 'Bee %i lost sight of the target (connection timed out running ab).' % params['i']
+            print 'Bee %i lost sight of the target (connection timed out running ab). ab command: [%s] \nresult: [%s]\nerror:[%s].' % (params['i'], benchmark_command, ab_results, ab_error)
             return None
 
         requests_per_second_search = re.search('Requests\ per\ second:\s+([0-9.]+)\ \[#\/sec\]\ \(mean\)', ab_results)
@@ -465,15 +476,29 @@ def _print_results(results, params, csv_filename, gnuplot_filename, stats_filena
     if gnuplot_filename:
         print 'Joining gnuplot files from all bees.'
         files = [r['tsv_filename'] for r in results if r is not None]
-        # using csvkit utils to join the tsv files from all of the bees, adding a column to show whic bee produced each line. using sort because of performance problems with csvsort.
-        command = "csvstack -t -n bee -g " + ",".join(["%(i)s" % p for p in complete_bees_params]) + " " + " ".join(files) + " | csvcut -c 2-7,1 | sort -nk 5 -t ',' | sed 's/,/\t/g' > " + gnuplot_filename
+        # using csvkit utils to join the tsv files from all of the bees, adding a column to show which bee produced each line. using sort because of performance problems with csvsort.
+        #command = "csvstack -t -n bee -g " + ",".join(["%(i)s" % p for p in complete_bees_params]) + " " + " ".join(files) + " | csvcut -c 2-7,1 | sort -nk 5 -t ',' | sed 's/,/\t/g' > " + gnuplot_filename
+        # csvkit took too long for joining files, using builtins instead
+        command = "head -1 " + files[0] + " > " + gnuplot_filename + " && cat " + " ".join(files) + " | grep -v starttime >> " + gnuplot_filename
         call(command, shell=True)
         # removing temp files
         call(["rm"] + files)
 
     if stats_filename:
         print 'Calculating statistics.'
-        csvstat_results = check_output(["csvstat", "-tc", "ttime", gnuplot_filename])
+        try:
+            csvstat_results = check_output(["csvstat", "-tc", "ttime", gnuplot_filename])
+        except CalledProcessError as e:
+            print 'Error running csvstat: %d output: [%s]' % (e.returncode, e.output)
+            csvstat_results = """
+                               Dummy values:
+                               Min: 0
+                               Max: 0
+                               Mean: 0
+                               Median: 0
+                               Standard Deviation: 0
+                              """
+
         min_search = re.search('\sMin:\s+([0-9]+)', csvstat_results)
         max_search = re.search('\sMax:\s+([0-9]+)', csvstat_results)
         mean_search = re.search('\sMean:\s+([0-9.]+)', csvstat_results)
@@ -557,14 +582,22 @@ def attack(urls, n, c, t, **options):
     instance_count = len(instances)
 
     if c < instance_count:
-        print 'bees: error: the number of concurrent requests must be at least %d (num. instances)' % instance_count
-        return
+        instance_count = c
+        del instances[c:]
+        print 'bees: warning: the number of concurrent requests is lower than the number of bees, only %d of the bees will be used' % instance_count
     connections_per_instance = int(float(c) / instance_count)
     if instance_count < len(urls):
         print "bees: error: the number of urls (%d) can't exceed the number of bees (%d)" % (len(urls), instance_count)
         return
     if instance_count % len(urls):
        print "bees: warning: the load will not be evenly distributed between the urls because they can't be evenly divided between the bees [(%d bees) mod (%d urls) != 0]" % (instance_count, len(urls))
+    post_files = options.get('post_files')
+    if post_files:
+        if instance_count < len(post_files):
+            print "bees: error: the number of post_files (%d) can't exceed the number of bees (%d)" % (len(post_files), instance_count)
+            return
+        if instance_count % len(post_files):
+            print "bees: warning: the load will not be evenly distributed between the post_files because they can't be evenly divided between the bees [(%d bees) mod (%d post_files) != 0]" % (instance_count, len(post_files))
     if t > 0:
         print 'Each of %i bees will fire for %s seconds, %s at a time.' % (instance_count, t, connections_per_instance)
         requests_per_instance = 50000;
@@ -583,31 +616,35 @@ def attack(urls, n, c, t, **options):
     params = []
 
     for i, instance in enumerate(instances):
+        post_file = False
+        if post_files:
+            post_file = post_files[len(post_files) - (i % len(post_files)) - 1] # reverse iteration so it won't coinside with the urls iteration
         params.append({
             'i': i,
             'instance_id': instance.id,
             'instance_name': instance.public_dns_name,
             'url': urls[i % len(urls)],
+            #'url': urls[i % len(urls)] + "?uuid=" + str(uuid4()),
             'concurrent_requests': connections_per_instance,
             'num_requests': requests_per_instance,
             'timelimit': t,
             'username': username,
             'key_name': key_name,
             'headers': headers,
-            'post_file': options.get('post_file'),
+            'post_file': post_file,
             'mime_type': options.get('mime_type', ''),
             'gnuplot_filename': gnuplot_filename,
         })
 
-    print 'Stinging URLs so they will be cached for the attack.'
+#    print 'Stinging URLs so they will be cached for the attack.'
 
     # Ping url so it will be cached for testing
-    dict_headers = {}
-    if headers is not '':
-        dict_headers = headers = dict(h.split(':') for h in headers.split(';'))
-    for url in urls:
-        request = urllib2.Request(url, headers=dict_headers)
-        urllib2.urlopen(request).read()
+#    dict_headers = {}
+#    if headers is not '':
+#        dict_headers = headers = dict(h.split(':') for h in headers.split(';'))
+#    for url in urls:
+#        request = urllib2.Request(url, headers=dict_headers)
+#        urllib2.urlopen(request).read()
 
     print 'Organizing the swarm.'
 
diff --git a/beeswithmachineguns/main.py b/beeswithmachineguns/main.py
index 264ba88..db0931d 100644
--- a/beeswithmachineguns/main.py
+++ b/beeswithmachineguns/main.py
@@ -62,7 +62,7 @@ def parse_options():
     # Required
     attack_cmd.add_argument('-u', '--url', metavar="URL", dest='urls', action='append', required=True, help="URL(s) of the target to attack.")
 
-    attack_cmd.add_argument('-p', '--post-file',  metavar="POST_FILE", dest='post_file', default=False, help="The POST file to deliver with the bee's payload.")
+    attack_cmd.add_argument('-p', '--post-file',  metavar="POST_FILE", dest='post_files', action='append', help="The POST file(s) to deliver with the bee's payload.")
     attack_cmd.add_argument('-m', '--mime-type',  metavar="MIME_TYPE", dest='mime_type', default='text/plain', help="The MIME type to send with the request.")
     attack_cmd.add_argument('-n', '--number', metavar="NUMBER", dest='number', type=int, default=1000, help="The number of total connections to make to the target (default: 1000).")
     attack_cmd.add_argument('-c', '--concurrent', metavar="CONCURRENT", dest='concurrent', type=int, default=100, help="The number of concurrent connections to make to the target (default: 100).")
@@ -100,7 +100,7 @@ def parse_options():
 
         additional_options = dict(
             headers=options.headers,
-            post_file=options.post_file,
+            post_files=options.post_files,
             mime_type=options.mime_type,
             csv_filename=options.csv_filename,
             gnuplot_filename=options.gnuplot_filename,
diff --git a/examples/LoadTest.gpi b/examples/LoadTest.gpi
index 9c356b8..38bddf6 100644
--- a/examples/LoadTest.gpi
+++ b/examples/LoadTest.gpi
@@ -9,7 +9,7 @@ set terminal jpeg size 1440,900
 
 # This sets the aspect ratio of the graph
 set size 1, 1
-set lmargin 10
+set lmargin 12
 set rmargin 10
 
 set output filename.'.jpg'
@@ -17,12 +17,13 @@ set output filename.'.jpg'
 # Where to place the legend/key
 set key left top
 
-set multiplot layout 2, 1 title filename
+set multiplot layout 2, 1 title filename font "Bold,20"
 
 # Draw gridlines oriented on the y axis
 set grid y
 # Label the x-axis
-set xlabel 'Concurrent Users'
+#set xlabel 'Iteration'
+set xlabel 'Concurrent Requests'
 # Tell gnuplot to use commas as the delimiter instead of spaces (default)
 set datafile separator ','
 set key autotitle columnhead
@@ -30,23 +31,34 @@ set key autotitle columnhead
 #
 # first graph
 #
-set title "Requests/Second and % Errors"
-set ytics nomirror 
-set y2tics 
-set ylabel 'Requests/Second' 
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
 set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
 
 # Plot the data
 plot filename.'.csv' using 1:7 with lines lt 5 lw 3 axes x1y1, \
-             ''      using 1:8 with lines lt 2 lw 3 axes x1y1, \
-             ''      using 1:($6*100) with lines lt 1 lw 3 axes x1y2
+             ''      using 1:($7-($7-$8)/2) with lines lt 2 lw 3 axes x1y1, \
+             ''      using 1:($6*50) with lines lt 1 lw 2 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+#             ''      using 1:8 with lines lt 2 lw 3 axes x1y1, \
+#             ''      using 1:($6*100) with lines lt 1 lw 3 axes x1y2
 unset y2tics 
 unset y2label
+set yrange [*:*]
 
 #
 # second graph
 #
-set title "Response Time"
+set title "Response Time" font "Bold,14"
+unset ylabel
 set ylabel "ms"
 
 set bars 4.0
diff --git a/examples/LoadTestIter.gpi b/examples/LoadTestIter.gpi
new file mode 100644
index 0000000..9197558
--- /dev/null
+++ b/examples/LoadTestIter.gpi
@@ -0,0 +1,71 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename=''" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 12
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename font "Bold,20"
+
+# Draw gridlines oriented on the y axis
+set grid y
+set xtics 1
+# Label the x-axis
+set xlabel 'Iteration'
+#set xlabel 'Concurrent Requests'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
+set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
+
+# Plot the data
+plot filename.'.csv' using 1:7 with lines lt 5 lw 3 axes x1y1, \
+             ''      using 1:($7-($7-$8)/2) with lines lt 2 lw 3 axes x1y1, \
+             ''      using 1:($6*50) with lines lt 1 lw 3 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+#             ''      using 1:8 with lines lt 2 lw 3 axes x1y1, \
+#             ''      using 1:($6*100) with lines lt 1 lw 3 axes x1y2
+unset y2tics 
+unset y2label
+set yrange [*:*]
+
+#
+# second graph
+#
+set title "Response Time" font "Bold,14"
+unset ylabel
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:15:9:32:31 with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+          ''         using 1:12:12:12:12 with candlesticks lt -1 notitle,\
+          ''         using 1:11 with lines lt -1 lw 3
diff --git a/examples/LoadTestNames.gpi b/examples/LoadTestNames.gpi
new file mode 100644
index 0000000..c1e6473
--- /dev/null
+++ b/examples/LoadTestNames.gpi
@@ -0,0 +1,71 @@
+# script to generate graphs from a load test done by Bees with Machine Guns
+#
+# usage:
+# gnuplot -e "filename=''" LoadTest.gpi
+#
+
+# output to a jpeg file
+set terminal jpeg size 1440,900
+
+# This sets the aspect ratio of the graph
+set size 1, 1
+set lmargin 12
+set rmargin 10
+
+set output filename.'.jpg'
+
+# Where to place the legend/key
+set key left top
+
+set multiplot layout 2, 1 title filename font "Bold,20"
+
+# Draw gridlines oriented on the y axis
+set grid y
+# Label the x-axis
+set xlabel 'Iteration'
+#set xlabel 'Concurrent Requests'
+# Tell gnuplot to use commas as the delimiter instead of spaces (default)
+set datafile separator ','
+set key autotitle columnhead
+
+#
+# first graph
+#
+set title "Requests/Second(green) and % Errors(red)" font "Bold,14"
+set ytics nomirror
+set y2tics
+set ylabel 'Requests/Second' textcolor lt 2
+set y2label 'Error Percentage' textcolor lt 1
+set decimal locale
+#set format "%'.0f"
+set format "%'g"
+set format y2 "%g %%"
+set yrange [0:]
+set y2range [0:10]
+#set boxwidth 0.5
+#set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:8:xtic(2) with lines lt 5 lw 3 axes x1y1, \
+             ''      using 1:($8-($8-$9)/2) with lines lt 2 lw 3 axes x1y1, \
+             ''      using 1:($7*50) with lines lt 1 lw 3 axes x1y2
+# the creative arithmetic above is done in order to overcome a bug in ab in which it counts each error twice, and since successful hits are calculated as total-bad it also has to be fixed.
+unset y2tics
+unset y2label
+set yrange [*:*]
+#unset boxwidth
+
+#
+# second graph
+#
+set title "Response Time" font "Bold,14"
+unset ylabel
+set ylabel "ms"
+
+set bars 4.0
+set style fill solid
+
+# Plot the data
+plot filename.'.csv' using 1:16:10:33:32:xtic(2) with candlesticks lt 2 title 'Min/P10/Med/P90/P95' whiskerbars 0.6, \
+          ''         using 1:13:13:13:13 with candlesticks lt -1 notitle,\
+          ''         using 1:12 with lines lt -1 lw 3