Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions Gemfile
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
source 'https://rubygems.org'

gem 'chef', '>= 11.18.6'
gem 'berkshelf', '~> 4.0'
gem 'berkshelf-api-client', '~> 2.0'
gem 'chef', '>= 11.18.6'
gem 'stove', '~> 3.2'

group :test do
gem 'chefspec', '~> 4.2'
gem 'rspec', '~> 3.2'
gem 'foodcritic', '~> 4.0'
gem 'rspec', '~> 3.2'
gem 'rubocop', '~> 0.27.1'
end

group :integration do
gem 'test-kitchen', '~> 1.4'
gem 'kitchen-vagrant'
gem 'kitchen-sync'
gem 'kitchen-ec2', '~> 0.10'
gem 'kitchen-sync'
gem 'kitchen-vagrant'
gem 'test-kitchen', '~> 1.4'
end

group :documentation do
Expand Down
16 changes: 9 additions & 7 deletions metadata.rb
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,17 @@
license 'Apache License 2.0'
description 'A cookbook to install and configure Apache Spark'
version '1.2.14'
source_url 'https://github.com/clearstorydata-cookbooks/apache_spark'
issues_url 'https://github.com/clearstorydata-cookbooks/apache_spark/issues'
chef_version '>= 11.18.6'

%w( debian ubuntu centos redhat fedora ).each do |os|
%w(debian ubuntu centos redhat fedora).each do |os|
supports os
end

depends 'apt', '~> 2.0'
depends 'java', '~> 1.0'
depends 'logrotate', '~> 1.0'
depends 'monit_wrapper', '~> 3.0'
depends 'apt'
depends 'java'
depends 'logrotate'
depends 'monit_wrapper'
depends 'tar'

source_url 'https://github.com/clearstorydata-cookbooks/apache_spark'
issues_url 'https://github.com/clearstorydata-cookbooks/apache_spark/issues'
2 changes: 1 addition & 1 deletion recipes/find-free-port.rb
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

template '/usr/local/bin/find-free-port.rb' do
source 'find-free-port.rb.erb'
mode 0755
mode '0755'
owner 'root'
group 'root'
variables ruby_interpreter: RbConfig.ruby
Expand Down
8 changes: 4 additions & 4 deletions recipes/spark-install.rb
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@
] + local_dirs.to_a
).each do |dir|
directory dir do
mode 0755
mode '0755'
owner spark_user
group spark_group
action :create
Expand All @@ -84,7 +84,7 @@

template "#{spark_conf_dir}/spark-env.sh" do
source 'spark-env.sh.erb'
mode 0644
mode '0644'
owner spark_user
group spark_group
variables node['apache_spark']['standalone']
Expand All @@ -97,7 +97,7 @@

template "#{spark_conf_dir}/log4j.properties" do
source 'spark_log4j.properties.erb'
mode 0644
mode '0644'
owner spark_user
group spark_group
variables node['apache_spark']['standalone']
Expand All @@ -110,7 +110,7 @@

template "#{spark_conf_dir}/spark-defaults.conf" do
source 'spark-defaults.conf.erb'
mode 0644
mode '0644'
owner spark_user
group spark_group
variables options: node['apache_spark']['conf'].to_hash.merge(
Expand Down
2 changes: 1 addition & 1 deletion recipes/spark-standalone-master.rb
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

template master_runner_script do
source 'spark_master_runner.sh.erb'
mode 0744
mode '0744'
owner spark_user
group spark_group
variables node['apache_spark']['standalone'].merge(
Expand Down
6 changes: 3 additions & 3 deletions recipes/spark-standalone-worker.rb
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

template worker_runner_script do
source 'spark_worker_runner.sh.erb'
mode 0744
mode '0744'
owner spark_user
group spark_group
variables node['apache_spark']['standalone'].merge(
Expand All @@ -40,7 +40,7 @@
end

directory node['apache_spark']['standalone']['worker_work_dir'] do
mode 0755
mode '0755'
owner spark_user
group spark_group
action :create
Expand All @@ -49,7 +49,7 @@

template '/usr/local/bin/clean_spark_worker_dir.rb' do
source 'clean_spark_worker_dir.rb.erb'
mode 0755
mode '0755'
owner 'root'
group 'root'
variables ruby_interpreter: RbConfig.ruby
Expand Down
2 changes: 1 addition & 1 deletion spec/spark-standalone-master_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
it 'adds the master runner script' do
expect(chef_run).to create_template('/usr/share/spark/bin/master_runner.sh').with(
source: 'spark_master_runner.sh.erb',
mode: 0744,
mode: '0744',
owner: 'spark',
group: 'spark'
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,20 +26,32 @@
stop_spark

start_monit_service('spark-standalone-master')
assert_equal('Running', get_stable_monit_service_status('spark-standalone-master'))
assert_equal(
'Running',
get_stable_monit_service_status('spark-standalone-master')
)

# The worker should still be down.
assert_equal('Not monitored', get_stable_monit_service_status('spark-standalone-worker'))
assert_equal(
'Not monitored',
get_stable_monit_service_status('spark-standalone-worker')
)
end

it 'allows starting Spark standalone worker' do
stop_spark

start_monit_service('spark-standalone-worker')
assert_equal('Running', get_stable_monit_service_status('spark-standalone-worker'))
assert_equal(
'Running',
get_stable_monit_service_status('spark-standalone-worker')
)

# The master should still be down.
assert_equal('Not monitored', get_stable_monit_service_status('spark-standalone-master'))
assert_equal(
'Not monitored',
get_stable_monit_service_status('spark-standalone-master')
)
end

it 'allows to run a Spark program (SparkPi)' do
Expand All @@ -63,6 +75,7 @@
assert(
spark_pi_result.stdout.include?(expected_msg),
"Expected stdout to say '#{expected_msg}'. " \
"Actual stdout:\n#{spark_pi_result.stdout}\n\nStderr:\n#{spark_pi_result.stderr}")
"Actual stdout:\n#{spark_pi_result.stdout}\n\nStderr:\n#{spark_pi_result.stderr}"
)
end
end
6 changes: 3 additions & 3 deletions test/integration/default/serverspec/default_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,17 @@
end

# Spark master port
describe port(7077) do
describe port(7_077) do
it { should be_listening }
end

# Spark master WebUI port
describe port('8081') do
describe port(8_081) do
it { should be_listening }
end

# Spark worker WebUI port
describe port('18080') do
describe port(18_080) do
it { should be_listening }
end

Expand Down