From 765f50bdb14a9d921524932b550c4419a3f772ba Mon Sep 17 00:00:00 2001 From: Max Lincoln Date: Tue, 27 Aug 2013 23:49:33 -0300 Subject: [PATCH 1/2] Multi-provider refactor --- Gemfile | 1 + lib/microstatic.rb | 1 - lib/microstatic/base_deployer.rb | 69 +++++++++++++++++++ lib/microstatic/s3_bucket_creator.rb | 17 ----- lib/microstatic/s3_deployer.rb | 47 ++----------- lib/microstatic/uses_fog.rb | 14 ++-- microstatic.gemspec | 2 +- spec/integration/fixtures/subdir/foo.txt | 1 + spec/integration/spec_helper.rb | 11 ++- .../integration/upload_to_test_bucket_spec.rb | 28 +++++--- 10 files changed, 111 insertions(+), 80 deletions(-) create mode 100644 lib/microstatic/base_deployer.rb delete mode 100644 lib/microstatic/s3_bucket_creator.rb diff --git a/Gemfile b/Gemfile index 0855092..6ab3724 100644 --- a/Gemfile +++ b/Gemfile @@ -2,3 +2,4 @@ source 'https://rubygems.org' # Specify your gem's dependencies in microstatic.gemspec gemspec +gem 'fog', :git => 'git://github.com/fog/fog', :branch => 'storage_mocks' diff --git a/lib/microstatic.rb b/lib/microstatic.rb index 93508e3..c8c6839 100644 --- a/lib/microstatic.rb +++ b/lib/microstatic.rb @@ -3,7 +3,6 @@ require 'microstatic/uses_fog' require "microstatic/s3_deployer" -require "microstatic/s3_bucket_creator" module Microstatic def self.aws_creds_from_env diff --git a/lib/microstatic/base_deployer.rb b/lib/microstatic/base_deployer.rb new file mode 100644 index 0000000..8b0f4d6 --- /dev/null +++ b/lib/microstatic/base_deployer.rb @@ -0,0 +1,69 @@ +require 'digest/md5' +require 'pathname' + +module Microstatic + +# The following is based on code generously +# shared by Giles Alexander (@gga) +class BaseDeployer + include UsesFog + def initialize( local_dir, bucket, creds ) + store_creds(creds) + + @local_dir = Pathname.new(local_dir) + @bucket = bucket + end + + def self.factory( local_dir, bucket, creds ) + case creds[:provider] + when 'AWS' + S3Deployer.new local_dir, bucket, creds + else + BaseDeployer.new local_dir, bucket, creds + end + end + + def upload + Pathname.glob(@local_dir+"**/*") do |child| + upload_file(child) unless child.directory? + end + end + + def file_metadata + # Override this if you need more! + { :public => true } + end + + def upload_file( file ) + file_key = file.relative_path_from(@local_dir).to_s + + begin + file_object = connection.head_object(@bucket,file_key) + rescue Excon::Errors::NotFound + file_object = false + end + + if !file_object + log_action('CREATE', file_key) + connection.put_object( @bucket, file_key, file.open, file_metadata ) + else + etag = file_object.headers['ETag'] || file_object.headers['Etag'] + file_md5 = etag.sub(/"(.*)"/,'\1') + local_md5 = Digest::MD5.hexdigest( file.read ) + + if( file_md5 == local_md5 ) + log_action('NO CHANGE', file_key) + else + log_action('UPDATE', file_key) + connection.put_object( @bucket, file_key, file.open ) + end + end + end + + def log_action(action,file) + message = action.to_s.rjust(10) + " " + file + puts message + end +end + +end diff --git a/lib/microstatic/s3_bucket_creator.rb b/lib/microstatic/s3_bucket_creator.rb deleted file mode 100644 index f55a662..0000000 --- a/lib/microstatic/s3_bucket_creator.rb +++ /dev/null @@ -1,17 +0,0 @@ -module Microstatic - -class S3BucketCreator - include UsesFog - - def initialize( aws_creds ) - check_and_store_aws_creds(aws_creds) - end - - def create( bucket_name ) - connection.put_bucket( bucket_name ) - connection.put_bucket_acl( bucket_name, 'public-read' ) - connection.put_bucket_website( bucket_name, 'index.html', :key => '404.html' ) - end -end - -end diff --git a/lib/microstatic/s3_deployer.rb b/lib/microstatic/s3_deployer.rb index 71ff374..93abc58 100644 --- a/lib/microstatic/s3_deployer.rb +++ b/lib/microstatic/s3_deployer.rb @@ -1,53 +1,16 @@ require 'digest/md5' require 'pathname' +require File.expand_path '../base_deployer', __FILE__ module Microstatic # The following is based on code generously # shared by Giles Alexander (@gga) -class S3Deployer +class S3Deployer < BaseDeployer include UsesFog - def initialize( local_dir, bucket, aws_creds ) - check_and_store_aws_creds(aws_creds) - - @local_dir = Pathname.new(local_dir) - @bucket = bucket - end - - def upload - Pathname.glob(@local_dir+"**/*") do |child| - upload_file(child) unless child.directory? - end - end - - def upload_file( file ) - s3_key = file.relative_path_from(@local_dir).to_s - - begin - s3_object = connection.head_object(@bucket,s3_key) - rescue Excon::Errors::NotFound - s3_object = false - end - - if !s3_object - log_action('CREATE', s3_key) - connection.put_object( @bucket, s3_key, file.open, 'x-amz-acl' => 'public-read' ) - else - s3_md5 = s3_object.headers['ETag'].sub(/"(.*)"/,'\1') - local_md5 = Digest::MD5.hexdigest( file.read ) - - if( s3_md5 == local_md5 ) - log_action('NO CHANGE', s3_key) - else - log_action('UPDATE', s3_key) - connection.put_object( @bucket, s3_key, file.open ) - end - end - end - - def log_action(action,file) - message = action.to_s.rjust(10) + " " + file - puts message + def initialize( local_dir, bucket, creds ) + check_and_store_aws_creds(creds) + super(local_dir, bucket, creds) end end diff --git a/lib/microstatic/uses_fog.rb b/lib/microstatic/uses_fog.rb index 079e4d3..de5673c 100644 --- a/lib/microstatic/uses_fog.rb +++ b/lib/microstatic/uses_fog.rb @@ -3,19 +3,19 @@ module Microstatic module UsesFog def check_and_store_aws_creds( aws_creds ) - [:access_key_id,:secret_access_key].each do |required_key| + [:aws_access_key_id,:aws_secret_access_key].each do |required_key| raise ArgumentError, "must supply :#{required_key}" unless aws_creds.key?(required_key) end - @aws_creds = aws_creds + store_creds(aws_creds.merge({:provider => 'AWS'})) + end + + def store_creds(creds) + @creds = creds end def connection - @_connection ||= Fog::Storage.new({ - :provider => 'AWS', - :aws_access_key_id => @aws_creds.fetch(:access_key_id), - :aws_secret_access_key => @aws_creds.fetch(:secret_access_key) - }) + @_connection ||= Fog::Storage.new @creds end end end diff --git a/microstatic.gemspec b/microstatic.gemspec index 311157c..9a0d167 100644 --- a/microstatic.gemspec +++ b/microstatic.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |gem| gem.require_paths = ["lib"] gem.version = Microstatic::VERSION - gem.add_runtime_dependency "fog", ">=1" + # gem.add_runtime_dependency "fog", ">=1" gem.add_development_dependency "rake" gem.add_development_dependency "bundler" diff --git a/spec/integration/fixtures/subdir/foo.txt b/spec/integration/fixtures/subdir/foo.txt index e69de29..8bd6648 100644 --- a/spec/integration/fixtures/subdir/foo.txt +++ b/spec/integration/fixtures/subdir/foo.txt @@ -0,0 +1 @@ +asdf diff --git a/spec/integration/spec_helper.rb b/spec/integration/spec_helper.rb index 477da5a..90792c4 100644 --- a/spec/integration/spec_helper.rb +++ b/spec/integration/spec_helper.rb @@ -2,9 +2,14 @@ # require_relative '../../lib/microstatic/uses_fog' if ENV['FOG_MOCK'] + %w{RAX_USERNAME RAX_API_KEY AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY}.each do |key| + ENV[key] = 'dummy' + end Fog.mock! - include Microstatic::UsesFog - check_and_store_aws_creds(Microstatic.aws_creds_from_env) - connection.directories.create(:key => 'microstatic-test-bucket') +end + +def create_bucket(bucket, creds) + puts "Creating #{bucket}" + Fog::Storage.new(creds).directories.create(:key => bucket) end require 'pry' diff --git a/spec/integration/upload_to_test_bucket_spec.rb b/spec/integration/upload_to_test_bucket_spec.rb index 8343d7d..d3fa459 100644 --- a/spec/integration/upload_to_test_bucket_spec.rb +++ b/spec/integration/upload_to_test_bucket_spec.rb @@ -1,4 +1,5 @@ require_relative 'spec_helper' +require_relative '../../lib/microstatic/base_deployer' require_relative '../../lib/microstatic/s3_deployer' module Microstatic @@ -11,17 +12,26 @@ def test_dir File.expand_path( "../fixtures", __FILE__ ) end - def aws_creds - { - :access_key_id => ENV.fetch('AWS_ACCESS_KEY_ID'), - :secret_access_key => ENV.fetch('AWS_SECRET_ACCESS_KEY') + creds = { + :aws => { + :provider => 'AWS', + :aws_access_key_id => ENV.fetch('AWS_ACCESS_KEY_ID'), + :aws_secret_access_key => ENV.fetch('AWS_SECRET_ACCESS_KEY') + }, + :rax => { + :provider => 'Rackspace', + :rackspace_username => ENV.fetch('RAX_USERNAME'), + :rackspace_api_key => ENV.fetch('RAX_API_KEY'), } - end + } - it 'succeeds' do - deployer = S3Deployer.new( test_dir, test_bucket, aws_creds ) - deployer.upload + creds.each do |provider, creds| + it "succeeds (#{provider})" do + create_bucket(test_bucket, creds) # if Fog.mock? + deployer = BaseDeployer.factory( test_dir, test_bucket, creds ) + deployer.upload + end end - + end end From 314b3d46381020625e90493f1532e170322efa3b Mon Sep 17 00:00:00 2001 From: Max Lincoln Date: Tue, 27 Aug 2013 23:51:25 -0300 Subject: [PATCH 2/2] Whoops - put fog dependencies back how they were --- Gemfile | 1 - microstatic.gemspec | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Gemfile b/Gemfile index 6ab3724..0855092 100644 --- a/Gemfile +++ b/Gemfile @@ -2,4 +2,3 @@ source 'https://rubygems.org' # Specify your gem's dependencies in microstatic.gemspec gemspec -gem 'fog', :git => 'git://github.com/fog/fog', :branch => 'storage_mocks' diff --git a/microstatic.gemspec b/microstatic.gemspec index 9a0d167..311157c 100644 --- a/microstatic.gemspec +++ b/microstatic.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |gem| gem.require_paths = ["lib"] gem.version = Microstatic::VERSION - # gem.add_runtime_dependency "fog", ">=1" + gem.add_runtime_dependency "fog", ">=1" gem.add_development_dependency "rake" gem.add_development_dependency "bundler"