From d61fcebc6636876bb51c54dfcfcc05fa0c60ccbf Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Wed, 23 Dec 2015 14:10:50 -0500 Subject: [PATCH 1/7] Add support for retries, retry delay and checksums --- Gemfile | 1 + README.md | 3 ++ libraries/github_archive.rb | 46 ++++++++++++++++++++++++++- libraries/github_asset.rb | 45 +++++++++++++++++++++++++- libraries/github_errors.rb | 10 ++++++ providers/archive.rb | 3 +- providers/asset.rb | 3 +- providers/deploy.rb | 3 ++ resources/archive.rb | 3 ++ resources/asset.rb | 4 +++ resources/deploy.rb | 3 ++ spec/libraries/github_archive_spec.rb | 8 ++--- spec/spec_helper.rb | 2 +- 13 files changed, 125 insertions(+), 9 deletions(-) diff --git a/Gemfile b/Gemfile index 47d05d6..7a6afba 100644 --- a/Gemfile +++ b/Gemfile @@ -10,4 +10,5 @@ group :test do gem 'guard-spork', platforms: :ruby gem 'coolline' gem 'fuubar' + gem 'rspec-its' end diff --git a/README.md b/README.md index f9bb571..7af82fb 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ Downloads an asset from a Github release - **owner** - owner of the downloaded asset on disk - **group** - group of the downloaded asset on disk - **force** - force downloading even if the asset already exists on disk +- **retries** - number of times to retry download +- **retry_delay** - number of seconds between attempts to download file +- **checksum** - SHA-256 of file, if the checksum does not match, the file is not used ## HTTP proxy support diff --git a/libraries/github_archive.rb b/libraries/github_archive.rb index 178e324..f51e54f 100644 --- a/libraries/github_archive.rb +++ b/libraries/github_archive.rb @@ -31,9 +31,24 @@ def initialize(fqrn, options = {}) @host = options[:host] ||= self.class.default_host end + # @param [String] filepath + def file_checksum filepath + puts filepath + Digest::SHA256.file(filepath.to_s).hexdigest + end + + # @param [String] expected + # @param [String] actual + def valid_checksum? expected, actual + expected == actual + end + # @option options [String] :user # @option options [String] :token # @option options [Boolean] :force + # @option options [Integer] :retries + # @option options [Integer] :retry_delay + # @option options [String] :checksum def download(options = {}) if options[:force] FileUtils.rm_rf(local_archive_path) @@ -45,12 +60,41 @@ def download(options = {}) open(download_uri, http_basic_authentication: [options[:user], options[:token]]) do |source| IO.copy_stream(source, file) end + + unless options[:checksum].nil? + checksum = file_checksum(options[:path]) + fail GithubCB::ChecksumMismatch.new(options[:checksum], checksum) unless valid_checksum? options[:checksum], checksum + end + + true rescue OpenURI::HTTPError => ex + FileUtils.rm_rf(options[:path]) case ex.message when /406 Not Acceptable/ raise GithubCB::AuthenticationError else - raise ex + if options[:retries] <= 0 + raise ex + else + options[:retries] -= 1 + puts "Retrying Download" + if options[:retry_delay] > 0 + sleep options[:retry_delay] + end + download options + end + end + rescue GithubCB::ChecksumMismatch => ex + puts "Failed Checksum, Retries left #{options[:retries]}" + if options[:retries] <= 0 + fail ex + else + options[:retries] -= 1 + puts "Retrying Download" + if options[:retry_delay] > 0 + sleep options[:retry_delay] + end + download options end ensure file.close unless file.nil? diff --git a/libraries/github_asset.rb b/libraries/github_asset.rb index 0046ec8..2cb1b3d 100644 --- a/libraries/github_asset.rb +++ b/libraries/github_asset.rb @@ -65,10 +65,25 @@ def asset_url(options) asset.rels[:self].href end + # @param [String] filepath + def file_checksum filepath + puts filepath + Digest::SHA256.file(filepath.to_s).hexdigest + end + + # @param [String] expected + # @param [String] actual + def valid_checksum? expected, actual + expected == actual + end + # @option options [String] :path # @option options [String] :user # @option options [String] :token # @option options [Boolean] :force + # @option options [Integer] :retries + # @option options [Integer] :retry_delay + # @option options [String] :checksum def download(options = {}) if options[:force] FileUtils.rm_rf(options[:path]) @@ -92,13 +107,41 @@ def download(options = {}) file = ::File.open(options[:path], "wb") open(res['location']) { |source| IO.copy_stream(source, file) } + + unless options[:checksum].nil? + checksum = file_checksum(options[:path]) + fail GithubCB::ChecksumMismatch.new(options[:checksum], checksum) unless valid_checksum? options[:checksum], checksum + end + true rescue OpenURI::HTTPError => ex + FileUtils.rm_rf(options[:path]) case ex.message when /406 Not Acceptable/ raise GithubCB::AuthenticationError else - raise ex + if options[:retries] <= 0 + raise ex + else + options[:retries] -= 1 + puts "Retrying Download" + if options[:retry_delay] > 0 + sleep options[:retry_delay] + end + download options + end + end + rescue GithubCB::ChecksumMismatch => ex + puts "Failed Checksum, Retries left #{options[:retries]}" + if options[:retries] <= 0 + fail ex + else + options[:retries] -= 1 + puts "Retrying Download" + if options[:retry_delay] > 0 + sleep options[:retry_delay] + end + download options end ensure file.close unless file.nil? diff --git a/libraries/github_errors.rb b/libraries/github_errors.rb index 57666b8..d0f7a86 100644 --- a/libraries/github_errors.rb +++ b/libraries/github_errors.rb @@ -13,6 +13,16 @@ def initialize end end + class ChecksumMismatch < GHError + attr_reader :expected + attr_reader :actual + def initialize(expected="", actual="") + @expected = expected + @actual = actual + super("expected: #{expected}, actual: #{actual}") + end + end + class ReleaseNotFound < GHError; end class AssetNotFound < GHError; end end diff --git a/providers/archive.rb b/providers/archive.rb index 453be13..86067ff 100644 --- a/providers/archive.rb +++ b/providers/archive.rb @@ -21,7 +21,8 @@ def load_current_resource unless !new_resource.force || archive.downloaded? Chef::Log.info "github_archive[#{new_resource.name}] downloading archive" archive.download(user: new_resource.github_user, token: new_resource.github_token, - force: new_resource.force) + force: new_resource.force, retries: new_resource.retries, + retry_delay: new_resource.retry_delay, checksum: new_resource.checksum) new_resource.updated_by_last_action(true) end diff --git a/providers/asset.rb b/providers/asset.rb index 25f9f9d..55b8584 100644 --- a/providers/asset.rb +++ b/providers/asset.rb @@ -27,7 +27,8 @@ def load_current_resource Chef::Log.info "github_asset[#{new_resource.name}] downloading asset" updated = asset.download(user: new_resource.github_user, token: new_resource.github_token, - force: new_resource.force, path: new_resource.asset_path) + force: new_resource.force, path: new_resource.asset_path, retries: new_resource.retries, + retry_delay: new_resource.retry_delay, checksum: new_resource.checksum) new_resource.updated_by_last_action(updated) end diff --git a/providers/deploy.rb b/providers/deploy.rb index e465923..3c55fb3 100644 --- a/providers/deploy.rb +++ b/providers/deploy.rb @@ -35,6 +35,9 @@ host new_resource.host extract_to new_resource.deploy_path force @should_force + retries new_resource.retries + retry_delay new_resource.retry_delay + checksum new_resource.checksum if @should_force action [ :delete, :extract ] diff --git a/resources/archive.rb b/resources/archive.rb index f25606c..23beea5 100644 --- a/resources/archive.rb +++ b/resources/archive.rb @@ -17,3 +17,6 @@ attribute :group, kind_of: String attribute :extract_to, kind_of: String, required: true attribute :force, kind_of: [TrueClass, FalseClass], default: false +attribute :retries, kind_of: Integer, default: 2 +attribute :retry_delay, kind_of: Integer, default: 1 +attribute :checksum, kind_of: String diff --git a/resources/asset.rb b/resources/asset.rb index d5e828d..864b128 100644 --- a/resources/asset.rb +++ b/resources/asset.rb @@ -16,6 +16,10 @@ attribute :owner, kind_of: String attribute :group, kind_of: String attribute :force, kind_of: [TrueClass, FalseClass], default: false +attribute :retries, kind_of: Integer, default: 2 +attribute :retry_delay, kind_of: Integer, default: 1 +attribute :checksum, kind_of: String + def asset_path GithubCB::Asset.asset_path(@repo, @release, @name) diff --git a/resources/deploy.rb b/resources/deploy.rb index fcdf36c..f0af569 100644 --- a/resources/deploy.rb +++ b/resources/deploy.rb @@ -18,6 +18,9 @@ attribute :group, kind_of: String attribute :shared_directories, kind_of: Array, default: [ "pids", "log" ] attribute :force, kind_of: [TrueClass, FalseClass], default: false +attribute :retries, kind_of: Integer, default: 2 +attribute :retry_delay, kind_of: Integer, default: 1 +attribute :checksum, kind_of: String attribute :configure, kind_of: Proc attribute :before_migrate, kind_of: Proc diff --git a/spec/libraries/github_archive_spec.rb b/spec/libraries/github_archive_spec.rb index 0c6c2dc..95a4e66 100644 --- a/spec/libraries/github_archive_spec.rb +++ b/spec/libraries/github_archive_spec.rb @@ -37,7 +37,7 @@ target = File.join(Chef::Config[:file_cache_path], "github_deploy", "archives", "berkshelf-cookbook-master.tar.gz") - expect(File.exist?(target)).to be_true + expect(File.exist?(target)).to be_truthy end end @@ -46,13 +46,13 @@ before { subject.download } it "returns true" do - expect(subject.downloaded?).to be_true + expect(subject.downloaded?).to be_truthy end end context "when it is not present on disk" do it "returns false" do - expect(subject.downloaded?).to be_false + expect(subject.downloaded?).to be_falsey end end end @@ -63,7 +63,7 @@ it "extracts the contents of the archive into the target directory" do subject.extract(target) - expect(File.exist?(File.join(target, "metadata.rb"))).to be_true + expect(File.exist?(File.join(target, "metadata.rb"))).to be_truthy end end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 599c9db..a392e09 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -3,6 +3,7 @@ require 'rspec' require 'spork' require 'chef' +require 'rspec/its' Spork.prefork do Dir[File.join(File.expand_path("../../spec/support/**/*.rb", __FILE__))].each { |f| require f } @@ -13,7 +14,6 @@ end config.mock_with :rspec - config.treat_symbols_as_metadata_keys_with_true_values = true config.filter_run focus: true config.run_all_when_everything_filtered = true From cb995893f5d079038d9be1318a17a57e2288c8b9 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Wed, 30 Dec 2015 03:49:36 -0500 Subject: [PATCH 2/7] remove a debug message and force remove file on checksum error --- libraries/github_archive.rb | 2 +- libraries/github_asset.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/github_archive.rb b/libraries/github_archive.rb index f51e54f..8bae7fc 100644 --- a/libraries/github_archive.rb +++ b/libraries/github_archive.rb @@ -33,7 +33,6 @@ def initialize(fqrn, options = {}) # @param [String] filepath def file_checksum filepath - puts filepath Digest::SHA256.file(filepath.to_s).hexdigest end @@ -85,6 +84,7 @@ def download(options = {}) end end rescue GithubCB::ChecksumMismatch => ex + FileUtils.rm_rf(options[:path]) puts "Failed Checksum, Retries left #{options[:retries]}" if options[:retries] <= 0 fail ex diff --git a/libraries/github_asset.rb b/libraries/github_asset.rb index 2cb1b3d..e009599 100644 --- a/libraries/github_asset.rb +++ b/libraries/github_asset.rb @@ -67,7 +67,6 @@ def asset_url(options) # @param [String] filepath def file_checksum filepath - puts filepath Digest::SHA256.file(filepath.to_s).hexdigest end @@ -132,6 +131,7 @@ def download(options = {}) end end rescue GithubCB::ChecksumMismatch => ex + FileUtils.rm_rf(options[:path]) puts "Failed Checksum, Retries left #{options[:retries]}" if options[:retries] <= 0 fail ex From 32466a1e7152f91a3c1de51a6e939bebc24455a6 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 9 Feb 2017 18:07:04 -0500 Subject: [PATCH 3/7] =?UTF-8?q?Switch=20to=20use=20Octokit.release=5Ffor?= =?UTF-8?q?=5Ftag=20since=20it=20doesn=E2=80=99t=20require=20paging.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 ++ .rspec | 2 ++ libraries/github_asset.rb | 8 ++----- spec/libraries/github_asset_spec.rb | 33 +++++++++++++++++++++++++++++ 4 files changed, 39 insertions(+), 6 deletions(-) create mode 100644 .rspec create mode 100644 spec/libraries/github_asset_spec.rb diff --git a/.gitignore b/.gitignore index 1fc8803..afab144 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,5 @@ bin/* .kitchen/ .kitchen.local.yml /tmp + +vendor/* diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..83e16f8 --- /dev/null +++ b/.rspec @@ -0,0 +1,2 @@ +--color +--require spec_helper diff --git a/libraries/github_asset.rb b/libraries/github_asset.rb index e009599..a3ec245 100644 --- a/libraries/github_asset.rb +++ b/libraries/github_asset.rb @@ -50,16 +50,12 @@ def asset_url(options) c.access_token = options[:token] end - release = Octokit.releases(fqrn).find do |release| - release.tag_name == tag_name - end - + release = Octokit.release_for_tag(fqrn, tag_name) raise GithubCB::ReleaseNotFound, "release not found" if release.nil? - asset = release.rels[:assets].get.data.find do |asset| + asset = release[:assets].find do |asset| asset.name == name end - raise GithubCB::AssetNotFound, "asset not found" if asset.nil? asset.rels[:self].href diff --git a/spec/libraries/github_asset_spec.rb b/spec/libraries/github_asset_spec.rb new file mode 100644 index 0000000..142212d --- /dev/null +++ b/spec/libraries/github_asset_spec.rb @@ -0,0 +1,33 @@ +require 'spec_helper' + +describe GithubCB::Asset do + describe "ClassMethods" do + describe "::new" do + let(:fqrn) { "berkshelf/berkshelf-cookbook" } + let(:options) { { + :release => "v0.3.1", + :name => "cookbooks.tar.gz" + } } + subject { described_class.new(fqrn, options) } + + its(:organization) { should eq("berkshelf") } + its(:repo) { should eq("berkshelf-cookbook") } + its(:host) { should eq("https://github.com") } + its(:tag_name) { should eq("v0.3.1") } + its(:name) { should eq("cookbooks.tar.gz") } + end + end + + subject { described_class.new("berkshelf/berkshelf-cookbook", { + :release => "v0.3.1", + :name => "cookbooks.tar.gz" + })} + + describe "#asset_url" do + let(:options) { { + } } + it "gets the url of the asset" do + expect(subject.asset_url(options)).not_to be_empty + end + end +end From 7f12565f15e78b9359b8e66bf24ebf24e193ba13 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Mon, 13 Feb 2017 13:50:59 -0500 Subject: [PATCH 4/7] Ignore vendor files and update chef super market link --- Berksfile | 2 +- chefignore | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Berksfile b/Berksfile index 26c74c6..967b9a7 100644 --- a/Berksfile +++ b/Berksfile @@ -1,3 +1,3 @@ -source "http://api.berkshelf.com" +source "https://supermarket.chef.io" metadata diff --git a/chefignore b/chefignore index a6de142..0332169 100644 --- a/chefignore +++ b/chefignore @@ -94,3 +94,6 @@ Vagrantfile # Travis # ########## .travis.yml + +vendor +vendor/* From aa353cd54c0eff2419e7d3d0c847d34513d2c1d1 Mon Sep 17 00:00:00 2001 From: Lance Hudson Date: Thu, 27 Jul 2017 17:52:22 -0400 Subject: [PATCH 5/7] Update metadata.rb --- metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.rb b/metadata.rb index bbd4f5b..de43f7c 100644 --- a/metadata.rb +++ b/metadata.rb @@ -8,4 +8,4 @@ supports "ubuntu" -depends "libarchive" +depends "libarchive", "~> 0.4.1" From 4a383e14f4541cd66ad8610dbe3e4cc885be5f76 Mon Sep 17 00:00:00 2001 From: mike-hightail Date: Wed, 28 Nov 2018 12:38:45 -0800 Subject: [PATCH 6/7] Change libarchive version --- metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.rb b/metadata.rb index de43f7c..bbd4f5b 100644 --- a/metadata.rb +++ b/metadata.rb @@ -8,4 +8,4 @@ supports "ubuntu" -depends "libarchive", "~> 0.4.1" +depends "libarchive" From ee17a6cc2931f81831a4d6e6ea49572f27d0e4ac Mon Sep 17 00:00:00 2001 From: mike-hightail Date: Wed, 28 Nov 2018 12:47:51 -0800 Subject: [PATCH 7/7] pin version to current latest --- metadata.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.rb b/metadata.rb index bbd4f5b..f8c62a7 100644 --- a/metadata.rb +++ b/metadata.rb @@ -8,4 +8,4 @@ supports "ubuntu" -depends "libarchive" +depends "libarchive", "~> 2.1.0"