From 913ea788d956c351152b0f1868625f531fc28d3e Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Sun, 17 Mar 2019 15:29:25 +0200 Subject: [PATCH 01/27] Update .travis.yml bump ruby and rails versions --- .travis.yml | 9 +++++---- Gemfile | 2 +- tasks/test.rake | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7b00bc4..e088332 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,9 @@ script: bundle exec rspec spec env: matrix: - - RAILS=5.0.0 - - RAILS=5.1.0 + - RAILS=5.1.6.2 + - RAILS=5.2.2.1 rvm: - - 2.2.5 - - 2.3.1 + - 2.3.8 + - 2.4.5 + - 2.5.5 diff --git a/Gemfile b/Gemfile index 57ec576..7ad241d 100644 --- a/Gemfile +++ b/Gemfile @@ -11,7 +11,7 @@ group :test do gem 'rails', rails_version gem 'rspec-rails' gem 'coveralls', require: false # Test coverage website. Go to https://coveralls.io - gem 'sqlite3' + gem 'sqlite3',"~> 1.3.6" gem 'launchy' gem 'database_cleaner' gem 'capybara' diff --git a/tasks/test.rake b/tasks/test.rake index 8def1e9..6699224 100644 --- a/tasks/test.rake +++ b/tasks/test.rake @@ -3,5 +3,5 @@ desc 'Creates a test rails app for the specs to run against' task :setup do require 'rails/version' system('mkdir spec/rails') unless File.exist?('spec/rails') - system "bundle exec rails new spec/rails/rails-#{Rails::VERSION::STRING} -m spec/support/rails_template.rb --skip-spring --skip-turbolinks" + system "bundle exec rails new spec/rails/rails-#{Rails::VERSION::STRING} -m spec/support/rails_template.rb --skip-spring --skip-turbolinks --skip-bootsnap" end From 1f674cc707b460c976cab89ec1cfde658e93a779 Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Sun, 17 Mar 2019 16:16:49 +0200 Subject: [PATCH 02/27] refs #163, handle errors on base --- lib/active_admin_import/import_result.rb | 5 ++++- spec/import_result_spec.rb | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/lib/active_admin_import/import_result.rb b/lib/active_admin_import/import_result.rb index 2556a95..8ff75c4 100644 --- a/lib/active_admin_import/import_result.rb +++ b/lib/active_admin_import/import_result.rb @@ -33,7 +33,10 @@ def failed_message(options = {}) limit = options[:limit] || failed.count failed.first(limit).map do |record| errors = record.errors - (errors.full_messages.zip errors.keys.map { |k| record.send k }).map { |ms| ms.join(' - ') }.join(', ') + failed_values = errors.keys.map do |key| + key == :base ? nil : record.public_send(key) + end + errors.full_messages.zip(failed_values).map { |ms| ms.compact.join(' - ') }.join(', ') end.join(' ; ') end end diff --git a/spec/import_result_spec.rb b/spec/import_result_spec.rb index 918eab4..266ceed 100644 --- a/spec/import_result_spec.rb +++ b/spec/import_result_spec.rb @@ -5,17 +5,18 @@ context 'failed_message' do let(:import_result) { ActiveAdminImport::ImportResult.new } - before do - Author.create(name: 'John', last_name: 'Doe') - Author.create(name: 'Jane', last_name: 'Roe') + let(:failed_instances) do + [ + Author.new(last_name: 'Doe').tap {|r| r.errors.add(:last_name, :taken) }, + Author.new(name: "", last_name: 'Doe').tap {|r| r.errors.add(:name, :blank); r.errors.add(:last_name, :taken) }, + Author.new.tap {|r| r.errors.add(:base, 'custom') } + ] + end + + before do @result = double \ - failed_instances: [ - # {:last_name=>["has already been taken"]} - Author.create(name: 'Jim', last_name: 'Doe'), - # {:name=>["can't be blank"], :last_name=>["has already been taken"]} - Author.create(name: nil, last_name: 'Doe') - ] + failed_instances: failed_instances end it 'should work without any failed instances' do @@ -26,7 +27,7 @@ import_result.add(@result, 4) expect(import_result.failed_message) .to eq( - "Last name has already been taken - Doe ; Name can't be blank - , Last name has already been taken - Doe" + "Last name has already been taken - Doe ; Name can't be blank - , Last name has already been taken - Doe ; custom" ) end From 5ccf95c7bd0bb8368aec235021e7087ccd11cc69 Mon Sep 17 00:00:00 2001 From: Peter Dore Date: Thu, 12 Sep 2019 10:19:14 -0700 Subject: [PATCH 03/27] Allow batch_slice_columns to work when batching Currently the `use_indexes` are set correctly only for the first batch. --- lib/active_admin_import/importer.rb | 22 ++++++++++++++-------- spec/fixtures/files/authors.csv | 6 +++--- spec/import_spec.rb | 29 +++++++++++++++++++++-------- 3 files changed, 38 insertions(+), 19 deletions(-) diff --git a/lib/active_admin_import/importer.rb b/lib/active_admin_import/importer.rb index 9d18e4e..495d691 100644 --- a/lib/active_admin_import/importer.rb +++ b/lib/active_admin_import/importer.rb @@ -70,7 +70,7 @@ def batch_replace(header_key, options) end end - # Use it when CSV file contain redundant columns + # Use this method when CSV file contains unnecessary columns # # Example: # @@ -81,16 +81,22 @@ def batch_replace(header_key, options) # end # def batch_slice_columns(slice_columns) - use_indexes = [] - headers.values.each_with_index do |val, index| - use_indexes << index if val.in?(slice_columns) + # Only set @use_indexes for the first batch so that @use_indexes are in correct + # position for subsequent batches + unless defined?(@use_indexes) + @use_indexes = [] + headers.values.each_with_index do |val, index| + @use_indexes << index if val.in?(slice_columns) + end + return csv_lines if @use_indexes.empty? + + # slice CSV headers + @headers = headers.to_a.values_at(*@use_indexes).to_h end - return csv_lines if use_indexes.empty? - # slice CSV headers - @headers = headers.to_a.values_at(*use_indexes).to_h + # slice CSV values csv_lines.map! do |line| - line.values_at(*use_indexes) + line.values_at(*@use_indexes) end end diff --git a/spec/fixtures/files/authors.csv b/spec/fixtures/files/authors.csv index b523140..a79bff3 100644 --- a/spec/fixtures/files/authors.csv +++ b/spec/fixtures/files/authors.csv @@ -1,3 +1,3 @@ -Name,Last name,Birthday -John,Doe,1986-05-01 -Jane,Roe,1988-11-16 \ No newline at end of file +Birthday,Name,Last name +1986-05-01,John,Doe +1988-11-16,Jane,Roe diff --git a/spec/import_spec.rb b/spec/import_spec.rb index 390edef..c8c2879 100644 --- a/spec/import_spec.rb +++ b/spec/import_spec.rb @@ -434,11 +434,14 @@ def upload_file!(name, ext = 'csv') end context "with slice_columns option" do + let(:batch_size) { 2 } + before do add_author_resource template_object: ActiveAdminImport::Model.new, before_batch_import: lambda { |importer| importer.batch_slice_columns(slice_columns) - } + }, + batch_size: batch_size visit "/admin/authors/import" upload_file!(:authors) end @@ -446,13 +449,23 @@ def upload_file!(name, ext = 'csv') context "slice last column and superfluous column" do let(:slice_columns) { %w(name last_name not_existing_column) } - it "should not fill `birthday` column" do - expect(Author.pluck(:name, :last_name, :birthday)).to match_array( - [ - ["Jane", "Roe", nil], - ["John", "Doe", nil] - ] - ) + shared_examples_for "birthday column removed" do + it "should not fill `birthday` column" do + expect(Author.pluck(:name, :last_name, :birthday)).to match_array( + [ + ["Jane", "Roe", nil], + ["John", "Doe", nil] + ] + ) + end + end + + it_behaves_like "birthday column removed" + + context "when doing more than one batch" do + let(:batch_size) { 1 } + + it_behaves_like "birthday column removed" end end From 5c43397081e850c8db490965a0d95aee7a3c3021 Mon Sep 17 00:00:00 2001 From: Gena M Date: Fri, 20 Sep 2019 08:18:04 +0300 Subject: [PATCH 04/27] bump 4.1.1 --- CHANGELOG.md | 4 ++++ lib/active_admin_import/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63495ef..4490aa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## [4.1.1] - 2019-09-20 +- Fix column slicing #168 | @doredesign +- Handle errors on base #163 + ## [4.1.0] - 2019-01-15 - Upgrade dependencies: `activerecord-import` to >=0.27.1 | @jkowens diff --git a/lib/active_admin_import/version.rb b/lib/active_admin_import/version.rb index 5f9317a..8984dbd 100644 --- a/lib/active_admin_import/version.rb +++ b/lib/active_admin_import/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActiveAdminImport - VERSION = '4.1.0' + VERSION = '4.1.1' end From ef68193c746b27f8bd2252c44035b5b36d665dcc Mon Sep 17 00:00:00 2001 From: hmalyi Date: Fri, 20 Sep 2019 08:26:37 +0300 Subject: [PATCH 05/27] Update .gitignore --- .gitignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 54cb8bb..b4fb8da 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,7 @@ capybara-*.html /spec/tmp/* **.orig rerun.txt -pickle-email-*.html \ No newline at end of file +pickle-email-*.html +pkg +spec/rails +Gemfile.lock From 08f5f8e8e3cebef975131234926cd67e86c28fdd Mon Sep 17 00:00:00 2001 From: Prevenios Marinos Date: Tue, 10 Dec 2019 10:31:58 +0200 Subject: [PATCH 06/27] Allow activerecord-import >= 0.27 --- active_admin_import.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/active_admin_import.gemspec b/active_admin_import.gemspec index fb2ffce..983fbb6 100644 --- a/active_admin_import.gemspec +++ b/active_admin_import.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |gem| gem.name = 'active_admin_import' gem.require_paths = ['lib'] gem.version = ActiveAdminImport::VERSION - gem.add_runtime_dependency 'activerecord-import', '~> 0.27' + gem.add_runtime_dependency 'activerecord-import', '>= 0.27' gem.add_runtime_dependency 'rchardet', '~> 1.6' gem.add_runtime_dependency 'rubyzip', '~> 1.2' gem.add_dependency 'activeadmin', '>= 1.0.0.pre2' From e4c42ce1ae8223e985bbcb73984d110796fe86c4 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 15 Dec 2019 18:34:52 +0200 Subject: [PATCH 07/27] allow application/octet-stream content-type --- lib/active_admin_import/model.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/active_admin_import/model.rb b/lib/active_admin_import/model.rb index 5927996..11db86e 100644 --- a/lib/active_admin_import/model.rb +++ b/lib/active_admin_import/model.rb @@ -21,6 +21,7 @@ module CONST application/csv application/vnd.ms-excel application/vnd.msexcel + application/octet-stream text/tsv text/x-tsv text/tab-separated-values From a2ebd55b12dbf9a4f7ac1da4a4adb78e15e6b8e5 Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Mon, 16 Dec 2019 17:13:25 +0200 Subject: [PATCH 08/27] bump 4.1.2 --- CHANGELOG.md | 3 +++ lib/active_admin_import/version.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4490aa5..2bd92af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## [4.1.2] - 2019-12-16 +- allow application/octet-stream content-type #172 | @dmitry-sinina +- Allow activerecord-import >= 0.27 #171 | @sagium ## [4.1.1] - 2019-09-20 - Fix column slicing #168 | @doredesign diff --git a/lib/active_admin_import/version.rb b/lib/active_admin_import/version.rb index 8984dbd..8011a2f 100644 --- a/lib/active_admin_import/version.rb +++ b/lib/active_admin_import/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActiveAdminImport - VERSION = '4.1.1' + VERSION = '4.1.2' end From d1fc7ff3cfe16dbe1ae7acc0a9a261e856745be7 Mon Sep 17 00:00:00 2001 From: linqueta Date: Sat, 1 Feb 2020 19:41:17 -0300 Subject: [PATCH 09/27] fix uninitialized constant Responders::FlashResponder::ActionView --- spec/spec_helper.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index aded259..c8fd58e 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -20,6 +20,7 @@ require 'active_model' # require ActiveRecord to ensure that Ransack loads correctly require 'active_record' +require 'action_view' require 'active_admin' ActiveAdmin.application.load_paths = [ENV['RAILS_ROOT'] + '/app/admin'] require ENV['RAILS_ROOT'] + '/config/environment.rb' From e4e23c9f482ecffb0a6cff552f3f48e1f6180d6a Mon Sep 17 00:00:00 2001 From: linqueta Date: Thu, 30 Jan 2020 15:15:51 -0300 Subject: [PATCH 10/27] add active admin import exception handle exception when import a file fix uninitialized constant Responders::FlashResponder::ActionView use file_error instead no_file_error fix uninitialized constant Responders::FlashResponder::ActionView fix uninitialized constant Responders::FlashResponder::ActionView --- lib/active_admin_import.rb | 1 + lib/active_admin_import/dsl.rb | 6 +++++- lib/active_admin_import/exception.rb | 5 +++++ spec/import_spec.rb | 22 ++++++++++++++++++++++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 lib/active_admin_import/exception.rb diff --git a/lib/active_admin_import.rb b/lib/active_admin_import.rb index 40adf2b..8654626 100644 --- a/lib/active_admin_import.rb +++ b/lib/active_admin_import.rb @@ -3,6 +3,7 @@ require 'active_admin' require 'active_admin_import/version' require 'active_admin_import/engine' +require 'active_admin_import/exception' require 'active_admin_import/import_result' require 'active_admin_import/options' require 'active_admin_import/dsl' diff --git a/lib/active_admin_import/dsl.rb b/lib/active_admin_import/dsl.rb index 6d4698f..1d5303b 100644 --- a/lib/active_admin_import/dsl.rb +++ b/lib/active_admin_import/dsl.rb @@ -92,7 +92,11 @@ def active_admin_import(options = {}, &block) else instance_exec result, options, &DEFAULT_RESULT_PROC end - rescue ActiveRecord::Import::MissingColumnError, NoMethodError, ActiveRecord::StatementInvalid, CSV::MalformedCSVError => e + rescue ActiveRecord::Import::MissingColumnError, + NoMethodError, + ActiveRecord::StatementInvalid, + CSV::MalformedCSVError, + ActiveAdminImport::Exception => e Rails.logger.error(I18n.t('active_admin_import.file_error', message: e.message)) Rails.logger.error(e.backtrace.join("\n")) flash[:error] = I18n.t('active_admin_import.file_error', message: e.message[0..200]) diff --git a/lib/active_admin_import/exception.rb b/lib/active_admin_import/exception.rb new file mode 100644 index 0000000..7d70a07 --- /dev/null +++ b/lib/active_admin_import/exception.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true +module ActiveAdminImport + class Exception < StandardError + end +end \ No newline at end of file diff --git a/spec/import_spec.rb b/spec/import_spec.rb index c8c2879..c3f1e38 100644 --- a/spec/import_spec.rb +++ b/spec/import_spec.rb @@ -430,6 +430,28 @@ def upload_file!(name, ext = 'csv') upload_file!(:authors) expect(Author.count).to eq(2) end + + context 'when the option before_import raises a ActiveAdminImport::Exception' do + let(:options) { { before_import: ->(_) { raise ActiveAdminImport::Exception, 'error message' } } } + + before { upload_file!(:authors) } + + it 'should show error' do + expect(page).to have_content I18n.t('active_admin_import.file_error', message: 'error message') + expect(Author.count).to eq(0) + end + end + + context 'when the option before_batch_import raises a ActiveAdminImport::Exception' do + let(:options) { { before_batch_import: ->(_) { raise ActiveAdminImport::Exception, 'error message' } } } + + before { upload_file!(:authors) } + + it 'should show error' do + expect(page).to have_content I18n.t('active_admin_import.file_error', message: 'error message') + expect(Author.count).to eq(0) + end + end end end From 5c08253392f4d489c991cabd0305990afae7837c Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Wed, 5 Feb 2020 18:47:30 +0200 Subject: [PATCH 11/27] bump 4.2.0 --- CHANGELOG.md | 3 +++ lib/active_admin_import/version.rb | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bd92af..e423ff6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,7 @@ # Changelog +## [4.2.0] - 2020-02-05 +- generic exception for import added #175 | @linqueta + ## [4.1.2] - 2019-12-16 - allow application/octet-stream content-type #172 | @dmitry-sinina - Allow activerecord-import >= 0.27 #171 | @sagium diff --git a/lib/active_admin_import/version.rb b/lib/active_admin_import/version.rb index 8011a2f..d6c6cec 100644 --- a/lib/active_admin_import/version.rb +++ b/lib/active_admin_import/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActiveAdminImport - VERSION = '4.1.2' + VERSION = '4.2.0' end From 4f2aadcf6791627a4a269682d1ed354a4d45ae1b Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Thu, 6 Feb 2020 10:50:48 +0200 Subject: [PATCH 12/27] bump ruby versions --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e088332..d4ef5fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ env: - RAILS=5.1.6.2 - RAILS=5.2.2.1 rvm: - - 2.3.8 - 2.4.5 - 2.5.5 + - 2.6.5 + - 2.7.0 From d0efd0985b8459752f4e1feb77fc39af4b5b1d84 Mon Sep 17 00:00:00 2001 From: Hai Phan Nguyen Date: Fri, 24 Jul 2020 20:48:49 +0700 Subject: [PATCH 13/27] Support for rails 6 (#183) * Update active_admin_import.gemspec * add 6.0.2.2 on 2.7.1 test * ignore ruby low version for rails 6 * fix test * fix bundler * fix ignore --- .travis.yml | 9 +++++++-- Gemfile | 4 ++-- active_admin_import.gemspec | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index d4ef5fe..4dc1668 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,15 @@ script: bundle exec rspec spec env: matrix: - - RAILS=5.1.6.2 - - RAILS=5.2.2.1 + - RAILS=5.2.4.3 + - RAILS=6.0.3.2 rvm: - 2.4.5 - 2.5.5 - 2.6.5 - 2.7.0 + - 2.7.1 +jobs: + exclude: + - rvm: 2.4.5 + env: RAILS=6.0.3.2 \ No newline at end of file diff --git a/Gemfile b/Gemfile index 7ad241d..c85c5be 100644 --- a/Gemfile +++ b/Gemfile @@ -6,12 +6,12 @@ gemspec group :test do - default_rails_version = "~> 5.1" + default_rails_version = "~> 5.2.4" rails_version = ENV['RAILS'] || default_rails_version gem 'rails', rails_version gem 'rspec-rails' gem 'coveralls', require: false # Test coverage website. Go to https://coveralls.io - gem 'sqlite3',"~> 1.3.6" + gem "sqlite3", "~> 1.4.0" gem 'launchy' gem 'database_cleaner' gem 'capybara' diff --git a/active_admin_import.gemspec b/active_admin_import.gemspec index 983fbb6..6176e3e 100644 --- a/active_admin_import.gemspec +++ b/active_admin_import.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |gem| gem.require_paths = ['lib'] gem.version = ActiveAdminImport::VERSION gem.add_runtime_dependency 'activerecord-import', '>= 0.27' - gem.add_runtime_dependency 'rchardet', '~> 1.6' - gem.add_runtime_dependency 'rubyzip', '~> 1.2' + gem.add_runtime_dependency 'rchardet', '>= 1.6' + gem.add_runtime_dependency 'rubyzip', '>= 1.2' gem.add_dependency 'activeadmin', '>= 1.0.0.pre2' end From 1c7601fcccff35bb4217430e3fd0626312f64f2f Mon Sep 17 00:00:00 2001 From: naokirin Date: Tue, 11 Aug 2020 01:11:30 +0900 Subject: [PATCH 14/27] Support for a non UTF-8 file when zip uploading (#185) --- lib/active_admin_import/model.rb | 2 +- spec/import_spec.rb | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/active_admin_import/model.rb b/lib/active_admin_import/model.rb index 11db86e..309528f 100644 --- a/lib/active_admin_import/model.rb +++ b/lib/active_admin_import/model.rb @@ -110,7 +110,7 @@ def encode_file def unzip_file Zip::File.open(file_path) do |zip_file| - self.file = Tempfile.new(CONST::TMP_FILE) + self.file = Tempfile.new(CONST::TMP_FILE, binmode: true) data = zip_file.entries.select(&:file?).first.get_input_stream.read file << data file.close diff --git a/spec/import_spec.rb b/spec/import_spec.rb index c3f1e38..172a6ba 100644 --- a/spec/import_spec.rb +++ b/spec/import_spec.rb @@ -371,6 +371,20 @@ def upload_file!(name, ext = 'csv') end end end + + context 'when zipped with Win1251 file' do + let(:options) do + attributes = { force_encoding: :auto } + { template_object: ActiveAdminImport::Model.new(attributes) } + end + it 'should import file' do + with_zipped_csv(:authors_win1251_win_endline) do + upload_file!(:authors_win1251_win_endline, :zip) + expect(page).to have_content 'Successfully imported 2 authors' + expect(Author.count).to eq(2) + end + end + end end context 'with different header attribute names' do From 30858d42a9aa790c2e723485fea245c2e16e1883 Mon Sep 17 00:00:00 2001 From: John Cline <2057878+clinejj@users.noreply.github.com> Date: Wed, 10 Nov 2021 02:05:01 -0500 Subject: [PATCH 15/27] Add compatibility for Ruby 3.0 (#190) * Splat csv options for CSV.parse * Update CI env * Add sassc-rails to gemfile for development * Exclude correct job --- .travis.yml | 20 +++++++++++--------- Gemfile | 1 + lib/active_admin_import/importer.rb | 4 ++-- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4dc1668..0489aec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,17 @@ script: bundle exec rspec spec env: matrix: - - RAILS=5.2.4.3 - - RAILS=6.0.3.2 + - RAILS=5.2.4.6 + - RAILS=6.0.3.7 rvm: - - 2.4.5 - - 2.5.5 - - 2.6.5 - - 2.7.0 - - 2.7.1 + - 2.4.10 + - 2.5.9 + - 2.6.7 + - 2.7.3 + - 3.0.1 jobs: exclude: - - rvm: 2.4.5 - env: RAILS=6.0.3.2 \ No newline at end of file + - rvm: 2.4.10 + env: RAILS=6.0.3.7 + - rvm: 3.0.1 + env: RAILS=5.2.4.6 diff --git a/Gemfile b/Gemfile index c85c5be..8dd493e 100644 --- a/Gemfile +++ b/Gemfile @@ -8,6 +8,7 @@ gemspec group :test do default_rails_version = "~> 5.2.4" rails_version = ENV['RAILS'] || default_rails_version + gem 'sassc-rails' gem 'rails', rails_version gem 'rspec-rails' gem 'coveralls', require: false # Test coverage website. Go to https://coveralls.io diff --git a/lib/active_admin_import/importer.rb b/lib/active_admin_import/importer.rb index 495d691..7c99b78 100644 --- a/lib/active_admin_import/importer.rb +++ b/lib/active_admin_import/importer.rb @@ -37,7 +37,7 @@ def file end def cycle(lines) - @csv_lines = CSV.parse(lines.join, @csv_options) + @csv_lines = CSV.parse(lines.join, **@csv_options) import_result.add(batch_import, lines.count) end @@ -115,7 +115,7 @@ def process_file batch_size = options[:batch_size].to_i File.open(file.path) do |f| # capture headers if not exist - prepare_headers { CSV.parse(f.readline, @csv_options).first } + prepare_headers { CSV.parse(f.readline, **@csv_options).first } f.each_line do |line| lines << line if line.present? if lines.size == batch_size || f.eof? From a81ce6653c2ea670dea3064fbbccd9c92634ded0 Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Wed, 10 Nov 2021 09:21:10 +0200 Subject: [PATCH 16/27] drop ruby 2.4 support --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0489aec..4139dea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,6 @@ env: - RAILS=5.2.4.6 - RAILS=6.0.3.7 rvm: - - 2.4.10 - 2.5.9 - 2.6.7 - 2.7.3 From d3322ea39b2f24b4cb422326cb5b3942b621d2cb Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Wed, 10 Nov 2021 09:22:03 +0200 Subject: [PATCH 17/27] bump activeadmin dependency --- active_admin_import.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/active_admin_import.gemspec b/active_admin_import.gemspec index 6176e3e..252c83d 100644 --- a/active_admin_import.gemspec +++ b/active_admin_import.gemspec @@ -18,5 +18,5 @@ Gem::Specification.new do |gem| gem.add_runtime_dependency 'activerecord-import', '>= 0.27' gem.add_runtime_dependency 'rchardet', '>= 1.6' gem.add_runtime_dependency 'rubyzip', '>= 1.2' - gem.add_dependency 'activeadmin', '>= 1.0.0.pre2' + gem.add_dependency 'activeadmin', '>= 1.0.0' end From 672c5c7b1c9bb6d8a647447c28f69693c82067a1 Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Wed, 10 Nov 2021 09:23:17 +0200 Subject: [PATCH 18/27] bump version --- lib/active_admin_import/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/active_admin_import/version.rb b/lib/active_admin_import/version.rb index d6c6cec..994e654 100644 --- a/lib/active_admin_import/version.rb +++ b/lib/active_admin_import/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActiveAdminImport - VERSION = '4.2.0' + VERSION = '5.0.0' end From 2adaa75bfdff9db75c20e21a06366e65e2a59533 Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Tue, 16 Nov 2021 10:51:35 +0200 Subject: [PATCH 19/27] ruby/rails versions in travis.yml updated --- .travis.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4139dea..f850708 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,14 @@ script: bundle exec rspec spec env: matrix: - - RAILS=5.2.4.6 - - RAILS=6.0.3.7 + - RAILS=5.2.6 + - RAILS=6.0.4 + - RAILS=6.1.4 rvm: - - 2.5.9 - 2.6.7 - - 2.7.3 - - 3.0.1 + - 2.7.4 + - 3.0.2 jobs: exclude: - - rvm: 2.4.10 - env: RAILS=6.0.3.7 - rvm: 3.0.1 - env: RAILS=5.2.4.6 + env: RAILS=5.2.6 From 90e82de06cc6cfa2af10953a4c9597eac62e6fe6 Mon Sep 17 00:00:00 2001 From: Igor Fedoronchuk Date: Tue, 16 Nov 2021 11:35:58 +0200 Subject: [PATCH 20/27] changelog updated --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e423ff6..458a8a7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,11 @@ # Changelog +## [5.0.0] - 2021-11-16 +- Ruby 3 compatibility added #190 | @clinejj +- Support for a non UTF-8 file when zip uploading #185| @naokirin +- Rails 6 supported #183 | @pnghai +- Drop ruby 2.4 support #192 | @Fivell + + ## [4.2.0] - 2020-02-05 - generic exception for import added #175 | @linqueta From 9ca311a774b076765cc3eb9f4a20beb470abbdd3 Mon Sep 17 00:00:00 2001 From: enomotodev Date: Thu, 30 Dec 2021 17:33:25 +0900 Subject: [PATCH 21/27] Migrate to GitHub Actions --- .github/workflows/test.yml | 40 ++++++++++++++++++++++++++++++++++++++ .travis.yml | 14 ------------- README.md | 6 +++--- 3 files changed, 43 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/test.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..e155366 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +name: Test + +on: + - push + - pull_request + +jobs: + build: + runs-on: ubuntu-latest + + strategy: + matrix: + ruby_version: + - '2.6' + - '2.7' + - '3.0' + rails_version: + - '5.2.6' + - '6.0.4' + - '6.1.4' + exclude: + - ruby_version: '3.0' + rails_version: '5.2.6' + + name: Ruby ${{ matrix.ruby_version }} / Rails ${{ matrix.rails_version }} + + env: + RAILS: ${{ matrix.rails_version }} + + steps: + - uses: actions/checkout@v2 + + - name: Setup Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby_version }} + bundler-cache: true + + - name: Test + run: bundle exec rspec spec diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f850708..0000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -script: bundle exec rspec spec -env: - matrix: - - RAILS=5.2.6 - - RAILS=6.0.4 - - RAILS=6.1.4 -rvm: - - 2.6.7 - - 2.7.4 - - 3.0.2 -jobs: - exclude: - - rvm: 3.0.1 - env: RAILS=5.2.6 diff --git a/README.md b/README.md index c492ee5..357db1b 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ActiveAdminImport -[![Travis Build ][build_badge]][build_link] +[![Build Status ][build_badge]][build_link] [![Coverage Status][coveralls_badge]][coveralls_link] [![Code Climate ][codeclimate_badge]][codeclimate_link] [![Gem Version ][rubygems_badge]][rubygems_link] @@ -91,8 +91,8 @@ Tool | Description [rchardet]: https://github.com/jmhodges/rchardet [activerecord-import]: https://github.com/zdennis/activerecord-import -[build_badge]: https://travis-ci.org/activeadmin-plugins/active_admin_import.svg?branch=master -[build_link]: https://travis-ci.org/activeadmin-plugins/active_admin_import +[build_badge]: https://github.com/activeadmin-plugins/active_admin_import/actions/workflows/test.yml/badge.svg +[build_link]: https://github.com/activeadmin-plugins/active_admin_import/actions [coveralls_badge]: https://coveralls.io/repos/activeadmin-plugins/active_admin_import/badge.svg [coveralls_link]: https://coveralls.io/github/activeadmin-plugins/active_admin_import [codeclimate_badge]: https://codeclimate.com/github/activeadmin-plugins/active_admin_import/badges/gpa.svg From 52325746513b1359aab607313fe2b8a189a73608 Mon Sep 17 00:00:00 2001 From: artemlutsenko Date: Tue, 10 Oct 2023 16:38:08 +0300 Subject: [PATCH 22/27] Fix error when passed empty empty csv and model has force_encoding=:auto --- lib/active_admin_import/model.rb | 2 ++ spec/import_spec.rb | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/lib/active_admin_import/model.rb b/lib/active_admin_import/model.rb index 309528f..f3fde01 100644 --- a/lib/active_admin_import/model.rb +++ b/lib/active_admin_import/model.rb @@ -103,6 +103,8 @@ def file_path def encode_file data = File.read(file_path) + return if data.empty? + File.open(file_path, 'w') do |f| f.write(encode(data)) end diff --git a/spec/import_spec.rb b/spec/import_spec.rb index 172a6ba..9bc6403 100644 --- a/spec/import_spec.rb +++ b/spec/import_spec.rb @@ -424,6 +424,22 @@ def upload_file!(name, ext = 'csv') expect(Author.count).to eq(2) end end + + context 'with empty csv and auto detect encoding' do + let(:options) do + attributes = { force_encoding: :auto } + { template_object: ActiveAdminImport::Model.new(attributes) } + end + + before do + upload_file!(:empty) + end + + it 'should render warning' do + expect(page).to have_content I18n.t('active_admin_import.file_empty_error') + expect(Author.count).to eq(0) + end + end end context 'with callback procs options' do From 4e83191ff49f9cf62b03dbb351afaf08b8992365 Mon Sep 17 00:00:00 2001 From: artemlutsenko Date: Tue, 10 Oct 2023 16:39:20 +0300 Subject: [PATCH 23/27] Fix error when active_model_import raise ArgumentError Number of values (n) exceeds number of columns (m) --- lib/active_admin_import/dsl.rb | 1 + .../files/authors_values_exceeded_headers.csv | 3 +++ spec/import_spec.rb | 14 ++++++++++++++ 3 files changed, 18 insertions(+) create mode 100644 spec/fixtures/files/authors_values_exceeded_headers.csv diff --git a/lib/active_admin_import/dsl.rb b/lib/active_admin_import/dsl.rb index 1d5303b..01adff6 100644 --- a/lib/active_admin_import/dsl.rb +++ b/lib/active_admin_import/dsl.rb @@ -94,6 +94,7 @@ def active_admin_import(options = {}, &block) end rescue ActiveRecord::Import::MissingColumnError, NoMethodError, + ArgumentError, ActiveRecord::StatementInvalid, CSV::MalformedCSVError, ActiveAdminImport::Exception => e diff --git a/spec/fixtures/files/authors_values_exceeded_headers.csv b/spec/fixtures/files/authors_values_exceeded_headers.csv new file mode 100644 index 0000000..e3ee481 --- /dev/null +++ b/spec/fixtures/files/authors_values_exceeded_headers.csv @@ -0,0 +1,3 @@ +Birthday,Name,Last name +1986-05-01,John,Doe +1988-11-16,Jane,Roe, exceeded value \ No newline at end of file diff --git a/spec/import_spec.rb b/spec/import_spec.rb index 9bc6403..1c360c3 100644 --- a/spec/import_spec.rb +++ b/spec/import_spec.rb @@ -440,6 +440,20 @@ def upload_file!(name, ext = 'csv') expect(Author.count).to eq(0) end end + + context 'with csv which has exceeded values' do + before do + upload_file!(:authors_values_exceeded_headers) + end + + it 'should render warning' do + # 5 columns: 'birthday, name, last_name, created_at, updated_at' + # 6 values: '"1988-11-16", "Jane", "Roe", " exceeded value", datetime, datetime' + msg = 'Number of values (6) exceeds number of columns (5)' + expect(page).to have_content I18n.t('active_admin_import.file_error', message: msg) + expect(Author.count).to eq(0) + end + end end context 'with callback procs options' do From a71035edfbd84f496fd1151a6875e5ddf855d3c4 Mon Sep 17 00:00:00 2001 From: artemlutsenko Date: Mon, 30 Oct 2023 15:16:58 +0200 Subject: [PATCH 24/27] Fix bugs with cached models --- lib/active_admin_import/dsl.rb | 14 ++++++++++--- lib/active_admin_import/options.rb | 2 +- spec/import_spec.rb | 32 ++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/lib/active_admin_import/dsl.rb b/lib/active_admin_import/dsl.rb index 01adff6..405caeb 100644 --- a/lib/active_admin_import/dsl.rb +++ b/lib/active_admin_import/dsl.rb @@ -54,11 +54,14 @@ def active_admin_import(options = {}, &block) options.assert_valid_keys(*Options::VALID_OPTIONS) options = Options.options_for(config, options) - params_key = ActiveModel::Naming.param_key(options[:template_object]) collection_action :import, method: :get do authorize!(ActiveAdminImport::Auth::IMPORT, active_admin_config.resource_class) - @active_admin_import_model = options[:template_object] + @active_admin_import_model = if options[:template_object].is_a?(Proc) + options[:template_object].call + else + options[:template_object] + end render template: options[:template] end @@ -75,7 +78,12 @@ def active_admin_import(options = {}, &block) authorize!(ActiveAdminImport::Auth::IMPORT, active_admin_config.resource_class) _params = params.respond_to?(:to_unsafe_h) ? params.to_unsafe_h : params params = ActiveSupport::HashWithIndifferentAccess.new _params - @active_admin_import_model = options[:template_object] + @active_admin_import_model = if options[:template_object].is_a?(Proc) + options[:template_object].call + else + options[:template_object] + end + params_key = ActiveModel::Naming.param_key(@active_admin_import_model.class) @active_admin_import_model.assign_attributes(params[params_key].try(:deep_symbolize_keys) || {}) # go back to form return render template: options[:template] unless @active_admin_import_model.valid? diff --git a/lib/active_admin_import/options.rb b/lib/active_admin_import/options.rb index 53f4a78..3ed1ce8 100644 --- a/lib/active_admin_import/options.rb +++ b/lib/active_admin_import/options.rb @@ -26,7 +26,7 @@ module Options def self.options_for(config, options = {}) unless options.key? :template_object - options[:template_object] = ActiveAdminImport::Model.new + options[:template_object] = -> { ActiveAdminImport::Model.new } end { diff --git a/spec/import_spec.rb b/spec/import_spec.rb index 1c360c3..cc4db50 100644 --- a/spec/import_spec.rb +++ b/spec/import_spec.rb @@ -96,6 +96,38 @@ def upload_file!(name, ext = 'csv') end include_examples 'successful inserts for author' end + + context 'when template object passed like proc' do + before do + add_post_resource(template_object: -> { ActiveAdminImport::Model.new(author_id: author.id) }, + validate: true, + before_batch_import: lambda do |importer| + importer.csv_lines.map! { |row| row << importer.model.author_id } + importer.headers.merge!(:'Author Id' => :author_id) + end + ) + + visit '/admin/posts/import' + upload_file!(:posts_for_author) + end + + include_examples 'successful inserts for author' + + context 'after successful import try without file' do + let(:after_successful_import_do!) do + # reload page + visit '/admin/posts/import' + # submit form without file + find_button('Import').click + end + + it 'should render validation error' do + after_successful_import_do! + + expect(page).to have_content I18n.t('active_admin_import.no_file_error') + end + end + end end context 'for csv with author name' do From 10132e1ef38b7af7703de304a77a865096b7f0e5 Mon Sep 17 00:00:00 2001 From: Igor Gonchar Date: Tue, 13 Jun 2023 16:26:30 +0300 Subject: [PATCH 25/27] rails v7.0 support --- .github/workflows/test.yml | 3 +++ lib/active_admin_import/import_result.rb | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e155366..52683d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,9 +18,12 @@ jobs: - '5.2.6' - '6.0.4' - '6.1.4' + - '7.0.0' exclude: - ruby_version: '3.0' rails_version: '5.2.6' + - ruby_version: '2.6' + rails_version: '7.0.0' name: Ruby ${{ matrix.ruby_version }} / Rails ${{ matrix.rails_version }} diff --git a/lib/active_admin_import/import_result.rb b/lib/active_admin_import/import_result.rb index 8ff75c4..9abcc6d 100644 --- a/lib/active_admin_import/import_result.rb +++ b/lib/active_admin_import/import_result.rb @@ -33,11 +33,21 @@ def failed_message(options = {}) limit = options[:limit] || failed.count failed.first(limit).map do |record| errors = record.errors - failed_values = errors.keys.map do |key| + failed_values = attribute_names_for(errors).map do |key| key == :base ? nil : record.public_send(key) end errors.full_messages.zip(failed_values).map { |ms| ms.compact.join(' - ') }.join(', ') end.join(' ; ') end + + private + + def attribute_names_for(errors) + if Gem::Version.new(Rails.version) >= Gem::Version.new('7.0') + errors.attribute_names + else + errors.keys + end + end end end From c3d74af474a29aabde5a1dae1a756e09f8b7d44d Mon Sep 17 00:00:00 2001 From: Igor Gonchar Date: Thu, 2 Nov 2023 11:42:21 +0200 Subject: [PATCH 26/27] bump v5.1.0 --- lib/active_admin_import/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/active_admin_import/version.rb b/lib/active_admin_import/version.rb index 994e654..6affade 100644 --- a/lib/active_admin_import/version.rb +++ b/lib/active_admin_import/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module ActiveAdminImport - VERSION = '5.0.0' + VERSION = '5.1.0' end From 27cb9af58b266284beda0c6f47feb88d786f2f23 Mon Sep 17 00:00:00 2001 From: Bence Kiss Date: Wed, 9 Aug 2017 15:57:37 +0200 Subject: [PATCH 27/27] add controller context --- lib/active_admin_import/dsl.rb | 1 + lib/active_admin_import/importer.rb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/active_admin_import/dsl.rb b/lib/active_admin_import/dsl.rb index 405caeb..ccfd4ab 100644 --- a/lib/active_admin_import/dsl.rb +++ b/lib/active_admin_import/dsl.rb @@ -76,6 +76,7 @@ def active_admin_import(options = {}, &block) collection_action :do_import, method: :post do authorize!(ActiveAdminImport::Auth::IMPORT, active_admin_config.resource_class) + options[:controller_context] = self _params = params.respond_to?(:to_unsafe_h) ? params.to_unsafe_h : params params = ActiveSupport::HashWithIndifferentAccess.new _params @active_admin_import_model = if options[:template_object].is_a?(Proc) diff --git a/lib/active_admin_import/importer.rb b/lib/active_admin_import/importer.rb index 7c99b78..ab337f6 100644 --- a/lib/active_admin_import/importer.rb +++ b/lib/active_admin_import/importer.rb @@ -18,7 +18,8 @@ class Importer :headers_rewrites, :batch_size, :batch_transaction, - :csv_options + :csv_options, + :controller_context ].freeze def initialize(resource, model, options)