Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 24 additions & 6 deletions app/models/site_upload.rb
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ class SiteUpload
MAX_FILE_SIZE = 5.megabytes
REQUIRED_HEADERS = ["url"].freeze
SUPPORTED_SEPARATORS = [",", ";"].freeze
FIRST_DATA_ROW_NUMBER = 2 # Row 1 contains CSV headers
BOM = /^\xEF\xBB\xBF/

attr_accessor :file, :team, :tag_ids, :tags, :new_sites, :existing_sites
Expand Down Expand Up @@ -67,13 +68,30 @@ def count
def parse_sites
require "csv"

CSV.foreach(file.path, headers: true, encoding: "bom|utf-8", col_sep:) do |row|
CSV.foreach(file.path, headers: true, encoding: "bom|utf-8", col_sep:).with_index(FIRST_DATA_ROW_NUMBER) do |row, line_number|
row = row.to_h.transform_keys { |header| header.to_s.downcase } # Case-insensitive headers

raw_url = row["url"].to_s.strip
next if raw_url.empty?
Comment thread
yann120 marked this conversation as resolved.

url = Link.normalize(raw_url)
begin
parsed_url = Link.parse(raw_url)
raise Link::InvalidUriError.new(raw_url) if parsed_url.relative?
Comment thread
yann120 marked this conversation as resolved.

url = Link.normalize(parsed_url)
rescue Link::InvalidUriError => error
Rails.logger.warn(
"site_upload_invalid_url " \
"team_id=#{team&.id} " \
"filename=#{file&.original_filename} " \
"line_number=#{line_number} " \
"raw_url=#{raw_url} " \
"error_class=#{error.class.name} " \
"error_message=#{error.message}"
)
errors.add(:file, :invalid_row_url, line_number:, url: raw_url)
next
end
name = row["nom"] || row["name"]
tag_names = row["tags"].present? ? row["tags"].split(",").map(&:strip).compact_blank.uniq : []

Expand All @@ -92,10 +110,10 @@ def parse_sites
rescue CSV::MalformedCSVError => error
Rails.logger.warn(
"site_upload_malformed_csv " \
"team_id=#{team&.id.inspect} " \
"filename=#{file&.original_filename.inspect} " \
"error_class=#{error.class.name.inspect} " \
"error_message=#{error.message.inspect}"
"team_id=#{team&.id} " \
"filename=#{file&.original_filename} " \
"error_class=#{error.class.name} " \
"error_message=#{error.message}"
)
errors.add(:file, :malformed_csv)
end
Expand Down
1 change: 1 addition & 0 deletions config/locales/fr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ fr:
file:
invalid_format: Le fichier n'est pas reconnu comme un fichier CSV.
invalid_headers: Le fichier n'a pas les bons en-têtes de colonne.
invalid_row_url: "Le fichier contient une URL invalide à la ligne %{line_number} : %{url}"
invalid_size: Le fichier est trop lourd. Merci de le découper en plusieurs fichiers.
malformed_csv: Le fichier CSV est mal formé. Veuillez ré-exporter votre fichier depuis votre tableur et vérifier sa structure.
activerecord:
Expand Down
32 changes: 31 additions & 1 deletion spec/models/site_upload_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,36 @@
expect(site_upload.new_sites.keys).to contain_exactly("https://example.com/", "https://test.com/")
end

it "logs an invalid URL row" do
csv.write("url,name\nhttps://example.com/,Example Site\nhttp://,Broken")
csv.rewind
allow(Rails.logger).to receive(:warn)

site_upload.parse_sites

expect(site_upload.errors.added?(:file, :invalid_row_url, line_number: 3, url: "http://")).to be(true)
expect(Rails.logger).to have_received(:warn).with(
include("site_upload_invalid_url", "line_number=3", "raw_url=http://", "filename=sites.csv")
)
end

it "continues parsing and collects multiple invalid URL rows" do
csv.write("url,name\nhttps://example.com/,Example Site\nhttp://,Broken\n/contact,Relative\nhttps://test.com/,Test Site")
csv.rewind
allow(Rails.logger).to receive(:warn)

site_upload.parse_sites

expect(site_upload.new_sites.keys).to contain_exactly("https://example.com/", "https://test.com/")
expect(site_upload.errors.details[:file]).to include(
error: :invalid_row_url, line_number: 3, url: "http://"
)
expect(site_upload.errors.details[:file]).to include(
error: :invalid_row_url, line_number: 4, url: "/contact"
)
expect(Rails.logger).to have_received(:warn).twice
end

it "ignores duplicate existing sites" do
existing_site = build(:site, url: "https://example.com/")
allow(team.sites).to receive(:find_by_url) { |args| existing_site if args[:url] == "https://example.com/" }
Expand Down Expand Up @@ -354,7 +384,7 @@
expect(site_upload.save).to be false
expect(site_upload.errors.added?(:file, :malformed_csv)).to be(true)
expect(Rails.logger).to have_received(:warn).with(
include("site_upload_malformed_csv", "filename=\"sites.csv\"")
include("site_upload_malformed_csv", "filename=sites.csv")
)
end
end
Expand Down
Loading