From 199b053fa3802099ee1037946ff7c9c8e3a071b7 Mon Sep 17 00:00:00 2001 From: nnouns Date: Tue, 25 Apr 2017 15:50:52 -0400 Subject: [PATCH 1/3] added avatar_remote_url method and wrote temp avatar scrape task --- app/models/player.rb | 9 ++++++++ lib/tasks/scrape_player_avatars.rake | 34 ++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 lib/tasks/scrape_player_avatars.rake diff --git a/app/models/player.rb b/app/models/player.rb index 49c03a2..59b309d 100644 --- a/app/models/player.rb +++ b/app/models/player.rb @@ -38,4 +38,13 @@ def strip_nationality self.save end end + + # avatar methods + def avatar_remote_url=(url_value) + if url_value.present? + self.avatar = URI.parse(url_value) + @avatar_remote_url = url_value + end + end + end \ No newline at end of file diff --git a/lib/tasks/scrape_player_avatars.rake b/lib/tasks/scrape_player_avatars.rake new file mode 100644 index 0000000..a6466f3 --- /dev/null +++ b/lib/tasks/scrape_player_avatars.rake @@ -0,0 +1,34 @@ +namespace :app do + task :scrape_player_avatars => :environment do + require 'nokogiri' + require 'open-uri' + require 'resolv' + require 'resolv-replace' + + url_base = 'http://www.transfermarkt.com' + + club_players = Player.all + + start = Time.now + + puts "Number of avatars to update: " + club_players.count.to_s + + club_players.each_with_index do |player, i| + + url = url_base + player.profile_link + doc = Nokogiri::HTML(open(url)) + + avatar_url = doc.at_css('.dataBild>img').attributes["src"].value + avatar_url.prepend("http:") + + player.avatar_remote_url=(avatar_url) + player.save! + + print "\rNumber of players birthplaces updated: #{i + 1}" + end + + finish = Time.now + + puts "\nIt took #{finish - start} seconds to get all player avatars" + end +end From a6873c8716b3b57bfd4d1488cc4f76b40703b1d4 Mon Sep 17 00:00:00 2001 From: nnouns Date: Tue, 25 Apr 2017 15:50:52 -0400 Subject: [PATCH 2/3] modified scrape_player_avatars task to use threading --- lib/tasks/scrape_player_avatars.rake | 47 ++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/lib/tasks/scrape_player_avatars.rake b/lib/tasks/scrape_player_avatars.rake index a6466f3..53e24c5 100644 --- a/lib/tasks/scrape_player_avatars.rake +++ b/lib/tasks/scrape_player_avatars.rake @@ -1,7 +1,10 @@ +# scrapes all players in database... arguments coming soon + namespace :app do task :scrape_player_avatars => :environment do require 'nokogiri' require 'open-uri' + require 'thread' require 'resolv' require 'resolv-replace' @@ -13,19 +16,37 @@ namespace :app do puts "Number of avatars to update: " + club_players.count.to_s - club_players.each_with_index do |player, i| - - url = url_base + player.profile_link - doc = Nokogiri::HTML(open(url)) - - avatar_url = doc.at_css('.dataBild>img').attributes["src"].value - avatar_url.prepend("http:") - - player.avatar_remote_url=(avatar_url) - player.save! - - print "\rNumber of players birthplaces updated: #{i + 1}" - end + work_q = Queue.new + club_players.each {|player| work_q << player} + + i = 0 + workers = (0..4).map do + Thread.new do + begin + while player = work_q.pop(true) + begin + doc = Nokogiri::HTML(open(url_base + player.profile_link)) + + avatar_url = doc.at_css('.dataBild>img').attributes["src"].value + avatar_url.prepend("http:") + + player.avatar_remote_url=(avatar_url) + player.save! + + print "\rNumber of players birthplaces updated: #{i + 1}" + i = i+1 + + rescue => e + puts "problem at #{profile_url}" + puts e.inspect + end + + end + rescue ThreadError + end + end + end + workers.map(&:join) finish = Time.now From 204422085f4b9c9389f250eed61a596eb8205fb8 Mon Sep 17 00:00:00 2001 From: nnouns Date: Wed, 17 May 2017 19:46:39 -0400 Subject: [PATCH 3/3] added zoom functionality shout out to wunderbart: https://jsfiddle.net/wunderbart/Lom3b0gb/ --- app/assets/stylesheets/index.css | 6 +- app/views/landings/index.html.erb | 267 ++++++++++++++++++++++++++---- 2 files changed, 235 insertions(+), 38 deletions(-) diff --git a/app/assets/stylesheets/index.css b/app/assets/stylesheets/index.css index f69c102..817c378 100644 --- a/app/assets/stylesheets/index.css +++ b/app/assets/stylesheets/index.css @@ -1,7 +1,9 @@ #container { - height: 100%; - width: 100%; + overflow: hidden; position: relative; + margin: auto; + height: 90vh; + width: 90vw; } #get_club_name { diff --git a/app/views/landings/index.html.erb b/app/views/landings/index.html.erb index 8092120..e6fe902 100644 --- a/app/views/landings/index.html.erb +++ b/app/views/landings/index.html.erb @@ -2,47 +2,242 @@ <%= stylesheet_link_tag 'application' %> -
- <%= select("get", "club_name", Club.all.collect {|p| [ p.name ] }, {include_blank: 'Select club:'}) %> +
+ <%= select("get", "club_name", Club.all.collect {|p| [ p.name ] }, {include_blank: 'Select club:'}) %>