From 3f12182c7719c2457cd4dae3b3fb6a7c8e2a56f2 Mon Sep 17 00:00:00 2001 From: "Vetrila, Dorian" Date: Thu, 27 Oct 2016 01:43:56 +0100 Subject: [PATCH] Finish off that thing --- Makefile | 17 ------------- Rakefile | 76 +++++++++++++++---------------------------------------- Readme.md | 4 +-- 3 files changed, 21 insertions(+), 76 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 98b6b08..0000000 --- a/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -fetch: - ruby scripts/fetch.rb - -parse: fetch - ruby scripts/parse.rb - -analyze: parse - ruby scripts/analyze.rb - -run: analyze - (python -m SimpleHTTPServer &) - sleep 1 - (open http://localhost:8000/visualization) - -racai: parse - ruby scripts/racai.rb - diff --git a/Rakefile b/Rakefile index 6b93845..4574f53 100644 --- a/Rakefile +++ b/Rakefile @@ -1,64 +1,28 @@ require_relative "./main" +sources = ['Timpul', 'Publika', 'Unimedia', 'ProTv', 'Agora', 'Prime'] + +desc "Run for debug mode" task :console do sh "ruby -e \"require('./main.rb');binding.pry\"" end -desc "Fetched only new content by default." -namespace :fetch do - desc "Fetch resources from Timpul Source" - task :timpul, [:first, :last] do |_task, options| - Fetchers::Timpul.new.run(options[:first], options[:last]) - end - - desc "Fetch resources from Publika Source" - task :publika, [:first, :last] do |_task, options| - Fetchers::Publika.new.run(options[:first], options[:last]) - end - - desc "Fetch resources from Unimedia Source" - task :unimedia, [:first, :last] do |_task, options| - Fetchers::Unimedia.new.run(options[:first], options[:last]) - end - - desc "Fetch resources from ProTV Source" - task :protv, [:first, :last] do |_task, options| - Fetchers::ProTv.new.run(options[:first], options[:last]) - end - - desc "Fetch resources from Agora Source" - task :agora, [:first, :last] do |_task, options| - Fetchers::Agora.new.run(options[:first], options[:last]) - end - - desc "Fetch resources from Prime Source" - task :prime, [:first, :last] do |_task, options| - Fetchers::Prime.new.run(options[:first], options[:last]) - end -end - -namespace :parse do - desc "Parse ProTV source and update DB" - task :protv do - Parsers::ProTv.new.run - end - - desc "Parse Timpul source and update DB" - task :timpul do - Parsers::Timpul.new.run - end - - desc "Parse Publika source and update DB" - task :publika do - Parsers::Publika.new.run - end - - desc "Parse Unimedia source and update DB" - task :unimedia do - Parsers::Unimedia.new.run - end - - task :agora do - Parsers::Agora.new.run +sources.each do |source| + namespace :fetch do + desc "Fetch resources from #{source} Source" + task source.downcase.to_sym, [:first, :last] do |_task, options| + if options.to_hash != {} + Kernel.const_get("Fetchers::#{source}").new.run(options[:first], options[:last]) + else + Kernel.const_get("Fetchers::#{source}").new.run + end + end + end + + namespace :parse do + desc "Parse resources from #{source} and update DB" + task source.downcase.to_sym do + Kernel.const_get("Parsers::#{source}").new.run + end end end diff --git a/Readme.md b/Readme.md index d52d9fb..8da7f55 100644 --- a/Readme.md +++ b/Readme.md @@ -18,9 +18,6 @@ There are several steps: - Analyze the full data set - Create a visualization -Run `make run`, wait around 6 hours and you'll have everything. -You'll also need around 7GB of disk space. - Use rake tasks to trigger fetchers and parsers. Ex. `rake fetch:publika` or `rake parse:unimedia` @@ -30,3 +27,4 @@ Currently in stock: - Unimedia - Publika - Timpul +- Agora