From 78ce811fea6826cecfdf9cd14a15453a68b8fbef Mon Sep 17 00:00:00 2001 From: sriram-rao Date: Sun, 30 Jun 2013 22:26:51 +0530 Subject: [PATCH 1/8] Nothing changed --- crawler/crawler.rb | 1 - spec/start_system.rb | 1 - 2 files changed, 2 deletions(-) diff --git a/crawler/crawler.rb b/crawler/crawler.rb index 8081ce0..faed3d0 100644 --- a/crawler/crawler.rb +++ b/crawler/crawler.rb @@ -4,7 +4,6 @@ require 'rubygems' require 'mechanize' require "sanitize" -require './models.rb' class Crawler #write code to make basic crawling and indexing diff --git a/spec/start_system.rb b/spec/start_system.rb index 7bc736e..00e86a9 100644 --- a/spec/start_system.rb +++ b/spec/start_system.rb @@ -16,7 +16,6 @@ it "should start crawling" do server = Server.new('test.db') server.start_crawling('https://www.coursera.org/') - server.start_crawling server.crawling_status.should be_true end From 848a4af410c2e82d33d81d0fbb1705e581122911 Mon Sep 17 00:00:00 2001 From: sriram-rao Date: Thu, 11 Jul 2013 00:07:53 +0530 Subject: [PATCH 2/8] Previous version --- Gemfile | 1 + Gemfile.lock | 3 +++ start.rb | 4 ++-- test.db | Bin 0 -> 3072 bytes 4 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 test.db diff --git a/Gemfile b/Gemfile index a83ee67..3196a9d 100644 --- a/Gemfile +++ b/Gemfile @@ -5,6 +5,7 @@ gem 'sanitize' gem 'sinatra' gem 'rails' gem 'sqlite3' +gem 'protected_attributes' group :test do gem 'rspec-given' diff --git a/Gemfile.lock b/Gemfile.lock index ee9e55a..0be424e 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -64,6 +64,8 @@ GEM mini_portile (~> 0.5.0) ntlm-http (0.1.1) polyglot (0.3.3) + protected_attributes (1.0.3) + activemodel (>= 4.0.0, < 5.0) rack (1.5.2) rack-protection (1.5.0) rack @@ -132,6 +134,7 @@ PLATFORMS DEPENDENCIES capybara mechanize + protected_attributes rack rack-test rails diff --git a/start.rb b/start.rb index bf74b00..318553c 100644 --- a/start.rb +++ b/start.rb @@ -1,11 +1,11 @@ require 'mechanize' require 'thread' require_relative './crawler/crawler.rb' -require_relative './crawler/models.rb' require_relative './dbconnect.rb' require 'sinatra' -require 'sinatra' +set :server, 'webrick' + get '/' do erb :form end diff --git a/test.db b/test.db new file mode 100644 index 0000000000000000000000000000000000000000..b5cfe77ba563bfa191ce5dc462c78a928510f07c GIT binary patch literal 3072 zcmWFz^vNtqRY=P(%1ta$FlJz3U}R))P*7lCU|<4bW*~+E5T6Cc0n&^}Y+Otx2Hi*M zyg*qdW@ZNFULY|FMnhn5g+L=a6T7&rK4YVPW?o8aMQL7UVQFf7adJj#Zeo0HW_nR# zNoIatF@^xMbC9cJh^s=VpJ!mGtAeMWi)(~}5-#mZ3jTfyN*IPHDQGB_r4|(fZBWuI zWMmiD2RcNsBrz!`6+ Date: Thu, 11 Jul 2013 00:15:09 +0530 Subject: [PATCH 3/8] Previous files --- start.rb | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 start.rb diff --git a/start.rb b/start.rb new file mode 100644 index 0000000..b2d794b --- /dev/null +++ b/start.rb @@ -0,0 +1,43 @@ +require 'mechanize' +require_relative './crawler/crawler.rb' +require_relative './dbconnect.rb' +require 'sinatra' + +set :server, 'webrick' + +require_relative './model/word.rb' +require 'sinatra' + +get '/' do + erb :form +end + +def app + Sinatra::Application +end + +class Server + attr_accessor :crawl + + def initialize table + + end + + def check_table table + return ActiveRecord::Base.connection.table_exists?(:words) + end + + def start_crawling link + @thr = Thread.new { + agent = Mechanize.new + page = agent.get(link) + crawl_bot= Crawler.new + crawl_bot.crawl_page(link) + } + end + + def crawling_status + return true if @thr.status + end + +end From 053190bdbf1c2248337f91b6f4aec90974f9fec6 Mon Sep 17 00:00:00 2001 From: sriram-rao Date: Tue, 16 Jul 2013 22:13:12 +0530 Subject: [PATCH 4/8] Added basic POST request handling --- spec/start_spec.rb | 15 +++++++++++++++ start.rb | 15 +++++++++++++++ test.db | Bin 0 -> 6144 bytes views/display.erb | 2 ++ views/form.erb | 2 +- 5 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 test.db create mode 100644 views/display.erb diff --git a/spec/start_spec.rb b/spec/start_spec.rb index e52239a..95b3600 100644 --- a/spec/start_spec.rb +++ b/spec/start_spec.rb @@ -43,4 +43,19 @@ page.has_selector?('input[name=\'q\']').should be_true end + describe "result of post request" do + let (:server) {Server.new('test.db')} + before {visit '/'} + + it "should respond with 200: positive test" do + fill_in 'q', :with=>"testing" + page.click_button 'search' + page.status_code.should == 200 + end + + it "should respond with 403 or 404: negative test" do + click_button 'search' + page.status_code.should == 404 or page.status_code.should == 403 + end + end end diff --git a/start.rb b/start.rb index b2d794b..a7c6e39 100644 --- a/start.rb +++ b/start.rb @@ -12,6 +12,21 @@ erb :form end +post '/form' do + @words = params[:words] + @link = params[:link_name] + + auth = Authentication.where(:authentication_token=>params[:auth_token]) + + if auth.nil? + status 403 + else + status 200 + end + + erb :display +end + def app Sinatra::Application end diff --git a/test.db b/test.db new file mode 100644 index 0000000000000000000000000000000000000000..e2f6bd45eb3b6b2f99d0a4f841afb96001a46cc4 GIT binary patch literal 6144 zcmeH}L2uJA6o75Nrj?MUic_w}5>lfvm9^S|0}=%y6w#$yNhQi9GL2!8)|4jc7`JtI zeh2@HAAp2}IKYKxw?pej*?qMoKfm{q=VyP2_X*l zD)x(}b#`IkNH6K@REO+*y|=~Ai9^o_d>>NI?)A;{&v&5ax-LC`RLZx<>f%U!5bn?1P{W)nNf^U zT_y~|=l*Lh{_&dNz3>uC9VNVZjn&-8O*(Iyh*_56cH&}mq{cEHr|)xFrjx9==743o zK^Pnkd@PE|x-NW?`r0JAq$esbFoW=>wB|N(A$J#aZ(IReAuRqo`N|3Sdp^7^L=W(t zz;F18gKd_8C2*Sw0I+%+7^}B{vij~F+$+F00>9t~4z^hWmcXqg;6RgC($;1pWZhS=^NX literal 0 HcmV?d00001 diff --git a/views/display.erb b/views/display.erb new file mode 100644 index 0000000..6c4d4ad --- /dev/null +++ b/views/display.erb @@ -0,0 +1,2 @@ +

Words are <%= @words.join %>

+

Link is <%= @link %>

\ No newline at end of file diff --git a/views/form.erb b/views/form.erb index 270153e..643fa4e 100644 --- a/views/form.erb +++ b/views/form.erb @@ -1,6 +1,6 @@
- +
From c748b36d54947d212d66f00ade1edfc65b30674e Mon Sep 17 00:00:00 2001 From: sriram-rao Date: Tue, 16 Jul 2013 22:29:52 +0530 Subject: [PATCH 5/8] Added migration for authentication table --- migrations/db/004_create_authentication_table.rb | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 migrations/db/004_create_authentication_table.rb diff --git a/migrations/db/004_create_authentication_table.rb b/migrations/db/004_create_authentication_table.rb new file mode 100644 index 0000000..6c873b2 --- /dev/null +++ b/migrations/db/004_create_authentication_table.rb @@ -0,0 +1,14 @@ +#the class name must be the camelcased version of the filename +class CreateAuthenticationTable < ActiveRecord::Migration + def up + create_table :authentication do |t| + t.string :server_name + t.string :server_address + t.string :authentication_token + end + end + + def down + drop_table :authentication + end +end \ No newline at end of file From 9278c7289f8637c80dff030ad5d101290b64f0b5 Mon Sep 17 00:00:00 2001 From: sriram-rao Date: Wed, 17 Jul 2013 00:10:58 +0530 Subject: [PATCH 6/8] Deleted conflict files --- dbconnect.rb | 6 +++++- start.rb | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/dbconnect.rb b/dbconnect.rb index 34ca40b..0fafb11 100644 --- a/dbconnect.rb +++ b/dbconnect.rb @@ -3,7 +3,11 @@ require 'active_record' #change for your actual setup -ActiveRecord::Base.establish_connection(:adapter=>'sqlite3', :database=>'test.db') +ActiveRecord::Base.establish_connection(:adapter=>'postgresql', :database=>'HEROKU_POSTGRESQL_AQUA') #personal preference +<<<<<<< HEAD ActiveRecord::Base.pluralize_table_names = true +======= +ActiveRecord::Base.pluralize_table_names = false +>>>>>>> 5d97379cdfad6b663d22b4f3f26550bc7394a9c3 diff --git a/start.rb b/start.rb index a7c6e39..be608e0 100644 --- a/start.rb +++ b/start.rb @@ -8,6 +8,8 @@ require_relative './model/word.rb' require 'sinatra' +set :server, 'webrick' + get '/' do erb :form end From cccfcd16241a3ff14d21b01b16d2ccd19524c3c6 Mon Sep 17 00:00:00 2001 From: sriram-rao Date: Wed, 17 Jul 2013 00:23:30 +0530 Subject: [PATCH 7/8] Deleted files --- Gemfile.lock.orig | 150 -------------------------------------------- Gemfile.orig | 32 ---------- dbconnect.rb.orig | 13 ---- start.rb.orig | 64 ------------------- views/form.erb.orig | 57 ----------------- 5 files changed, 316 deletions(-) delete mode 100644 Gemfile.lock.orig delete mode 100644 Gemfile.orig delete mode 100644 dbconnect.rb.orig delete mode 100644 start.rb.orig delete mode 100644 views/form.erb.orig diff --git a/Gemfile.lock.orig b/Gemfile.lock.orig deleted file mode 100644 index dee46e7..0000000 --- a/Gemfile.lock.orig +++ /dev/null @@ -1,150 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - actionmailer (4.0.0) - actionpack (= 4.0.0) - mail (~> 2.5.3) - actionpack (4.0.0) - activesupport (= 4.0.0) - builder (~> 3.1.0) - erubis (~> 2.7.0) - rack (~> 1.5.2) - rack-test (~> 0.6.2) - activemodel (4.0.0) - activesupport (= 4.0.0) - builder (~> 3.1.0) - activerecord (4.0.0) - activemodel (= 4.0.0) - activerecord-deprecated_finders (~> 1.0.2) - activesupport (= 4.0.0) - arel (~> 4.0.0) - activerecord-deprecated_finders (1.0.3) - activesupport (4.0.0) - i18n (~> 0.6, >= 0.6.4) - minitest (~> 4.2) - multi_json (~> 1.3) - thread_safe (~> 0.1) - tzinfo (~> 0.3.37) - arel (4.0.0) - atomic (1.1.10) - builder (3.1.4) - capybara (2.1.0) - mime-types (>= 1.16) - nokogiri (>= 1.3.3) - rack (>= 1.0.0) - rack-test (>= 0.5.4) - xpath (~> 2.0) - diff-lcs (1.2.4) - domain_name (0.5.12) - unf (>= 0.0.5, < 1.0.0) - erubis (2.7.0) - hike (1.2.3) - http-cookie (1.0.1) - domain_name (~> 0.5) - i18n (0.6.4) - mail (2.5.4) - mime-types (~> 1.16) - treetop (~> 1.4.8) - mechanize (2.7.1) - domain_name (~> 0.5, >= 0.5.1) - http-cookie (~> 1.0.0) - mime-types (~> 1.17, >= 1.17.2) - net-http-digest_auth (~> 1.1, >= 1.1.1) - net-http-persistent (~> 2.5, >= 2.5.2) - nokogiri (~> 1.4) - ntlm-http (~> 0.1, >= 0.1.1) - webrobots (>= 0.0.9, < 0.2) - mime-types (1.23) - mini_portile (0.5.1) - minitest (4.7.5) - multi_json (1.7.7) - net-http-digest_auth (1.3) - net-http-persistent (2.8) - nokogiri (1.6.0) - mini_portile (~> 0.5.0) - ntlm-http (0.1.1) - pg (0.15.1) - polyglot (0.3.3) - protected_attributes (1.0.3) - activemodel (>= 4.0.0, < 5.0) - rack (1.5.2) - rack-protection (1.5.0) - rack - rack-test (0.6.2) - rack (>= 1.0) - rails (4.0.0) - actionmailer (= 4.0.0) - actionpack (= 4.0.0) - activerecord (= 4.0.0) - activesupport (= 4.0.0) - bundler (>= 1.3.0, < 2.0) - railties (= 4.0.0) - sprockets-rails (~> 2.0.0) - railties (4.0.0) - actionpack (= 4.0.0) - activesupport (= 4.0.0) - rake (>= 0.8.7) - thor (>= 0.18.1, < 2.0) - rake (10.1.0) - rspec (2.14.0) - rspec-core (~> 2.14.0) - rspec-expectations (~> 2.14.0) - rspec-mocks (~> 2.14.0) - rspec-core (2.14.2) - rspec-expectations (2.14.0) - diff-lcs (>= 1.1.3, < 2.0) - rspec-given (2.4.4) - rspec (>= 2.12) - sorcerer (>= 0.3.7) - rspec-mocks (2.14.1) - sanitize (2.0.5) - nokogiri (>= 1.4.4) - sinatra (1.4.3) - rack (~> 1.4) - rack-protection (~> 1.4) - tilt (~> 1.3, >= 1.3.4) - sorcerer (0.3.10) - sprockets (2.10.0) - hike (~> 1.2) - multi_json (~> 1.0) - rack (~> 1.0) - tilt (~> 1.1, != 1.3.0) - sprockets-rails (2.0.0) - actionpack (>= 3.0) - activesupport (>= 3.0) - sprockets (~> 2.8) - sqlite3 (1.3.7) - thor (0.18.1) - thread_safe (0.1.0) - atomic - tilt (1.4.1) - treetop (1.4.14) - polyglot - polyglot (>= 0.3.1) - tzinfo (0.3.37) - unf (0.1.1) - unf_ext - unf_ext (0.0.6) - webrobots (0.1.1) - xpath (2.0.0) - nokogiri (~> 1.3) - -PLATFORMS - ruby - -DEPENDENCIES - capybara - mechanize -<<<<<<< HEAD - protected_attributes -======= - pg ->>>>>>> upstream/master - rack - rack-test - rails - rspec - rspec-given - sanitize - sinatra - sqlite3 diff --git a/Gemfile.orig b/Gemfile.orig deleted file mode 100644 index 9cc38c1..0000000 --- a/Gemfile.orig +++ /dev/null @@ -1,32 +0,0 @@ -source "https://rubygems.org" - -ruby '2.0.0' - -gem 'mechanize' -gem 'sanitize' -gem 'sinatra' -gem 'rails' -<<<<<<< HEAD -gem 'sqlite3' -gem 'protected_attributes' -======= - -group :development do - gem 'sqlite3' -end ->>>>>>> upstream/master - -group :test do - gem 'rspec-given' - gem 'rspec' - gem 'rack' - gem 'rack-test' - gem 'capybara' -end - -group :production do - gem 'pg' -end - - - diff --git a/dbconnect.rb.orig b/dbconnect.rb.orig deleted file mode 100644 index 0fafb11..0000000 --- a/dbconnect.rb.orig +++ /dev/null @@ -1,13 +0,0 @@ -# default boilerplate -require 'rubygems' -require 'active_record' - -#change for your actual setup -ActiveRecord::Base.establish_connection(:adapter=>'postgresql', :database=>'HEROKU_POSTGRESQL_AQUA') - -#personal preference -<<<<<<< HEAD -ActiveRecord::Base.pluralize_table_names = true -======= -ActiveRecord::Base.pluralize_table_names = false ->>>>>>> 5d97379cdfad6b663d22b4f3f26550bc7394a9c3 diff --git a/start.rb.orig b/start.rb.orig deleted file mode 100644 index 20b5344..0000000 --- a/start.rb.orig +++ /dev/null @@ -1,64 +0,0 @@ -require 'mechanize' -require_relative './crawler/crawler.rb' -require_relative './dbconnect.rb' -require 'sinatra' - -set :server, 'webrick' - -require_relative './model/word.rb' -require 'sinatra' - -set :server, 'webrick' - -get '/' do - erb :form -end - -<<<<<<< HEAD -post '/form' do - @words = params[:words] - @link = params[:link_name] - - auth = Authentication.where(:authentication_token=>params[:auth_token]) - - if auth.nil? - status 403 - else - status 200 - end - - erb :display -end -======= - ->>>>>>> upstream/master - -def app - Sinatra::Application -end - -class Server - attr_accessor :crawl - - def initialize table - - end - - def check_table table - return ActiveRecord::Base.connection.table_exists?(:words) - end - - def start_crawling link - @thr = Thread.new { - agent = Mechanize.new - page = agent.get(link) - crawl_bot= Crawler.new - crawl_bot.crawl_page(link) - } - end - - def crawling_status - return true if @thr.status - end - -end diff --git a/views/form.erb.orig b/views/form.erb.orig deleted file mode 100644 index dccd3a0..0000000 --- a/views/form.erb.orig +++ /dev/null @@ -1,57 +0,0 @@ - - - -
-<<<<<<< HEAD - - -======= - -
- - -
->>>>>>> upstream/master -
- - From a9eb5566cee6549f1f80f089554251bc68016dcf Mon Sep 17 00:00:00 2001 From: sriram-rao Date: Wed, 17 Jul 2013 00:39:06 +0530 Subject: [PATCH 8/8] changed dbconnect --- dbconnect.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dbconnect.rb b/dbconnect.rb index 0fafb11..729da78 100644 --- a/dbconnect.rb +++ b/dbconnect.rb @@ -6,8 +6,4 @@ ActiveRecord::Base.establish_connection(:adapter=>'postgresql', :database=>'HEROKU_POSTGRESQL_AQUA') #personal preference -<<<<<<< HEAD -ActiveRecord::Base.pluralize_table_names = true -======= -ActiveRecord::Base.pluralize_table_names = false ->>>>>>> 5d97379cdfad6b663d22b4f3f26550bc7394a9c3 +ActiveRecord::Base.pluralize_table_names = false \ No newline at end of file