SuperAI-Search/python-crawler-test2.py at master · stingraze/SuperAI-Search · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from bs4 import BeautifulSoup
from time import sleep
import urllib2
#input from STDIN
input_url = raw_input()
#UserAgent info
req = urllib2.Request(input_url, headers={ 'User-Agent' : 'Mozilla/5.0' } )

#open URL for BeautifulSoup to parse
resource = urllib2.urlopen(req).read()

soup = BeautifulSoup(resource, "html.parser")

links = soup.find_all("a")

for a in links:
	href = a.attrs['href']
	text = a.string
	print href.encode('utf-8')
	#skips twitter
	if 'twitter.com' in href:
		continue
	#skips linkedin
	if 'linkedin.com' in href:
		continue

	if "http" in href:
		try:
			#UserAgent Info
			req2 = urllib2.Request(href, headers={ 'User-Agent' : 'Mozilla/5.0' } )

			new_resource = urllib2.urlopen(req2).read()
			new_soup = BeautifulSoup(new_resource, "html.parser")
			links2 = new_soup.find_all("a")
			print "New Resource"
			sleep(1)
			for b in links2:
				try:
					href2 = b.attrs['href']
					text2 = b.string
					print href2.encode('utf-8')
					sleep(1)
				except:
					continue

		except urllib2.HTTPError as err:
			if err.code == 404:
    				print "404 Error, Skipped"
				continue
			if err.code == 999:
				print "999 (LinkedIn?) Error, Skipped"
				continue
			else:
				continue