Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions .gitpod.Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
FROM gitpod/workspace-full

# Install custom tools, runtimes, etc.
# For example "bastet", a command-line tetris clone:
# RUN brew install bastet
#
# More information: https://www.gitpod.io/docs/config-docker/
5 changes: 5 additions & 0 deletions .gitpod.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
image:
file: .gitpod.Dockerfile

tasks:
- init: pip install -r ./requirements.txt
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/gabrielfroes/webscraping_python_selenium)

# Web Scraping JavaScript Generated Pages with Python

This project was created just for educational proposes.
Expand Down
10 changes: 9 additions & 1 deletion webscraping.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# -*- encoding: utf-8 -*-

import time
import requests
import pandas as pd
from bs4 import BeautifulSoup
Expand All @@ -20,6 +20,11 @@
'blocks': {'field': 'BLK', 'label': 'BLK'},
}

def acceptTerms():
acceptBt = driver.find_element_by_id('onetrust-accept-btn-handler')
acceptBt.click()
return


def buildrank(type):

Expand Down Expand Up @@ -53,6 +58,9 @@ def buildrank(type):
driver.get(url)
driver.implicitly_wait(10) # in seconds

acceptTerms()
time.sleep(10) #in seconds

for k in rankings:
top10ranking[k] = buildrank(k)

Expand Down