From 6b70db5bc08cff5d99bd6cbbe328c56d2fe29521 Mon Sep 17 00:00:00 2001 From: silvagpe Date: Mon, 27 Jul 2020 10:05:43 -0300 Subject: [PATCH 1/2] aceitar os termos aguardar o reload e iniciar --- webscraping.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/webscraping.py b/webscraping.py index 3cdbba9..97e8969 100644 --- a/webscraping.py +++ b/webscraping.py @@ -1,5 +1,5 @@ # -*- encoding: utf-8 -*- - +import time import requests import pandas as pd from bs4 import BeautifulSoup @@ -20,6 +20,11 @@ 'blocks': {'field': 'BLK', 'label': 'BLK'}, } +def acceptTerms(): + acceptBt = driver.find_element_by_id('onetrust-accept-btn-handler') + acceptBt.click() + return + def buildrank(type): @@ -53,6 +58,9 @@ def buildrank(type): driver.get(url) driver.implicitly_wait(10) # in seconds +acceptTerms() +time.sleep(10) #in seconds + for k in rankings: top10ranking[k] = buildrank(k) From 1b01d0c9fd65bf0e99f58c195f2ab4102d2cab48 Mon Sep 17 00:00:00 2001 From: silvagpe Date: Thu, 27 Aug 2020 13:01:23 +0000 Subject: [PATCH 2/2] Fully automate dev setup with Gitpod This commit implements a fully-automated development setup using Gitpod.io, an online IDE for GitHub and GitLab that enables Dev-Environments-As-Code. This makes it easy for anyone to get a ready-to-code workspace for any branch, issue or pull request almost instantly with a single click. --- .gitpod.Dockerfile | 7 +++++++ .gitpod.yml | 5 +++++ README.md | 2 ++ 3 files changed, 14 insertions(+) create mode 100644 .gitpod.Dockerfile create mode 100644 .gitpod.yml diff --git a/.gitpod.Dockerfile b/.gitpod.Dockerfile new file mode 100644 index 0000000..f8e49f2 --- /dev/null +++ b/.gitpod.Dockerfile @@ -0,0 +1,7 @@ +FROM gitpod/workspace-full + +# Install custom tools, runtimes, etc. +# For example "bastet", a command-line tetris clone: +# RUN brew install bastet +# +# More information: https://www.gitpod.io/docs/config-docker/ diff --git a/.gitpod.yml b/.gitpod.yml new file mode 100644 index 0000000..44bd2e7 --- /dev/null +++ b/.gitpod.yml @@ -0,0 +1,5 @@ +image: + file: .gitpod.Dockerfile + +tasks: + - init: pip install -r ./requirements.txt diff --git a/README.md b/README.md index 24efb47..4f0ec54 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +[![Gitpod ready-to-code](https://img.shields.io/badge/Gitpod-ready--to--code-blue?logo=gitpod)](https://gitpod.io/#https://github.com/gabrielfroes/webscraping_python_selenium) + # Web Scraping JavaScript Generated Pages with Python This project was created just for educational proposes.