diff --git a/your-code/Lab_main_Matheus_Freire.ipynb b/your-code/Lab_main_Matheus_Freire.ipynb new file mode 100644 index 0000000..6ee7998 --- /dev/null +++ b/your-code/Lab_main_Matheus_Freire.ipynb @@ -0,0 +1,4794 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Web Scraping Lab\n", + "\n", + "You will find in this notebook some scrapy exercises to practise your scraping skills.\n", + "\n", + "**Tips:**\n", + "\n", + "- Check the response status code for each request to ensure you have obtained the intended content.\n", + "- Print the response text in each request to understand the kind of info you are getting and its format.\n", + "- Check for patterns in the response text to extract the data/info requested in each question.\n", + "- Visit the urls below and take a look at their source code through Chrome DevTools. You'll need to identify the html tags, special class names, etc used in the html content you are expected to extract.\n", + "\n", + "**Resources**:\n", + "- [Requests library](http://docs.python-requests.org/en/master/#the-user-guide)\n", + "- [Beautiful Soup Doc](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)\n", + "- [Urllib](https://docs.python.org/3/library/urllib.html#module-urllib)\n", + "- [re lib](https://docs.python.org/3/library/re.html)\n", + "- [lxml lib](https://lxml.de/)\n", + "- [Scrapy](https://scrapy.org/)\n", + "- [List of HTTP status codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes)\n", + "- [HTML basics](http://www.simplehtmlguide.com/cheatsheet.php)\n", + "- [CSS basics](https://www.cssbasics.com/#page_start)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Below are the libraries and modules you may need. `requests`, `BeautifulSoup` and `pandas` are already imported for you. If you prefer to use additional libraries feel free to do it." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from bs4 import BeautifulSoup\n", + "import pandas as pd" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Download, parse (using BeautifulSoup), and print the content from the Trending Developers page from GitHub:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "200" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# This is the url you will scrape in this exercise\n", + "url = 'https://github.com/trending/developers'\n", + "response = requests.get(url)\n", + "response.status_code" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "Trending developers on GitHub today · GitHub\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "Skip to content\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + " Sign up\n", + " \n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " Sign in\n", + " \n", + "
\n", + "\n", + " Sign up\n", + " \n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "

Trending

\n", + "

\n", + " These are the developers building the hot tools today.\n", + "

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + " Language:\n", + "\n", + " \n", + " Any\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + " Select a language\n", + "\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + " Unknown languages\n", + " \n", + "\n", + " 1C Enterprise\n", + " \n", + "\n", + " 2-Dimensional Array\n", + " \n", + "\n", + " 4D\n", + " \n", + "\n", + " ABAP\n", + " \n", + "\n", + " ABAP CDS\n", + " \n", + "\n", + " ABNF\n", + " \n", + "\n", + " ActionScript\n", + " \n", + "\n", + " Ada\n", + " \n", + "\n", + " Adblock Filter List\n", + " \n", + "\n", + " Adobe Font Metrics\n", + " \n", + "\n", + " Agda\n", + " \n", + "\n", + " AGS Script\n", + " \n", + "\n", + " AIDL\n", + " \n", + "\n", + " AL\n", + " \n", + "\n", + " Alloy\n", + " \n", + "\n", + " Alpine Abuild\n", + " \n", + "\n", + " Altium Designer\n", + " \n", + "\n", + " AMPL\n", + " \n", + "\n", + " AngelScript\n", + " \n", + "\n", + " Ant Build System\n", + " \n", + "\n", + " Antlers\n", + " \n", + "\n", + " ANTLR\n", + " \n", + "\n", + " ApacheConf\n", + " \n", + "\n", + " Apex\n", + " \n", + "\n", + " API Blueprint\n", + " \n", + "\n", + " APL\n", + " \n", + "\n", + " Apollo Guidance Computer\n", + " \n", + "\n", + " AppleScript\n", + " \n", + "\n", + " Arc\n", + " \n", + "\n", + " AsciiDoc\n", + " \n", + "\n", + " ASL\n", + " \n", + "\n", + " ASN.1\n", + " \n", + "\n", + " Classic ASP\n", + " \n", + "\n", + " ASP.NET\n", + " \n", + "\n", + " AspectJ\n", + " \n", + "\n", + " Assembly\n", + " \n", + "\n", + " Astro\n", + " \n", + "\n", + " Asymptote\n", + " \n", + "\n", + " ATS\n", + " \n", + "\n", + " Augeas\n", + " \n", + "\n", + " AutoHotkey\n", + " \n", + "\n", + " AutoIt\n", + " \n", + "\n", + " Avro IDL\n", + " \n", + "\n", + " Awk\n", + " \n", + "\n", + " Ballerina\n", + " \n", + "\n", + " BASIC\n", + " \n", + "\n", + " Batchfile\n", + " \n", + "\n", + " Beef\n", + " \n", + "\n", + " Befunge\n", + " \n", + "\n", + " Berry\n", + " \n", + "\n", + " BibTeX\n", + " \n", + "\n", + " Bicep\n", + " \n", + "\n", + " Bikeshed\n", + " \n", + "\n", + " Bison\n", + " \n", + "\n", + " BitBake\n", + " \n", + "\n", + " Blade\n", + " \n", + "\n", + " BlitzBasic\n", + " \n", + "\n", + " BlitzMax\n", + " \n", + "\n", + " Bluespec\n", + " \n", + "\n", + " Boo\n", + " \n", + "\n", + " Boogie\n", + " \n", + "\n", + " Brainfuck\n", + " \n", + "\n", + " BrighterScript\n", + " \n", + "\n", + " Brightscript\n", + " \n", + "\n", + " Zeek\n", + " \n", + "\n", + " Browserslist\n", + " \n", + "\n", + " C\n", + " \n", + "\n", + " C#\n", + " \n", + "\n", + " C++\n", + " \n", + "\n", + " C-ObjDump\n", + " \n", + "\n", + " C2hs Haskell\n", + " \n", + "\n", + " Cabal Config\n", + " \n", + "\n", + " Cadence\n", + " \n", + "\n", + " Cairo\n", + " \n", + "\n", + " CameLIGO\n", + " \n", + "\n", + " CAP CDS\n", + " \n", + "\n", + " Cap'n Proto\n", + " \n", + "\n", + " CartoCSS\n", + " \n", + "\n", + " Ceylon\n", + " \n", + "\n", + " Chapel\n", + " \n", + "\n", + " Charity\n", + " \n", + "\n", + " Checksums\n", + " \n", + "\n", + " ChucK\n", + " \n", + "\n", + " CIL\n", + " \n", + "\n", + " Circom\n", + " \n", + "\n", + " Cirru\n", + " \n", + "\n", + " Clarion\n", + " \n", + "\n", + " Clarity\n", + " \n", + "\n", + " Classic ASP\n", + " \n", + "\n", + " Clean\n", + " \n", + "\n", + " Click\n", + " \n", + "\n", + " CLIPS\n", + " \n", + "\n", + " Clojure\n", + " \n", + "\n", + " Closure Templates\n", + " \n", + "\n", + " Cloud Firestore Security Rules\n", + " \n", + "\n", + " CMake\n", + " \n", + "\n", + " COBOL\n", + " \n", + "\n", + " CODEOWNERS\n", + " \n", + "\n", + " CodeQL\n", + " \n", + "\n", + " CoffeeScript\n", + " \n", + "\n", + " ColdFusion\n", + " \n", + "\n", + " ColdFusion CFC\n", + " \n", + "\n", + " COLLADA\n", + " \n", + "\n", + " Common Lisp\n", + " \n", + "\n", + " Common Workflow Language\n", + " \n", + "\n", + " Component Pascal\n", + " \n", + "\n", + " CoNLL-U\n", + " \n", + "\n", + " Cool\n", + " \n", + "\n", + " Coq\n", + " \n", + "\n", + " Cpp-ObjDump\n", + " \n", + "\n", + " Creole\n", + " \n", + "\n", + " Crystal\n", + " \n", + "\n", + " CSON\n", + " \n", + "\n", + " Csound\n", + " \n", + "\n", + " Csound Document\n", + " \n", + "\n", + " Csound Score\n", + " \n", + "\n", + " CSS\n", + " \n", + "\n", + " CSV\n", + " \n", + "\n", + " Cuda\n", + " \n", + "\n", + " CUE\n", + " \n", + "\n", + " Cue Sheet\n", + " \n", + "\n", + " cURL Config\n", + " \n", + "\n", + " Curry\n", + " \n", + "\n", + " CWeb\n", + " \n", + "\n", + " Cycript\n", + " \n", + "\n", + " Cypher\n", + " \n", + "\n", + " Cython\n", + " \n", + "\n", + " D\n", + " \n", + "\n", + " D-ObjDump\n", + " \n", + "\n", + " Dafny\n", + " \n", + "\n", + " Darcs Patch\n", + " \n", + "\n", + " Dart\n", + " \n", + "\n", + " DataWeave\n", + " \n", + "\n", + " Debian Package Control File\n", + " \n", + "\n", + " DenizenScript\n", + " \n", + "\n", + " desktop\n", + " \n", + "\n", + " Dhall\n", + " \n", + "\n", + " Diff\n", + " \n", + "\n", + " DIGITAL Command Language\n", + " \n", + "\n", + " dircolors\n", + " \n", + "\n", + " DirectX 3D File\n", + " \n", + "\n", + " DM\n", + " \n", + "\n", + " DNS Zone\n", + " \n", + "\n", + " Dockerfile\n", + " \n", + "\n", + " Dogescript\n", + " \n", + "\n", + " Dotenv\n", + " \n", + "\n", + " DTrace\n", + " \n", + "\n", + " Dylan\n", + " \n", + "\n", + " E\n", + " \n", + "\n", + " E-mail\n", + " \n", + "\n", + " Eagle\n", + " \n", + "\n", + " Earthly\n", + " \n", + "\n", + " Easybuild\n", + " \n", + "\n", + " EBNF\n", + " \n", + "\n", + " eC\n", + " \n", + "\n", + " Ecere Projects\n", + " \n", + "\n", + " ECL\n", + " \n", + "\n", + " ECLiPSe\n", + " \n", + "\n", + " Ecmarkup\n", + " \n", + "\n", + " EditorConfig\n", + " \n", + "\n", + " Edje Data Collection\n", + " \n", + "\n", + " edn\n", + " \n", + "\n", + " Eiffel\n", + " \n", + "\n", + " EJS\n", + " \n", + "\n", + " Elixir\n", + " \n", + "\n", + " Elm\n", + " \n", + "\n", + " Elvish\n", + " \n", + "\n", + " Elvish Transcript\n", + " \n", + "\n", + " Emacs Lisp\n", + " \n", + "\n", + " EmberScript\n", + " \n", + "\n", + " E-mail\n", + " \n", + "\n", + " EQ\n", + " \n", + "\n", + " Erlang\n", + " \n", + "\n", + " Euphoria\n", + " \n", + "\n", + " F#\n", + " \n", + "\n", + " F*\n", + " \n", + "\n", + " Factor\n", + " \n", + "\n", + " Fancy\n", + " \n", + "\n", + " Fantom\n", + " \n", + "\n", + " Faust\n", + " \n", + "\n", + " Fennel\n", + " \n", + "\n", + " FIGlet Font\n", + " \n", + "\n", + " Filebench WML\n", + " \n", + "\n", + " Filterscript\n", + " \n", + "\n", + " fish\n", + " \n", + "\n", + " Fluent\n", + " \n", + "\n", + " FLUX\n", + " \n", + "\n", + " Formatted\n", + " \n", + "\n", + " Forth\n", + " \n", + "\n", + " Fortran\n", + " \n", + "\n", + " Fortran Free Form\n", + " \n", + "\n", + " FreeBasic\n", + " \n", + "\n", + " FreeMarker\n", + " \n", + "\n", + " Frege\n", + " \n", + "\n", + " Futhark\n", + " \n", + "\n", + " G-code\n", + " \n", + "\n", + " Game Maker Language\n", + " \n", + "\n", + " GAML\n", + " \n", + "\n", + " GAMS\n", + " \n", + "\n", + " GAP\n", + " \n", + "\n", + " GCC Machine Description\n", + " \n", + "\n", + " GDB\n", + " \n", + "\n", + " GDScript\n", + " \n", + "\n", + " GEDCOM\n", + " \n", + "\n", + " Gemfile.lock\n", + " \n", + "\n", + " Gemini\n", + " \n", + "\n", + " Genero\n", + " \n", + "\n", + " Genero Forms\n", + " \n", + "\n", + " Genie\n", + " \n", + "\n", + " Genshi\n", + " \n", + "\n", + " Gentoo Ebuild\n", + " \n", + "\n", + " Gentoo Eclass\n", + " \n", + "\n", + " Gerber Image\n", + " \n", + "\n", + " Gettext Catalog\n", + " \n", + "\n", + " Gherkin\n", + " \n", + "\n", + " Git Attributes\n", + " \n", + "\n", + " Git Config\n", + " \n", + "\n", + " Git Revision List\n", + " \n", + "\n", + " Gleam\n", + " \n", + "\n", + " GLSL\n", + " \n", + "\n", + " Glyph\n", + " \n", + "\n", + " Glyph Bitmap Distribution Format\n", + " \n", + "\n", + " GN\n", + " \n", + "\n", + " Gnuplot\n", + " \n", + "\n", + " Go\n", + " \n", + "\n", + " Go Checksums\n", + " \n", + "\n", + " Go Module\n", + " \n", + "\n", + " Godot Resource\n", + " \n", + "\n", + " Golo\n", + " \n", + "\n", + " Gosu\n", + " \n", + "\n", + " Grace\n", + " \n", + "\n", + " Gradle\n", + " \n", + "\n", + " Grammatical Framework\n", + " \n", + "\n", + " Graph Modeling Language\n", + " \n", + "\n", + " GraphQL\n", + " \n", + "\n", + " Graphviz (DOT)\n", + " \n", + "\n", + " Groovy\n", + " \n", + "\n", + " Groovy Server Pages\n", + " \n", + "\n", + " GSC\n", + " \n", + "\n", + " Hack\n", + " \n", + "\n", + " Haml\n", + " \n", + "\n", + " Handlebars\n", + " \n", + "\n", + " HAProxy\n", + " \n", + "\n", + " Harbour\n", + " \n", + "\n", + " Haskell\n", + " \n", + "\n", + " Haxe\n", + " \n", + "\n", + " HCL\n", + " \n", + "\n", + " HiveQL\n", + " \n", + "\n", + " HLSL\n", + " \n", + "\n", + " HOCON\n", + " \n", + "\n", + " HolyC\n", + " \n", + "\n", + " hoon\n", + " \n", + "\n", + " HTML\n", + " \n", + "\n", + " Jinja\n", + " \n", + "\n", + " HTML+ECR\n", + " \n", + "\n", + " HTML+EEX\n", + " \n", + "\n", + " HTML+ERB\n", + " \n", + "\n", + " HTML+PHP\n", + " \n", + "\n", + " HTML+Razor\n", + " \n", + "\n", + " HTTP\n", + " \n", + "\n", + " HXML\n", + " \n", + "\n", + " Hy\n", + " \n", + "\n", + " HyPhy\n", + " \n", + "\n", + " IDL\n", + " \n", + "\n", + " Idris\n", + " \n", + "\n", + " Ignore List\n", + " \n", + "\n", + " IGOR Pro\n", + " \n", + "\n", + " ImageJ Macro\n", + " \n", + "\n", + " Imba\n", + " \n", + "\n", + " Inform 7\n", + " \n", + "\n", + " INI\n", + " \n", + "\n", + " Ink\n", + " \n", + "\n", + " Inno Setup\n", + " \n", + "\n", + " Io\n", + " \n", + "\n", + " Ioke\n", + " \n", + "\n", + " IRC log\n", + " \n", + "\n", + " Isabelle\n", + " \n", + "\n", + " Isabelle ROOT\n", + " \n", + "\n", + " J\n", + " \n", + "\n", + " Janet\n", + " \n", + "\n", + " JAR Manifest\n", + " \n", + "\n", + " Jasmin\n", + " \n", + "\n", + " Java\n", + " \n", + "\n", + " Java Properties\n", + " \n", + "\n", + " Java Server Pages\n", + " \n", + "\n", + " JavaScript\n", + " \n", + "\n", + " JavaScript+ERB\n", + " \n", + "\n", + " JCL\n", + " \n", + "\n", + " Jest Snapshot\n", + " \n", + "\n", + " JetBrains MPS\n", + " \n", + "\n", + " JFlex\n", + " \n", + "\n", + " Jinja\n", + " \n", + "\n", + " Jison\n", + " \n", + "\n", + " Jison Lex\n", + " \n", + "\n", + " Jolie\n", + " \n", + "\n", + " jq\n", + " \n", + "\n", + " JSON\n", + " \n", + "\n", + " JSON with Comments\n", + " \n", + "\n", + " JSON5\n", + " \n", + "\n", + " JSONiq\n", + " \n", + "\n", + " JSONLD\n", + " \n", + "\n", + " Jsonnet\n", + " \n", + "\n", + " Julia\n", + " \n", + "\n", + " Jupyter Notebook\n", + " \n", + "\n", + " Just\n", + " \n", + "\n", + " Kaitai Struct\n", + " \n", + "\n", + " KakouneScript\n", + " \n", + "\n", + " KerboScript\n", + " \n", + "\n", + " KiCad Layout\n", + " \n", + "\n", + " KiCad Legacy Layout\n", + " \n", + "\n", + " KiCad Schematic\n", + " \n", + "\n", + " Kickstart\n", + " \n", + "\n", + " Kit\n", + " \n", + "\n", + " Kotlin\n", + " \n", + "\n", + " KRL\n", + " \n", + "\n", + " Kusto\n", + " \n", + "\n", + " kvlang\n", + " \n", + "\n", + " LabVIEW\n", + " \n", + "\n", + " Lark\n", + " \n", + "\n", + " Lasso\n", + " \n", + "\n", + " Latte\n", + " \n", + "\n", + " Lean\n", + " \n", + "\n", + " Less\n", + " \n", + "\n", + " Lex\n", + " \n", + "\n", + " LFE\n", + " \n", + "\n", + " LigoLANG\n", + " \n", + "\n", + " LilyPond\n", + " \n", + "\n", + " Limbo\n", + " \n", + "\n", + " Linker Script\n", + " \n", + "\n", + " Linux Kernel Module\n", + " \n", + "\n", + " Liquid\n", + " \n", + "\n", + " Literate Agda\n", + " \n", + "\n", + " Literate CoffeeScript\n", + " \n", + "\n", + " Literate Haskell\n", + " \n", + "\n", + " LiveScript\n", + " \n", + "\n", + " LLVM\n", + " \n", + "\n", + " Logos\n", + " \n", + "\n", + " Logtalk\n", + " \n", + "\n", + " LOLCODE\n", + " \n", + "\n", + " LookML\n", + " \n", + "\n", + " LoomScript\n", + " \n", + "\n", + " LSL\n", + " \n", + "\n", + " LTspice Symbol\n", + " \n", + "\n", + " Lua\n", + " \n", + "\n", + " M\n", + " \n", + "\n", + " M4\n", + " \n", + "\n", + " M4Sugar\n", + " \n", + "\n", + " Macaulay2\n", + " \n", + "\n", + " Makefile\n", + " \n", + "\n", + " Mako\n", + " \n", + "\n", + " Markdown\n", + " \n", + "\n", + " Marko\n", + " \n", + "\n", + " Mask\n", + " \n", + "\n", + " Mathematica\n", + " \n", + "\n", + " MATLAB\n", + " \n", + "\n", + " Maven POM\n", + " \n", + "\n", + " Max\n", + " \n", + "\n", + " MAXScript\n", + " \n", + "\n", + " mcfunction\n", + " \n", + "\n", + " Wikitext\n", + " \n", + "\n", + " Mercury\n", + " \n", + "\n", + " Mermaid\n", + " \n", + "\n", + " Meson\n", + " \n", + "\n", + " Metal\n", + " \n", + "\n", + " Microsoft Developer Studio Project\n", + " \n", + "\n", + " Microsoft Visual Studio Solution\n", + " \n", + "\n", + " MiniD\n", + " \n", + "\n", + " MiniYAML\n", + " \n", + "\n", + " Mint\n", + " \n", + "\n", + " Mirah\n", + " \n", + "\n", + " mIRC Script\n", + " \n", + "\n", + " MLIR\n", + " \n", + "\n", + " Modelica\n", + " \n", + "\n", + " Modula-2\n", + " \n", + "\n", + " Modula-3\n", + " \n", + "\n", + " Module Management System\n", + " \n", + "\n", + " Monkey\n", + " \n", + "\n", + " Monkey C\n", + " \n", + "\n", + " Moocode\n", + " \n", + "\n", + " MoonScript\n", + " \n", + "\n", + " Motoko\n", + " \n", + "\n", + " Motorola 68K Assembly\n", + " \n", + "\n", + " Move\n", + " \n", + "\n", + " MQL4\n", + " \n", + "\n", + " MQL5\n", + " \n", + "\n", + " MTML\n", + " \n", + "\n", + " MUF\n", + " \n", + "\n", + " mupad\n", + " \n", + "\n", + " Muse\n", + " \n", + "\n", + " Mustache\n", + " \n", + "\n", + " Myghty\n", + " \n", + "\n", + " nanorc\n", + " \n", + "\n", + " Nasal\n", + " \n", + "\n", + " NASL\n", + " \n", + "\n", + " NCL\n", + " \n", + "\n", + " Nearley\n", + " \n", + "\n", + " Nemerle\n", + " \n", + "\n", + " NEON\n", + " \n", + "\n", + " nesC\n", + " \n", + "\n", + " NetLinx\n", + " \n", + "\n", + " NetLinx+ERB\n", + " \n", + "\n", + " NetLogo\n", + " \n", + "\n", + " NewLisp\n", + " \n", + "\n", + " Nextflow\n", + " \n", + "\n", + " Nginx\n", + " \n", + "\n", + " Nim\n", + " \n", + "\n", + " Ninja\n", + " \n", + "\n", + " Nit\n", + " \n", + "\n", + " Nix\n", + " \n", + "\n", + " NL\n", + " \n", + "\n", + " NPM Config\n", + " \n", + "\n", + " NSIS\n", + " \n", + "\n", + " Nu\n", + " \n", + "\n", + " NumPy\n", + " \n", + "\n", + " Nunjucks\n", + " \n", + "\n", + " NWScript\n", + " \n", + "\n", + " OASv2-json\n", + " \n", + "\n", + " OASv2-yaml\n", + " \n", + "\n", + " OASv3-json\n", + " \n", + "\n", + " OASv3-yaml\n", + " \n", + "\n", + " ObjDump\n", + " \n", + "\n", + " Object Data Instance Notation\n", + " \n", + "\n", + " Objective-C\n", + " \n", + "\n", + " Objective-C++\n", + " \n", + "\n", + " Objective-J\n", + " \n", + "\n", + " ObjectScript\n", + " \n", + "\n", + " OCaml\n", + " \n", + "\n", + " Odin\n", + " \n", + "\n", + " Omgrofl\n", + " \n", + "\n", + " ooc\n", + " \n", + "\n", + " Opa\n", + " \n", + "\n", + " Opal\n", + " \n", + "\n", + " Open Policy Agent\n", + " \n", + "\n", + " OpenAPI Specification v2\n", + " \n", + "\n", + " OpenAPI Specification v3\n", + " \n", + "\n", + " OpenCL\n", + " \n", + "\n", + " OpenEdge ABL\n", + " \n", + "\n", + " OpenQASM\n", + " \n", + "\n", + " OpenRC runscript\n", + " \n", + "\n", + " OpenSCAD\n", + " \n", + "\n", + " OpenStep Property List\n", + " \n", + "\n", + " OpenType Feature File\n", + " \n", + "\n", + " Option List\n", + " \n", + "\n", + " Org\n", + " \n", + "\n", + " Ox\n", + " \n", + "\n", + " Oxygene\n", + " \n", + "\n", + " Oz\n", + " \n", + "\n", + " P4\n", + " \n", + "\n", + " Pan\n", + " \n", + "\n", + " Papyrus\n", + " \n", + "\n", + " Parrot\n", + " \n", + "\n", + " Parrot Assembly\n", + " \n", + "\n", + " Parrot Internal Representation\n", + " \n", + "\n", + " Pascal\n", + " \n", + "\n", + " Pawn\n", + " \n", + "\n", + " PDDL\n", + " \n", + "\n", + " PEG.js\n", + " \n", + "\n", + " Pep8\n", + " \n", + "\n", + " Perl\n", + " \n", + "\n", + " PHP\n", + " \n", + "\n", + " Pic\n", + " \n", + "\n", + " Pickle\n", + " \n", + "\n", + " PicoLisp\n", + " \n", + "\n", + " PigLatin\n", + " \n", + "\n", + " Pike\n", + " \n", + "\n", + " PlantUML\n", + " \n", + "\n", + " PLpgSQL\n", + " \n", + "\n", + " PLSQL\n", + " \n", + "\n", + " Pod\n", + " \n", + "\n", + " Pod 6\n", + " \n", + "\n", + " PogoScript\n", + " \n", + "\n", + " Polar\n", + " \n", + "\n", + " Pony\n", + " \n", + "\n", + " Portugol\n", + " \n", + "\n", + " PostCSS\n", + " \n", + "\n", + " PostScript\n", + " \n", + "\n", + " POV-Ray SDL\n", + " \n", + "\n", + " PowerBuilder\n", + " \n", + "\n", + " PowerShell\n", + " \n", + "\n", + " Prisma\n", + " \n", + "\n", + " Processing\n", + " \n", + "\n", + " Procfile\n", + " \n", + "\n", + " Proguard\n", + " \n", + "\n", + " Prolog\n", + " \n", + "\n", + " Promela\n", + " \n", + "\n", + " Propeller Spin\n", + " \n", + "\n", + " Protocol Buffer\n", + " \n", + "\n", + " Protocol Buffer Text Format\n", + " \n", + "\n", + " Public Key\n", + " \n", + "\n", + " Pug\n", + " \n", + "\n", + " Puppet\n", + " \n", + "\n", + " Pure Data\n", + " \n", + "\n", + " PureBasic\n", + " \n", + "\n", + " PureScript\n", + " \n", + "\n", + " Pyret\n", + " \n", + "\n", + " Python\n", + " \n", + "\n", + " Python console\n", + " \n", + "\n", + " Python traceback\n", + " \n", + "\n", + " q\n", + " \n", + "\n", + " Q#\n", + " \n", + "\n", + " QMake\n", + " \n", + "\n", + " QML\n", + " \n", + "\n", + " Qt Script\n", + " \n", + "\n", + " Quake\n", + " \n", + "\n", + " R\n", + " \n", + "\n", + " Racket\n", + " \n", + "\n", + " Ragel\n", + " \n", + "\n", + " Raku\n", + " \n", + "\n", + " RAML\n", + " \n", + "\n", + " Rascal\n", + " \n", + "\n", + " Raw token data\n", + " \n", + "\n", + " RDoc\n", + " \n", + "\n", + " Readline Config\n", + " \n", + "\n", + " REALbasic\n", + " \n", + "\n", + " Reason\n", + " \n", + "\n", + " ReasonLIGO\n", + " \n", + "\n", + " Rebol\n", + " \n", + "\n", + " Record Jar\n", + " \n", + "\n", + " Red\n", + " \n", + "\n", + " Redcode\n", + " \n", + "\n", + " Redirect Rules\n", + " \n", + "\n", + " Regular Expression\n", + " \n", + "\n", + " Ren'Py\n", + " \n", + "\n", + " RenderScript\n", + " \n", + "\n", + " ReScript\n", + " \n", + "\n", + " reStructuredText\n", + " \n", + "\n", + " REXX\n", + " \n", + "\n", + " Rich Text Format\n", + " \n", + "\n", + " Ring\n", + " \n", + "\n", + " Riot\n", + " \n", + "\n", + " RMarkdown\n", + " \n", + "\n", + " RobotFramework\n", + " \n", + "\n", + " robots.txt\n", + " \n", + "\n", + " Roff\n", + " \n", + "\n", + " Roff Manpage\n", + " \n", + "\n", + " Rouge\n", + " \n", + "\n", + " RouterOS Script\n", + " \n", + "\n", + " RPC\n", + " \n", + "\n", + " RPGLE\n", + " \n", + "\n", + " RPM Spec\n", + " \n", + "\n", + " Ruby\n", + " \n", + "\n", + " RUNOFF\n", + " \n", + "\n", + " Rust\n", + " \n", + "\n", + " Sage\n", + " \n", + "\n", + " SaltStack\n", + " \n", + "\n", + " SAS\n", + " \n", + "\n", + " Sass\n", + " \n", + "\n", + " Scala\n", + " \n", + "\n", + " Scaml\n", + " \n", + "\n", + " Scenic\n", + " \n", + "\n", + " Scheme\n", + " \n", + "\n", + " Scilab\n", + " \n", + "\n", + " SCSS\n", + " \n", + "\n", + " sed\n", + " \n", + "\n", + " Self\n", + " \n", + "\n", + " SELinux Policy\n", + " \n", + "\n", + " ShaderLab\n", + " \n", + "\n", + " Shell\n", + " \n", + "\n", + " ShellCheck Config\n", + " \n", + "\n", + " ShellSession\n", + " \n", + "\n", + " Shen\n", + " \n", + "\n", + " Sieve\n", + " \n", + "\n", + " Simple File Verification\n", + " \n", + "\n", + " Singularity\n", + " \n", + "\n", + " Slash\n", + " \n", + "\n", + " Slice\n", + " \n", + "\n", + " Slim\n", + " \n", + "\n", + " Smali\n", + " \n", + "\n", + " Smalltalk\n", + " \n", + "\n", + " Smarty\n", + " \n", + "\n", + " Smithy\n", + " \n", + "\n", + " SmPL\n", + " \n", + "\n", + " SMT\n", + " \n", + "\n", + " Snakemake\n", + " \n", + "\n", + " Solidity\n", + " \n", + "\n", + " Soong\n", + " \n", + "\n", + " SourcePawn\n", + " \n", + "\n", + " SPARQL\n", + " \n", + "\n", + " Spline Font Database\n", + " \n", + "\n", + " SQF\n", + " \n", + "\n", + " SQL\n", + " \n", + "\n", + " SQLPL\n", + " \n", + "\n", + " Squirrel\n", + " \n", + "\n", + " SRecode Template\n", + " \n", + "\n", + " SSH Config\n", + " \n", + "\n", + " Stan\n", + " \n", + "\n", + " Standard ML\n", + " \n", + "\n", + " STAR\n", + " \n", + "\n", + " Starlark\n", + " \n", + "\n", + " Stata\n", + " \n", + "\n", + " STL\n", + " \n", + "\n", + " STON\n", + " \n", + "\n", + " StringTemplate\n", + " \n", + "\n", + " Stylus\n", + " \n", + "\n", + " SubRip Text\n", + " \n", + "\n", + " SugarSS\n", + " \n", + "\n", + " SuperCollider\n", + " \n", + "\n", + " Svelte\n", + " \n", + "\n", + " SVG\n", + " \n", + "\n", + " Sway\n", + " \n", + "\n", + " Swift\n", + " \n", + "\n", + " SWIG\n", + " \n", + "\n", + " SystemVerilog\n", + " \n", + "\n", + " Talon\n", + " \n", + "\n", + " Tcl\n", + " \n", + "\n", + " Tcsh\n", + " \n", + "\n", + " Tea\n", + " \n", + "\n", + " Terra\n", + " \n", + "\n", + " TeX\n", + " \n", + "\n", + " Texinfo\n", + " \n", + "\n", + " Text\n", + " \n", + "\n", + " Textile\n", + " \n", + "\n", + " TextMate Properties\n", + " \n", + "\n", + " Thrift\n", + " \n", + "\n", + " TI Program\n", + " \n", + "\n", + " TLA\n", + " \n", + "\n", + " TOML\n", + " \n", + "\n", + " TSQL\n", + " \n", + "\n", + " TSV\n", + " \n", + "\n", + " TSX\n", + " \n", + "\n", + " Turing\n", + " \n", + "\n", + " Turtle\n", + " \n", + "\n", + " Twig\n", + " \n", + "\n", + " TXL\n", + " \n", + "\n", + " Type Language\n", + " \n", + "\n", + " TypeScript\n", + " \n", + "\n", + " Unified Parallel C\n", + " \n", + "\n", + " Unity3D Asset\n", + " \n", + "\n", + " Unix Assembly\n", + " \n", + "\n", + " Uno\n", + " \n", + "\n", + " UnrealScript\n", + " \n", + "\n", + " UrWeb\n", + " \n", + "\n", + " V\n", + " \n", + "\n", + " Vala\n", + " \n", + "\n", + " Valve Data Format\n", + " \n", + "\n", + " VBA\n", + " \n", + "\n", + " VBScript\n", + " \n", + "\n", + " VCL\n", + " \n", + "\n", + " Velocity Template Language\n", + " \n", + "\n", + " Verilog\n", + " \n", + "\n", + " VHDL\n", + " \n", + "\n", + " Vim Help File\n", + " \n", + "\n", + " Vim Script\n", + " \n", + "\n", + " Vim Snippet\n", + " \n", + "\n", + " Visual Basic .NET\n", + " \n", + "\n", + " Visual Basic .NET\n", + " \n", + "\n", + " Visual Basic 6.0\n", + " \n", + "\n", + " Volt\n", + " \n", + "\n", + " Vue\n", + " \n", + "\n", + " Vyper\n", + " \n", + "\n", + " Wavefront Material\n", + " \n", + "\n", + " Wavefront Object\n", + " \n", + "\n", + " wdl\n", + " \n", + "\n", + " Web Ontology Language\n", + " \n", + "\n", + " WebAssembly\n", + " \n", + "\n", + " WebIDL\n", + " \n", + "\n", + " WebVTT\n", + " \n", + "\n", + " Wget Config\n", + " \n", + "\n", + " Whiley\n", + " \n", + "\n", + " Wikitext\n", + " \n", + "\n", + " Win32 Message File\n", + " \n", + "\n", + " Windows Registry Entries\n", + " \n", + "\n", + " wisp\n", + " \n", + "\n", + " Witcher Script\n", + " \n", + "\n", + " Wollok\n", + " \n", + "\n", + " World of Warcraft Addon Data\n", + " \n", + "\n", + " Wren\n", + " \n", + "\n", + " X BitMap\n", + " \n", + "\n", + " X Font Directory Index\n", + " \n", + "\n", + " X PixMap\n", + " \n", + "\n", + " X10\n", + " \n", + "\n", + " xBase\n", + " \n", + "\n", + " XC\n", + " \n", + "\n", + " XCompose\n", + " \n", + "\n", + " XML\n", + " \n", + "\n", + " XML Property List\n", + " \n", + "\n", + " Xojo\n", + " \n", + "\n", + " Xonsh\n", + " \n", + "\n", + " XPages\n", + " \n", + "\n", + " XProc\n", + " \n", + "\n", + " XQuery\n", + " \n", + "\n", + " XS\n", + " \n", + "\n", + " XSLT\n", + " \n", + "\n", + " Xtend\n", + " \n", + "\n", + " Yacc\n", + " \n", + "\n", + " YAML\n", + " \n", + "\n", + " YANG\n", + " \n", + "\n", + " YARA\n", + " \n", + "\n", + " YASnippet\n", + " \n", + "\n", + " Yul\n", + " \n", + "\n", + " ZAP\n", + " \n", + "\n", + " Zeek\n", + " \n", + "\n", + " ZenScript\n", + " \n", + "\n", + " Zephir\n", + " \n", + "\n", + " Zig\n", + " \n", + "\n", + " ZIL\n", + " \n", + "\n", + " Zimpl\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + " Date range:\n", + "\n", + " \n", + " Today\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + " Adjust time span\n", + "\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + " Today\n", + " \n", + "\n", + "\n", + "\n", + "\n", + " This week\n", + " \n", + "\n", + "\n", + "\n", + "\n", + " This month\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + " Sponsorable:\n", + " All\n", + "\n", + "\n", + "
\n", + "GitHub Sponsors participation\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "Sponsorable developers\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "All developers\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 1\n", + "\n", + "
\n", + "\n", + "\"@hwchase17\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Harrison Chase\n", + "

\n", + "

\n", + "\n", + " hwchase17\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 2\n", + "\n", + "
\n", + "\n", + "\"@merrymercy\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Lianmin Zheng\n", + "

\n", + "

\n", + "\n", + " merrymercy\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 3\n", + "\n", + "
\n", + "\n", + "\"@hrydgard\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Henrik Rydgård\n", + "

\n", + "

\n", + "\n", + " hrydgard\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "\n", + " 5\n", + "\n", + "
\n", + "\n", + "\"@Ebazhanov\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Evgenii Bazhanov\n", + "

\n", + "

\n", + "\n", + " Ebazhanov\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "\n", + " 7\n", + "\n", + "
\n", + "\n", + "\"@oobabooga\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " oobabooga\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 8\n", + "\n", + "
\n", + "\n", + "\"@MaikuB\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Michael Bui\n", + "

\n", + "

\n", + "\n", + " MaikuB\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 9\n", + "\n", + "
\n", + "\n", + "\"@fonsp\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Fons van der Plas\n", + "

\n", + "

\n", + "\n", + " fonsp\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 10\n", + "\n", + "
\n", + "\n", + "\"@jerryjliu\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Jerry Liu\n", + "

\n", + "

\n", + "\n", + " jerryjliu\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + " 14\n", + "\n", + "
\n", + "\n", + "\"@ammarahm-ed\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Ammar Ahmed\n", + "

\n", + "

\n", + "\n", + " ammarahm-ed\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 15\n", + "\n", + "
\n", + "\n", + "\"@hughsie\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Richard Hughes\n", + "

\n", + "

\n", + "\n", + " hughsie\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 16\n", + "\n", + "
\n", + "\n", + "\"@sunfishcode\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Dan Gohman\n", + "

\n", + "

\n", + "\n", + " sunfishcode\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 17\n", + "\n", + "
\n", + "\n", + "\"@wader\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Mattias Wadman\n", + "

\n", + "

\n", + "\n", + " wader\n", + "

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "Popular repo
\n", + "

\n", + "\n", + "\n", + "\n", + "\n", + " fq\n", + "

\n", + "
\n", + " jq for binary formats - tool, language and decoders for working with binary and text formats\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "\n", + " 19\n", + "\n", + "
\n", + "\n", + "\"@chfast\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Paweł Bylica\n", + "

\n", + "

\n", + "\n", + " chfast\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 20\n", + "\n", + "
\n", + "\n", + "\"@hkirat\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Kirat\n", + "

\n", + "

\n", + "\n", + " hkirat\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 21\n", + "\n", + "
\n", + "\n", + "\"@LaurentMazare\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Laurent Mazare\n", + "

\n", + "

\n", + "\n", + " LaurentMazare\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 22\n", + "\n", + "
\n", + "\n", + "\"@buger\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Leonid Bugaev\n", + "

\n", + "

\n", + "\n", + " buger\n", + "

\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "Popular repo
\n", + "

\n", + "\n", + "\n", + "\n", + "\n", + " goreplay\n", + "

\n", + "
\n", + " GoReplay is an open-source tool for capturing and replaying live HTTP traffic into a test environment in order to continuously test your …\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 23\n", + "\n", + "
\n", + "\n", + "\"@resir014\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Resi Respati\n", + "

\n", + "

\n", + "\n", + " resir014\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 24\n", + "\n", + "
\n", + "\n", + "\"@sirbrillig\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Payton Swick\n", + "

\n", + "

\n", + "\n", + " sirbrillig\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + " 25\n", + "\n", + "
\n", + "\n", + "\"@KodrAus\"\n", + "
\n", + "
\n", + "
\n", + "
\n", + "

\n", + "\n", + " Ashley Mannix\n", + "

\n", + "

\n", + "\n", + " KodrAus\n", + "

\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "Follow\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "
\n", + "
\n", + "
\n", + "\n", + "
\n", + "
\n", + "\n", + "\n", + "\n" + ] + } + ], + "source": [ + "soup = BeautifulSoup(response.content)\n", + "print(soup)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1. Display the names of the trending developers retrieved in the previous step.\n", + "\n", + "Your output should be a Python list of developer names. Each name should not contain any html tag.\n", + "\n", + "**Instructions:**\n", + "\n", + "1. Find out the html tag and class names used for the developer names. You can achieve this using Chrome DevTools or clicking in 'Inspect' on any browser. Here is an example:\n", + "\n", + "![title](example_1.png)\n", + "\n", + "2. Use BeautifulSoup `find_all()` to extract all the html elements that contain the developer names. Hint: pass in the `attrs` parameter to specify the class.\n", + "\n", + "3. Loop through the elements found and get the text for each of them.\n", + "\n", + "4. While you are at it, use string manipulation techniques to replace whitespaces and linebreaks (i.e. `\\n`) in the *text* of each html element. Use a list to store the clean names. Hint: you may also use `.get_text()` instead of `.text` and pass in the desired parameters to do some string manipulation (check the documentation).\n", + "\n", + "5. Print the list of names.\n", + "\n", + "Your output should look like below:\n", + "\n", + "```\n", + "['trimstray (@trimstray)',\n", + " 'joewalnes (JoeWalnes)',\n", + " 'charlax (Charles-AxelDein)',\n", + " 'ForrestKnight (ForrestKnight)',\n", + " 'revery-ui (revery-ui)',\n", + " 'alibaba (Alibaba)',\n", + " 'Microsoft (Microsoft)',\n", + " 'github (GitHub)',\n", + " 'facebook (Facebook)',\n", + " 'boazsegev (Bo)',\n", + " 'google (Google)',\n", + " 'cloudfetch',\n", + " 'sindresorhus (SindreSorhus)',\n", + " 'tensorflow',\n", + " 'apache (TheApacheSoftwareFoundation)',\n", + " 'DevonCrawford (DevonCrawford)',\n", + " 'ARMmbed (ArmMbed)',\n", + " 'vuejs (vuejs)',\n", + " 'fastai (fast.ai)',\n", + " 'QiShaoXuan (Qi)',\n", + " 'joelparkerhenderson (JoelParkerHenderson)',\n", + " 'torvalds (LinusTorvalds)',\n", + " 'CyC2018',\n", + " 'komeiji-satori (神楽坂覚々)',\n", + " 'script-8']\n", + " ```" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "hwchase17\n", + "merrymercy\n", + "hrydgard\n", + "PeerRich\n", + "Ebazhanov\n", + "chenrui333\n", + "MaikuB\n", + "fonsp\n", + "jerryjliu\n", + "mattleibow\n", + "bluwy\n", + "sbc100\n", + "ammarahm-ed\n", + "hughsie\n", + "sunfishcode\n", + "wader\n", + "chfast\n", + "hkirat\n", + "LaurentMazare\n", + "buger\n", + "resir014\n", + "sirbrillig\n", + "KodrAus\n" + ] + } + ], + "source": [ + "box = soup.find_all('div', attrs = {\"class\":\"Box\"})\n", + "\n", + "article = box[1].find_all('a', attrs = {\"class\":\"Link--secondary\"})\n", + "\n", + "for profile in article: \n", + " print(profile.get_text().replace(\"\\n\", \"\").strip())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1.1. Display the trending Python repositories in GitHub.\n", + "\n", + "The steps to solve this problem is similar to the previous one except that you need to find out the repository names instead of developer names." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the url you will scrape in this exercise\n", + "url = 'https://github.com/trending/python?since=daily'" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "AIGC-Audio / AudioGPT\n", + "deep-floyd / IF\n", + "nlpxucan / WizardLM\n", + "UX-Decoder / Segment-Everything-Everywhere-All-At-Once\n", + "lamini-ai / lamini\n", + "xtekky / gpt4free\n", + "gventuri / pandas-ai\n", + "ZrrSkywalker / LLaMA-Adapter\n", + "pytube / pytube\n", + "Rapptz / discord.py\n", + "goauthentik / authentik\n", + "deforum-art / deforum-stable-diffusion\n", + "spotDL / spotify-downloader\n", + "lm-sys / FastChat\n", + "litanlitudan / skyagi\n", + "isaiahbjork / Auto-GPT-Crypto-Plugin\n", + "farizrahman4u / loopgpt\n", + "declare-lab / tango\n", + "donnemartin / system-design-primer\n", + "alaeddine-13 / thinkgpt\n", + "X-PLUG / mPLUG-Owl\n", + "mikumifa / biliTickerBuy\n", + "ytdl-org / youtube-dl\n", + "Nriver / trilium-translation\n", + "itamargol / openai\n" + ] + } + ], + "source": [ + "response = requests.get(url)\n", + "response\n", + "\n", + "soup = BeautifulSoup(response.content)\n", + "trending = soup.find_all(\"h2\", attrs = {\"class\": \"h3 lh-condensed\"})\n", + "trending\n", + "for repo in trending: \n", + " print(repo.get_text().replace(\"\\n\", \"\").strip())" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 2. Display all the image links from Walt Disney wikipedia page.\n", + "Hint: use `.get()` to access information inside tags. Check out the documentation." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the url you will scrape in this exercise\n", + "url = 'https://en.wikipedia.org/wiki/Walt_Disney'" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": {}, + "outputs": [ + { + "ename": "IndexError", + "evalue": "list index out of range", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mIndexError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_9372\\286651990.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[0;32m 4\u001b[0m \u001b[0msoup\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcontent\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 5\u001b[0m \u001b[0msoup1\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msoup\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"td\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mattrs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;34m\"class\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[1;34m\"infobox-image\"\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m \u001b[0mphoto\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msoup1\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"img\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 7\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mimg\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mphoto\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 8\u001b[0m \u001b[0mprint\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"src\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mIndexError\u001b[0m: list index out of range" + ] + } + ], + "source": [ + "response = requests.get(url)\n", + "response\n", + "\n", + "soup = BeautifulSoup(response.content)\n", + "soup1 = soup.find_all(\"td\", attrs = {\"class\": \"infobox-image\"})\n", + "photo = soup1[0].find_all(\"img\")\n", + "for img in photo:\n", + " print(img.get(\"src\"))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 2.1. List all language names and number of related articles in the order they appear in wikipedia.org." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the url you will scrape in this exercise\n", + "url = 'https://www.wikipedia.org/'" + ] + }, + { + "cell_type": "code", + "execution_count": 40, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "English 6 644 000\n", + "Русский 1 909 000\n", + "日本語 1 370 000\n", + "Deutsch 2 792 000\n", + "Español 1 854 000\n", + "Français 2 514 000\n", + "Italiano 1 806 000\n", + "中文 1 347 000\n", + "فارسی فارسی\n", + "Português 1 101 000\n" + ] + }, + { + "data": { + "text/plain": [ + "'Português'" + ] + }, + "execution_count": 40, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response = requests.get(url)\n", + "response\n", + "\n", + "soup = BeautifulSoup(response.content)\n", + "n_language = soup.find_all(\"div\", attrs = {\"class\": \"central-featured-lang\"})\n", + "for i in n_language:\n", + " n_language = i.find(\"strong\").get_text().strip()\n", + " n_article = i.find(\"bdi\").text.strip().split(\"+\")[0]\n", + " print(n_language, n_article)\n", + "n_language" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 2.2. Display the top 10 languages by number of native speakers stored in a pandas dataframe.\n", + "Hint: After finding the correct table you want to analyse, you can use a nested **for** loop to find the elements row by row (check out the 'td' and 'tr' tags).
An easier way to do it is using pd.read_html(), check out documentation [here](https://pandas.pydata.org/pandas-docs/version/0.23.4/generated/pandas.read_html.html)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the url you will scrape in this exercise\n", + "url = 'https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers'" + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [ + { + "ename": "ValueError", + "evalue": "No tables found", + "output_type": "error", + "traceback": [ + "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[1;31mValueError\u001b[0m Traceback (most recent call last)", + "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_9372\\3772482250.py\u001b[0m in \u001b[0;36m\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_html\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 2\u001b[0m \u001b[0mtables\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_html\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0murl\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 3\u001b[0m \u001b[0mtables\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\util\\_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m 309\u001b[0m \u001b[0mstacklevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 310\u001b[0m )\n\u001b[1;32m--> 311\u001b[1;33m \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 312\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 313\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\html.py\u001b[0m in \u001b[0;36mread_html\u001b[1;34m(io, match, flavor, header, index_col, skiprows, attrs, parse_dates, thousands, encoding, decimal, converters, na_values, keep_default_na, displayed_only)\u001b[0m\n\u001b[0;32m 1111\u001b[0m \u001b[0mio\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mstringify_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mio\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1112\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1113\u001b[1;33m return _parse(\n\u001b[0m\u001b[0;32m 1114\u001b[0m \u001b[0mflavor\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mflavor\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 1115\u001b[0m \u001b[0mio\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mio\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\html.py\u001b[0m in \u001b[0;36m_parse\u001b[1;34m(flavor, io, match, attrs, encoding, displayed_only, **kwargs)\u001b[0m\n\u001b[0;32m 937\u001b[0m \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 938\u001b[0m \u001b[1;32massert\u001b[0m \u001b[0mretained\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m \u001b[1;31m# for mypy\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 939\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mretained\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 940\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 941\u001b[0m \u001b[0mret\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\html.py\u001b[0m in \u001b[0;36m_parse\u001b[1;34m(flavor, io, match, attrs, encoding, displayed_only, **kwargs)\u001b[0m\n\u001b[0;32m 917\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 918\u001b[0m \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 919\u001b[1;33m \u001b[0mtables\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparse_tables\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 920\u001b[0m \u001b[1;32mexcept\u001b[0m \u001b[0mValueError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mcaught\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 921\u001b[0m \u001b[1;31m# if `io` is an io-like object, check if it's seekable\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\html.py\u001b[0m in \u001b[0;36mparse_tables\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m 237\u001b[0m \u001b[0mlist\u001b[0m \u001b[0mof\u001b[0m \u001b[0mparsed\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mheader\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mbody\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfooter\u001b[0m\u001b[1;33m)\u001b[0m \u001b[0mtuples\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0mtables\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 238\u001b[0m \"\"\"\n\u001b[1;32m--> 239\u001b[1;33m \u001b[0mtables\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parse_tables\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_build_doc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mattrs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 240\u001b[0m \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_parse_thead_tbody_tfoot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtable\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mtable\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtables\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 241\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\pandas\\io\\html.py\u001b[0m in \u001b[0;36m_parse_tables\u001b[1;34m(self, doc, match, attrs)\u001b[0m\n\u001b[0;32m 567\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 568\u001b[0m \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mtables\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 569\u001b[1;33m \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"No tables found\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m 570\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m 571\u001b[0m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n", + "\u001b[1;31mValueError\u001b[0m: No tables found" + ] + } + ], + "source": [ + "pd.read_html(url)\n", + "tables = pd.read_html(url)\n", + "tables" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3. Display IMDB's top 250 data (movie name, initial release, director name and stars) as a pandas dataframe.\n", + "Hint: If you hover over the title of the movie, you should see the director's name. Can you find where it's stored in the html?" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the url you will scrape in this exercise \n", + "url = 'https://www.imdb.com/chart/top'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3.1. Display the movie name, year and a brief summary of the top 10 random movies (IMDB) as a pandas dataframe." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#This is the url you will scrape in this exercise\n", + "url = 'https://www.imdb.com/list/ls009796553/'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Bonus" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Find the live weather report (temperature, wind speed, description and weather) of a given city." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#https://openweathermap.org/current\n", + "city = input('Enter the city: ')\n", + "url = 'http://api.openweathermap.org/data/2.5/weather?'+'q='+city+'&APPID=b35975e18dc93725acb092f7272cc6b8&units=metric'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Find the book name, price and stock availability as a pandas dataframe." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the url you will scrape in this exercise. \n", + "# It is a fictional bookstore created to be scraped. \n", + "url = 'http://books.toscrape.com/'" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Display the 100 latest earthquakes info (date, time, latitude, longitude and region name) by the EMSC as a pandas dataframe.\n", + "***Hint:*** Here the displayed number of earthquakes per page is 20, but you can easily move to the next page by looping through the desired number of pages and adding it to the end of the url." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# This is the url you will scrape in this exercise\n", + "url = 'https://www.emsc-csem.org/Earthquake/?view='\n", + "\n", + "# This is how you will loop through each page:\n", + "number_of_pages = int(100/20)\n", + "each_page_urls = []\n", + "\n", + "for n in range(1, number_of_pages+1):\n", + " link = url+str(n)\n", + " each_page_urls.append(link)\n", + " \n", + "each_page_urls" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# your code here" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/your-code/main.ipynb b/your-code/main.ipynb deleted file mode 100755 index 1fe9046..0000000 --- a/your-code/main.ipynb +++ /dev/null @@ -1,417 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Web Scraping Lab\n", - "\n", - "You will find in this notebook some scrapy exercises to practise your scraping skills.\n", - "\n", - "**Tips:**\n", - "\n", - "- Check the response status code for each request to ensure you have obtained the intended content.\n", - "- Print the response text in each request to understand the kind of info you are getting and its format.\n", - "- Check for patterns in the response text to extract the data/info requested in each question.\n", - "- Visit the urls below and take a look at their source code through Chrome DevTools. You'll need to identify the html tags, special class names, etc used in the html content you are expected to extract.\n", - "\n", - "**Resources**:\n", - "- [Requests library](http://docs.python-requests.org/en/master/#the-user-guide)\n", - "- [Beautiful Soup Doc](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)\n", - "- [Urllib](https://docs.python.org/3/library/urllib.html#module-urllib)\n", - "- [re lib](https://docs.python.org/3/library/re.html)\n", - "- [lxml lib](https://lxml.de/)\n", - "- [Scrapy](https://scrapy.org/)\n", - "- [List of HTTP status codes](https://en.wikipedia.org/wiki/List_of_HTTP_status_codes)\n", - "- [HTML basics](http://www.simplehtmlguide.com/cheatsheet.php)\n", - "- [CSS basics](https://www.cssbasics.com/#page_start)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Below are the libraries and modules you may need. `requests`, `BeautifulSoup` and `pandas` are already imported for you. If you prefer to use additional libraries feel free to do it." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import requests\n", - "from bs4 import BeautifulSoup\n", - "import pandas as pd" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Download, parse (using BeautifulSoup), and print the content from the Trending Developers page from GitHub:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise\n", - "url = 'https://github.com/trending/developers'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 1. Display the names of the trending developers retrieved in the previous step.\n", - "\n", - "Your output should be a Python list of developer names. Each name should not contain any html tag.\n", - "\n", - "**Instructions:**\n", - "\n", - "1. Find out the html tag and class names used for the developer names. You can achieve this using Chrome DevTools or clicking in 'Inspect' on any browser. Here is an example:\n", - "\n", - "![title](example_1.png)\n", - "\n", - "2. Use BeautifulSoup `find_all()` to extract all the html elements that contain the developer names. Hint: pass in the `attrs` parameter to specify the class.\n", - "\n", - "3. Loop through the elements found and get the text for each of them.\n", - "\n", - "4. While you are at it, use string manipulation techniques to replace whitespaces and linebreaks (i.e. `\\n`) in the *text* of each html element. Use a list to store the clean names. Hint: you may also use `.get_text()` instead of `.text` and pass in the desired parameters to do some string manipulation (check the documentation).\n", - "\n", - "5. Print the list of names.\n", - "\n", - "Your output should look like below:\n", - "\n", - "```\n", - "['trimstray (@trimstray)',\n", - " 'joewalnes (JoeWalnes)',\n", - " 'charlax (Charles-AxelDein)',\n", - " 'ForrestKnight (ForrestKnight)',\n", - " 'revery-ui (revery-ui)',\n", - " 'alibaba (Alibaba)',\n", - " 'Microsoft (Microsoft)',\n", - " 'github (GitHub)',\n", - " 'facebook (Facebook)',\n", - " 'boazsegev (Bo)',\n", - " 'google (Google)',\n", - " 'cloudfetch',\n", - " 'sindresorhus (SindreSorhus)',\n", - " 'tensorflow',\n", - " 'apache (TheApacheSoftwareFoundation)',\n", - " 'DevonCrawford (DevonCrawford)',\n", - " 'ARMmbed (ArmMbed)',\n", - " 'vuejs (vuejs)',\n", - " 'fastai (fast.ai)',\n", - " 'QiShaoXuan (Qi)',\n", - " 'joelparkerhenderson (JoelParkerHenderson)',\n", - " 'torvalds (LinusTorvalds)',\n", - " 'CyC2018',\n", - " 'komeiji-satori (神楽坂覚々)',\n", - " 'script-8']\n", - " ```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 1.1. Display the trending Python repositories in GitHub.\n", - "\n", - "The steps to solve this problem is similar to the previous one except that you need to find out the repository names instead of developer names." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise\n", - "url = 'https://github.com/trending/python?since=daily'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 2. Display all the image links from Walt Disney wikipedia page.\n", - "Hint: use `.get()` to access information inside tags. Check out the documentation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise\n", - "url = 'https://en.wikipedia.org/wiki/Walt_Disney'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 2.1. List all language names and number of related articles in the order they appear in wikipedia.org." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise\n", - "url = 'https://www.wikipedia.org/'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 2.2. Display the top 10 languages by number of native speakers stored in a pandas dataframe.\n", - "Hint: After finding the correct table you want to analyse, you can use a nested **for** loop to find the elements row by row (check out the 'td' and 'tr' tags).
An easier way to do it is using pd.read_html(), check out documentation [here](https://pandas.pydata.org/pandas-docs/version/0.23.4/generated/pandas.read_html.html)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise\n", - "url = 'https://en.wikipedia.org/wiki/List_of_languages_by_number_of_native_speakers'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 3. Display IMDB's top 250 data (movie name, initial release, director name and stars) as a pandas dataframe.\n", - "Hint: If you hover over the title of the movie, you should see the director's name. Can you find where it's stored in the html?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise \n", - "url = 'https://www.imdb.com/chart/top'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### 3.1. Display the movie name, year and a brief summary of the top 10 random movies (IMDB) as a pandas dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#This is the url you will scrape in this exercise\n", - "url = 'https://www.imdb.com/list/ls009796553/'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Bonus" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Find the live weather report (temperature, wind speed, description and weather) of a given city." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#https://openweathermap.org/current\n", - "city = input('Enter the city: ')\n", - "url = 'http://api.openweathermap.org/data/2.5/weather?'+'q='+city+'&APPID=b35975e18dc93725acb092f7272cc6b8&units=metric'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Find the book name, price and stock availability as a pandas dataframe." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise. \n", - "# It is a fictional bookstore created to be scraped. \n", - "url = 'http://books.toscrape.com/'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Display the 100 latest earthquakes info (date, time, latitude, longitude and region name) by the EMSC as a pandas dataframe.\n", - "***Hint:*** Here the displayed number of earthquakes per page is 20, but you can easily move to the next page by looping through the desired number of pages and adding it to the end of the url." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# This is the url you will scrape in this exercise\n", - "url = 'https://www.emsc-csem.org/Earthquake/?view='\n", - "\n", - "# This is how you will loop through each page:\n", - "number_of_pages = int(100/20)\n", - "each_page_urls = []\n", - "\n", - "for n in range(1, number_of_pages+1):\n", - " link = url+str(n)\n", - " each_page_urls.append(link)\n", - " \n", - "each_page_urls" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# your code here" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -}