scraper.py

#!./venv/Scripts/python

import os
import requests
import textwrap
from argparse import ArgumentParser
from bs4 import BeautifulSoup
from urllib.parse import urlparse


def main():
    parser = ArgumentParser()
    parser.add_argument("-m", "--margin", type=int)
    parser.add_argument("-t", "--target", type=str)
    parser.add_argument("-w", "--width", type=int)
    args = parser.parse_args()
    Scraper(args.target).set_margin(args.margin).set_width(args.width).write_to_file()


class Scraper:

    def __init__(self, url):
        self.margin = "\n"*2
        self.soup = BeautifulSoup(requests.get(url).text, "html.parser")
        self.tags = self.__get_tags()
        self.url = url
        self.width = 80

    def write_to_file(self):
        pr = urlparse(self.url)
        filename = "./out/" + pr.netloc + pr.path[:-1] + ".txt"
        if not os.path.exists(os.path.dirname(filename)):
            os.makedirs(os.path.dirname(filename))
        open(filename, 'w', encoding="utf-8").write(self.get_text())

    def __get_tags(self):
        return self.soup.find_all(lambda tag: tag.name.startswith('h') and len(tag.name) <= 2 or tag.name == 'p')

    def get_text(self):
        temp = "Generated by PyScraper" + self.margin
        for tag in self.tags:
            link = tag.find('a')
            if link is not None:
                link.replaceWith("%s" % link.string + '[' + link.get("href") + ']')
            temp += "\n".join(textwrap.wrap(tag.getText(), self.width)) + self.margin
        return temp

    def set_margin(self, margin):
        if margin is not None:
            self.margin = "\n"*margin
        return self

    def set_width(self, width):
        if width is not None:
            self.width = width
        return self


main()