curl-adapter trying new curl adapter + some fixes

.github/workflows/python-package.yml

@@ -61,6 +61,7 @@ jobs:
       with:
         name: Test result
         path: report.html
+      if: ${{ always() }}
 
 
 

moddb/__init__.py

@@ -1,13 +1,15 @@
+from curl_adapter import CurlCffiAdapter
 import requests
 
 from .base import front_page, login, logout, parse_page, parse_results, rss, search, search_tags
 from .client import Client, Thread
 from .enums import *
 from .pages import *
-from .utils import BASE_URL, LOGGER, Object, get_page, request, soup, SSLAdapter
+from .utils import BASE_URL, LOGGER, Object, get_page, request, soup
 
 SESSION = requests.Session()
-SESSION.mount("https://", SSLAdapter())
+SESSION.mount("http://", CurlCffiAdapter())
+SESSION.mount("https://", CurlCffiAdapter())
 
 __version__ = "0.13.0"
 

moddb/boxes.py

@@ -208,19 +208,8 @@ def __init__(self, html: BeautifulSoup):
             "div", class_="table tablemenu"
         )
         self.contact = join(html.find("h5", string="Contact").parent.span.a["href"])
-        self.follow = join(
-            profile_raw.find_all(
-                "h5",
-                string=[
-                    "Mod watch",
-                    "Game watch",
-                    "Group watch",
-                    "Engine watch",
-                    "Hardware watch",
-                    "Software watch",
-                ],
-            )[0].parent.span.a["href"]
-        )
+
+        self.follow = join(html.find("a", title="Follow")["href"])
 
         try:
             share = profile_raw.find("h5", string="Share").parent.span.find_all("a")
@@ -874,8 +863,8 @@ def __init__(self, html: BeautifulSoup):
             )
 
         try:
-            self.follow = join(profile_raw.find("h5", string="Member watch").parent.span.a["href"])
-        except AttributeError:
+            self.follow = join(html.find("a", title="Follow")["href"])
+        except TypeError:
             LOGGER.info(
                 "Can't watch yourself, narcissist...", exc_info=LOGGER.level >= logging.DEBUG
             )
@@ -1073,7 +1062,9 @@ def __init__(self, **kwargs):
     def __repr__(self):
         return f"<Option text={self.text}>"
 
-T = TypeVar('T')
+
+T = TypeVar("T")
+
 
 class ModDBList(collections.abc.MutableSequence[T], Generic[T]):
     """Base List type for the lib

moddb/client.py

@@ -5,6 +5,7 @@
 import sys
 from typing import TYPE_CHECKING, Any, List, Tuple, Union
 
+from curl_adapter import CurlCffiAdapter
 import requests
 from bs4 import BeautifulSoup
 from requests import utils
@@ -20,7 +21,6 @@
     GLOBAL_LIMITER,
     GLOBAL_THROTLE,
     LOGGER,
-    SSLAdapter,
     concat_docs,
     generate_hash,
     generate_login_cookies,
@@ -295,7 +295,8 @@ class Client:
 
     def __init__(self, username: str, password: str):
         session = requests.Session()
-        session.mount("https://", SSLAdapter())
+        session.mount("http://", CurlCffiAdapter())
+        session.mount("https://", CurlCffiAdapter())
         session.cookies = generate_login_cookies(username, password, session=session)
         self._session = session
         LOGGER.info("Login successful for %s", username)

moddb/pages/article.py

@@ -129,7 +129,7 @@ def __init__(self, html: bs4.BeautifulSoup):
         self.summary = html.find("p", class_="introductiontext").string
 
         if self.category == ArticleCategory.tutorials:
-            cat = html.find("span", itemprop="proficiencyLevel").nextSibling.strip()
+            cat = html.find("span", itemprop="proficiencyLevel").next_sibling.strip()
             self.tutorial_category = TutorialCategory[
                 cat.replace("/", "_").replace(" ", "_").lower()
             ]

moddb/pages/fp.py

@@ -4,7 +4,7 @@
 
 from ..boxes import Thumbnail
 from ..enums import ThumbnailType
-from ..utils import LOGGER, get_page, get_page_type
+from ..utils import LOGGER, get_page, get_page_type, join
 from . import opinion
 
 
@@ -35,6 +35,8 @@ class FrontPage:
     files : List[Thumbnail]
         A list of file like thumbnail objects representing the suggested
         files on the front page.
+    poll_url : str
+        The url to the poll on the front page
     """
 
     def __init__(self, html: bs4.BeautifulSoup):
@@ -146,7 +148,15 @@ def __init__(self, html: bs4.BeautifulSoup):
             for x in files
         ]
 
-        self._poll_url = html.find("div", class_="poll").form["action"]
+        try:
+            # maybe they haven't voted yet and the vote bar is there
+            self.poll_url = html.find("div", class_="poll").form["action"]
+        except AttributeError:
+            # maybe they've already voted and the result link is there
+            self.poll_url = join(
+                html.find("div", class_="poll").find("a", class_="results")["href"]
+            )
+
         self._html = html
         self._poll = None
 
@@ -163,6 +173,6 @@ def get_poll(self) -> opinion.Poll:
             The returned poll
         """
         if self._poll is None:
-            self._poll = opinion.Poll(get_page(self._poll_url))
+            self._poll = opinion.Poll(get_page(self.poll_url))
 
         return self._poll

requirements.txt

@@ -1,2 +1,3 @@
 beautifulsoup4 == 4.*
 requests == 2.*
+curl_adapter == 1.*

run_lint.sh

@@ -1,2 +1,2 @@
-black --diff  moddb/ tests/ --line-length 100
+black --check --diff  moddb/ tests/ --line-length 100
 flakeheaven lint moddb/

tests/test_main.py

@@ -31,10 +31,12 @@
 handler.setFormatter(logging.Formatter("%(asctime)s:%(levelname)s:%(name)s: %(message)s"))
 logger.addHandler(handler)
 
-@pytest.fixture(scope='class')
+
+@pytest.fixture(scope="class")
 def delay():
     time.sleep(60)
 
+
 @pytest.mark.vcr
 class TestMods(test_mod.TestMod):
     @pytest.fixture(params=utils.mod_urls, autouse=True)