From bf3432368595267e37c5796c9b82b81f860b06cf Mon Sep 17 00:00:00 2001 From: Tosox <57193602+Tosox@users.noreply.github.com> Date: Fri, 14 Jun 2024 13:21:52 +0200 Subject: [PATCH 1/6] add test upload_addon method --- moddb/client.py | 67 ++++++++++++++++++++++++++++++++++++-- moddb/enums.py | 38 +++++++++++++++++++++ scripts/data/enums_base.py | 38 +++++++++++++++++++++ 3 files changed, 141 insertions(+), 2 deletions(-) diff --git a/moddb/client.py b/moddb/client.py index bb6b857..29afe27 100644 --- a/moddb/client.py +++ b/moddb/client.py @@ -1,5 +1,7 @@ from __future__ import annotations +import mimetypes +import os import random import re import sys @@ -11,7 +13,7 @@ from .base import parse_page from .boxes import ResultList, Thumbnail, _parse_results -from .enums import Status, ThumbnailType +from .enums import AddonCategory, Licence, PlatformCategory, Status, ThumbnailType from .errors import ModdbException from .pages import Member from .utils import ( @@ -324,7 +326,7 @@ def _request(self, method, url, **kwargs): } req = requests.Request( - method, url, headers=headers, cookies=cookies, data=kwargs.pop("data", {}) + method, url, headers=headers, cookies=cookies, data=kwargs.pop("data", {}), files=kwargs.pop("files", {}) ) prepped = self._session.prepare_request(req) LOGGER.info("Request: %s", prepped.url) @@ -1173,3 +1175,64 @@ def downvote_tag(self, tag: Tag) -> bool: Whether the downvote was successful """ return self._vote_tag(tag, 1) + + def upload_addon(self, mod: Mod, filepath: str, thumbnail_filepath: str, category: AddonCategory, + name: str, summary: str, description: str, platforms: List[PlatformCategory], + licence: Licence = Licence.proprietary, credits: str = "", tags: List[str] = []): + # Do some client-sided checks + cwd = os.getcwd() + abs_filepath = os.path.join(cwd, filepath) + if not os.path.isfile(abs_filepath): + raise ModdbException("Please select a file before uploading") + + # Values copied from ModDB + # https://static.moddb.com/html/external/min/index.php?b=cutoff&f=js/jquery.ajaxuploader.js,js/jquery.form.js,js/jquery.multiselects.js&1 + filesize = os.path.getsize(abs_filepath) + if filesize <= 0: + raise ModdbException("Your file cannot be empty") + elif filesize > (52428800 * 1024): + raise ModdbException(f"Your file must be less then {52428800 * 1024}") + + abs_thumbnail_filepath = os.path.join(cwd, thumbnail_filepath) + + form = self._request("GET", f"{mod.url}/addons/add") + html = soup(form.text) + + formhash = html.find("input", { "name": "formhash" })["value"] + mod_id = html.find("select", class_="right select").find_all("option")[1]["value"] + + addon_file = { + "filedata": open(abs_filepath, 'rb') + } + upload_resp = self._request("POST", f"https://upload.moddb.com/downloads/ajax/upload/{formhash}", files=addon_file) + with open(f"{cwd}/resp1.html", 'w+') as f: + f.write(upload_resp.text) + if upload_resp.json()["error"]: + raise ModdbException("An error occurred while trying to upload the add-on") + + logo_file = { + "logo": open(abs_thumbnail_filepath, 'rb') + } + data = { + "formhash": formhash, + "legacy": 0, + "platformstemp": 1, + "filedataUp": os.path.basename(filepath), + "category": category.value, + "licence": licence.value, + "credits": credits, + "tags": ",".join(tags), + "name": name, + "summary": summary, + "description": description, # Must be html + "downloads": "Please wait uploading file", + "links[]": [] + } + data["links[]"].extend([platform.value for platform in platforms]) + data["links[]"].append(mod_id) + + resp2 = self._request("POST", f"{mod.url}/addons/add", data=data, files=logo_file) + print(f"Status code {resp2.status_code}") + with open(f"{cwd}/resp2.html", 'w+') as f: + f.write(resp2.text) + # TODO: check resp diff --git a/moddb/enums.py b/moddb/enums.py index 4f59f44..da3863a 100644 --- a/moddb/enums.py +++ b/moddb/enums.py @@ -170,6 +170,44 @@ class Month(enum.Enum): december = "12" +class PlatformCategory(enum.Enum): + """The category of the platform""" + + windows = "Windows|platforms1" + mac = "Mac|platforms8" + linux = "Linux|platforms7" + vr = "VR|platforms35" + ar = "AR|platforms36" + web = "Web|platforms24" + rtx = "RTX|platforms40" + flash = "Flash|platforms23" + dos = "DOS|platforms19" + steamdeck = "SteamDeck|platforms41" + ios = "iOS|platforms20" + android = "Android|platforms22" + metro = "Metro|platforms25" + xsx = "XSX|platforms39" + xone = "XONE|platforms34" + x360 = "X360|platforms2" + xbox = "XBOX|platforms18" + ps5 = "PS5|platforms38" + ps4 = "PS4|platforms32" + ps3 = "PS3|platforms4" + ps2 = "PS2|platforms17" + ps1 = "PS1|platforms16" + vita = "VITA|platforms28" + psp = "PSP|platforms5" + switch = "Switch|platforms37" + wiiu = "WiiU|platforms31" + wii = "Wii|platforms3" + gcn = "GCN|platforms15" + n64 = "N64|platforms14" + snes = "SNES|platforms13" + nes = "NES|platforms12" + ds = "DS|platforms6" + gba = "GBA|platforms11" + + # BELOW THIS LINE ENUMS ARE GENERATED AUTOMATICALLY # PR changes to scripts/generate_enums.py if you want to # change something diff --git a/scripts/data/enums_base.py b/scripts/data/enums_base.py index 6976cd3..24aecba 100644 --- a/scripts/data/enums_base.py +++ b/scripts/data/enums_base.py @@ -169,6 +169,44 @@ class Month(enum.Enum): december = "12" +class PlatformCategory(enum.Enum): + """The category of the platform""" + + windows = "Windows|platforms1" + mac = "Mac|platforms8" + linux = "Linux|platforms7" + vr = "VR|platforms35" + ar = "AR|platforms36" + web = "Web|platforms24" + rtx = "RTX|platforms40" + flash = "Flash|platforms23" + dos = "DOS|platforms19" + steamdeck = "SteamDeck|platforms41" + ios = "iOS|platforms20" + android = "Android|platforms22" + metro = "Metro|platforms25" + xsx = "XSX|platforms39" + xone = "XONE|platforms34" + x360 = "X360|platforms2" + xbox = "XBOX|platforms18" + ps5 = "PS5|platforms38" + ps4 = "PS4|platforms32" + ps3 = "PS3|platforms4" + ps2 = "PS2|platforms17" + ps1 = "PS1|platforms16" + vita = "VITA|platforms28" + psp = "PSP|platforms5" + switch = "Switch|platforms37" + wiiu = "WiiU|platforms31" + wii = "Wii|platforms3" + gcn = "GCN|platforms15" + n64 = "N64|platforms14" + snes = "SNES|platforms13" + nes = "NES|platforms12" + ds = "DS|platforms6" + gba = "GBA|platforms11" + + # BELOW THIS LINE ENUMS ARE GENERATED AUTOMATICALLY # PR changes to scripts/generate_enums.py if you want to # change something From 898315968ef0a9562aeb3c89420bef16e4f61379 Mon Sep 17 00:00:00 2001 From: Tosox <57193602+Tosox@users.noreply.github.com> Date: Fri, 21 Jun 2024 18:55:04 +0200 Subject: [PATCH 2/6] refactor: upload addon method --- moddb/client.py | 98 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/moddb/client.py b/moddb/client.py index 29afe27..77fad94 100644 --- a/moddb/client.py +++ b/moddb/client.py @@ -1,6 +1,5 @@ from __future__ import annotations -import mimetypes import os import random import re @@ -40,7 +39,7 @@ if TYPE_CHECKING: from .boxes import Comment, Tag from .enums import WatchType - from .pages import Engine, Game, Group, Mod, Review, Team + from .pages import Addon, Engine, Game, Group, Mod, Review, Team class Message: @@ -1176,63 +1175,104 @@ def downvote_tag(self, tag: Tag) -> bool: """ return self._vote_tag(tag, 1) - def upload_addon(self, mod: Mod, filepath: str, thumbnail_filepath: str, category: AddonCategory, + def _validate_file(self, path: str, max_mbytes: int, accepted_extensions: List[str]): + # Check if is file + if not os.path.isfile(path): + raise ModdbException("Please select a valid file before uploading") + + # Check file size + file_size = os.path.getsize(path) / 2048 # b -> mb + if file_size <= 0: + raise ModdbException("Your file cannot be empty") + elif file_size > max_mbytes: + raise ModdbException(f"Your file must be less then {max_mbytes}mb") + + # Check file extension + file_ext = os.path.splitext(path)[1] + if file_ext not in accepted_extensions: + raise ModdbException(f"You cannot select a {file_ext} file only ({', '.join(accepted_extensions)})") + + def _validate_summary(self, text: str): + if len(text) < 50 or len(text) > 1000: + raise ModdbException("The summary must contain at least 50 and at most 1000 characters") + + def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: AddonCategory, name: str, summary: str, description: str, platforms: List[PlatformCategory], licence: Licence = Licence.proprietary, credits: str = "", tags: List[str] = []): - # Do some client-sided checks + form = self._request("GET", f"{mod.url}/addons/add") + html = soup(form.text) + cwd = os.getcwd() - abs_filepath = os.path.join(cwd, filepath) - if not os.path.isfile(abs_filepath): - raise ModdbException("Please select a file before uploading") - - # Values copied from ModDB - # https://static.moddb.com/html/external/min/index.php?b=cutoff&f=js/jquery.ajaxuploader.js,js/jquery.form.js,js/jquery.multiselects.js&1 - filesize = os.path.getsize(abs_filepath) - if filesize <= 0: - raise ModdbException("Your file cannot be empty") - elif filesize > (52428800 * 1024): - raise ModdbException(f"Your file must be less then {52428800 * 1024}") + abs_addon_path = os.path.join(cwd, addon_path) + abs_thumbnail_path = os.path.join(cwd, thumbnail_path) - abs_thumbnail_filepath = os.path.join(cwd, thumbnail_filepath) + # Do some client-sided checks - form = self._request("GET", f"{mod.url}/addons/add") - html = soup(form.text) + # Validate summary + self._validate_summary(summary) + + # Validate add-on file + addon_exts = html.find("input", id="downloadsfiledata")["accept"].split(",") + self._validate_file(abs_addon_path, 52428800, addon_exts) + # Validate thumbnail file + thumbnail_exts = html.find("input", id="downloadslogo")["accept"].split(",") + self._validate_file(abs_thumbnail_path, 8192, thumbnail_exts) + + # Retrieve data for payloads formhash = html.find("input", { "name": "formhash" })["value"] mod_id = html.find("select", class_="right select").find_all("option")[1]["value"] addon_file = { - "filedata": open(abs_filepath, 'rb') + "filedata": open(abs_addon_path, 'rb') } upload_resp = self._request("POST", f"https://upload.moddb.com/downloads/ajax/upload/{formhash}", files=addon_file) - with open(f"{cwd}/resp1.html", 'w+') as f: - f.write(upload_resp.text) if upload_resp.json()["error"]: raise ModdbException("An error occurred while trying to upload the add-on") logo_file = { - "logo": open(abs_thumbnail_filepath, 'rb') + "logo": open(abs_thumbnail_path, 'rb') } data = { "formhash": formhash, "legacy": 0, "platformstemp": 1, - "filedataUp": os.path.basename(filepath), + "filedataUp": os.path.basename(addon_path), "category": category.value, "licence": licence.value, "credits": credits, "tags": ",".join(tags), "name": name, "summary": summary, - "description": description, # Must be html + "description": description, # Must be html. Usually text enclosed with p-tag "downloads": "Please wait uploading file", "links[]": [] } data["links[]"].extend([platform.value for platform in platforms]) data["links[]"].append(mod_id) - resp2 = self._request("POST", f"{mod.url}/addons/add", data=data, files=logo_file) - print(f"Status code {resp2.status_code}") - with open(f"{cwd}/resp2.html", 'w+') as f: - f.write(resp2.text) - # TODO: check resp + post_resp = self._request("POST", f"{mod.url}/addons/add", data=data, files=logo_file) + + # Check if ModDB reports errors + post_resp_html = soup(post_resp.text) + download_button = post_resp_html.find("a", id="downloadmirrorstoggle") + if not download_button: + # We are still on the upload form + error_tooltip = post_resp_html.find("div", class_="tooltip errortooltip clear") + if error_tooltip: + if error_tooltip.ul: + error_list = error_tooltip.ul.find_all("li", recursive=False) + errors = "\n".join([f"- {error.text}" for error in error_list]) + else: + # p-tag contains a space at the beginning and a new line at the end + errors = f"-{error_tooltip.p.text[:-1]}" + raise ModdbException(f"Please correct the following: \n{errors}") + + def update_addon(self, addon: Addon): + form = self._request("GET", f"{addon.url}/edit") + html = soup(form.text) + + if not html.find("input", { "name": "formhash" }): + raise ModdbException("You do not have permission to edit the requested downloads content") + + From 50a26ac275c45ecfaac070a34bb4f54b28561d8e Mon Sep 17 00:00:00 2001 From: Tosox <57193602+Tosox@users.noreply.github.com> Date: Sun, 23 Jun 2024 14:25:31 +0200 Subject: [PATCH 3/6] improve validations --- moddb/client.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/moddb/client.py b/moddb/client.py index 77fad94..4761f84 100644 --- a/moddb/client.py +++ b/moddb/client.py @@ -6,6 +6,13 @@ import sys from typing import TYPE_CHECKING, Any, List, Tuple, Union +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.firefox.service import Service +import time + import requests from bs4 import BeautifulSoup from requests import utils @@ -1195,6 +1202,18 @@ def _validate_file(self, path: str, max_mbytes: int, accepted_extensions: List[s def _validate_summary(self, text: str): if len(text) < 50 or len(text) > 1000: raise ModdbException("The summary must contain at least 50 and at most 1000 characters") + + def _validate_description(self, text: str): + if len(text) < 100 or len(text) > 100000: + raise ModdbException("The description must contain at least 100 and at most 100000 characters") + + def normalize_description(self, description: str, editor_url: str): + # TODO: implement tinymce checks + return description + + def _validate_platforms(self, platforms: List[PlatformCategory]): + if len(platforms) == 0: + raise ModdbException("Select the platforms the linked mods relate to") def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: AddonCategory, name: str, summary: str, description: str, platforms: List[PlatformCategory], @@ -1211,6 +1230,15 @@ def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: # Validate summary self._validate_summary(summary) + # Validate description + self._validate_description(description) + + # Normalize description + description = self.normalize_description(description, f"{mod.url}/addons/add") + + # Validate platforms + self._validate_platforms(platforms) + # Validate add-on file addon_exts = html.find("input", id="downloadsfiledata")["accept"].split(",") self._validate_file(abs_addon_path, 52428800, addon_exts) @@ -1237,14 +1265,14 @@ def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: "formhash": formhash, "legacy": 0, "platformstemp": 1, - "filedataUp": os.path.basename(addon_path), + "filedataUp": os.path.basename(abs_addon_path), "category": category.value, "licence": licence.value, "credits": credits, "tags": ",".join(tags), "name": name, "summary": summary, - "description": description, # Must be html. Usually text enclosed with p-tag + "description": description, "downloads": "Please wait uploading file", "links[]": [] } From d217bcfc958206959577471128486079608c2e30 Mon Sep 17 00:00:00 2001 From: Tosox <57193602+Tosox@users.noreply.github.com> Date: Tue, 25 Jun 2024 19:42:13 +0200 Subject: [PATCH 4/6] remove unused imports --- moddb/client.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/moddb/client.py b/moddb/client.py index 4761f84..ced049e 100644 --- a/moddb/client.py +++ b/moddb/client.py @@ -6,13 +6,6 @@ import sys from typing import TYPE_CHECKING, Any, List, Tuple, Union -from selenium import webdriver -from selenium.webdriver.common.by import By -from selenium.webdriver.support.ui import WebDriverWait -from selenium.webdriver.support import expected_conditions as EC -from selenium.webdriver.firefox.service import Service -import time - import requests from bs4 import BeautifulSoup from requests import utils From 9a5cf652d9faa0a5de5bd7ecdaa4d0233fbf11dc Mon Sep 17 00:00:00 2001 From: Tosox <57193602+Tosox@users.noreply.github.com> Date: Thu, 4 Jul 2024 22:57:46 +0200 Subject: [PATCH 5/6] add update_addon function --- moddb/client.py | 170 +++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 146 insertions(+), 24 deletions(-) diff --git a/moddb/client.py b/moddb/client.py index ced049e..bb6cd0a 100644 --- a/moddb/client.py +++ b/moddb/client.py @@ -1200,13 +1200,89 @@ def _validate_description(self, text: str): if len(text) < 100 or len(text) > 100000: raise ModdbException("The description must contain at least 100 and at most 100000 characters") - def normalize_description(self, description: str, editor_url: str): + def _normalize_description(self, description: str): # TODO: implement tinymce checks return description def _validate_platforms(self, platforms: List[PlatformCategory]): if len(platforms) == 0: raise ModdbException("Select the platforms the linked mods relate to") + + def _validate_post_response(self, response_text: str): + post_resp_html = soup(response_text) + download_button = post_resp_html.find("a", id="downloadmirrorstoggle") + if not download_button: + # We are still on the upload form + error_tooltip = post_resp_html.find("div", class_="tooltip errortooltip clear") + if error_tooltip: + if error_tooltip.ul: + error_list = error_tooltip.ul.find_all("li", recursive=False) + errors = "\n".join([f"- {error.text}" for error in error_list]) + else: + # p-tag contains a space at the beginning and a new line at the end + errors = f"-{error_tooltip.p.text[:-1]}" + raise ModdbException(f"Please correct the following: \n{errors}") + + def _upload_addon_file(self, addon_path: str, url: str): + addon_file = { + "filedata": open(addon_path, 'rb') + } + upload_resp = self._request("POST", url, files=addon_file) + if upload_resp.json()["error"]: + raise ModdbException("An error occurred while trying to upload the add-on") + + class EditAddonInfo: + def __init__(self, category: AddonCategory, name: str, summary: str, description: str, + platforms: List[PlatformCategory], licence: Licence, credits: str, + url: str, tags: List[str]): + self.category = category + self.name = name + self.summary = summary + self.description = description + self.platforms = platforms + self.licence = licence + self.credits = credits + self.url = url + self.tags = tags + + def __str__(self): + platforms_str = ", ".join([str(platform) for platform in self.platforms]) + tags_str = ", ".join(self.tags) + return ( + f"EditAddonInfo(\n" + f" Category: {self.category},\n" + f" Name: {self.name},\n" + f" Summary: {self.summary},\n" + f" Description: {self.description},\n" + f" Platforms: {platforms_str},\n" + f" Licence: {self.licence},\n" + f" Credits: {self.credits},\n" + f" URL: {self.url},\n" + f" Tags: {tags_str}\n" + f")" + ) + + def get_edit_addon_info(self, addon: Addon): + form = self._request("GET", f"{addon.url}/edit") + html = soup(form.text) + + if not html.find("input", { "name": "formhash" }): + raise ModdbException("You do not have permission to edit the requested downloads content") + + category = AddonCategory(int(html.find("select", id="downloadscategory") + .find_all("option", { "selected": "selected" })[0]["value"])) + name = html.find("input", id="downloadsname")["value"] + summary = html.find("textarea", id="downloadssummary").text + description = html.find("textarea", id="downloadsdescription").text + platforms = list(map(lambda c: PlatformCategory(c["value"]), html.find("select", id="downloadsplatforms") + .find_all("option", { "selected": "selected" }))) + licence = Licence(int(html.find("select", id="downloadslicence") + .find_all("option", { "selected": "selected" })[0]["value"])) + credits = html.find("input", id="downloadscredit")["value"] + url = html.find("input", id="downloadsnameid")["value"] + tags = html.find("input", id="downloadstags")["value"].split(",") + + return self.EditAddonInfo(category, name, summary, description, platforms, licence, credits, url, tags) def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: AddonCategory, name: str, summary: str, description: str, platforms: List[PlatformCategory], @@ -1227,7 +1303,7 @@ def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: self._validate_description(description) # Normalize description - description = self.normalize_description(description, f"{mod.url}/addons/add") + description = self._normalize_description(description) # Validate platforms self._validate_platforms(platforms) @@ -1244,12 +1320,8 @@ def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: formhash = html.find("input", { "name": "formhash" })["value"] mod_id = html.find("select", class_="right select").find_all("option")[1]["value"] - addon_file = { - "filedata": open(abs_addon_path, 'rb') - } - upload_resp = self._request("POST", f"https://upload.moddb.com/downloads/ajax/upload/{formhash}", files=addon_file) - if upload_resp.json()["error"]: - raise ModdbException("An error occurred while trying to upload the add-on") + # Upload addon file + self._upload_addon_file(abs_addon_path, f"https://upload.moddb.com/downloads/ajax/upload/{formhash}") logo_file = { "logo": open(abs_thumbnail_path, 'rb') @@ -1261,7 +1333,7 @@ def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: "filedataUp": os.path.basename(abs_addon_path), "category": category.value, "licence": licence.value, - "credits": credits, + "credit": credits, "tags": ",".join(tags), "name": name, "summary": summary, @@ -1274,26 +1346,76 @@ def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: post_resp = self._request("POST", f"{mod.url}/addons/add", data=data, files=logo_file) - # Check if ModDB reports errors - post_resp_html = soup(post_resp.text) - download_button = post_resp_html.find("a", id="downloadmirrorstoggle") - if not download_button: - # We are still on the upload form - error_tooltip = post_resp_html.find("div", class_="tooltip errortooltip clear") - if error_tooltip: - if error_tooltip.ul: - error_list = error_tooltip.ul.find_all("li", recursive=False) - errors = "\n".join([f"- {error.text}" for error in error_list]) - else: - # p-tag contains a space at the beginning and a new line at the end - errors = f"-{error_tooltip.p.text[:-1]}" - raise ModdbException(f"Please correct the following: \n{errors}") + # Check if ModDB reports error + self._validate_post_response(post_resp.text) - def update_addon(self, addon: Addon): + def update_addon(self, addon: Addon, addon_path: str = None, thumbnail_path: str = None, category: AddonCategory = None, + name: str = None, summary: str = None, description: str = None, platforms: List[PlatformCategory] = None, + licence: Licence = None, credits: str = None, tags: List[str] = None, url: str = None): form = self._request("GET", f"{addon.url}/edit") html = soup(form.text) if not html.find("input", { "name": "formhash" }): raise ModdbException("You do not have permission to edit the requested downloads content") + cwd = os.getcwd() + addon_infos = self.get_edit_addon_info(addon) + + if summary: + self._validate_summary(summary) + + if description: + self._validate_description(description) + description = self._normalize_description(description) + + if platforms: + self._validate_platforms(platforms) + + # Retrieve data for payloads + formhash = html.find("input", { "name": "formhash" })["value"] + mod_id = html.find("select", class_="right select").find_all("option")[1]["value"] + + if addon_path: + # Validate add-on file + abs_addon_path = os.path.join(cwd, addon_path) + addon_exts = html.find("input", id="downloadsfiledata")["accept"].split(",") + self._validate_file(abs_addon_path, 52428800, addon_exts) + + # Upload file + self._upload_addon_file(abs_addon_path, f"https://upload.moddb.com/downloads/ajax/upload/{formhash}") + + if thumbnail_path: + # Validate thumbnail file + abs_thumbnail_path = os.path.join(cwd, thumbnail_path) + thumbnail_exts = html.find("input", id="downloadslogo")["accept"].split(",") + self._validate_file(abs_thumbnail_path, 8192, thumbnail_exts) + + logo_file = { + "logo": open(abs_thumbnail_path, 'rb') if thumbnail_path else None + } + data = { + "formhash": formhash, + "legacy": 0, + "platformstemp": 1, + "filedataUp": os.path.basename(abs_addon_path) if addon_path else None, + "category": category.value if category else addon_infos.category.value, + "licence": licence.value if licence else addon_infos.licence.value, + "credit": credits if credits else addon_infos.credits, + "tags": ",".join(tags) if tags else addon_infos.tags, + "nameid": url if url else addon_infos.url, + "name": name if name else addon_infos.name, + "summary": summary if summary else addon_infos.summary, + "description": description if description else addon_infos.description, + "downloads": "Please wait uploading file", + "links[]": [] + } + if platforms: + data["links[]"].extend([platform.value for platform in platforms]) + else: + data["links[]"].extend([platform.value for platform in addon_infos.platforms]) + data["links[]"].append(mod_id) + + post_resp = self._request("POST", f"{addon.url}/edit", data=data, files=logo_file) + # Check if ModDB reports error + self._validate_post_response(post_resp.text) From 3a7d26326653c5954c43cacb1a24bf7aaff23348 Mon Sep 17 00:00:00 2001 From: Tosox <57193602+Tosox@users.noreply.github.com> Date: Mon, 14 Apr 2025 17:13:48 +0200 Subject: [PATCH 6/6] chore: Clean up code --- moddb/client.py | 265 ++++++++++++++++++++++-------------------------- 1 file changed, 121 insertions(+), 144 deletions(-) diff --git a/moddb/client.py b/moddb/client.py index b27b584..793c252 100644 --- a/moddb/client.py +++ b/moddb/client.py @@ -4,7 +4,7 @@ import random import re import sys -from typing import TYPE_CHECKING, Any, List, Tuple, Union +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union import requests from bs4 import BeautifulSoup @@ -1177,16 +1177,16 @@ def downvote_tag(self, tag: Tag) -> bool: """ return self._vote_tag(tag, 1) - def _validate_file(self, path: str, max_mbytes: int, accepted_extensions: List[str]): + def _validate_file(self, path: str, max_mbytes: float, accepted_extensions: List[str]): # Check if is file if not os.path.isfile(path): raise ModdbException("Please select a valid file before uploading") # Check file size - file_size = os.path.getsize(path) / 2048 # b -> mb - if file_size <= 0: + file_size_mb = os.path.getsize(path) / (1024 * 1024) # b -> mb + if file_size_mb <= 0: raise ModdbException("Your file cannot be empty") - elif file_size > max_mbytes: + elif file_size_mb > max_mbytes: raise ModdbException(f"Your file must be less then {max_mbytes}mb") # Check file extension @@ -1195,11 +1195,11 @@ def _validate_file(self, path: str, max_mbytes: int, accepted_extensions: List[s raise ModdbException(f"You cannot select a {file_ext} file only ({', '.join(accepted_extensions)})") def _validate_summary(self, text: str): - if len(text) < 50 or len(text) > 1000: + if not (50 <= len(text) <= 1000): raise ModdbException("The summary must contain at least 50 and at most 1000 characters") def _validate_description(self, text: str): - if len(text) < 100 or len(text) > 100000: + if not (100 <= len(text) <= 100000): raise ModdbException("The description must contain at least 100 and at most 100000 characters") def _normalize_description(self, description: str): @@ -1207,67 +1207,38 @@ def _normalize_description(self, description: str): return description def _validate_platforms(self, platforms: List[PlatformCategory]): - if len(platforms) == 0: + if not platforms: raise ModdbException("Select the platforms the linked mods relate to") - def _validate_post_response(self, response_text: str): - post_resp_html = soup(response_text) - download_button = post_resp_html.find("a", id="downloadmirrorstoggle") - if not download_button: - # We are still on the upload form - error_tooltip = post_resp_html.find("div", class_="tooltip errortooltip clear") - if error_tooltip: - if error_tooltip.ul: - error_list = error_tooltip.ul.find_all("li", recursive=False) - errors = "\n".join([f"- {error.text}" for error in error_list]) - else: - # p-tag contains a space at the beginning and a new line at the end - errors = f"-{error_tooltip.p.text[:-1]}" - raise ModdbException(f"Please correct the following: \n{errors}") + def _validate_post_response(self, html_str: str): + soup_obj = soup(html_str) + if soup_obj.find("a", id="downloadmirrorstoggle"): + return # Upload successful + + # We are still on the upload form + error_tooltip = soup_obj.find("div", class_="tooltip errortooltip clear") + if error_tooltip: + if error_tooltip.ul: + error_list = error_tooltip.ul.find_all("li", recursive=False) + errors = "\n".join([f"- {error.text}" for error in error_list]) + else: + # p-tag contains a space at the beginning and a new line at the end + errors = f"- {error_tooltip.p.text.strip()}" + raise ModdbException(f"Please correct the following: \n{errors}") - def _upload_addon_file(self, addon_path: str, url: str): - addon_file = { - "filedata": open(addon_path, 'rb') - } - upload_resp = self._request("POST", url, files=addon_file) - if upload_resp.json()["error"]: - raise ModdbException("An error occurred while trying to upload the add-on") + def _upload_file(self, path: str, url: str): + with open(path, 'rb') as f: + resp = self._request("POST", url, files={"filedata": f}) + if resp.json()["error"]: + raise ModdbException("An error occurred while trying to upload the add-on") + + def _prepare_file(self, file_path: str, accepted_exts: List[str], max_size_mb: float): + abs_path = os.path.abspath(file_path) + self._validate_file(abs_path, max_size_mb, accepted_exts) + return abs_path - class EditAddonInfo: - def __init__(self, category: AddonCategory, name: str, summary: str, description: str, - platforms: List[PlatformCategory], licence: Licence, credits: str, - url: str, tags: List[str]): - self.category = category - self.name = name - self.summary = summary - self.description = description - self.platforms = platforms - self.licence = licence - self.credits = credits - self.url = url - self.tags = tags - - def __str__(self): - platforms_str = ", ".join([str(platform) for platform in self.platforms]) - tags_str = ", ".join(self.tags) - return ( - f"EditAddonInfo(\n" - f" Category: {self.category},\n" - f" Name: {self.name},\n" - f" Summary: {self.summary},\n" - f" Description: {self.description},\n" - f" Platforms: {platforms_str},\n" - f" Licence: {self.licence},\n" - f" Credits: {self.credits},\n" - f" URL: {self.url},\n" - f" Tags: {tags_str}\n" - f")" - ) - - def get_edit_addon_info(self, addon: Addon): - form = self._request("GET", f"{addon.url}/edit") - html = soup(form.text) - + def get_addon_edit_data(self, addon: Addon): + html = soup(self._request("GET", f"{addon.url}/edit").text) if not html.find("input", { "name": "formhash" }): raise ModdbException("You do not have permission to edit the requested downloads content") @@ -1289,41 +1260,26 @@ def get_edit_addon_info(self, addon: Addon): def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: AddonCategory, name: str, summary: str, description: str, platforms: List[PlatformCategory], licence: Licence = Licence.proprietary, credits: str = "", tags: List[str] = []): - form = self._request("GET", f"{mod.url}/addons/add") - html = soup(form.text) + html = soup(self._request("GET", f"{mod.url}/addons/add").text) - cwd = os.getcwd() - abs_addon_path = os.path.join(cwd, addon_path) - abs_thumbnail_path = os.path.join(cwd, thumbnail_path) - - # Do some client-sided checks + # Retrieve data for payloads + formhash = html.find("input", { "name": "formhash" })["value"] + mod_id = html.find("select", class_="right select").find_all("option")[1]["value"] - # Validate summary + # Validations self._validate_summary(summary) - - # Validate description self._validate_description(description) - - # Normalize description - description = self._normalize_description(description) - - # Validate platforms self._validate_platforms(platforms) + description = self._normalize_description(description) - # Validate add-on file + # Files addon_exts = html.find("input", id="downloadsfiledata")["accept"].split(",") - self._validate_file(abs_addon_path, 52428800, addon_exts) - - # Validate thumbnail file thumbnail_exts = html.find("input", id="downloadslogo")["accept"].split(",") - self._validate_file(abs_thumbnail_path, 8192, thumbnail_exts) - - # Retrieve data for payloads - formhash = html.find("input", { "name": "formhash" })["value"] - mod_id = html.find("select", class_="right select").find_all("option")[1]["value"] + abs_addon_path = self._prepare_file(addon_path, addon_exts, 52428800 / 1024) + abs_thumbnail_path = self._prepare_file(thumbnail_path, thumbnail_exts, 8192 / 1024) # Upload addon file - self._upload_addon_file(abs_addon_path, f"https://upload.moddb.com/downloads/ajax/upload/{formhash}") + self._upload_file(abs_addon_path, f"https://upload.moddb.com/downloads/ajax/upload/{formhash}") logo_file = { "logo": open(abs_thumbnail_path, 'rb') @@ -1341,83 +1297,104 @@ def upload_addon(self, mod: Mod, addon_path: str, thumbnail_path: str, category: "summary": summary, "description": description, "downloads": "Please wait uploading file", - "links[]": [] + "links[]": [*map(lambda p: p.value, platforms), mod_id] } - data["links[]"].extend([platform.value for platform in platforms]) - data["links[]"].append(mod_id) - - post_resp = self._request("POST", f"{mod.url}/addons/add", data=data, files=logo_file) - - # Check if ModDB reports error - self._validate_post_response(post_resp.text) - - def update_addon(self, addon: Addon, addon_path: str = None, thumbnail_path: str = None, category: AddonCategory = None, - name: str = None, summary: str = None, description: str = None, platforms: List[PlatformCategory] = None, - licence: Licence = None, credits: str = None, tags: List[str] = None, url: str = None): - form = self._request("GET", f"{addon.url}/edit") - html = soup(form.text) + resp = self._request("POST", f"{mod.url}/addons/add", data=data, files=logo_file) + self._validate_post_response(resp.text) + if logo_file["logo"]: + logo_file["logo"].close() + + def update_addon(self, addon: Addon, addon_path: Optional[str] = None, thumbnail_path: Optional[str] = None, + category: Optional[AddonCategory] = None, name: Optional[str] = None, summary: Optional[str] = None, + description: Optional[str] = None, platforms: Optional[List[PlatformCategory]] = None, + licence: Optional[Licence] = None, credits: Optional[str] = None, tags: Optional[List[str]] = None, + url: Optional[str] = None): + html = soup(self._request("GET", f"{addon.url}/edit").text) if not html.find("input", { "name": "formhash" }): raise ModdbException("You do not have permission to edit the requested downloads content") - cwd = os.getcwd() - addon_infos = self.get_edit_addon_info(addon) - - if summary: - self._validate_summary(summary) - - if description: - self._validate_description(description) - description = self._normalize_description(description) - - if platforms: - self._validate_platforms(platforms) - # Retrieve data for payloads formhash = html.find("input", { "name": "formhash" })["value"] mod_id = html.find("select", class_="right select").find_all("option")[1]["value"] + addon_data = self.get_addon_edit_data(addon) + + # Use existing values if not overridden + summary = summary or addon_data.summary + description = self._normalize_description(description or addon_data.description) + name = name or addon_data.name + url = url or addon_data.url + credits = credits or addon_data.credits + tags = tags or addon_data.tags + platforms = platforms or addon_data.platforms + category = category or addon_data.category + licence = licence or addon_data.licence + self._validate_summary(summary) + self._validate_description(description) + self._validate_platforms(platforms) + + abs_addon_path = None if addon_path: - # Validate add-on file - abs_addon_path = os.path.join(cwd, addon_path) addon_exts = html.find("input", id="downloadsfiledata")["accept"].split(",") - self._validate_file(abs_addon_path, 52428800, addon_exts) - - # Upload file - self._upload_addon_file(abs_addon_path, f"https://upload.moddb.com/downloads/ajax/upload/{formhash}") + abs_addon_path = self._prepare_file(addon_path, addon_exts, 52428800 / 1024) + self._upload_file(abs_addon_path, f"https://upload.moddb.com/downloads/ajax/upload/{formhash}") + abs_thumbnail_path = None if thumbnail_path: - # Validate thumbnail file - abs_thumbnail_path = os.path.join(cwd, thumbnail_path) thumbnail_exts = html.find("input", id="downloadslogo")["accept"].split(",") - self._validate_file(abs_thumbnail_path, 8192, thumbnail_exts) + abs_thumbnail_path = self._prepare_file(thumbnail_path, thumbnail_exts, 8192 / 1024) logo_file = { - "logo": open(abs_thumbnail_path, 'rb') if thumbnail_path else None + "logo": open(abs_thumbnail_path, 'rb') if abs_addon_path else None } data = { "formhash": formhash, "legacy": 0, "platformstemp": 1, - "filedataUp": os.path.basename(abs_addon_path) if addon_path else None, - "category": category.value if category else addon_infos.category.value, - "licence": licence.value if licence else addon_infos.licence.value, - "credit": credits if credits else addon_infos.credits, - "tags": ",".join(tags) if tags else addon_infos.tags, - "nameid": url if url else addon_infos.url, - "name": name if name else addon_infos.name, - "summary": summary if summary else addon_infos.summary, - "description": description if description else addon_infos.description, + "filedataUp": os.path.basename(abs_addon_path) if abs_addon_path else None, + "category": category.value, + "licence": licence.value, + "credit": credits, + "tags": ",".join(tags), + "nameid": url, + "name": name, + "summary": summary, + "description": description, "downloads": "Please wait uploading file", - "links[]": [] + "links[]": [*map(lambda p: p.value, platforms), mod_id] } - if platforms: - data["links[]"].extend([platform.value for platform in platforms]) - else: - data["links[]"].extend([platform.value for platform in addon_infos.platforms]) - data["links[]"].append(mod_id) - post_resp = self._request("POST", f"{addon.url}/edit", data=data, files=logo_file) + resp = self._request("POST", f"{addon.url}/edit", data=data, files=logo_file) + self._validate_post_response(resp.text) + if logo_file["logo"]: + logo_file["logo"].close() + + class EditAddonInfo: + def __init__(self, category: AddonCategory, name: str, summary: str, description: str, + platforms: List[PlatformCategory], licence: Licence, credits: str, + url: str, tags: List[str]): + self.category = category + self.name = name + self.summary = summary + self.description = description + self.platforms = platforms + self.licence = licence + self.credits = credits + self.url = url + self.tags = tags - # Check if ModDB reports error - self._validate_post_response(post_resp.text) + def __str__(self): + return ( + f"EditAddonInfo(\n" + f" Category: {self.category},\n" + f" Name: {self.name},\n" + f" Summary: {self.summary},\n" + f" Description: {self.description},\n" + f" Platforms: {', '.join(map(str, self.platforms))},\n" + f" Licence: {self.licence},\n" + f" Credits: {self.credits},\n" + f" URL: {self.url},\n" + f" Tags: {', '.join(self.tags)}\n" + f")" + )