From e0adb3743cf4a05172caf3ca48946c685f560a42 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:33:22 +0900 Subject: [PATCH 01/29] create separate session class --- scripts/mediawiki_session.py | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 scripts/mediawiki_session.py diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py new file mode 100644 index 00000000000..8b3cc39ddd3 --- /dev/null +++ b/scripts/mediawiki_session.py @@ -0,0 +1,92 @@ +import contextlib +import functools +import http.cookiejar +import os +import time + +import requests + +from typing import Any, Optional + +from deploy_util import HEADER, SLEEP_DURATION + +USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" +WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") +WIKI_USER = os.getenv("WIKI_USER") +WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") +HEADER = { + "User-Agent": USER_AGENT, + "accept": "application/json", + "Accept-Encoding": "gzip", +} + + +class MediaWikiSessionError(IOError): + pass + + +class MediaWikiSession(contextlib.AbstractContextManager): + __cookie_jar: http.cookiejar.FileCookieJar + __session: requests.Session + __wiki: str + + def __init__(self, wiki: str): + self.__wiki = wiki + self.__cookie_jar = self.__read_cookie_jar() + self.__session = requests.session() + self.__session.cookies = self.__cookie_jar + self.__session.headers.update(HEADER) + + def __read_cookie_jar(self) -> http.cookiejar.FileCookieJar: + ckf = f"cookie_{self.__wiki}.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + with contextlib.suppress(OSError): + cookie_jar.load(ignore_discard=True) + return cookie_jar + + @functools.cache + def __get_wiki_api_url(self): + return f"{WIKI_BASE_URL}/{self.__wiki}/api.php" + + def _login(self): + token_response = self.make_action( + "query", params={"meta": "tokens", "type": "login"} + ) + self.make_action( + "login", + data={ + "lgname": WIKI_USER, + "lgpassword": WIKI_PASSWORD, + "lgtoken": token_response["query"]["tokens"]["logintoken"], + }, + ) + self.__cookie_jar.save(ignore_discard=True) + self.cooldown() + + @functools.cached_property + def token(self) -> str: + self._login() + return self.make_action("query", params={"meta": "tokens"})["tokens"]["csrftoken"] + + def make_action( + self, action: str, params: Optional[dict] = None, data: Optional[dict] = None + ) -> dict[str, Any]: + merged_params = {"format": "json", "action": action} + if params is not None: + merged_params = merged_params | params + response: dict = self.__session.post( + self.__get_wiki_api_url(), params=merged_params, data=data + ).json() + if "error" in response.keys(): + raise MediaWikiSessionError(response["error"]["info"]) + return response[action] + + def cooldown(self): + time.sleep(SLEEP_DURATION) + + def close(self): + self.__cookie_jar.save(ignore_discard=True) + self.__session.close() + + def __exit__(self, exc_type, exc_value, traceback): + self.close() From 57bc7b89feeea23fc860f7e7f6785e37856bc402 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:36:05 +0900 Subject: [PATCH 02/29] cache get_wikis output --- scripts/deploy_util.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index dc4060f635e..f0c15e8256c 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -33,14 +33,15 @@ SLEEP_DURATION = 4 -def get_wikis() -> set[str]: +@functools.cache +def get_wikis() -> frozenset[str]: response = requests.get( "https://liquipedia.net/api.php", headers=HEADER, ) wikis = response.json() time.sleep(SLEEP_DURATION) - return set(wikis["allwikis"].keys()) + return frozenset(wikis["allwikis"].keys()) @functools.cache From f57b3286a41ce002b9290a0465bea1c7185b39fb Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:37:47 +0900 Subject: [PATCH 03/29] __all__ --- scripts/mediawiki_session.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 8b3cc39ddd3..8e80c649859 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -10,6 +10,11 @@ from deploy_util import HEADER, SLEEP_DURATION +__all__ = [ + "MediaWikiSession", + "MediaWikiSessionError", +] + USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") WIKI_USER = os.getenv("WIKI_USER") @@ -66,7 +71,9 @@ def _login(self): @functools.cached_property def token(self) -> str: self._login() - return self.make_action("query", params={"meta": "tokens"})["tokens"]["csrftoken"] + return self.make_action("query", params={"meta": "tokens"})["tokens"][ + "csrftoken" + ] def make_action( self, action: str, params: Optional[dict] = None, data: Optional[dict] = None From d037056aa94e764d79f0df4cc7762d585f408051 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:42:08 +0900 Subject: [PATCH 04/29] make pylance happy --- scripts/mediawiki_session.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 8e80c649859..53eb7b88e5e 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -95,5 +95,8 @@ def close(self): self.__cookie_jar.save(ignore_discard=True) self.__session.close() + def __enter__(self): + return self + def __exit__(self, exc_type, exc_value, traceback): self.close() From 6b36d6fd0089fa112aa0137ee5969bd0ac86c57d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 12:57:28 +0900 Subject: [PATCH 05/29] refactor resource deploy --- scripts/deploy_res.py | 113 ++++++++++++++++------------------- scripts/mediawiki_session.py | 59 +++++++++++++++++- 2 files changed, 110 insertions(+), 62 deletions(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 1a0020715bc..3f4fdd19cee 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -5,66 +5,56 @@ from typing import Iterable -import requests - from deploy_util import ( - HEADER, get_git_deploy_reason, - get_wiki_api_url, - deploy_file_to_wiki, - read_cookie_jar, read_file_from_path, ) -from login_and_get_token import get_token +from mediawiki_session import MediaWikiSession def deploy_resources( - res_type: str, file_paths: Iterable[pathlib.Path], deploy_reason: str + session: MediaWikiSession, + res_type: str, + file_paths: Iterable[pathlib.Path], + deploy_reason: str, ) -> tuple[bool, bool]: all_deployed = True changes_made = False - token = get_token("commons") - with requests.Session() as session: - session.cookies = read_cookie_jar("commons") - for file_path in file_paths: - print(f"::group::Checking {str(file_path)}") - file_content = read_file_from_path(file_path) - page = ( - f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" + for file_path in file_paths: + print(f"::group::Checking {str(file_path)}") + file_content = read_file_from_path(file_path) + page = ( + f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" + "/".join(file_path.parts[2:]) - ) - print(f"...page = {page}") - deploy_result = deploy_file_to_wiki( - session, file_path, file_content, "commons", page, token, deploy_reason - ) - all_deployed = all_deployed and deploy_result[0] - changes_made = changes_made or deploy_result[1] - print("::endgroup::") + ) + print(f"...page = {page}") + deploy_result = session.deploy_file( + file_path, file_content, page, deploy_reason + ) + all_deployed = all_deployed and deploy_result[0] + changes_made = changes_made or deploy_result[1] + print("::endgroup::") return (all_deployed, changes_made) -def update_cache(): - with requests.Session() as session: - session.cookies = read_cookie_jar("commons") - cache_result = session.post( - get_wiki_api_url("commons"), - headers=HEADER, - params={"format": "json", "action": "updatelpmwmessageapi"}, - data={ - "messagename": "Resourceloaderarticles-cacheversion", - "value": subprocess.check_output(["git", "log", "-1", "--pretty=%h"]) - .decode() - .strip(), - }, - ).json() - if ( - cache_result["updatelpmwmessageapi"].get("message") - == "Successfully changed the message value" - ): - print("Resource cache version updated succesfully!") - else: - print("::error::Resource cache version unable to be updated!") - exit(1) +def update_cache(session: MediaWikiSession): + cache_result = session.make_action( + "updatelpmwmessageapi", + data={ + "messagename": "Resourceloaderarticles-cacheversion", + "value": subprocess.check_output(["git", "log", "-1", "--pretty=%h"]) + .decode() + .strip(), + }, + ) + if ( + cache_result["updatelpmwmessageapi"].get("message") + == "Successfully changed the message value" + ): + print("Resource cache version updated succesfully!") + else: + print("::error::Resource cache version unable to be updated!") + exit(1) def main(): @@ -82,22 +72,23 @@ def main(): resource_files = [pathlib.Path(arg) for arg in sys.argv[1:]] git_deploy_reason = get_git_deploy_reason() - for res_type, files in itertools.groupby( - sorted(resource_files), lambda path: path.suffix - ): - group_all_deployed, group_changes_made = deploy_resources( - res_type, list(files), git_deploy_reason - ) - all_deployed = all_deployed and group_all_deployed - changes_made = changes_made or group_changes_made + with MediaWikiSession("commons") as commons_session: + for res_type, files in itertools.groupby( + sorted(resource_files), lambda path: path.suffix + ): + group_all_deployed, group_changes_made = deploy_resources( + commons_session, res_type, list(files), git_deploy_reason + ) + all_deployed = all_deployed and group_all_deployed + changes_made = changes_made or group_changes_made - if not all_deployed: - print( - "::error::Some files were not deployed; resource cache version not updated!" - ) - exit(1) - elif changes_made: - update_cache() + if not all_deployed: + print( + "::error::Some files were not deployed; resource cache version not updated!" + ) + exit(1) + elif changes_made: + update_cache(commons_session) if __name__ == "__main__": diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 53eb7b88e5e..380fa17465f 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -2,13 +2,19 @@ import functools import http.cookiejar import os +import pathlib import time import requests from typing import Any, Optional -from deploy_util import HEADER, SLEEP_DURATION +from deploy_util import ( + DEPLOY_TRIGGER, + HEADER, + SLEEP_DURATION, + write_to_github_summary_file, +) __all__ = [ "MediaWikiSession", @@ -91,6 +97,57 @@ def make_action( def cooldown(self): time.sleep(SLEEP_DURATION) + def deploy_file( + self, + file_path: pathlib.Path, + file_content: str, + target_page: str, + deploy_reason: str, + ) -> tuple[bool, bool]: + try: + change_made = False + deployed = True + response = self.make_action( + "edit", + data={ + "title": target_page, + "text": file_content, + "summary": f"Git: {deploy_reason}", + "bot": "true", + "recreate": "true", + "token": self.token, + }, + ) + result = response.get("result") + new_rev_id = response.get("newrevid") + if result == "Success": + if new_rev_id is not None: + change_made = True + if DEPLOY_TRIGGER != "push": + print(f"::warning file={str(file_path)}::File changed") + print(f"...{result}") + print("...done") + write_to_github_summary_file( + f":information_source: {str(file_path)} successfully deployed" + ) + + else: + print(f"::warning file={str(file_path)}::failed to deploy") + write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy" + ) + deployed = False + time.sleep(SLEEP_DURATION) + return deployed, change_made + except MediaWikiSessionError as e: + print(f"::warning file={str(file_path)}::failed to deploy (API error)") + write_to_github_summary_file( + f":warning: {str(file_path)} failed to deploy due to API error: {str(e)}" + ) + deployed = False + time.sleep(SLEEP_DURATION) + return deployed, change_made + def close(self): self.__cookie_jar.save(ignore_discard=True) self.__session.close() From 5f81933652d05aad1de4221a65ccce1a3d3baeb8 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:05:43 +0900 Subject: [PATCH 06/29] refactor module deploy --- scripts/deploy.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/scripts/deploy.py b/scripts/deploy.py index 662139e7f3b..dffd1854605 100644 --- a/scripts/deploy.py +++ b/scripts/deploy.py @@ -6,16 +6,12 @@ from typing import Iterable -import requests - from deploy_util import ( get_git_deploy_reason, - deploy_file_to_wiki, - read_cookie_jar, read_file_from_path, write_to_github_summary_file, ) -from login_and_get_token import get_token +from mediawiki_session import MediaWikiSession HEADER_PATTERN = re.compile( r"\A---\n" r"-- @Liquipedia\n" r"-- page=(?P[^\n]*)\n" @@ -26,9 +22,7 @@ def deploy_all_files_for_wiki( wiki: str, file_paths: Iterable[pathlib.Path], deploy_reason: str ) -> bool: all_modules_deployed = True - token = get_token(wiki) - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) + with MediaWikiSession(wiki) as session: for file_path in file_paths: print(f"::group::Checking {str(file_path)}") file_content = read_file_from_path(file_path) @@ -40,8 +34,8 @@ def deploy_all_files_for_wiki( page = header_match.groupdict()["pageName"] + ( os.getenv("LUA_DEV_ENV_NAME") or "" ) - module_deployed, _ = deploy_file_to_wiki( - session, file_path, file_content, wiki, page, token, deploy_reason + module_deployed, _ = session.deploy_file( + file_path, file_content, page, deploy_reason ) all_modules_deployed &= module_deployed print("::endgroup::") From 125a0c3e023e829bebfb3ba4ba26e039de85b0f8 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:29:26 +0900 Subject: [PATCH 07/29] refactor dev remove --- scripts/mediawiki_session.py | 4 ++ scripts/remove_dev.py | 95 +++++++++++++++--------------------- 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 380fa17465f..15ab76ab043 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -81,6 +81,10 @@ def token(self) -> str: "csrftoken" ] + @property + def wiki(self) -> str: + return self.__wiki + def make_action( self, action: str, params: Optional[dict] = None, data: Optional[dict] = None ) -> dict[str, Any]: diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index c8918c96652..4c8fa610b83 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -1,85 +1,70 @@ import os -import time - -import requests from deploy_util import ( - HEADER, - SLEEP_DURATION, - get_wiki_api_url, get_wikis, - read_cookie_jar, write_to_github_summary_file, ) -from login_and_get_token import get_token, login +from mediawiki_session import MediaWikiSession, MediaWikiSessionError LUA_DEV_ENV_NAME = os.getenv("LUA_DEV_ENV_NAME") remove_errors: list[str] = list() -def remove_page(session: requests.Session, page: str, wiki: str): - print(f"deleting {wiki}:{page}") - token = get_token(wiki) - - result = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={ - "format": "json", - "action": "delete", - }, - data={ - "title": page, - "reason": f"Remove {LUA_DEV_ENV_NAME}", - "token": token, - }, - ).json() - time.sleep(SLEEP_DURATION) - - if "delete" not in result.keys(): - print(f"::warning::could not delete {page} on {wiki}") - write_to_github_summary_file(f":warning: could not delete {page} on {wiki}") - remove_errors.append(f"{wiki}:{page}") +def remove_page(session: MediaWikiSession, page: str): + print(f"deleting {session.wiki}:{page}") - -def search_and_remove(wiki: str): - with requests.Session() as session: - search_result = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "query"}, + try: + session.post( + "delete", data={ - "list": "search", - "srsearch": f"intitle:{LUA_DEV_ENV_NAME}", - "srnamespace": 828, - "srlimit": 5000, - "srprop": "", + "title": page, + "reason": f"Remove {LUA_DEV_ENV_NAME}", + "token": session.token, }, - ).json() - time.sleep(SLEEP_DURATION) + ) + except MediaWikiSessionError as e: + print(f"::warning::could not delete {page} on {session.wiki}") + write_to_github_summary_file( + f":warning: could not delete {page} on {session.wiki}" + ) + remove_errors.append(f"{session.wiki}:{page}") + finally: + session.cooldown() - # Handle API error responses and missing or empty search results safely. - if "error" in search_result: - error_info = search_result.get("error") - print(f"::warning::search API error on {wiki}: {error_info}") + +def search_and_remove(wiki: str): + with MediaWikiSession(wiki) as session: + search_result: dict + try: + search_result = session.make_action( + "query", + data={ + "list": "search", + "srsearch": f"intitle:{LUA_DEV_ENV_NAME}", + "srnamespace": 828, + "srlimit": 5000, + "srprop": "", + }, + ) + except MediaWikiSessionError as e: + print(f"::warning::search API error on {wiki}: {str(e)}") write_to_github_summary_file( - f":warning: search API error on {wiki}: {error_info}" + f":warning: search API error on {wiki}: {str(e)}" ) return + finally: + session.cooldown() - pages = search_result.get("query", {}).get("search") or [] + pages = search_result.get("search") or [] if len(pages) == 0: return - login(wiki) - session.cookies = read_cookie_jar(wiki) - for page in pages: if os.getenv("INCLUDE_SUB_ENVS") == "true" or page["title"].endswith( LUA_DEV_ENV_NAME ): - remove_page(session, page["title"], wiki) + remove_page(session, page["title"]) def main(): From 3be29243c116c1f65641325cec2f91b55cc9d722 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:29:57 +0900 Subject: [PATCH 08/29] remove duplicate code --- scripts/mediawiki_session.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 15ab76ab043..9c5e07de95f 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -141,7 +141,6 @@ def deploy_file( f":warning: {str(file_path)} failed to deploy" ) deployed = False - time.sleep(SLEEP_DURATION) return deployed, change_made except MediaWikiSessionError as e: print(f"::warning file={str(file_path)}::failed to deploy (API error)") @@ -149,8 +148,9 @@ def deploy_file( f":warning: {str(file_path)} failed to deploy due to API error: {str(e)}" ) deployed = False - time.sleep(SLEEP_DURATION) return deployed, change_made + finally: + self.cooldown() def close(self): self.__cookie_jar.save(ignore_discard=True) From ad6df70e6351beb26e0a58708f47cc9d6331ccb0 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 13:32:20 +0900 Subject: [PATCH 09/29] cleanup --- scripts/deploy_util.py | 85 +----------------------------------- scripts/mediawiki_session.py | 44 +++++++++---------- 2 files changed, 24 insertions(+), 105 deletions(-) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index f0c15e8256c..e53e8c800fc 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -1,3 +1,4 @@ +import contextlib import functools import http.cookiejar import os @@ -8,10 +9,8 @@ import requests __all__ = [ - "DEPLOY_TRIGGER", "HEADER", "SLEEP_DURATION", - "deploy_file_to_wiki", "get_git_deploy_reason", "get_wiki_api_url", "get_wikis", @@ -20,8 +19,6 @@ "write_to_github_summary_file", ] -DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") -DRY_RUN = bool(int(os.getenv("DRY_RUN", 0))) GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") @@ -57,89 +54,11 @@ def get_git_deploy_reason(): ) -def deploy_file_to_wiki( - session: requests.Session, - file_path: pathlib.Path, - file_content: str, - wiki: str, - target_page: str, - token: str, - deploy_reason: str, -) -> tuple[bool, bool]: - payload = { - "title": target_page, - "text": file_content, - "summary": f"Git: {deploy_reason}", - "bot": "true", - "recreate": "true", - "token": token, - } - if DRY_RUN: - print(f"HEADER: {HEADER}") - print(f"PARAM: { {'format': 'json', 'action': 'edit'} }") - print(f"DATA: {payload}") - return True, False - - change_made = False - deployed = True - response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "edit"}, - data=payload, - ).json() - edit_info = response.get("edit") - error_info = response.get("error") - - # Handle API errors or unexpected response structure - if error_info is not None or edit_info is None: - print(f"::warning file={str(file_path)}::failed to deploy (API error)") - details = "" - if isinstance(error_info, dict): - code = error_info.get("code") - info = error_info.get("info") - detail_parts = [] - if code: - detail_parts.append(f"code={code}") - if info: - detail_parts.append(f"info={info}") - if detail_parts: - details = " (" + ", ".join(detail_parts) + ")" - write_to_github_summary_file( - f":warning: {str(file_path)} failed to deploy due to API error{details}" - ) - deployed = False - time.sleep(SLEEP_DURATION) - return deployed, change_made - - result = edit_info.get("result") - new_rev_id = edit_info.get("newrevid") - if result == "Success": - if new_rev_id is not None: - change_made = True - if DEPLOY_TRIGGER != "push": - print(f"::warning file={str(file_path)}::File changed") - print(f"...{result}") - print("...done") - write_to_github_summary_file( - f":information_source: {str(file_path)} successfully deployed" - ) - - else: - print(f"::warning file={str(file_path)}::failed to deploy") - write_to_github_summary_file(f":warning: {str(file_path)} failed to deploy") - deployed = False - time.sleep(SLEEP_DURATION) - return deployed, change_made - - def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: ckf = f"cookie_{wiki}.ck" cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - try: + with contextlib.suppress(OSError): cookie_jar.load(ignore_discard=True) - except OSError: - pass return cookie_jar diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 9c5e07de95f..cfd179efa2a 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -10,9 +10,10 @@ from typing import Any, Optional from deploy_util import ( - DEPLOY_TRIGGER, HEADER, SLEEP_DURATION, + WIKI_BASE_URL, + read_cookie_jar, write_to_github_summary_file, ) @@ -21,15 +22,10 @@ "MediaWikiSessionError", ] -USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" -WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") +DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") +DRY_RUN = bool(int(os.getenv("DRY_RUN", 0))) WIKI_USER = os.getenv("WIKI_USER") WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") -HEADER = { - "User-Agent": USER_AGENT, - "accept": "application/json", - "Accept-Encoding": "gzip", -} class MediaWikiSessionError(IOError): @@ -49,15 +45,11 @@ def __init__(self, wiki: str): self.__session.headers.update(HEADER) def __read_cookie_jar(self) -> http.cookiejar.FileCookieJar: - ckf = f"cookie_{self.__wiki}.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - with contextlib.suppress(OSError): - cookie_jar.load(ignore_discard=True) - return cookie_jar + return read_cookie_jar(self.wiki) @functools.cache def __get_wiki_api_url(self): - return f"{WIKI_BASE_URL}/{self.__wiki}/api.php" + return f"{WIKI_BASE_URL}/{self.wiki}/api.php" def _login(self): token_response = self.make_action( @@ -76,6 +68,8 @@ def _login(self): @functools.cached_property def token(self) -> str: + if DRY_RUN: + return "DRY_RUN_DUMMY_TOKEN" self._login() return self.make_action("query", params={"meta": "tokens"})["tokens"][ "csrftoken" @@ -108,19 +102,25 @@ def deploy_file( target_page: str, deploy_reason: str, ) -> tuple[bool, bool]: + payload = { + "title": target_page, + "text": file_content, + "summary": f"Git: {deploy_reason}", + "bot": "true", + "recreate": "true", + "token": self.token, + } + if DRY_RUN: + print(f"HEADER: {HEADER}") + print(f"PARAM: { {'format': 'json', 'action': 'edit'} }") + print(f"DATA: {payload}") + return True, False try: change_made = False deployed = True response = self.make_action( "edit", - data={ - "title": target_page, - "text": file_content, - "summary": f"Git: {deploy_reason}", - "bot": "true", - "recreate": "true", - "token": self.token, - }, + data=payload, ) result = response.get("result") new_rev_id = response.get("newrevid") From c4acd0ba4d5c153e6f3140cbc05ba8be0b7542b7 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:09:19 +0900 Subject: [PATCH 10/29] overhaul module protect --- scripts/protect.py | 100 ++++++++++++++++++++++++++-------------- scripts/protect_page.py | 70 +++++++++++++++++++++++++++- 2 files changed, 134 insertions(+), 36 deletions(-) diff --git a/scripts/protect.py b/scripts/protect.py index 562544b6c75..9a0bb9ab4f1 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -1,58 +1,88 @@ +import itertools import os import pathlib import sys -from typing import Iterable - from deploy_util import get_wikis +from mediawiki_session import MediaWikiSession from protect_page import ( - protect_non_existing_page, - protect_existing_page, + protect_non_existing_pages, + protect_existing_pages, handle_protect_errors, ) WIKI_TO_PROTECT = os.getenv("WIKI_TO_PROTECT") -def check_for_local_version(module: str, wiki: str): - if wiki == "commons": - return False - return pathlib.Path(f"./lua/wikis/{wiki}/{module}.lua").exists() +def protect_new_wiki(wiki_to_protect: str): + lua_files = itertools.chain( + pathlib.Path("./lua/wikis/commons/").rglob("*.lua"), + pathlib.Path("./lua/wikis/" + wiki_to_protect + "/").rglob("*.lua"), + ) + + commons_modules: set[str] = set() + local_modules: set[str] = set() + + for file_to_protect in sorted(lua_files): + wiki = file_to_protect.parts[2] + module = "/".join(file_to_protect.parts[3:])[:-4] + page = "Module:" + module + + if wiki == wiki_to_protect: + local_modules.add(page) + elif wiki == "commons": + commons_modules.add(page) + with MediaWikiSession(wiki_to_protect) as session: + protect_non_existing_pages(session, commons_modules - local_modules) + protect_existing_pages(session, local_modules) -def protect_if_has_no_local_version(module: str, wiki: str): - page = "Module:" + module - if not check_for_local_version(module, wiki): - protect_non_existing_page(page, wiki) + handle_protect_errors() def main(): - lua_files: Iterable[pathlib.Path] - if len(sys.argv[1:]) > 0: - lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] - elif WIKI_TO_PROTECT: - lua_files = pathlib.Path("./lua/wikis/").rglob("*.lua") - else: + if WIKI_TO_PROTECT: + protect_new_wiki(WIKI_TO_PROTECT) + return + elif len(sys.argv[1:]) == 0: print("Nothing to protect") exit(0) - for file_to_protect in sorted(lua_files): - print(f"::group::Checking {str(file_to_protect)}") - wiki = file_to_protect.parts[2] - module = "/".join(file_to_protect.parts[3:])[:-4] - page = "Module:" + module - if WIKI_TO_PROTECT: - if wiki == WIKI_TO_PROTECT: - protect_existing_page(page, wiki) - elif wiki == "commons": - protect_if_has_no_local_version(module, WIKI_TO_PROTECT) - elif wiki != "commons": - protect_existing_page(page, wiki) - else: # commons case - protect_existing_page(page, wiki) - for deploy_wiki in get_wikis() - {"commons"}: - protect_if_has_no_local_version(module, deploy_wiki) - print("::endgroup::") + lua_files = [pathlib.Path(arg) for arg in sys.argv[1:]] + + files_to_protect_by_wiki: dict[str, set[str]] = dict() + + for wiki, files_to_protect in itertools.groupby( + sorted(lua_files), lambda path: path.parts[2] + ): + files_to_protect_by_wiki[wiki] = set( + [ + "Module:" + "/".join(file_to_protect.parts[3:])[:-4] + for file_to_protect in files_to_protect + ] + ) + + new_commons_modules = files_to_protect_by_wiki.get("commons") + + if new_commons_modules: + for wiki in get_wikis(): + with MediaWikiSession(wiki) as session: + if wiki == "commons": + protect_existing_pages(session, new_commons_modules) + else: + new_local_modules = files_to_protect_by_wiki.get(wiki) + if new_local_modules: + protect_existing_pages(session, new_local_modules) + protect_non_existing_pages( + session, new_commons_modules - new_local_modules + ) + else: + protect_non_existing_pages(session, new_commons_modules) + else: + for wiki, new_modules in files_to_protect_by_wiki: + with MediaWikiSession(wiki) as session: + protect_existing_pages(session, new_modules) + handle_protect_errors() diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 3d34c9a21ae..a5de8597e44 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -1,6 +1,6 @@ import time -from typing import Literal +from typing import Iterable, Literal import requests @@ -12,10 +12,13 @@ write_to_github_summary_file, ) from login_and_get_token import get_token +from mediawiki_session import MediaWikiSession, MediaWikiSessionError __all__ = [ "protect_non_existing_page", + "protect_non_existing_pages", "protect_existing_page", + "protect_existing_pages", "handle_protect_errors", ] @@ -64,6 +67,49 @@ def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): protect_errors.append(f"{protect_mode}:{wiki}:{page}") +def protect_pages( + session: MediaWikiSession, + pages: Iterable[str], + protect_mode: Literal["edit", "create"], +): + protect_options: str + if protect_mode == "edit": + protect_options = "edit=allow-only-sysop|move=allow-only-sysop" + elif protect_mode == "create": + protect_options = "create=allow-only-sysop" + else: + raise ValueError(f"invalid protect mode: {protect_mode}") + print(f"...wiki = {session.wiki}") + for page in pages: + print(f"...page = {page}") + try: + protections = session.make_action( + "protect", + data={ + "title": page, + "protections": protect_options, + "reason": "Git maintained", + "expiry": "infinite", + "bot": "true", + "token": session.token, + }, + ) + for protection in protections: + if protection[protect_mode] == "allow-only-sysop": + return + print( + f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}" + ) + protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") + except MediaWikiSessionError as e: + print( + f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}: {str(e)}" + ) + protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") + finally: + session.cooldown() + + def check_if_page_exists(page: str, wiki: str) -> bool: with requests.Session() as session: session.cookies = read_cookie_jar(wiki) @@ -87,10 +133,32 @@ def protect_non_existing_page(page: str, wiki: str): protect_page(page, wiki, "create") +def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): + def filter_non_existing_pages(page: str) -> bool: + try: + result = session.post( + "query", + data={"titles": page, "prop": "info"}, + ) + if "-1" in result["pages"]: + return True + print(f"::warning::{page} already exists on {session.wiki}") + protect_errors.append(f"create:{session.wiki}:{page}") + return False + finally: + time.sleep(SLEEP_DURATION) + + protect_pages(session, filter(filter_non_existing_pages, pages), "create") + + def protect_existing_page(page: str, wiki: str): protect_page(page, wiki, "edit") +def protect_existing_pages(session: MediaWikiSession, pages: Iterable[str]): + protect_pages(session, pages, "edit") + + def handle_protect_errors(): if len(protect_errors) == 0: return From d3d4a047fd321b201e6c41f147bb32239ad51592 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:23:33 +0900 Subject: [PATCH 11/29] overhaul template protect --- scripts/protect_page.py | 76 ------------------------------------ scripts/protect_templates.py | 23 ++++++----- 2 files changed, 14 insertions(+), 85 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index a5de8597e44..7bac8ae638d 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -2,22 +2,15 @@ from typing import Iterable, Literal -import requests from deploy_util import ( - HEADER, SLEEP_DURATION, - get_wiki_api_url, - read_cookie_jar, write_to_github_summary_file, ) -from login_and_get_token import get_token from mediawiki_session import MediaWikiSession, MediaWikiSessionError __all__ = [ - "protect_non_existing_page", "protect_non_existing_pages", - "protect_existing_page", "protect_existing_pages", "handle_protect_errors", ] @@ -25,48 +18,6 @@ protect_errors = list() -def protect_page(page: str, wiki: str, protect_mode: Literal["edit", "create"]): - protect_options: str - if protect_mode == "edit": - protect_options = "edit=allow-only-sysop|move=allow-only-sysop" - elif protect_mode == "create": - protect_options = "create=allow-only-sysop" - else: - raise ValueError(f"invalid protect mode: {protect_mode}") - print(f"...wiki = {wiki}") - print(f"...page = {page}") - token = get_token(wiki) - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) - response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "protect"}, - data={ - "title": page, - "protections": protect_options, - "reason": "Git maintained", - "expiry": "infinite", - "bot": "true", - "token": token, - }, - ).json() - - time.sleep(SLEEP_DURATION) - if response.get("error"): - print( - f"::warning::could not ({protect_mode}) protect {page} on {wiki}: {response['error']['info']}" - ) - protect_errors.append(f"{protect_mode}:{wiki}:{page}") - return - protections = response["protect"].get("protections") - for protection in protections: - if protection[protect_mode] == "allow-only-sysop": - return - print(f"::warning::could not ({protect_mode}) protect {page} on {wiki}") - protect_errors.append(f"{protect_mode}:{wiki}:{page}") - - def protect_pages( session: MediaWikiSession, pages: Iterable[str], @@ -110,29 +61,6 @@ def protect_pages( session.cooldown() -def check_if_page_exists(page: str, wiki: str) -> bool: - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) - - result = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={"format": "json", "action": "query"}, - data={"titles": page, "prop": "info"}, - ).json() - - time.sleep(SLEEP_DURATION) - return "-1" not in result["query"]["pages"] - - -def protect_non_existing_page(page: str, wiki: str): - if check_if_page_exists(page, wiki): - print(f"::warning::{page} already exists on {wiki}") - protect_errors.append(f"create:{wiki}:{page}") - else: - protect_page(page, wiki, "create") - - def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): def filter_non_existing_pages(page: str) -> bool: try: @@ -151,10 +79,6 @@ def filter_non_existing_pages(page: str) -> bool: protect_pages(session, filter(filter_non_existing_pages, pages), "create") -def protect_existing_page(page: str, wiki: str): - protect_page(page, wiki, "edit") - - def protect_existing_pages(session: MediaWikiSession, pages: Iterable[str]): protect_pages(session, pages, "edit") diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index 488b93be20b..c26fa3fbfe8 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -1,8 +1,9 @@ import os +from mediawiki_session import MediaWikiSession from protect_page import ( - protect_non_existing_page, - protect_existing_page, + protect_non_existing_pages, + protect_existing_pages, handle_protect_errors, ) @@ -11,15 +12,19 @@ def main(): with open("./templates/templatesToProtect", "r") as templates_to_protect: - for template_name in templates_to_protect.read().splitlines(): - if len(template_name.strip()) == 0: - continue - template = "Template:" + template_name - print(f"::group::Checking {WIKI_TO_PROTECT}:{template}") + templates = [ + "Template:" + template_name + for template_name in filter( + lambda template: len(template.strip()) > 0, + templates_to_protect.read().splitlines(), + ) + ] + with MediaWikiSession(WIKI_TO_PROTECT) as session: + print(f"::group::Checking {WIKI_TO_PROTECT}") if WIKI_TO_PROTECT == "commons": - protect_existing_page(template, WIKI_TO_PROTECT) + protect_existing_pages(session, templates) else: - protect_non_existing_page(template, WIKI_TO_PROTECT) + protect_non_existing_pages(session, templates) print("::endgroup::") handle_protect_errors() From 7f9f7fb6031a28948b11e4c3d732e965c103e30f Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:29:38 +0900 Subject: [PATCH 12/29] clean up unused --- scripts/deploy_util.py | 19 +-------- scripts/login_and_get_token.py | 70 ---------------------------------- scripts/mediawiki_session.py | 9 +++-- 3 files changed, 7 insertions(+), 91 deletions(-) delete mode 100644 scripts/login_and_get_token.py diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index e53e8c800fc..b2e617ec46a 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -1,6 +1,4 @@ -import contextlib import functools -import http.cookiejar import os import pathlib import subprocess @@ -12,16 +10,14 @@ "HEADER", "SLEEP_DURATION", "get_git_deploy_reason", - "get_wiki_api_url", "get_wikis", - "read_cookie_jar", "read_file_from_path", "write_to_github_summary_file", ] GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" -WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") + HEADER = { "User-Agent": USER_AGENT, "accept": "application/json", @@ -41,11 +37,6 @@ def get_wikis() -> frozenset[str]: return frozenset(wikis["allwikis"].keys()) -@functools.cache -def get_wiki_api_url(wiki: str) -> str: - return f"{WIKI_BASE_URL}/{wiki}/api.php" - - def get_git_deploy_reason(): return ( subprocess.check_output(["git", "log", "-1", "--pretty='%h %s'"]) @@ -54,14 +45,6 @@ def get_git_deploy_reason(): ) -def read_cookie_jar(wiki: str) -> http.cookiejar.FileCookieJar: - ckf = f"cookie_{wiki}.ck" - cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) - with contextlib.suppress(OSError): - cookie_jar.load(ignore_discard=True) - return cookie_jar - - def read_file_from_path(file_path: pathlib.Path) -> str: with file_path.open("r") as file: return file.read() diff --git a/scripts/login_and_get_token.py b/scripts/login_and_get_token.py deleted file mode 100644 index 2f6afa81b33..00000000000 --- a/scripts/login_and_get_token.py +++ /dev/null @@ -1,70 +0,0 @@ -import functools -import os -import time - -import requests - -from deploy_util import HEADER, SLEEP_DURATION, get_wiki_api_url, read_cookie_jar - -__all__ = ["get_token"] - -DRY_RUN = bool(int(os.getenv("DRY_RUN", 0))) -WIKI_USER = os.getenv("WIKI_USER") -WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") - -loggedin: set[str] = set() - - -def login(wiki: str): - if wiki in loggedin: - return - cookie_jar = read_cookie_jar(wiki) - print(f"...logging in on {wiki}") - if DRY_RUN: - return - with requests.Session() as session: - session.cookies = cookie_jar - token_response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={ - "format": "json", - "action": "query", - "meta": "tokens", - "type": "login", - }, - ).json() - session.post( - get_wiki_api_url(wiki), - headers=HEADER, - data={ - "lgname": WIKI_USER, - "lgpassword": WIKI_PASSWORD, - "lgtoken": token_response["query"]["tokens"]["logintoken"], - }, - params={"format": "json", "action": "login"}, - ) - loggedin.add(wiki) - cookie_jar.save(ignore_discard=True) - time.sleep(SLEEP_DURATION) - - -@functools.cache -def get_token(wiki: str) -> str: - login(wiki) - - if DRY_RUN: - return "DRY_RUN_DUMMY_TOKEN" - - with requests.Session() as session: - session.cookies = read_cookie_jar(wiki) - token_response = session.post( - get_wiki_api_url(wiki), - headers=HEADER, - params={ - "format": "json", - "action": "query", - "meta": "tokens", - }, - ).json() - return token_response["query"]["tokens"]["csrftoken"] diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index cfd179efa2a..b205d036b50 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -12,8 +12,6 @@ from deploy_util import ( HEADER, SLEEP_DURATION, - WIKI_BASE_URL, - read_cookie_jar, write_to_github_summary_file, ) @@ -24,6 +22,7 @@ DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") DRY_RUN = bool(int(os.getenv("DRY_RUN", 0))) +WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") WIKI_USER = os.getenv("WIKI_USER") WIKI_PASSWORD = os.getenv("WIKI_PASSWORD") @@ -45,7 +44,11 @@ def __init__(self, wiki: str): self.__session.headers.update(HEADER) def __read_cookie_jar(self) -> http.cookiejar.FileCookieJar: - return read_cookie_jar(self.wiki) + ckf = f"cookie_{self.wiki}.ck" + cookie_jar = http.cookiejar.LWPCookieJar(filename=ckf) + with contextlib.suppress(OSError): + cookie_jar.load(ignore_discard=True) + return cookie_jar @functools.cache def __get_wiki_api_url(self): From 24a0834179e50e20c57b82a2d8d1683061e03ed4 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:31:09 +0900 Subject: [PATCH 13/29] fix indexing --- scripts/mediawiki_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index b205d036b50..09259618400 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -63,7 +63,7 @@ def _login(self): data={ "lgname": WIKI_USER, "lgpassword": WIKI_PASSWORD, - "lgtoken": token_response["query"]["tokens"]["logintoken"], + "lgtoken": token_response["tokens"]["logintoken"], }, ) self.__cookie_jar.save(ignore_discard=True) From 8471b26c43ee507232463885256c2fc728552163 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:43:18 +0900 Subject: [PATCH 14/29] fix calls --- scripts/protect_page.py | 2 +- scripts/remove_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 7bac8ae638d..f33244ba17d 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -64,7 +64,7 @@ def protect_pages( def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): def filter_non_existing_pages(page: str) -> bool: try: - result = session.post( + result = session.make_action( "query", data={"titles": page, "prop": "info"}, ) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 4c8fa610b83..93a74a2c091 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -15,7 +15,7 @@ def remove_page(session: MediaWikiSession, page: str): print(f"deleting {session.wiki}:{page}") try: - session.post( + session.make_action( "delete", data={ "title": page, From 4c6e906299d5dc23eb571909a07da8d01b6b7b4d Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:45:21 +0900 Subject: [PATCH 15/29] fix key --- scripts/protect_page.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index f33244ba17d..de67edf9392 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -44,7 +44,8 @@ def protect_pages( "bot": "true", "token": session.token, }, - ) + )["protections"] + for protection in protections: if protection[protect_mode] == "allow-only-sysop": return From dfb24356e8331b48a6a0429cc635b66e27adc810 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:48:04 +0900 Subject: [PATCH 16/29] fix incorrect return --- scripts/protect_page.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index de67edf9392..5118e2f0cfe 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -45,14 +45,15 @@ def protect_pages( "token": session.token, }, )["protections"] - + for protection in protections: if protection[protect_mode] == "allow-only-sysop": - return - print( - f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}" - ) - protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") + break + else: + print( + f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}" + ) + protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") except MediaWikiSessionError as e: print( f"::warning::could not ({protect_mode}) protect {page} on {session.wiki}: {str(e)}" From 0232322b6c8191ed8aa76dc6a7abbafdb2cde11a Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 15:59:18 +0900 Subject: [PATCH 17/29] adjust GHA output --- scripts/protect.py | 2 ++ scripts/protect_templates.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/protect.py b/scripts/protect.py index 9a0bb9ab4f1..25a0c3a01e3 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -34,8 +34,10 @@ def protect_new_wiki(wiki_to_protect: str): commons_modules.add(page) with MediaWikiSession(wiki_to_protect) as session: + print(f"::group::Protecting {WIKI_TO_PROTECT}") protect_non_existing_pages(session, commons_modules - local_modules) protect_existing_pages(session, local_modules) + print("::endgroup::") handle_protect_errors() diff --git a/scripts/protect_templates.py b/scripts/protect_templates.py index c26fa3fbfe8..2bf92cc487f 100644 --- a/scripts/protect_templates.py +++ b/scripts/protect_templates.py @@ -20,7 +20,7 @@ def main(): ) ] with MediaWikiSession(WIKI_TO_PROTECT) as session: - print(f"::group::Checking {WIKI_TO_PROTECT}") + print(f"::group::Protecting {WIKI_TO_PROTECT}") if WIKI_TO_PROTECT == "commons": protect_existing_pages(session, templates) else: From aa38333c395448793fc354ddaa7e40fa15ae3d71 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:12:22 +0900 Subject: [PATCH 18/29] add GHA debug support --- scripts/mediawiki_session.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 09259618400..84eed4cbd17 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -20,6 +20,7 @@ "MediaWikiSessionError", ] +ACTIONS_STEP_DEBUG = os.getenv("ACTIONS_STEP_DEBUG") == "true" DEPLOY_TRIGGER = os.getenv("DEPLOY_TRIGGER") DRY_RUN = bool(int(os.getenv("DRY_RUN", 0))) WIKI_BASE_URL = os.getenv("WIKI_BASE_URL") @@ -74,9 +75,12 @@ def token(self) -> str: if DRY_RUN: return "DRY_RUN_DUMMY_TOKEN" self._login() - return self.make_action("query", params={"meta": "tokens"})["tokens"][ + token = self.make_action("query", params={"meta": "tokens"})["tokens"][ "csrftoken" ] + if ACTIONS_STEP_DEBUG: + print(f"::add-mask::{token}") + return token @property def wiki(self) -> str: @@ -88,12 +92,18 @@ def make_action( merged_params = {"format": "json", "action": action} if params is not None: merged_params = merged_params | params - response: dict = self.__session.post( + response = self.__session.post( self.__get_wiki_api_url(), params=merged_params, data=data - ).json() - if "error" in response.keys(): - raise MediaWikiSessionError(response["error"]["info"]) - return response[action] + ) + if ACTIONS_STEP_DEBUG: + print(f"params: {merged_params}") + print(f"data: {data}") + print(f"HTTP Status: {response.status_code}") + print(f"Raw response: \"{response}\"") + parsed_response: dict[str, Any] = response.json() + if "error" in parsed_response.keys(): + raise MediaWikiSessionError(parsed_response["error"]["info"]) + return parsed_response[action] def cooldown(self): time.sleep(SLEEP_DURATION) From b99d884977de557851d81cd720fbc51ff5f81f73 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:16:11 +0900 Subject: [PATCH 19/29] shorthand --- scripts/mediawiki_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 84eed4cbd17..4b8ae26c719 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -91,7 +91,7 @@ def make_action( ) -> dict[str, Any]: merged_params = {"format": "json", "action": action} if params is not None: - merged_params = merged_params | params + merged_params |= params response = self.__session.post( self.__get_wiki_api_url(), params=merged_params, data=data ) From c7795340f480444d18fdcc7a99d1df9ca69bcbb7 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:17:37 +0900 Subject: [PATCH 20/29] use cooldown --- scripts/protect_page.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 5118e2f0cfe..3dd5299b9b0 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -3,10 +3,7 @@ from typing import Iterable, Literal -from deploy_util import ( - SLEEP_DURATION, - write_to_github_summary_file, -) +from deploy_util import write_to_github_summary_file from mediawiki_session import MediaWikiSession, MediaWikiSessionError __all__ = [ @@ -76,7 +73,7 @@ def filter_non_existing_pages(page: str) -> bool: protect_errors.append(f"create:{session.wiki}:{page}") return False finally: - time.sleep(SLEEP_DURATION) + session.cooldown() protect_pages(session, filter(filter_non_existing_pages, pages), "create") From 8c9c236f6c43a1e33b87de80547968275e338534 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:17:45 +0900 Subject: [PATCH 21/29] lint --- scripts/mediawiki_session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 4b8ae26c719..7564334e5ec 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -99,7 +99,7 @@ def make_action( print(f"params: {merged_params}") print(f"data: {data}") print(f"HTTP Status: {response.status_code}") - print(f"Raw response: \"{response}\"") + print(f'Raw response: "{response}"') parsed_response: dict[str, Any] = response.json() if "error" in parsed_response.keys(): raise MediaWikiSessionError(parsed_response["error"]["info"]) From 76b44e2dbfa7911bfb4300cd63d34af6a2dbcba3 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Thu, 5 Mar 2026 16:21:17 +0900 Subject: [PATCH 22/29] lint --- scripts/protect_page.py | 2 -- scripts/remove_dev.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 3dd5299b9b0..e915cbe436d 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -1,5 +1,3 @@ -import time - from typing import Iterable, Literal diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 93a74a2c091..8deec4dba4b 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -23,7 +23,7 @@ def remove_page(session: MediaWikiSession, page: str): "token": session.token, }, ) - except MediaWikiSessionError as e: + except MediaWikiSessionError: print(f"::warning::could not delete {page} on {session.wiki}") write_to_github_summary_file( f":warning: could not delete {page} on {session.wiki}" From 7d236bafa29fbb93687666271cddfeee66735f51 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 9 Mar 2026 18:56:32 +0900 Subject: [PATCH 23/29] sort --- scripts/protect_page.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index e915cbe436d..55e031dde71 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -26,7 +26,7 @@ def protect_pages( else: raise ValueError(f"invalid protect mode: {protect_mode}") print(f"...wiki = {session.wiki}") - for page in pages: + for page in sorted(pages): print(f"...page = {page}") try: protections = session.make_action( From bc64e8a32bbf77a36cbfef800f24bb7517e7bf59 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:12:48 +0900 Subject: [PATCH 24/29] group messages --- scripts/protect_page.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/protect_page.py b/scripts/protect_page.py index 55e031dde71..a7ff00b9584 100644 --- a/scripts/protect_page.py +++ b/scripts/protect_page.py @@ -25,7 +25,7 @@ def protect_pages( protect_options = "create=allow-only-sysop" else: raise ValueError(f"invalid protect mode: {protect_mode}") - print(f"...wiki = {session.wiki}") + print(f"::group::Protecting {session.wiki}") for page in sorted(pages): print(f"...page = {page}") try: @@ -56,6 +56,7 @@ def protect_pages( protect_errors.append(f"{protect_mode}:{session.wiki}:{page}") finally: session.cooldown() + print("::endgroup::") def protect_non_existing_pages(session: MediaWikiSession, pages: Iterable[str]): From 99c91f74820a75f7495a4c90d64a6d027ca63450 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Mon, 9 Mar 2026 19:13:01 +0900 Subject: [PATCH 25/29] sort --- scripts/protect.py | 2 +- scripts/remove_dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/protect.py b/scripts/protect.py index 25a0c3a01e3..fa2455abf21 100644 --- a/scripts/protect.py +++ b/scripts/protect.py @@ -67,7 +67,7 @@ def main(): new_commons_modules = files_to_protect_by_wiki.get("commons") if new_commons_modules: - for wiki in get_wikis(): + for wiki in sorted(get_wikis()): with MediaWikiSession(wiki) as session: if wiki == "commons": protect_existing_pages(session, new_commons_modules) diff --git a/scripts/remove_dev.py b/scripts/remove_dev.py index 8deec4dba4b..8b0a80ecd95 100644 --- a/scripts/remove_dev.py +++ b/scripts/remove_dev.py @@ -68,7 +68,7 @@ def search_and_remove(wiki: str): def main(): - for wiki in get_wikis(): + for wiki in sorted(get_wikis()): if wiki == "commons" and os.getenv("INCLUDE_COMMONS") != "true": continue search_and_remove(wiki) From 243d0f5ea380fe19af0ebd26f53618d21d461af4 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:14:45 +0900 Subject: [PATCH 26/29] better dry mode --- scripts/mediawiki_session.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/mediawiki_session.py b/scripts/mediawiki_session.py index 7564334e5ec..badba1976a1 100644 --- a/scripts/mediawiki_session.py +++ b/scripts/mediawiki_session.py @@ -92,6 +92,11 @@ def make_action( merged_params = {"format": "json", "action": action} if params is not None: merged_params |= params + if DRY_RUN: + print(f"HEADER: {HEADER}") + print(f"PARAM: {merged_params}") + print(f"DATA: {data}") + return True, False response = self.__session.post( self.__get_wiki_api_url(), params=merged_params, data=data ) From 647f99add30a6fb2f351328d390a55e24d2ede2a Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:14:51 +0900 Subject: [PATCH 27/29] lint --- scripts/deploy_res.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/deploy_res.py b/scripts/deploy_res.py index 3f4fdd19cee..0f32bd30f31 100644 --- a/scripts/deploy_res.py +++ b/scripts/deploy_res.py @@ -23,9 +23,8 @@ def deploy_resources( for file_path in file_paths: print(f"::group::Checking {str(file_path)}") file_content = read_file_from_path(file_path) - page = ( - f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" - + "/".join(file_path.parts[2:]) + page = f"MediaWiki:Common.{'js' if res_type == '.js' else 'css'}/" + "/".join( + file_path.parts[2:] ) print(f"...page = {page}") deploy_result = session.deploy_file( From 7401a61c44e8477b9233bcc27992f8388595ebb5 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:15:39 +0900 Subject: [PATCH 28/29] version bump in user agent --- scripts/deploy_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/deploy_util.py b/scripts/deploy_util.py index b2e617ec46a..c4261ce0ef2 100644 --- a/scripts/deploy_util.py +++ b/scripts/deploy_util.py @@ -16,7 +16,7 @@ ] GITHUB_STEP_SUMMARY_FILE = os.getenv("GITHUB_STEP_SUMMARY") -USER_AGENT = f"GitHub Autodeploy Bot/2.0.0 ({os.getenv('WIKI_UA_EMAIL')})" +USER_AGENT = f"GitHub Autodeploy Bot/2.1.0 ({os.getenv('WIKI_UA_EMAIL')})" HEADER = { "User-Agent": USER_AGENT, From 5e19d5c7f9251f58f623a3447ce5008fad058b33 Mon Sep 17 00:00:00 2001 From: ElectricalBoy <15651807+ElectricalBoy@users.noreply.github.com> Date: Fri, 27 Mar 2026 14:29:03 +0900 Subject: [PATCH 29/29] update protect_pages_across_wikis.py --- scripts/protect_page_across_wikis.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/protect_page_across_wikis.py b/scripts/protect_page_across_wikis.py index 43a49d42c32..3f98d876115 100644 --- a/scripts/protect_page_across_wikis.py +++ b/scripts/protect_page_across_wikis.py @@ -1,9 +1,10 @@ import os from deploy_util import get_wikis +from mediawiki_session import MediaWikiSession from protect_page import ( - protect_non_existing_page, - protect_existing_page, + protect_non_existing_pages, + protect_existing_pages, handle_protect_errors, ) @@ -11,13 +12,12 @@ def main(): - for wiki in get_wikis(): - print(f"::group::Checking {wiki}:{PAGE_TO_PROTECT}") - if wiki == "commons": - protect_existing_page(PAGE_TO_PROTECT, wiki) - else: - protect_non_existing_page(PAGE_TO_PROTECT, wiki) - print("::endgroup::") + for wiki in sorted(get_wikis()): + with MediaWikiSession(wiki) as session: + if wiki == "commons": + protect_existing_pages(session, [PAGE_TO_PROTECT]) + else: + protect_non_existing_pages(session, [PAGE_TO_PROTECT]) handle_protect_errors()