From 9964a65c316da29935dd1989f234ff504f3f4561 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Sun, 19 Jun 2022 14:33:43 +0200 Subject: [PATCH 1/5] wxGUI/extensions: fix reinstall/uninstall multi-addon e.g. wx.metadata --- gui/wxpython/modules/extensions.py | 3 +- python/grass/script/core.py | 72 +++++++++++ python/grass/utils/download.py | 105 ++++++++++++++++ scripts/g.extension.all/g.extension.all.py | 133 +-------------------- 4 files changed, 180 insertions(+), 133 deletions(-) diff --git a/gui/wxpython/modules/extensions.py b/gui/wxpython/modules/extensions.py index 84d9aeba315..a3897e916b3 100644 --- a/gui/wxpython/modules/extensions.py +++ b/gui/wxpython/modules/extensions.py @@ -24,6 +24,7 @@ import wx from grass.script import task as gtask +from grass.script import find_addon_name from core import globalvar from core.gcmd import GError, RunCommand, GException, GMessage @@ -477,7 +478,7 @@ def _getSelectedExtensions(self): GMessage(_("No extension selected. " "Operation canceled."), parent=self) return [] - return eList + return find_addon_name(addons=eList) def OnUninstall(self, event): """Uninstall selected extensions""" diff --git a/python/grass/script/core.py b/python/grass/script/core.py index d027f941309..f6c95894d1f 100644 --- a/python/grass/script/core.py +++ b/python/grass/script/core.py @@ -32,10 +32,14 @@ import io from tempfile import NamedTemporaryFile from pathlib import Path +from urllib.parse import urlparse + +import xml.etree.ElementTree as etree from .utils import KeyValue, parse_key_val, basename, encode, decode, try_remove from grass.exceptions import ScriptError, CalledModuleError from grass.grassdb.manage import resolve_mapset_path +from grass.utils.download import download_file # subprocess wrapper that uses shell on Windows @@ -2054,6 +2058,74 @@ def create_environment(gisdbase, location, mapset, env=None): return f.name, env +def find_addon_name(addons, url=None): + """Find correct addon name if addon is a multi-addon + e.g. wx.metadata contains multiple modules g.gui.cswbrowser etc. + + Examples: + - for the g.gui.cswbrowser module the wx.metadata addon name is + returned + - for the i.sentinel.download module the i.sentinel addon name is + returned + etc. + + :param list addons: list of individual addon modules + :param str url: Addons modules.xml file URL + + :return: list of unique simple and multi addons + + >>> addons = find_addon_name( + ... addons=[ + ... "g.gui.metadata", + ... "g.gui.cswbrowser", + ... "db.csw.run", + ... "db.csw.harvest", + ... "db.csw.admin", + ... "v.info.iso", + ... "r.info.iso", + ... "t.info.iso", + ... "db.join", + ... ] + ... ) # doctest: +SKIP + >>> addons.sort() # doctest: +SKIP + >>> addons # doctest: +SKIP + ['db.join', 'wx.metadata'] + """ + if not url: + grass_version = os.getenv("GRASS_VERSION", "unknown") + if grass_version != "unknown": + major, minor, patch = grass_version.split(".") + else: + fatal(_("Unable to get GRASS GIS version.")) + url = "https://grass.osgeo.org/addons/grass{major}/modules.xml".format( + major=major, + ) + response = download_file( + url=url, + response_format="application/xml", + file_name=os.path.basename(urlparse(url).path), + ) + tree = etree.fromstring(response.read()) + result = [] + for addon in addons: + found = False + addon_pattern = re.compile(r".*{}$".format(addon)) + for i in tree: + for f in i.findall(".//binary/file"): + if re.match(addon_pattern, f.text): + result.append(i.attrib["name"]) + found = True + break + if not found: + warning( + _( + "The addon <{}> was not found among the official" + " addons.".format(addon) + ), + ) + return list(set(result)) + + if __name__ == "__main__": import doctest diff --git a/python/grass/utils/download.py b/python/grass/utils/download.py index f6b734b7291..e39422ef4ca 100644 --- a/python/grass/utils/download.py +++ b/python/grass/utils/download.py @@ -12,6 +12,7 @@ """Download and extract various archives""" +import http import os import shutil import tarfile @@ -21,6 +22,12 @@ from urllib.error import HTTPError, URLError from urllib.parse import urlparse from urllib.request import urlretrieve +from urllib import request as urlrequest + +HEADERS = { + "User-Agent": "Mozilla/5.0", +} +HTTP_STATUS_CODES = list(http.HTTPStatus) def debug(*args, **kwargs): @@ -206,3 +213,101 @@ def name_from_url(url): # Special treatment of .tar.gz extension. return os.path.splitext(name)[0] return name + + +def urlopen(url, *args, **kwargs): + """Wrapper around urlopen. Same function as 'urlopen', but with the + ability to define headers. + + :param str url: URL + + :return: urllib.request.urlopen response object + """ + proxy = kwargs.get("proxy") + if proxy: + del kwargs["proxy"] + PROXIES = {} + for ptype, purl in (p.split("=") for p in proxy.split(",")): + PROXIES[ptype] = purl + proxy = urlrequest.ProxyHandler(PROXIES) + opener = urlrequest.build_opener(proxy) + urlrequest.install_opener(opener) + request = urlrequest.Request(url, headers=HEADERS) + return urlrequest.urlopen(request, *args, **kwargs) + + +def download_file(url, response_format, file_name, *args, **kwargs): + """Download file + + :param str url: file URL address + :param str response_format: content type of downloaded file + :param str file_name: downloaded file name + + :return: urllib.request.urlopen response object + + >>> grass_version = os.getenv("GRASS_VERSION", "unknown") + >>> if grass_version != "unknown": + ... major, minor, patch = grass_version.split(".") + ... url = ( + ... "https://grass.osgeo.org/addons/grass{}/" + ... "modules.xml".format(major) + ... ) + ... response = download_file( + ... url=url, + ... response_format="application/xml", + ... file_name=os.path.basename(urlparse(url).path), + ... ) # doctest: +SKIP + ... response.code # doctest: +SKIP + 200 + """ + import grass.script as gs + + try: + response = urlopen(url, *args, **kwargs) + + if not response.code == 200: + index = HTTP_STATUS_CODES.index(response.code) + desc = HTTP_STATUS_CODES[index].description + gs.fatal( + _( + "The download of the <{file_name}> file " + " from the server <{url}> was not successful" + " return status code {code}," + " {desc}".format( + file_name=file_name, + url=url, + code=response.code, + desc=desc, + ), + ), + ) + if response_format not in response.getheader("Content-Type"): + gs.fatal( + _( + "Wrong downloaded <{file_name}> format." + " Check url <{url}>. Allowed file format is" + " {response_format}.".format( + file_name=file_name, + url=url, + response_format=response_format, + ), + ), + ) + return response + except (HTTPError, URLError) as err: + desc = "" + if hasattr(err, "code"): + index = HTTP_STATUS_CODES.index(err.code) + desc = ", {}".format(HTTP_STATUS_CODES[index].description) + gs.fatal( + _( + "The download of the <{file_name}> file" + " from the server <{url}> was not successful, " + " {err}{desc}.".format( + file_name=file_name, + url=url, + err=err, + desc=desc, + ), + ), + ) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index e8a24f86fd0..1e85e17fc7b 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -40,20 +40,11 @@ import os import re import sys - import xml.etree.ElementTree as etree -from urllib import request as urlrequest -from urllib.error import HTTPError, URLError - import grass.script as gscript from grass.exceptions import CalledModuleError -HEADERS = { - "User-Agent": "Mozilla/5.0", -} -HTTP_STATUS_CODES = list(http.HTTPStatus) - def get_extensions(): addon_base = os.getenv("GRASS_ADDON_BASE") @@ -84,128 +75,6 @@ def get_extensions(): return ret -def urlopen(url, *args, **kwargs): - """Wrapper around urlopen. Same function as 'urlopen', but with the - ability to define headers. - """ - request = urlrequest.Request(url, headers=HEADERS) - return urlrequest.urlopen(request, *args, **kwargs) - - -def download_modules_xml_file(url, response_format, *args, **kwargs): - """Generates JSON file containing the download URLs of the official - Addons - - :param str url: url address - :param str response_format: content type - - :return response: urllib.request.urlopen response object or None - """ - try: - response = urlopen(url, *args, **kwargs) - - if not response.code == 200: - index = HTTP_STATUS_CODES.index(response.code) - desc = HTTP_STATUS_CODES[index].description - gscript.fatal( - _( - "Download file from <{url}>, " - "return status code {code}, " - "{desc}".format( - url=url, - code=response.code, - desc=desc, - ), - ), - ) - if response_format not in response.getheader("Content-Type"): - gscript.fatal( - _( - "Wrong file format downloaded. " - "Check url <{url}>. Allowed file format is " - "{response_format}.".format( - url=url, - response_format=response_format, - ), - ), - ) - return response - - except HTTPError as err: - if err.code == 404: - gscript.fatal( - _( - "The download of the modules.xml file " - "from the server was not successful. " - "File on the server <{url}> doesn't " - "exists.".format(url=url), - ), - ) - else: - return download_modules_xml_file( - url=url, - response_format=response_format, - ) - except URLError: - gscript.fatal( - _( - "Download file from <{url}>, " - "failed. Check internet connection.".format( - url=url, - ), - ), - ) - - -def find_addon_name(addons): - """Find correct addon name if addon is a multi-addon - e.g. wx.metadata contains multiple modules g.gui.cswbrowser etc. - - Examples: - - for the g.gui.cswbrowser module the wx.metadata addon name is - returned - - for the i.sentinel.download module the i.sentinel addon name is - returned - etc. - - :param list addons: list of individual addon modules to be reinstalled - - :return set result: set of unique addon names to be reinstalled - """ - grass_version = os.getenv("GRASS_VERSION", "unknown") - if grass_version != "unknown": - major, minor, patch = grass_version.split(".") - else: - gscript.fatal(_("Unable to get GRASS GIS version.")) - url = "https://grass.osgeo.org/addons/grass{major}/modules.xml".format( - major=major, - ) - response = download_modules_xml_file( - url=url, - response_format="application/xml", - ) - tree = etree.fromstring(response.read()) - result = [] - for addon in addons: - found = False - addon_pattern = re.compile(r".*{}$".format(addon)) - for i in tree: - for f in i.findall(".//binary/file"): - if re.match(addon_pattern, f.text): - result.append(i.attrib["name"]) - found = True - break - if not found: - gscript.warning( - _( - "The <{}> addon cannot be reinstalled. " - "Addon wasn't found among the official " - "addons.".format(addon) - ), - ) - return set(result) - - def main(): remove = options["operation"] == "remove" if remove or flags["f"]: @@ -232,7 +101,7 @@ def main(): ) return 0 - for ext in find_addon_name(addons=extensions): + for ext in gscript.find_addon_name(addons=extensions): gscript.message("-" * 60) if remove: gscript.message(_("Removing extension <%s>...") % ext) From e5df0b5378965390bb431d2e78ae41576fc97787 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Sun, 19 Jun 2022 19:56:48 +0200 Subject: [PATCH 2/5] Remove unused imports --- scripts/g.extension.all/g.extension.all.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/g.extension.all/g.extension.all.py b/scripts/g.extension.all/g.extension.all.py index 1e85e17fc7b..9311086579b 100644 --- a/scripts/g.extension.all/g.extension.all.py +++ b/scripts/g.extension.all/g.extension.all.py @@ -36,9 +36,7 @@ # % label: Force operation (required for removal) # % end -import http import os -import re import sys import xml.etree.ElementTree as etree From 6afb98b9be3d394fcfee44ae793afb7e5ff36393 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 5 Jun 2024 09:31:48 +0200 Subject: [PATCH 3/5] Enable doc test --- python/grass/script/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/grass/script/core.py b/python/grass/script/core.py index f6c95894d1f..2dd9e7e1125 100644 --- a/python/grass/script/core.py +++ b/python/grass/script/core.py @@ -2086,9 +2086,9 @@ def find_addon_name(addons, url=None): ... "t.info.iso", ... "db.join", ... ] - ... ) # doctest: +SKIP - >>> addons.sort() # doctest: +SKIP - >>> addons # doctest: +SKIP + ... ) + >>> addons.sort() + >>> addons ['db.join', 'wx.metadata'] """ if not url: From 1df9699d1730f2c48d5685dbe212577ed549873a Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 5 Jun 2024 09:37:20 +0200 Subject: [PATCH 4/5] Add missing Python re module --- python/grass/script/core.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/grass/script/core.py b/python/grass/script/core.py index 2dd9e7e1125..27f8aac5009 100644 --- a/python/grass/script/core.py +++ b/python/grass/script/core.py @@ -19,6 +19,7 @@ """ import os +import re import sys import atexit import subprocess From faac751d56b3d24f4d5a7ca40e8fc4925a33be45 Mon Sep 17 00:00:00 2001 From: Tomas Zigo Date: Wed, 5 Jun 2024 09:40:31 +0200 Subject: [PATCH 5/5] Enable doc test --- python/grass/utils/download.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/grass/utils/download.py b/python/grass/utils/download.py index e39422ef4ca..407c02c0417 100644 --- a/python/grass/utils/download.py +++ b/python/grass/utils/download.py @@ -256,8 +256,8 @@ def download_file(url, response_format, file_name, *args, **kwargs): ... url=url, ... response_format="application/xml", ... file_name=os.path.basename(urlparse(url).path), - ... ) # doctest: +SKIP - ... response.code # doctest: +SKIP + ... ) + ... response.code 200 """ import grass.script as gs