From dfe55b1e7276c20ec382533d5a1260bd72529f54 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Mon, 9 Sep 2024 16:22:29 +0200 Subject: [PATCH 1/9] add dissect-update --- dissect/__init__.py | 0 dissect/update/__init__.py | 135 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 11 +++ tox.ini | 22 ++++++ 4 files changed, 168 insertions(+) create mode 100644 dissect/__init__.py create mode 100644 dissect/update/__init__.py diff --git a/dissect/__init__.py b/dissect/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py new file mode 100644 index 0000000..3b0f181 --- /dev/null +++ b/dissect/update/__init__.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import argparse +import logging +import os +import subprocess +import urllib.request +from pathlib import Path + +from pip._vendor import tomli + +try: + import structlog + + structlog.configure( + processors=[ + structlog.contextvars.merge_contextvars, + structlog.processors.add_log_level, + structlog.processors.StackInfoRenderer(), + structlog.dev.set_exc_info, + structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S UTC", utc=True), + structlog.dev.ConsoleRenderer(), + ], + wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), + ) + log = structlog.get_logger() + +except ImportError: + logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") + log = logging.getLogger(__name__) + + +PYPROJECT_FILE_PATHS = [ + # wheels will have a toml file at dissect/update/pyproject.toml + str(Path(os.path.realpath(__file__)).parent) + "/pyproject.toml", + # tgz dist files will have a toml file at dissect/pyproject.toml + str(Path(os.path.realpath(__file__)).parent.parent) + "/pyproject.toml", + # git source repositories will have a toml file inside the repository root. + str(Path(os.path.realpath(__file__)).parent.parent.parent) + "/pyproject.toml", +] + +PYPROJECT_ONLINE_URL = os.getenv( + "DISSECT_PYPROJECT_URL", "https://raw.githubusercontent.com/fox-it/dissect/main/pyproject.toml" +) + + +def _run(cmd: str, verbose: int) -> subprocess.CompletedProcess: + """Wrapper for subprocess run command.""" + res = subprocess.run(cmd, shell=True, capture_output=True) + if verbose or res.returncode != 0: + print(res.stdout.decode("utf-8")) + if res.stderr != b"": + print(res.stderr.decode("utf-8")) + return res + + +def main(): + help_formatter = argparse.ArgumentDefaultsHelpFormatter + parser = argparse.ArgumentParser( + description="Update your Dissect installation.", + fromfile_prefix_chars="@", + formatter_class=help_formatter, + ) + parser.add_argument("-u", "--do-not-upgrade-pip", action="store_true", help="do not upgrade pip", default=False) + parser.add_argument("-o", "--online", action="store_true", help="use the latest pyproject.toml from GitHub.com") + parser.add_argument("-f", "--file", default=False, action="store", help="path to a custom pyproject.toml file") + parser.add_argument("-v", "--verbose", action="store_true", help="show output of pip", default=False) + args = parser.parse_args() + + if not args.do_not_upgrade_pip: + log.info("Updating pip..") + _run("pip install --upgrade pip", args.verbose) + + if args.online: + args.file = PYPROJECT_ONLINE_URL + log.info(f"The following url will be used to determine dependencies: {args.file}") + try: + input("Press ENTER to continue..") + except KeyboardInterrupt: + print() + return + + pyproject = load_pyproject_toml(args.file) + + if not pyproject: + log.error("No pyproject.toml found, exiting..") + return + + modules = pyproject["project"]["dependencies"] + log.info(f"Found {str(len(modules))} dependencies") + + for module in modules: + pretty_module_name = module.split(">")[0].split("=")[0] + log.info(f"Updating dependency {pretty_module_name}") + # --pre does not do anything if pyproject.toml defines its dependencies strict like foo==1.0.0, + # so this only affects loose custom dependency definitions with, e.g. foo>1.0.0,<2.0.0. + # TODO: figure out if this is a git repository, then just git pull! + _run(f"pip install '{module.strip(',')}' --upgrade --no-cache-dir --pre", args.verbose) + + log.info("Finished updating all dependencies!") + + if args.verbose: + log.info("Currently installed dependencies listed below:") + _run("pip freeze", args.verbose) + + +def load_pyproject_toml(custom_path: str | None) -> dict | None: + """Attempt to load a pyproject.toml file and return the parsed dictionary.""" + + if custom_path: + log.info(f"Using {custom_path} as pyproject.toml source.") + path = Path(custom_path) + + if path.exists(): + with open(custom_path, mode="rb") as f: + return tomli.load(f) + + elif custom_path.startswith("https://"): + try: + content = urllib.request.urlopen(custom_path).read().decode() + return tomli.loads(content) + except Exception as e: + log.error(f"Unable to fetch {custom_path}: {str(e)}") + return + + for toml_file in PYPROJECT_FILE_PATHS: + try: + with open(toml_file, mode="rb") as f: + log.info(f"Found file {toml_file} to read dependencies from.") + return tomli.load(f) + except FileNotFoundError: + log.debug(f"File {toml_file} not found!") + continue + + log.error("No pyproject.toml files found to read dependencies from!") diff --git a/pyproject.toml b/pyproject.toml index 90cd14d..d01204e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,3 +84,14 @@ license-files = ["LICENSE", "COPYRIGHT"] include = ["dissect.*"] [tool.setuptools_scm] + +[tool.setuptools.package-data] +"dissect" = ["pyproject.toml"] + +[project.scripts] +# TODO: pick one :) +dissect-update = "dissect.update:main" +dissect-upgrade = "dissect.update:main" +update-dissect = "dissect.update:main" +upgrade-dissect = "dissect.update:main" +target-update = "dissect.update:main" diff --git a/tox.ini b/tox.ini index edbb142..24008c1 100644 --- a/tox.ini +++ b/tox.ini @@ -28,6 +28,28 @@ deps = commands = pyproject-build +[testenv:fix] +package = skip +deps = + black==23.1.0 + isort==5.11.4 +commands = + black dissect + isort dissect + +[testenv:lint] +package = skip +deps = + black==23.1.0 + flake8 + flake8-black + flake8-isort + isort==5.11.4 + vermin +commands = + flake8 dissect + vermin -t=3.9- --no-tips --lint dissect + [flake8] max-line-length = 120 extend-ignore = From fdfc4d75470a785cd65b08e6b2775b2a0d31845f Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:49:08 +0200 Subject: [PATCH 2/9] add support for editable installs and version diff output --- dissect/update/__init__.py | 112 ++++++++++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 13 deletions(-) diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py index 3b0f181..c7a2250 100644 --- a/dissect/update/__init__.py +++ b/dissect/update/__init__.py @@ -1,11 +1,13 @@ from __future__ import annotations import argparse +import json import logging import os import subprocess import urllib.request from pathlib import Path +from typing import Iterator from pip._vendor import tomli @@ -24,10 +26,12 @@ wrapper_class=structlog.make_filtering_bound_logger(logging.INFO), ) log = structlog.get_logger() + HAS_STRUCTLOG = True except ImportError: logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") log = logging.getLogger(__name__) + HAS_STRUCTLOG = False PYPROJECT_FILE_PATHS = [ @@ -50,11 +54,20 @@ def _run(cmd: str, verbose: int) -> subprocess.CompletedProcess: if verbose or res.returncode != 0: print(res.stdout.decode("utf-8")) if res.stderr != b"": + log.error("Process returned stderr output:") print(res.stderr.decode("utf-8")) return res def main(): + try: + actual_main() + except KeyboardInterrupt: + print() + return + + +def actual_main(): help_formatter = argparse.ArgumentDefaultsHelpFormatter parser = argparse.ArgumentParser( description="Update your Dissect installation.", @@ -67,10 +80,24 @@ def main(): parser.add_argument("-v", "--verbose", action="store_true", help="show output of pip", default=False) args = parser.parse_args() + if args.verbose: + if HAS_STRUCTLOG: + structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.DEBUG)) + else: + logging.getLogger().setLevel(logging.DEBUG) + + # By default we want to ensure we have the latest pip version since outdated pip + # versions can be troublesome with dependency version resolving and matching. if not args.do_not_upgrade_pip: log.info("Updating pip..") _run("pip install --upgrade pip", args.verbose) + # We collect the current state of the installed modules so we can compare versions later. + initial_modules = environment_modules(args.verbose) + + # If the user requested an online pyproject.toml update we mis-use the args.file flag + # and ask the user to confirm the URL for safety since we use dependency module names + # from that file inside subprocess.run calls. if args.online: args.file = PYPROJECT_ONLINE_URL log.info(f"The following url will be used to determine dependencies: {args.file}") @@ -80,24 +107,57 @@ def main(): print() return + # We attempt to obtain our dependencies from a pyproject.toml file. pyproject = load_pyproject_toml(args.file) - if not pyproject: - log.error("No pyproject.toml found, exiting..") + # We check if the current environment has any git editable install locations. + editable_installs = list(find_editable_installs(args.verbose)) + + if not pyproject and not editable_installs: + log.error("No pyproject.toml or editable installs found, exiting..") return - modules = pyproject["project"]["dependencies"] - log.info(f"Found {str(len(modules))} dependencies") + if pyproject: + modules = pyproject["project"]["dependencies"] + log.info(f"Found {str(len(modules))} dependencies") - for module in modules: - pretty_module_name = module.split(">")[0].split("=")[0] - log.info(f"Updating dependency {pretty_module_name}") - # --pre does not do anything if pyproject.toml defines its dependencies strict like foo==1.0.0, - # so this only affects loose custom dependency definitions with, e.g. foo>1.0.0,<2.0.0. - # TODO: figure out if this is a git repository, then just git pull! - _run(f"pip install '{module.strip(',')}' --upgrade --no-cache-dir --pre", args.verbose) + for module in modules: + pretty_module_name = module.split(">")[0].split("=")[0] + + # If this module is also in the editable installs we found we skip them here. + if pretty_module_name in [m.get("name") for m in editable_installs]: + log.debug(f"Not updating module {pretty_module_name} as it is installed as editable") + continue + + log.info(f"Updating dependency using pip: {pretty_module_name}") + # --pre does not do anything if pyproject.toml defines its dependencies strict like foo==1.0.0, + # so this only affects loose custom dependency definitions with, e.g. foo>1.0.0,<2.0.0. + _run(f"pip install '{module.strip(',')}' --upgrade --no-cache-dir --pre", args.verbose) + + if editable_installs: + log.info(f"Found {str(len(editable_installs))} editable installs in current environment") + + for module in editable_installs: + module_name = module.get("name") + module_path = module.get("editable_project_location") + log.info(f"Updating local dependency: {module_name} @ {module_path}") + # We assume that this is a git repository and we have git available to us. + _run(f"cd {module_path} && git pull && pip install -e .", args.verbose) + + log.info("Finished updating all dependencies, see below for changes") + + # Display the version differences between the dependencies. + current_modules = environment_modules(args.verbose) + if initial_modules and current_modules: + for module in current_modules: + previous_module_version = next( + filter(lambda prev_module: prev_module["name"] == module["name"], initial_modules) + ) - log.info("Finished updating all dependencies!") + if previous_module_version.get("version") != module.get("version"): + print("\x1b[92m\x1b[1m", end="") + + print(f'{module.get("name")} {previous_module_version.get("version")} -> {module.get("version")}\x1b[0m') if args.verbose: log.info("Currently installed dependencies listed below:") @@ -132,4 +192,30 @@ def load_pyproject_toml(custom_path: str | None) -> dict | None: log.debug(f"File {toml_file} not found!") continue - log.error("No pyproject.toml files found to read dependencies from!") + log.error("No pyproject.toml files found to read dependencies from! Consider using --file or --online") + + +def environment_modules(verbose: bool) -> list[dict] | None: + """Wrapper around pip list command.""" + + try: + modules = json.loads(_run("pip list --format=json", verbose).stdout.decode()) + return modules + + except Exception as e: + log.error("Failed to parse current environment using pip!") + log.debug("", exc_info=e) + return + + +def find_editable_installs(verbose: bool) -> Iterator[dict] | None: + """Attempt to find editable installs in the current environment.""" + + modules = environment_modules(verbose) + + if not modules: + return + + for module in modules: + if module.get("editable_project_location"): + yield module From e3142863010bb3259415af2e28c214616707e66e Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:55:51 +0200 Subject: [PATCH 3/9] only output changes in version diff --- dissect/update/__init__.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py index c7a2250..d780b1a 100644 --- a/dissect/update/__init__.py +++ b/dissect/update/__init__.py @@ -144,20 +144,21 @@ def actual_main(): # We assume that this is a git repository and we have git available to us. _run(f"cd {module_path} && git pull && pip install -e .", args.verbose) - log.info("Finished updating all dependencies, see below for changes") + log.info("Finished updating dependencies") # Display the version differences between the dependencies. current_modules = environment_modules(args.verbose) if initial_modules and current_modules: for module in current_modules: - previous_module_version = next( + previous_module = next( filter(lambda prev_module: prev_module["name"] == module["name"], initial_modules) ) + module_name = module.get("name") + previous_version = previous_module.get("version") + current_version = module.get("version") - if previous_module_version.get("version") != module.get("version"): - print("\x1b[92m\x1b[1m", end="") - - print(f'{module.get("name")} {previous_module_version.get("version")} -> {module.get("version")}\x1b[0m') + if previous_version != current_version: + print(f'{module_name} \x1b[31m{previous_version}\x1b[0m -> \x1b[32m\x1b[1m{current_version}\x1b[0m') if args.verbose: log.info("Currently installed dependencies listed below:") From e6b8e07561be11fe62857e9c9fbd18e96f88c8f0 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:56:02 +0200 Subject: [PATCH 4/9] fix linter --- dissect/update/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py index d780b1a..c6cc995 100644 --- a/dissect/update/__init__.py +++ b/dissect/update/__init__.py @@ -150,15 +150,13 @@ def actual_main(): current_modules = environment_modules(args.verbose) if initial_modules and current_modules: for module in current_modules: - previous_module = next( - filter(lambda prev_module: prev_module["name"] == module["name"], initial_modules) - ) + previous_module = next(filter(lambda prev_module: prev_module["name"] == module["name"], initial_modules)) module_name = module.get("name") previous_version = previous_module.get("version") current_version = module.get("version") if previous_version != current_version: - print(f'{module_name} \x1b[31m{previous_version}\x1b[0m -> \x1b[32m\x1b[1m{current_version}\x1b[0m') + print(f"{module_name} \x1b[31m{previous_version}\x1b[0m -> \x1b[32m\x1b[1m{current_version}\x1b[0m") if args.verbose: log.info("Currently installed dependencies listed below:") From f13d33cbc958d8770f2c74d2c3a9498892688b41 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:53:10 +0200 Subject: [PATCH 5/9] Lisan al-Gaib --- pyproject.toml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index d01204e..edb8405 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,9 +89,4 @@ include = ["dissect.*"] "dissect" = ["pyproject.toml"] [project.scripts] -# TODO: pick one :) dissect-update = "dissect.update:main" -dissect-upgrade = "dissect.update:main" -update-dissect = "dissect.update:main" -upgrade-dissect = "dissect.update:main" -target-update = "dissect.update:main" From f33fe950397641ae095e35784184012501b254c1 Mon Sep 17 00:00:00 2001 From: JSCU-CNI <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:57:57 +0200 Subject: [PATCH 6/9] use interpreter pip module instead of global pip --- dissect/update/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py index c6cc995..7c99bdf 100644 --- a/dissect/update/__init__.py +++ b/dissect/update/__init__.py @@ -5,6 +5,7 @@ import logging import os import subprocess +import sys import urllib.request from pathlib import Path from typing import Iterator @@ -90,7 +91,7 @@ def actual_main(): # versions can be troublesome with dependency version resolving and matching. if not args.do_not_upgrade_pip: log.info("Updating pip..") - _run("pip install --upgrade pip", args.verbose) + _run(f"{sys.executable} -m pip install --upgrade pip", args.verbose) # We collect the current state of the installed modules so we can compare versions later. initial_modules = environment_modules(args.verbose) @@ -132,7 +133,7 @@ def actual_main(): log.info(f"Updating dependency using pip: {pretty_module_name}") # --pre does not do anything if pyproject.toml defines its dependencies strict like foo==1.0.0, # so this only affects loose custom dependency definitions with, e.g. foo>1.0.0,<2.0.0. - _run(f"pip install '{module.strip(',')}' --upgrade --no-cache-dir --pre", args.verbose) + _run(f"{sys.executable} -m pip install '{module.strip(',')}' --upgrade --no-cache-dir --pre", args.verbose) if editable_installs: log.info(f"Found {str(len(editable_installs))} editable installs in current environment") @@ -142,7 +143,7 @@ def actual_main(): module_path = module.get("editable_project_location") log.info(f"Updating local dependency: {module_name} @ {module_path}") # We assume that this is a git repository and we have git available to us. - _run(f"cd {module_path} && git pull && pip install -e .", args.verbose) + _run(f"cd {module_path} && git pull && {sys.executable} -m pip install -e .", args.verbose) log.info("Finished updating dependencies") @@ -160,7 +161,7 @@ def actual_main(): if args.verbose: log.info("Currently installed dependencies listed below:") - _run("pip freeze", args.verbose) + _run(f"{sys.executable} -m pip freeze", args.verbose) def load_pyproject_toml(custom_path: str | None) -> dict | None: @@ -198,7 +199,7 @@ def environment_modules(verbose: bool) -> list[dict] | None: """Wrapper around pip list command.""" try: - modules = json.loads(_run("pip list --format=json", verbose).stdout.decode()) + modules = json.loads(_run(f"{sys.executable} -m pip list --format=json", verbose).stdout.decode()) return modules except Exception as e: From 581a4d41892ed42d3e17929705f0d1df888614ad Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:58:43 +0200 Subject: [PATCH 7/9] Apply suggestion from @Schamper Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/update/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py index 7c99bdf..c000302 100644 --- a/dissect/update/__init__.py +++ b/dissect/update/__init__.py @@ -53,7 +53,7 @@ def _run(cmd: str, verbose: int) -> subprocess.CompletedProcess: """Wrapper for subprocess run command.""" res = subprocess.run(cmd, shell=True, capture_output=True) if verbose or res.returncode != 0: - print(res.stdout.decode("utf-8")) + print(res.stdout.decode()) if res.stderr != b"": log.error("Process returned stderr output:") print(res.stderr.decode("utf-8")) From d0f958827cb4d069dd669c62004ae761f871591f Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 15 Jul 2025 12:58:54 +0200 Subject: [PATCH 8/9] Apply suggestion from @Schamper Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/update/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py index c000302..e2a97c1 100644 --- a/dissect/update/__init__.py +++ b/dissect/update/__init__.py @@ -54,7 +54,7 @@ def _run(cmd: str, verbose: int) -> subprocess.CompletedProcess: res = subprocess.run(cmd, shell=True, capture_output=True) if verbose or res.returncode != 0: print(res.stdout.decode()) - if res.stderr != b"": + if res.stderr: log.error("Process returned stderr output:") print(res.stderr.decode("utf-8")) return res From 8f53ba1cd4c72b6beaab0d59aa693e3c6763ec85 Mon Sep 17 00:00:00 2001 From: Computer Network Investigation <121175071+JSCU-CNI@users.noreply.github.com> Date: Tue, 15 Jul 2025 13:00:12 +0200 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: Erik Schamper <1254028+Schamper@users.noreply.github.com> --- dissect/update/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dissect/update/__init__.py b/dissect/update/__init__.py index e2a97c1..1e3a50d 100644 --- a/dissect/update/__init__.py +++ b/dissect/update/__init__.py @@ -56,7 +56,7 @@ def _run(cmd: str, verbose: int) -> subprocess.CompletedProcess: print(res.stdout.decode()) if res.stderr: log.error("Process returned stderr output:") - print(res.stderr.decode("utf-8")) + print(res.stderr.decode()) return res @@ -90,13 +90,13 @@ def actual_main(): # By default we want to ensure we have the latest pip version since outdated pip # versions can be troublesome with dependency version resolving and matching. if not args.do_not_upgrade_pip: - log.info("Updating pip..") + log.info("Updating pip...") _run(f"{sys.executable} -m pip install --upgrade pip", args.verbose) # We collect the current state of the installed modules so we can compare versions later. initial_modules = environment_modules(args.verbose) - # If the user requested an online pyproject.toml update we mis-use the args.file flag + # If the user requested an online pyproject.toml update we re-use the args.file flag # and ask the user to confirm the URL for safety since we use dependency module names # from that file inside subprocess.run calls. if args.online: @@ -115,7 +115,7 @@ def actual_main(): editable_installs = list(find_editable_installs(args.verbose)) if not pyproject and not editable_installs: - log.error("No pyproject.toml or editable installs found, exiting..") + log.error("No pyproject.toml or editable installs found, exiting...") return if pyproject: