From 67bc73dfb5c385c48838f8421a501cd3db9ecfa4 Mon Sep 17 00:00:00 2001 From: Spencer Rak Date: Mon, 10 Nov 2025 12:42:11 -0500 Subject: [PATCH 1/5] Create carthage.libvirt plugin * Move vm.py -> libvirt/base.py * Create import shim with warnings for importers of carthage.vm * Add libvirt plugin initialization * Move libvirt ConfigSchema to libvirt plugin * Move LibvirtDeployableFinder to libvirt plugin init --- carthage/libvirt/__init__.py | 125 +++++++++++++++++++++++++++ carthage/{vm.py => libvirt/base.py} | 53 ------------ carthage/libvirt/carthage_plugin.yml | 7 ++ carthage/vm/__init__.py | 27 ++++++ 4 files changed, 159 insertions(+), 53 deletions(-) create mode 100644 carthage/libvirt/__init__.py rename carthage/{vm.py => libvirt/base.py} (91%) create mode 100644 carthage/libvirt/carthage_plugin.yml create mode 100644 carthage/vm/__init__.py diff --git a/carthage/libvirt/__init__.py b/carthage/libvirt/__init__.py new file mode 100644 index 00000000..912031f4 --- /dev/null +++ b/carthage/libvirt/__init__.py @@ -0,0 +1,125 @@ +# Copyright (C) 2025, Hadron Industries, Inc. +# Carthage is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. It is distributed +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the file +# LICENSE for details. + +import logging +logger = logging.getLogger("carthage.libvirt") + +from carthage import deployment +from carthage.config import ConfigSchema, ConfigLayout +from carthage.dependency_injection import inject, Injector + +from .base import * + +class LibvirtSchema(ConfigSchema, prefix='libvirt'): + + #: The preferred format for newly created disk images + preferred_format: str = 'raw' + + #: When creating a format like qcow2 that can be represented as a + # delta on top of another file, should we use such a backing + # file. If true, then that file must remain unmodified. Generally + # it is better to use OS-level facilities like reflinks to obtain + # copy-on-write. + use_backing_file: bool = False + + #: Set whether Carthage defines libvirt domains by default + # defaults to not defining domains + # may be overridden on models + should_define: bool = False + + #: Default disk size in mebibytes + # defaults to 10GiB + # may be overridden on models + image_size_mib: int = 10485760 + + #: Default image location + # defaults to a place libvirt can access + image_dir: str = "/srv/carthage/libvirt" + + #: Default vm memory in MB + # defaults to 2G + # may be overridden on models + memory_mb: int = 2048 + + #: Default vm vcpu count + # defaults to 2 + # may be overridden on models + cpus: int = 2 + + #: Default image to use + # Defaults to None, which will raise an error if not provided elsewhere + # MUST be set in the config, layout, or on the model + # this is a fallback provided for convenience + image: str = None + + #: Is Carthage running on the hypervisor + # may be overridden in the layout + local_hypervisor: bool = False + +class LibvirtDeployableFinder(deployment.DeployableFinder): + + name = 'libvirt' + + async def find(self, ainjector): + ''' + MachineDeployableFinder already finds Vms. + ''' + return [] + + async def find_orphans(self, deployables): + try: + import libvirt + import carthage.modeling + except ImportError: + logger.debug('Not looking for libvirt orphans because libvirt API is not available') + return [] + con = libvirt.open('') + vm_names = [v.full_name for v in deployables if isinstance(v, Vm)] + try: + layout = await self.ainjector.get_instance_async(carthage.modeling.CarthageLayout) + layout_name = layout.layout_name + except KeyError: + layout_name = None + if layout_name is None: + logger.info('Unable to find libvirt orphans because layout name not set') + return [] + results = [] + for d in con.listAllDomains(): + try: + metadata_str = d.metadata(libvirt.VIR_DOMAIN_METADATA_ELEMENT, 'https://github.com/hadron/carthage') + except libvirt.libvirtError: continue + metadata = xml.etree.ElementTree.fromstring(metadata_str) + if metadata.attrib['layout'] != layout_name: continue + if d.name() in vm_names: + continue + with instantiation_not_ready(): + vm = await self.ainjector( + Vm, + name=d.name(), + image=None, + ) + vm.injector.add_provider(deployment.orphan_policy, deployment.DeletionPolicy[metadata.attrib['orphan_policy']]) + if await vm.find(): + results.append(vm) + return results + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.injector.add_provider(ConfigLayout) + cl = self.injector.get_instance(ConfigLayout) + cl.container_prefix = "" + +@inject(injector=Injector) +def carthage_plugin(injector): + # this is done in case carthage.vm is loaded first which already sets up our provider + from carthage.dependency_injection.base import ExistingProvider + try: + injector.add_provider(LibvirtDeployableFinder) + except ExistingProvider: + pass + diff --git a/carthage/vm.py b/carthage/libvirt/base.py similarity index 91% rename from carthage/vm.py rename to carthage/libvirt/base.py index 9573457b..ceb7064b 100644 --- a/carthage/vm.py +++ b/carthage/libvirt/base.py @@ -544,59 +544,6 @@ async def populate(self): ''' await self._build_image() -class LibvirtDeployableFinder(carthage.deployment.DeployableFinder): - - name = 'libvirt' - - async def find(self, ainjector): - ''' - MachineDeployableFinder already finds Vms. - ''' - return [] - - async def find_orphans(self, deployables): - try: - import libvirt - import carthage.modeling - except ImportError: - logger.debug('Not looking for libvirt orphans because libvirt API is not available') - return [] - con = libvirt.open('') - vm_names = [v.full_name for v in deployables if isinstance(v, Vm)] - try: - layout = await self.ainjector.get_instance_async(carthage.modeling.CarthageLayout) - layout_name = layout.layout_name - except KeyError: - layout_name = None - if layout_name is None: - logger.info('Unable to find libvirt orphans because layout name not set') - return [] - results = [] - for d in con.listAllDomains(): - try: - metadata_str = d.metadata(libvirt.VIR_DOMAIN_METADATA_ELEMENT, 'https://github.com/hadron/carthage') - except libvirt.libvirtError: continue - metadata = xml.etree.ElementTree.fromstring(metadata_str) - if metadata.attrib['layout'] != layout_name: continue - if d.name() in vm_names: - continue - with instantiation_not_ready(): - vm = await self.ainjector( - Vm, - name=d.name(), - image=None, - ) - vm.injector.add_provider(deployment.orphan_policy, deployment.DeletionPolicy[metadata.attrib['orphan_policy']]) - if await vm.find(): - results.append(vm) - return results - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.injector.add_provider(ConfigLayout) - cl = self.injector.get_instance(ConfigLayout) - cl.container_prefix = "" - def vm_as_image(key): ''' Return the volume of a VM to be used for cloning. Typical usage:: diff --git a/carthage/libvirt/carthage_plugin.yml b/carthage/libvirt/carthage_plugin.yml new file mode 100644 index 00000000..ba8e0077 --- /dev/null +++ b/carthage/libvirt/carthage_plugin.yml @@ -0,0 +1,7 @@ +name: carthage.libvirt +dependencies: + - deb: python3-libvirt + pypi: libvirt-python + - deb: zstd + - deb: dosfstools + - deb: fai-server diff --git a/carthage/vm/__init__.py b/carthage/vm/__init__.py new file mode 100644 index 00000000..4bef26ad --- /dev/null +++ b/carthage/vm/__init__.py @@ -0,0 +1,27 @@ +# Copyright (C) 2025, Hadron Industries, Inc. +# Carthage is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. It is distributed +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the file +# LICENSE for details. + +import inspect +def get_caller(): + for sf in inspect.stack(): + if sf.filename.startswith("<"): continue + if sf.filename.endswith("/carthage/vm/__init__.py"): continue + return (sf.filename, sf.lineno) +file, line = get_caller() +import warnings +warnings.filterwarnings("default", category=DeprecationWarning, module=__name__) +warnings.warn(f"\n\n\ +=============================================\n\ +carthage.vm has migrated to carthage.libvirt.\n\ +File: '{file}' has imported carthage.vm at line: {line}\n\ +Please import carthage.libvirt in the future.\n\ +=============================================\n\ +", category=DeprecationWarning) + +from carthage.libvirt import * +from carthage.libvirt.base import vm_image_key, LibvirtCreatedImage From 537e3d6a91aef4c823dd8814798b5cba1432104b Mon Sep 17 00:00:00 2001 From: Spencer Rak Date: Mon, 10 Nov 2025 12:43:19 -0500 Subject: [PATCH 2/5] Move libvirt templates to libvirt --- carthage/{ => libvirt}/resources/templates/vm-config.mako | 0 carthage/{ => libvirt}/resources/templates/vm-console.mako | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename carthage/{ => libvirt}/resources/templates/vm-config.mako (100%) rename carthage/{ => libvirt}/resources/templates/vm-console.mako (100%) diff --git a/carthage/resources/templates/vm-config.mako b/carthage/libvirt/resources/templates/vm-config.mako similarity index 100% rename from carthage/resources/templates/vm-config.mako rename to carthage/libvirt/resources/templates/vm-config.mako diff --git a/carthage/resources/templates/vm-console.mako b/carthage/libvirt/resources/templates/vm-console.mako similarity index 100% rename from carthage/resources/templates/vm-console.mako rename to carthage/libvirt/resources/templates/vm-console.mako From 6850e6b3c51daeffdfbd292383714b2c42f36221 Mon Sep 17 00:00:00 2001 From: Spencer Rak Date: Mon, 10 Nov 2025 12:43:56 -0500 Subject: [PATCH 3/5] Remove libvirt schema from config/base.py --- carthage/config/base.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/carthage/config/base.py b/carthage/config/base.py index c9619af2..2cf90174 100644 --- a/carthage/config/base.py +++ b/carthage/config/base.py @@ -76,16 +76,3 @@ class DebianConfig(ConfigSchema, prefix="debian"): #: Any debootstrap option to include debootstrap_options: str = "" - -class LibvirtSchema(ConfigSchema, prefix='libvirt'): - - #: The preferred format for newly created disk images - preferred_format: str = 'raw' - - #: When creating a format like qcow2 that can be represented as a - #delta on top of another file, should we use such a backing - #file. If true, then that file must remain unmodified. Generally - #it is better to use OS-level facilities like reflinks to obtain - #copy-on-write. - use_backing_file: bool = False - From 3ce2b21e2e8484b97679eebb1e39b0e7e248f9f4 Mon Sep 17 00:00:00 2001 From: Spencer Rak Date: Mon, 10 Nov 2025 12:45:45 -0500 Subject: [PATCH 4/5] Move LibvirtImageModel to libvirt.modeling --- carthage/libvirt/modeling.py | 35 +++++++++++++++++++++++++++++++++++ carthage/modeling/base.py | 20 -------------------- 2 files changed, 35 insertions(+), 20 deletions(-) create mode 100644 carthage/libvirt/modeling.py diff --git a/carthage/libvirt/modeling.py b/carthage/libvirt/modeling.py new file mode 100644 index 00000000..e9379c15 --- /dev/null +++ b/carthage/libvirt/modeling.py @@ -0,0 +1,35 @@ +# Copyright (C) 2025, Hadron Industries, Inc. +# Carthage is free software; you can redistribute it and/or modify +# it under the terms of the GNU Lesser General Public License version 3 +# as published by the Free Software Foundation. It is distributed +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the file +# LICENSE for details. + +from carthage.dependency_injection import InjectionKey +from carthage.machine import BaseCustomization +from carthage.modeling import ImageRole +from carthage.utils import memoproperty + +from .base import LibvirtCreatedImage + + +__all__ = [] + +class LibvirtImageModel(LibvirtCreatedImage, ImageRole): + + ''' + A :class:`carthage.libvirt.LibvirtCreatedImage` that is a modeling class, so modeling language constructs work. In addition, any customization in the class is included in the default *vm_customizations*. + ''' + + disk_cache = 'unsafe' #Volume is destroyed on failure + @classmethod + def our_key(cls): + return InjectionKey(LibvirtCreatedImage, name=cls.name) + + @memoproperty + def vm_customizations(self): + return [x[1] for x in self.injector.filter_instantiate( + BaseCustomization, + ['description'], stop_at=self.injector)] +__all__ += ["LibvirtImageModel"] diff --git a/carthage/modeling/base.py b/carthage/modeling/base.py index 6e37d9a0..92007748 100644 --- a/carthage/modeling/base.py +++ b/carthage/modeling/base.py @@ -21,7 +21,6 @@ import carthage.kvstore import carthage.network import carthage.machine -import carthage.vm from .utils import * logger = logging.getLogger(__name__) @@ -706,22 +705,3 @@ class model(*dynamic_set_of_bases): transclusion_overrides=True) __all__ += ['add_dynamic_machine_model'] - -class LibvirtImageModel(carthage.vm.LibvirtCreatedImage, ImageRole): - - ''' - A :class:`carthage.vm.LibvirtCreatedImage` that is a modeling class, so modeling language constructs work. In addition, any customization in the class is included in the default *vm_customizations*. - ''' - - disk_cache = 'unsafe' #Volume is destroyed on failure - @classmethod - def our_key(cls): - return InjectionKey(carthage.vm.LibvirtCreatedImage, name=cls.name) - - @memoproperty - def vm_customizations(self): - return [x[1] for x in self.injector.filter_instantiate( - carthage.machine.BaseCustomization, - ['description'], stop_at=self.injector)] - -__all__ += ['LibvirtImageModel'] From 83bfce5bf6317144981af1f010ae2d6bf124a430 Mon Sep 17 00:00:00 2001 From: Spencer Rak Date: Mon, 10 Nov 2025 12:49:24 -0500 Subject: [PATCH 5/5] Don't use relative imports in plugin --- carthage/libvirt/base.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/carthage/libvirt/base.py b/carthage/libvirt/base.py index ceb7064b..2e3664bf 100644 --- a/carthage/libvirt/base.py +++ b/carthage/libvirt/base.py @@ -6,31 +6,28 @@ # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the file # LICENSE for details. +import logging +logger = logging.getLogger("carthage.libvirt") + import asyncio import json -import logging import os import os.path import shutil import types import uuid -import xml.etree.ElementTree -import mako import mako.lookup -import mako.template -from pathlib import Path -from .dependency_injection import * -from . import deployment -from .utils import when_needed, memoproperty -from .setup_tasks import SetupTaskMixin, setup_task -from .image import ImageVolume -from .machine import Machine, SshMixin, ContainerCustomization, disk_config_from_model, AbstractMachineModel -from . import sh -from .config import ConfigLayout -from .ports import PortReservation + import carthage.network -logger = logging.getLogger('carthage.vm') +from carthage import deployment, sh +from carthage.config import ConfigLayout +from carthage.dependency_injection import * +from carthage.image import ImageVolume +from carthage.machine import disk_config_from_model, Machine, SshMixin, ContainerCustomization, AbstractMachineModel +from carthage.ports import PortReservation +from carthage.setup_tasks import SetupTaskMixin, setup_task +from carthage.utils import when_needed, memoproperty _resources_path = os.path.join(os.path.dirname(__file__), "resources") _templates = mako.lookup.TemplateLookup([_resources_path + '/templates']) @@ -89,7 +86,7 @@ def __init__(self, name, *, console_needed=None, @memoproperty def uuid(self): - from .modeling import CarthageLayout + from carthage.modeling import CarthageLayout layout = self.injector.get_instance(InjectionKey(CarthageLayout, _optional=True)) if layout: layout_uuid = layout.layout_uuid @@ -125,7 +122,7 @@ async def find_or_create(self): await self.start_machine() async def write_config(self): - from .modeling import CarthageLayout + from carthage.modeling import CarthageLayout template = _templates.get_template("vm-config.mako") await self.resolve_networking() for i, link in self.network_links.items():