From 86e2b09b9718ad41270c6fb9581554b8155f0a57 Mon Sep 17 00:00:00 2001 From: matthiasL-scality Date: Mon, 26 Jan 2026 10:58:57 +0100 Subject: [PATCH] (PTFE-2972) Add search from marketplace image by name --- runner_manager/backend/scaleway.py | 118 +++++++++++++++++++++++++--- tests/unit/backend/test_scaleway.py | 115 ++++++++++++++++++++++++--- 2 files changed, 210 insertions(+), 23 deletions(-) diff --git a/runner_manager/backend/scaleway.py b/runner_manager/backend/scaleway.py index 67d98694..5cb5f098 100644 --- a/runner_manager/backend/scaleway.py +++ b/runner_manager/backend/scaleway.py @@ -17,6 +17,7 @@ from scaleway.instance.v1.custom_api import ( InstanceUtilsV1API, # type: ignore[import-untyped] ) +from scaleway.marketplace.v2 import MarketplaceV2API # type: ignore[import-untyped] from runner_manager.backend.base import BaseBackend from runner_manager.models.backend import ( @@ -74,25 +75,122 @@ def sanitize_tags(self, tags: List[str]) -> List[str]: return sanitized def get_image(self, image_name: str) -> Image: - """Get image by name or ID.""" + """Get image by name or ID. + + Supports three lookup methods in order: + 1. Direct UUID lookup (for explicit image IDs) + 2. User's custom images by name (e.g., Packer-built images) + 3. Scaleway Marketplace images by label + + Args: + image_name: Image UUID, custom image name, or marketplace label + + Returns: + Image object from Scaleway Instance API + + Raises: + ValueError: If image is not found + """ + import re + + # Check if it's a UUID format + uuid_pattern = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + re.IGNORECASE, + ) + is_uuid = bool(uuid_pattern.match(image_name)) + + # 1. Try direct UUID lookup + if is_uuid: + try: + return self.client.get_image( + zone=self.config.zone, + image_id=image_name, + ).image + except Exception as e: + log.debug(f"Image ID lookup failed: {e}") + raise ValueError( + f"Image with ID '{image_name}' not found in zone {self.config.zone}" + ) + + # 2. Try to find in user's custom images by name (e.g., Packer images) try: - # Try to get by ID first - return self.client.get_image( - zone=self.config.zone, - image_id=image_name, - ).image - except Exception: - # Otherwise, list images and find by name images = self.client.list_images( zone=self.config.zone, name=image_name, ).images if images: + log.info(f"Found user image '{image_name}': {images[0].id}") return images[0] - raise ValueError( - f"Image '{image_name}' not found in zone {self.config.zone}" + except Exception as e: + log.debug(f"User images lookup failed: {e}") + + # 3. Try Scaleway Marketplace + try: + # Create marketplace client + scw_client = Client( + access_key=self.config.access_key or os.getenv("SCW_ACCESS_KEY"), + secret_key=self.config.secret_key or os.getenv("SCW_SECRET_KEY"), + default_project_id=self.config.project_id, + default_zone=self.config.zone, + default_region=self.config.region, + ) + marketplace_client = MarketplaceV2API(scw_client) + + # List all marketplace images with pagination + all_images = [] + page = 1 + page_size = 100 + + while True: + images_result = marketplace_client.list_images( + include_eol=True, + page=page, + page_size=page_size, + ) + all_images.extend(images_result.images) + + if len(images_result.images) < page_size: + break + page += 1 + + # Find image by label + marketplace_image = None + for img in all_images: + if img.label == image_name: + marketplace_image = img + break + + if not marketplace_image: + raise ValueError(f"Image label '{image_name}' not found in marketplace") + + log.info(f"Found marketplace image '{image_name}': {marketplace_image.id}") + + # Get the local version for the current zone + local_images = marketplace_client.list_local_images( + image_id=marketplace_image.id, + zone=self.config.zone, ) + if local_images.local_images: + for local_img in local_images.local_images: + if local_img.zone == self.config.zone: + log.info(f"Resolved to local image ID: {local_img.id}") + return self.client.get_image( + zone=self.config.zone, + image_id=local_img.id, + ).image + + except Exception as marketplace_error: + log.debug( + f"Marketplace lookup failed for '{image_name}': {marketplace_error}" + ) + + raise ValueError( + f"Image '{image_name}' not found in zone {self.config.zone}. " + f"Tried: UUID lookup, user images, and marketplace." + ) + def wait_for_server_state( self, server_id: str, diff --git a/tests/unit/backend/test_scaleway.py b/tests/unit/backend/test_scaleway.py index 81ecc563..3cb0e6fe 100644 --- a/tests/unit/backend/test_scaleway.py +++ b/tests/unit/backend/test_scaleway.py @@ -124,7 +124,7 @@ def test_backend_name(fake_scaleway_group): def test_get_image(fake_scaleway_group): - """Test getting image by name.""" + """Test getting image by name from user images.""" backend = fake_scaleway_group.backend image = backend.get_image("ubuntu_jammy") @@ -132,6 +132,107 @@ def test_get_image(fake_scaleway_group): assert image.name == "ubuntu_jammy" +def test_get_image_by_uuid(fake_scaleway_group): + """Test getting image by UUID.""" + backend = fake_scaleway_group.backend + + # UUID format should trigger direct get_image call + uuid = "ec31d73d-ca36-4536-adf4-0feb76d30379" + image = backend.get_image(uuid) + + assert image.id == "test-image-id" + + +def test_get_image_marketplace(fake_scaleway_group, monkeypatch): + """Test getting image from Scaleway Marketplace.""" + # Mock marketplace image + mock_marketplace_img = MagicMock() + mock_marketplace_img.id = "marketplace-img-id" + mock_marketplace_img.label = "ubuntu_noble" + + # Mock local image + mock_local_img = MagicMock() + mock_local_img.id = "local-img-uuid" + mock_local_img.zone = "fr-par-1" + + # Mock marketplace API + mock_marketplace_api = MagicMock() + mock_marketplace_api.list_images.return_value = MagicMock( + images=[mock_marketplace_img] + ) + mock_marketplace_api.list_local_images.return_value = MagicMock( + local_images=[mock_local_img] + ) + + # Mock MarketplaceV2API where it's imported in the backend module + monkeypatch.setattr( + "runner_manager.backend.scaleway.MarketplaceV2API", + lambda client: mock_marketplace_api, + ) + + # Mock list_images to return empty (force marketplace lookup) + backend = fake_scaleway_group.backend + mock_client = backend.client + mock_client.list_images.return_value = MagicMock(images=[]) + + # Get image from marketplace + image = backend.get_image("ubuntu_noble") + + # Verify it called marketplace API + assert mock_marketplace_api.list_images.called + assert mock_marketplace_api.list_local_images.called + assert image.id == "test-image-id" + + +def test_get_image_not_found(fake_scaleway_group, monkeypatch): + """Test error when image is not found anywhere.""" + backend = fake_scaleway_group.backend + + # Mock all lookups to fail + mock_client = backend.client + mock_client.list_images.return_value = MagicMock(images=[]) + + # Mock marketplace to also fail + mock_marketplace_api = MagicMock() + mock_marketplace_api.list_images.return_value = MagicMock(images=[]) + + monkeypatch.setattr( + "runner_manager.backend.scaleway.MarketplaceV2API", + lambda client: mock_marketplace_api, + ) + + with pytest.raises(ValueError, match="not found in zone"): + backend.get_image("non-existent-image") + + +def test_get_image_user_priority(fake_scaleway_group, monkeypatch): + """Test that user images take priority over marketplace images.""" + backend = fake_scaleway_group.backend + + # Mock user image found + mock_user_image = MagicMock() + mock_user_image.id = "user-custom-image-id" + mock_user_image.name = "my-custom-ubuntu" + + mock_client = backend.client + mock_client.list_images.return_value = MagicMock(images=[mock_user_image]) + + # Mock marketplace (should not be called) + mock_marketplace_api = MagicMock() + monkeypatch.setattr( + "runner_manager.backend.scaleway.MarketplaceV2API", + lambda client: mock_marketplace_api, + ) + + image = backend.get_image("my-custom-ubuntu") + + # Verify user image was returned + assert image.id == "user-custom-image-id" + + # Verify marketplace was NOT called (user image found first) + assert not mock_marketplace_api.list_images.called + + def test_create_instance_mock(scaleway_runner, fake_scaleway_group): """Test instance creation with mocked client.""" backend = fake_scaleway_group.backend @@ -438,18 +539,6 @@ def test_get_image_by_id(fake_scaleway_group): assert image.id == "test-image-id" -def test_get_image_not_found(fake_scaleway_group, monkeypatch): - """Test get_image when image is not found.""" - backend = fake_scaleway_group.backend - - mock_client = backend.client - mock_client.get_image.side_effect = Exception("Image not found") - mock_client.list_images.return_value = MagicMock(images=[]) - - with pytest.raises(ValueError, match="not found in zone"): - backend.get_image("non-existent-image") - - # Real API tests (skipped if credentials not available)