diff --git a/README.md b/README.md index 9b6c26c..1562a1d 100644 --- a/README.md +++ b/README.md @@ -88,10 +88,14 @@ All endpoints are under `/api/v1/`: |----------------------------|--------|------------------------------------------| | `/health/` | GET | API health check | | `/list/?filter=...` | GET | List available core packages | -| `//get/` | GET | Download a `.core` file | +| `/get/?core=...` | GET | Download a `.core` file by VLNV name | | `/validate/` | POST | Validate a core file (`multipart/form`) | | `/publish/` | POST | Publish a core file to GitHub | +- **Download (`/get/`)**: Provide the `core` query parameter with the full VLNV (e.g., `acme:lib1:foo:1.0.0`). +- **Validation and publishing**: Upload core files (and optional signatures) via `multipart/form-data`. +- **OpenAPI/Swagger docs**: Interactive documentation is available at `/api/v1/docs/swagger/` and `/api/v1/docs/redoc/`. + --- ## Web UI diff --git a/core_directory/models.py b/core_directory/models.py index 4889bb5..3de4526 100644 --- a/core_directory/models.py +++ b/core_directory/models.py @@ -144,6 +144,19 @@ def is_signed(self): """ return bool(self.sig_url) + @property + def sanitized_vlnv(self): + """ + Returns a filesystem- and URL-safe version of the core's VLNV (Vendor:Library:Name:Version), + with colons and other problematic characters replaced by underscores. + """ + return ( + f'{self.project.vendor.sanitized_name}_' + f'{self.project.library.sanitized_name}_' + f'{self.project.sanitized_name}_' + f'{self.sanitized_name}' + ) + class Meta: """Ensure version is unique per project.""" unique_together = ('project', 'version') diff --git a/core_directory/tests/api/test_cores.py b/core_directory/tests/api/test_cores.py index e8ce98a..ca5aa03 100644 --- a/core_directory/tests/api/test_cores.py +++ b/core_directory/tests/api/test_cores.py @@ -1,74 +1,106 @@ -from unittest import mock import pytest from django.urls import reverse +from core_directory.models import Vendor, Library, Project, CorePackage @pytest.mark.django_db -@mock.patch("core_directory.views.api_views.Github") -def test_cores_success(mock_github, client, mocker): +def test_cores_success(client, mocker): url = reverse('core_directory:core_list') - # Mock GitHub repo and contents - mock_repo = mock.Mock() - mock_content = mock.Mock() - mock_content.type = "file" - mock_content.path = "foo.core" - mock_repo.get_contents.return_value = [mock_content] - mock_github.return_value.get_repo.return_value = mock_repo - mocker.patch("os.getenv", side_effect=lambda key, default=None: "dummy_token" if key == "GITHUB_ACCESS_TOKEN" else default) + # Mock the queryset to return a list of vlnv_names + mock_qs = mocker.Mock() + mock_qs.order_by.return_value = mock_qs + mock_qs.values_list.return_value = ["acme:lib1:core1:1.0.0"] + mock_filter = mocker.patch("core_directory.models.CorePackage.objects.filter", return_value=mock_qs) response = client.get(url) assert response.status_code == 200 - assert response.json() == ["foo"] + assert response.json() == ["acme:lib1:core1:1.0.0"] + mock_filter.assert_called_once_with(vlnv_name__icontains="") @pytest.mark.django_db -def test_cores_github_exception(client, mocker): - url = reverse('core_directory:core_list') - mock_repo = mocker.Mock() - # Define a mock exception with a .data attribute - class GithubException(Exception): - def __init__(self): - self.data = "fail" - mock_repo.get_contents.side_effect = GithubException() - mock_github = mocker.patch("core_directory.views.api_views.Github") - mock_github.return_value.get_repo.return_value = mock_repo - mocker.patch("core_directory.views.api_views.GithubException", GithubException) - mocker.patch("os.getenv", side_effect=lambda key, default=None: "dummy_token" if key == "GITHUB_ACCESS_TOKEN" else default) +def test_multiple_cores_success(client): + # Set up test data: two cores in the database + vendor = Vendor.objects.create(name="Acme") + library = Library.objects.create(vendor=vendor, name="Lib1") + project1 = Project.objects.create(vendor=vendor, library=library, name="Core1", description="desc") + project2 = Project.objects.create(vendor=vendor, library=library, name="Core2", description="desc") + CorePackage.objects.create( + project=project1, + vlnv_name="acme:lib1:core1:1.0.0", + version="1.0.0", + version_major=1, + version_minor=0, + version_patch=0, + core_url="https://example.com/core1", + description="desc" + ) + CorePackage.objects.create( + project=project2, + vlnv_name="acme:lib1:core2:1.0.0", + version="1.0.0", + version_major=1, + version_minor=0, + version_patch=0, + core_url="https://example.com/core2_v1.0.0", + description="desc" + ) + CorePackage.objects.create( + project=project2, + vlnv_name="acme:lib1:core2:0.1.0", + version="0.1.0", + version_major=0, + version_minor=1, + version_patch=0, + core_url="https://example.com/core2_v0.1.0", + description="desc" + ) + url = reverse('core_directory:core_list') response = client.get(url) - assert response.status_code == 500 - assert "GitHub error" in str(response.content) - assert "fail" in str(response.content) - -import pytest -from django.urls import reverse + assert response.status_code == 200 + # The API returns a list of vlnv_names + assert set(response.json()) == {"acme:lib1:core1:1.0.0", "acme:lib1:core2:0.1.0", "acme:lib1:core2:1.0.0"} @pytest.mark.django_db -def test_cores_with_filter(client, mocker): +def test_cores_with_filter(client): + # Set up test data: two cores in the database + vendor = Vendor.objects.create(name="Acme") + library = Library.objects.create(vendor=vendor, name="Lib1") + project1 = Project.objects.create(vendor=vendor, library=library, name="foo_core", description="desc") + project2 = Project.objects.create(vendor=vendor, library=library, name="bar_core", description="desc") + cp1 = CorePackage.objects.create( + project=project1, + vlnv_name="acme:lib1:foo_core:1.0.0", + version="1.0.0", + version_major=1, + version_minor=0, + version_patch=0, + core_url="https://example.com/foo_core", + description="desc" + ) + cp2 = CorePackage.objects.create( + project=project2, + vlnv_name="acme:lib1:bar_core:1.0.0", + version="1.0.0", + version_major=1, + version_minor=0, + version_patch=0, + core_url="https://example.com/bar_core", + description="desc" + ) + url = reverse('core_directory:core_list') - mock_repo = mocker.Mock() - # Simulate two core files, only one matches the filter - mock_content1 = mocker.Mock() - mock_content1.type = "file" - mock_content1.path = "foo.core" - mock_content2 = mocker.Mock() - mock_content2.type = "file" - mock_content2.path = "bar.core" - mock_repo.get_contents.return_value = [mock_content1, mock_content2] - mock_github = mocker.patch("core_directory.views.api_views.Github") - mock_github.return_value.get_repo.return_value = mock_repo - mocker.patch("os.getenv", side_effect=lambda key, default=None: "dummy_token" if key == "GITHUB_ACCESS_TOKEN" else default) # Apply filter 'foo' response = client.get(url, {"filter": "foo"}) assert response.status_code == 200 - data = response.json() - assert data == ["foo"] + assert response.json() == ["acme:lib1:foo_core:1.0.0"] + # Apply filter 'bar' response = client.get(url, {"filter": "bar"}) assert response.status_code == 200 - data = response.json() - assert data == ["bar"] + assert response.json() == ["acme:lib1:bar_core:1.0.0"] + # Apply filter that matches nothing response = client.get(url, {"filter": "baz"}) assert response.status_code == 200 - data = response.json() - assert data == [] + assert response.json() == [] \ No newline at end of file diff --git a/core_directory/tests/api/test_get_core.py b/core_directory/tests/api/test_get_core.py index 57e8185..5fed934 100644 --- a/core_directory/tests/api/test_get_core.py +++ b/core_directory/tests/api/test_get_core.py @@ -1,61 +1,78 @@ import pytest from django.urls import reverse +from core_directory.models import Vendor, Library, Project, CorePackage @pytest.mark.django_db def test_getcore_success(client, mocker): - url = reverse('core_directory:core_get', kwargs={"package_name": "foo"}) - # Mock the repo and content - mock_repo = mocker.Mock() - mock_content = mocker.Mock() - mock_content.decoded_content = b"core content" - mock_repo.get_contents.return_value = mock_content - mock_github = mocker.patch("core_directory.views.api_views.Github") - mock_github.return_value.get_repo.return_value = mock_repo - mocker.patch("os.getenv", side_effect=lambda key, default=None: "dummy_token" if key == "GITHUB_ACCESS_TOKEN" else default) + # Set up test data + vendor = Vendor.objects.create(name="Acme") + library = Library.objects.create(vendor=vendor, name="Lib1") + project = Project.objects.create(vendor=vendor, library=library, name="foo", description="desc") + core_package = CorePackage.objects.create( + project=project, + vlnv_name="acme:lib1:foo:1.0.0", + version="1.0.0", + version_major=1, + version_minor=0, + version_patch=0, + core_url="https://example.com/foo.core", + description="desc" + ) - response = client.get(url) + # Mock requests.get to return a fake file + mock_response = mocker.Mock() + mock_response.status_code = 200 + mock_response.content = b"core file content" + mocker.patch("requests.get", return_value=mock_response) + + url = reverse('core_directory:core_get') + response = client.get(url, {"core": "acme:lib1:foo:1.0.0"}) assert response.status_code == 200 - assert b"core content" in response.content - assert response["Content-Disposition"].endswith("foo.core") + assert b'core file content' in response.content + assert response["Content-Disposition"].endswith("acme_lib1_foo_1.0.0.core") @pytest.mark.django_db -def test_getcore_not_found(client, mocker): - url = reverse('core_directory:core_get', kwargs={"package_name": "foo"}) - mock_repo = mocker.Mock() - class NotFound(Exception): - status = 404 - data = "not found" - mock_repo.get_contents.side_effect = NotFound() - mock_github = mocker.patch("core_directory.views.api_views.Github") - mock_github.return_value.get_repo.return_value = mock_repo - mocker.patch("core_directory.views.api_views.GithubException", NotFound) - mocker.patch("os.getenv", side_effect=lambda key, default=None: "dummy_token" if key == "GITHUB_ACCESS_TOKEN" else default) - - response = client.get(url) +def test_getcore_not_found(client): + url = reverse('core_directory:core_get') + response = client.get(url, {"core": "acme:lib1:doesnotexist:1.0.0"}) assert response.status_code == 404 - assert "not found" in str(response.content) + assert b"not available" in response.content or b"not available" in response.json().get("error", "").lower() import pytest from django.urls import reverse +from core_directory.models import Vendor, Library, Project, CorePackage @pytest.mark.django_db -def test_getcore_github_exception(client, mocker): - url = reverse('core_directory:core_get', kwargs={"package_name": "foo"}) - - # Define a mock GithubException with .status and .data attributes - class GithubException(Exception): - def __init__(self, status=500, data="fail"): - self.status = status - self.data = data - - mock_repo = mocker.Mock() - mock_repo.get_contents.side_effect = GithubException(500, "fail") - mock_github = mocker.patch("core_directory.views.api_views.Github") - mock_github.return_value.get_repo.return_value = mock_repo - mocker.patch("core_directory.views.api_views.GithubException", GithubException) - mocker.patch("os.getenv", side_effect=lambda key, default=None: "dummy_token" if key == "GITHUB_ACCESS_TOKEN" else default) +def test_getcore_file_not_found(client, mocker): + # Set up test data: core exists in DB + vendor = Vendor.objects.create(name="Acme") + library = Library.objects.create(vendor=vendor, name="Lib1") + project = Project.objects.create(vendor=vendor, library=library, name="foo", description="desc") + core_package = CorePackage.objects.create( + project=project, + vlnv_name="acme:lib1:foo:1.0.0", + version="1.0.0", + version_major=1, + version_minor=0, + version_patch=0, + core_url="https://example.com/foo.core", + description="desc" + ) + # Mock requests.get to simulate file not found (404) + mock_response = mocker.Mock() + mock_response.status_code = 404 + mock_response.content = b"" + mocker.patch("requests.get", return_value=mock_response) + + url = reverse('core_directory:core_get') + response = client.get(url, {"core": "acme:lib1:foo:1.0.0"}) + assert response.status_code == 404 + assert b"not found" in response.content or b"not found" in response.json().get("error", "").lower() + +@pytest.mark.django_db +def test_getcore_missing_param(client): + url = reverse("core_directory:core_get") response = client.get(url) - assert response.status_code == 500 - assert b"GitHub error" in response.content - assert b"fail" in response.content \ No newline at end of file + assert response.status_code == 400 + assert b"missing" in response.content or b"required" in response.content \ No newline at end of file diff --git a/core_directory/tests/api/test_publish.py b/core_directory/tests/api/test_publish.py index 5704d8b..c2924c2 100644 --- a/core_directory/tests/api/test_publish.py +++ b/core_directory/tests/api/test_publish.py @@ -1,7 +1,12 @@ import pytest + +from io import BytesIO + from django.urls import reverse from django.core.files.uploadedfile import SimpleUploadedFile +from core_directory.models import Vendor, Library, Project, CorePackage + @pytest.mark.django_db def test_publish_success(client, mocker): url = reverse('core_directory:publish') @@ -28,11 +33,44 @@ class GithubException(Exception): mocker.patch("os.getenv", side_effect=lambda key, default=None: "dummy_token" if key == "GITHUB_ACCESS_TOKEN" else default) response = client.post(url, data={"core_file": SimpleUploadedFile("test.core", b"dummy")}) - assert response.status_code in (200, 201) + assert response.status_code is 201 assert b"published" in response.content or b"valid" in response.content @pytest.mark.django_db -def test_publish_already_exists(client, mocker): +def test_publish_core_already_exists_in_db(client, mocker): + # Set up test data: create a core with the same VLNV in the database + vendor = Vendor.objects.create(name="Acme") + library = Library.objects.create(vendor=vendor, name="Lib1") + project = Project.objects.create(vendor=vendor, library=library, name="foo", description="desc") + CorePackage.objects.create( + project=project, + vlnv_name="acme:lib1:foo:1.0.0", + version="1.0.0", + version_major=1, + version_minor=0, + version_patch=0, + core_url="https://example.com/foo.core", + description="desc" + ) + + url = reverse('core_directory:publish') + # Mock serializer + mock_serializer = mocker.patch("core_directory.views.api_views.CoreSerializer") + instance = mock_serializer.return_value + instance.is_valid.return_value = True + instance.validated_data = { + "vlnv_name": "acme:lib1:foo:1.0.0", + "core_file": SimpleUploadedFile("test.core", b"dummy"), + "sanitized_name": "core", + "signature_file": None, + } + + response = client.post(url, data={"core_file": SimpleUploadedFile("test.core", b"dummy")}) + assert response.status_code == 409 + assert b"already exists" in response.content + +@pytest.mark.django_db +def test_publish_already_exists_on_github(client, mocker): url = reverse('core_directory:publish') mock_serializer = mocker.patch("core_directory.views.api_views.CoreSerializer") instance = mock_serializer.return_value diff --git a/core_directory/tests/test_urls.py b/core_directory/tests/test_urls.py index bc03e34..d2c7cdb 100644 --- a/core_directory/tests/test_urls.py +++ b/core_directory/tests/test_urls.py @@ -6,7 +6,7 @@ ("redirect_to_docs", {}, 301, False, "get"), ("health_check", {}, 200, False, "get"), ("core_list", {}, 200, False, "get"), - ("core_get", {"package_name": "example"}, 200, False, "get"), + ("core_get", {}, 400, False, "get"), ("validate", {}, 400, False, "post"), ("publish", {}, 400, False, "post"), ("api_docs_landing", {}, 200, False, "get"), diff --git a/core_directory/urls.py b/core_directory/urls.py index fd106f6..44ca24e 100644 --- a/core_directory/urls.py +++ b/core_directory/urls.py @@ -41,7 +41,7 @@ path('', RedirectView.as_view(url='docs/', permanent=True), name='redirect_to_docs'), path('health/', HealthCheckView.as_view(), name='health_check'), path('list/', Cores.as_view(), name='core_list'), - path('/get/', GetCore.as_view(), name='core_get'), + path('get/', GetCore.as_view(), name='core_get'), path('validate/', Validate.as_view(), name='validate'), path('publish/', Publish.as_view(), name='publish'), diff --git a/core_directory/views/api_views.py b/core_directory/views/api_views.py index c0ac9c0..3da3e68 100644 --- a/core_directory/views/api_views.py +++ b/core_directory/views/api_views.py @@ -3,6 +3,8 @@ from dataclasses import dataclass +import requests + from django.http import HttpResponse from django.views.generic import TemplateView @@ -18,6 +20,7 @@ from rest_framework.response import Response from rest_framework.views import APIView +from ..models import CorePackage from ..serializers import CoreSerializer class APIDocsLandingPageView(TemplateView): @@ -63,10 +66,13 @@ def get(self, request, *args, **kwargs): return Response({'status': 'ok'}, status=200) class Cores(APIView): - """Endpoint for listing all available cores.""" + """Endpoint for listing all available core packages in the database.""" @extend_schema_with_429( summary='List all cores', - description='Returns a list of all cores available in FuseSoC-PD.', + description=( + 'turns a list of all core packages available in FuseSoC-PD, ' + 'optionally filtered by a keyword in the VLNV name.' + ), responses={200: OpenApiResponse(description='List of all available cores')}, parameters=[ OpenApiParameter( @@ -78,96 +84,86 @@ class Cores(APIView): ) ] ) - def get(self, request): - """List all available cores, optionally filtered by a keyword. + def get(self, request, *args, **kwargs): + """List all available core packages, optionally filtered by a keyword. Returns: - Response: List of core names or error message. + Response: List of core VLNV names or error message. """ - repo_name = os.getenv('GITHUB_REPO') - filter_keyword = request.query_params.get('filter', '') - try: - # Initialize GitHub client - g = Github(auth=GitHubAuthToken(os.getenv('GITHUB_ACCESS_TOKEN'))) - repo = g.get_repo(repo_name) - - contents = repo.get_contents('') - core_files = [ - content.path.removesuffix('.core') - for content in contents - if content.type == 'file' and content.path.endswith('.core') - ] - - # Apply filter if filter_keyword is provided - if filter_keyword: - core_files = [core for core in core_files if filter_keyword.lower() in core.lower()] + available_cores = ( + CorePackage.objects + .filter(vlnv_name__icontains=filter_keyword) + .order_by('vlnv_name') + .values_list('vlnv_name', flat=True) + ) + return Response(available_cores, status=status.HTTP_200_OK) - return Response(core_files, status=status.HTTP_200_OK) - except GithubException as err: - # Handle specific GitHub API errors - return Response({'error': f'GitHub error: {err.data}'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) class GetCore(APIView): - """Endpoint for downloading a FuseSoC core package.""" + """Endpoint for downloading a FuseSoC core package file by VLNV name.""" @extend_schema_with_429( - summary='Download a FuseSoC Core Package', - description='Provide the FuseSoC Core Package as a YAML file to the user.', - parameters=[ - OpenApiParameter( - name='package_name', - description='The name of the FuseSoC Core Package', - required=True, - type=OpenApiTypes.STR, - location=OpenApiParameter.PATH - ) - ], - responses={ - 200: OpenApiResponse(description='FuseSoC Core Package successfully retrieved'), - 404: OpenApiResponse(description='FuseSoC Core Package not found') - } + summary='Download a FuseSoC Core Package', + description='Provide the FuseSoC Core Package as a .core file to the user.', + parameters=[ + OpenApiParameter( + name='core', + description='Downloads the `.core` file for a given core package (identified by its VLNV name).', + required=True, + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY + ) + ], + responses={ + 200: OpenApiResponse(description='FuseSoC Core Package successfully retrieved'), + 404: OpenApiResponse(description='FuseSoC Core Package not found') + } ) - def get(self, request, package_name): - """Download a FuseSoC core package by name. + def get(self, request): + """Download a FuseSoC core package file by VLNV name. - Args: - package_name (str): The name of the core package to download. + Query Parameters: + core (str): The VLNV name of the core package to download (e.g., 'acme:lib1:foo:1.0.0'). Returns: - HttpResponse: The core file as an attachment, or error message. + HttpResponse: The core file as an attachment, or error message if not found. """ - repo_name = os.getenv('GITHUB_REPO') - - try: - # Initialize GitHub client - g = Github(auth=GitHubAuthToken(os.getenv('GITHUB_ACCESS_TOKEN'))) - repo = g.get_repo(repo_name) - - contents = contents = repo.get_contents(f'{package_name}.core') - # Decode the content from base64 - file_content = contents.decoded_content.decode('utf-8') + requested_core_vlnv = request.query_params.get('core', '') - response = HttpResponse(file_content, content_type='application/octet-stream') - response['Content-Disposition'] = f'attachment; filename={package_name}.core' - return response - except GithubException as err: - if err.status == 404: - return Response( - {'error': f'FuseSoC Core Package {package_name} not found'}, - status=status.HTTP_404_NOT_FOUND - ) - return Response({'error': f'GitHub error: {err.data}'}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + if not requested_core_vlnv: + return Response( + {'error': 'Missing required "core" query parameter.'}, + status=status.HTTP_400_BAD_REQUEST + ) + try: + core_object = CorePackage.objects.get(vlnv_name=requested_core_vlnv) + + requested_file = requests.get(core_object.core_url, timeout=10) + if requested_file.status_code == 200: + response = HttpResponse(requested_file.content, content_type='application/octet-stream') + response['Content-Disposition'] = f'attachment; filename={core_object.sanitized_vlnv}.core' + return response + return Response( + {'error': f'FuseSoC Core Package {requested_core_vlnv} not found.'}, + status=status.HTTP_404_NOT_FOUND + ) + except CorePackage.DoesNotExist: + return Response( + {'error': f'FuseSoC Core Package {requested_core_vlnv} not available.'}, + status=status.HTTP_404_NOT_FOUND + ) class Publish(APIView): - """Endpoint for publishing a core file.""" + """Endpoint for publishing a new core file to FuseSoC Package Directory.""" parser_classes = (MultiPartParser, FormParser) @extend_schema_with_429( summary='Publish a core file', description=( - 'Validates and publish a core file to github. ' - 'The core file should be uploaded as a multipart/form-data request.' + 'Validates and publishes a core file to FuseSoC Package Directory. ' + 'The core file should be uploaded as a multipart/form-data request. ' + 'On success, the directory is updated with the new core package.' ), request={ 'multipart/form-data': { @@ -190,12 +186,12 @@ class Publish(APIView): } }, responses={ - 200: OpenApiResponse(description='Core file is valid'), + 201: OpenApiResponse(description='Core published successfully'), 400: OpenApiResponse(description='Error message indicating why the validation failed') } ) def post(self, request, *args, **kwargs): - """Validate and publish a core file to GitHub. + """Validate and publish a core file to FuseSoC Package Directory. Returns: Response: Success message or error message. @@ -245,6 +241,15 @@ def read_signature_content(self): serializer = CoreSerializer(data=request.data) if serializer.is_valid(): + + vlnv_name = serializer.validated_data['vlnv_name'] + # Check if a core with this VLNV already exists in the database + if CorePackage.objects.filter(vlnv_name=vlnv_name).exists(): + return Response( + {'error': f'Core \'{vlnv_name}\' already exists in FuseSoC Package Directory.'}, + status=status.HTTP_409_CONFLICT + ) + core_data = CoreData( vlnv_name = serializer.validated_data['vlnv_name'], core_file = serializer.validated_data['core_file'], @@ -264,7 +269,7 @@ def read_signature_content(self): _ = repo.get_contents(core_data.core_file_name) # The core already exists -> do not create again return Response( - {'message': f'Core \'{core_data.vlnv_name}\' already exists in the database.'}, + {'message': f'Core \'{core_data.vlnv_name}\' already exists in FuseSoC Package Directory.'}, status=status.HTTP_409_CONFLICT ) except (UnknownObjectException, IndexError, GithubException): @@ -307,7 +312,7 @@ def read_signature_content(self): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) class Validate(APIView): - """Endpoint for validating a core file.""" + """Endpoint for validating a core file (before publishing).""" @extend_schema_with_429( summary='Validate a core file', description=( @@ -338,7 +343,7 @@ class Validate(APIView): } ) def post(self, request, *args, **kwargs): - """Validate a core file against a predefined JSON schema. + """Validate a core file before publishing. Returns: Response: Validation success or error message.