From cb021bf7bc29a22b15a342ebdb964bf30a5ff9cc Mon Sep 17 00:00:00 2001 From: Arpad Borsos Date: Mon, 28 Apr 2025 11:23:14 +0200 Subject: [PATCH] Turn `labelanalysis` and `staticanalysis` views into noops As part of deprecating ATS, this turns all the views into noops that return empty objects. Checking in with the requests that the CLI sends, returning an empty `external_id` and `absent_labels` for the labelanalysis will result in the CLI falling back to using the clientside requested labels. Similarly, returning an empty list of `filepaths` means that the CLI will not do any further uploads. --- apps/codecov-api/labelanalysis/serializers.py | 89 ---- .../tests/integration/test_views.py | 413 +----------------- apps/codecov-api/labelanalysis/urls.py | 9 +- apps/codecov-api/labelanalysis/views.py | 71 ++- .../codecov-api/staticanalysis/serializers.py | 157 ------- .../staticanalysis/tests/test_views.py | 63 +-- .../staticanalysis/tests/unit/__init__.py | 0 .../tests/unit/test_serializers.py | 331 -------------- apps/codecov-api/staticanalysis/urls.py | 19 +- apps/codecov-api/staticanalysis/views.py | 47 +- 10 files changed, 73 insertions(+), 1126 deletions(-) delete mode 100644 apps/codecov-api/labelanalysis/serializers.py delete mode 100644 apps/codecov-api/staticanalysis/serializers.py delete mode 100644 apps/codecov-api/staticanalysis/tests/unit/__init__.py delete mode 100644 apps/codecov-api/staticanalysis/tests/unit/test_serializers.py diff --git a/apps/codecov-api/labelanalysis/serializers.py b/apps/codecov-api/labelanalysis/serializers.py deleted file mode 100644 index a75237a1c5..0000000000 --- a/apps/codecov-api/labelanalysis/serializers.py +++ /dev/null @@ -1,89 +0,0 @@ -from rest_framework import exceptions, serializers - -from core.models import Commit -from labelanalysis.models import ( - LabelAnalysisProcessingError, - LabelAnalysisRequest, - LabelAnalysisRequestState, -) - - -class CommitFromShaSerializerField(serializers.Field): - def __init__(self, *args, **kwargs): - self.accepts_fallback = kwargs.pop("accepts_fallback", False) - super().__init__(*args, **kwargs) - - def to_representation(self, commit): - return commit.commitid - - def to_internal_value(self, commit_sha): - commit = Commit.objects.filter( - repository__in=self.context["request"].auth.get_repositories(), - commitid=commit_sha, - ).first() - if commit is None: - raise exceptions.NotFound(f"Commit {commit_sha[:7]} not found.") - if commit.staticanalysissuite_set.exists(): - return commit - if not self.accepts_fallback: - raise serializers.ValidationError("No static analysis found") - attempted_commits = [] - for _ in range(10): - attempted_commits.append(commit.commitid) - commit = commit.parent_commit - if commit is None: - raise serializers.ValidationError( - f"No possible commits have static analysis sent. Attempted commits: {','.join(attempted_commits)}" - ) - if commit.staticanalysissuite_set.exists(): - return commit - raise serializers.ValidationError( - f"No possible commits have static analysis sent. Attempted too many commits: {','.join(attempted_commits)}" - ) - - -class LabelAnalysisProcessingErrorSerializer(serializers.ModelSerializer): - class Meta: - model = LabelAnalysisProcessingError - fields = ("error_code", "error_params") - read_only_fields = ("error_code", "error_params") - - -class ProcessingErrorList(serializers.ListField): - child = LabelAnalysisProcessingErrorSerializer() - - def to_representation(self, data): - data = data.select_related( - "label_analysis_request", - ).all() - return super().to_representation(data) - - -class LabelAnalysisRequestSerializer(serializers.ModelSerializer): - base_commit = CommitFromShaSerializerField(required=True, accepts_fallback=True) - head_commit = CommitFromShaSerializerField(required=True, accepts_fallback=False) - state = serializers.SerializerMethodField() - errors = ProcessingErrorList(required=False) - - def validate(self, data): - if data["base_commit"] == data["head_commit"]: - raise serializers.ValidationError( - {"base_commit": "Base and head must be different commits"} - ) - return data - - class Meta: - model = LabelAnalysisRequest - fields = ( - "base_commit", - "head_commit", - "requested_labels", - "result", - "state", - "external_id", - "errors", - ) - read_only_fields = ("result", "external_id", "errors") - - def get_state(self, obj): - return LabelAnalysisRequestState.enum_from_int(obj.state_id).name.lower() diff --git a/apps/codecov-api/labelanalysis/tests/integration/test_views.py b/apps/codecov-api/labelanalysis/tests/integration/test_views.py index b81c9a0ea9..ad733b9ac5 100644 --- a/apps/codecov-api/labelanalysis/tests/integration/test_views.py +++ b/apps/codecov-api/labelanalysis/tests/integration/test_views.py @@ -1,30 +1,16 @@ -from uuid import uuid4 - from django.urls import reverse from rest_framework.test import APIClient -from labelanalysis.models import ( - LabelAnalysisProcessingError, - LabelAnalysisRequest, - LabelAnalysisRequestState, -) -from labelanalysis.tests.factories import LabelAnalysisRequestFactory -from services.task import TaskService -from shared.celery_config import label_analysis_task_name +from labelanalysis.views import EMPTY_RESPONSE from shared.django_apps.core.tests.factories import ( CommitFactory, - RepositoryFactory, RepositoryTokenFactory, ) -from staticanalysis.tests.factories import StaticAnalysisSuiteFactory -def test_simple_label_analysis_call_flow(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") +def test_simple_label_analysis_call_flow(db): commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) @@ -42,364 +28,30 @@ def test_simple_label_analysis_call_flow(db, mocker): format="json", ) assert response.status_code == 201 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object - assert produced_object.base_commit == base_commit - assert produced_object.head_commit == commit - assert produced_object.requested_labels is None - assert produced_object.state_id == LabelAnalysisRequestState.CREATED.db_id - assert produced_object.result is None - response_json = response.json() - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } - assert response_json == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": produced_object.id}, - apply_async_kwargs={}, - ) - get_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 200 - assert response.json() == expected_response_json - + assert response.json() == EMPTY_RESPONSE -def test_simple_label_analysis_call_flow_same_commit_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - expected_response_json = { - "base_commit": ["Base and head must be different commits"] - } - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_call_flow_with_fallback_on_base(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_parent_parent = CommitFactory.create(repository=commit.repository) - base_commit_parent = CommitFactory.create( - parent_commit_id=base_commit_parent_parent.commitid, - repository=commit.repository, - ) - base_commit = CommitFactory.create( - parent_commit_id=base_commit_parent.commitid, repository=commit.repository - ) - StaticAnalysisSuiteFactory.create(commit=base_commit_parent_parent) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 201 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object - assert produced_object.base_commit == base_commit_parent_parent - assert produced_object.head_commit == commit - assert produced_object.requested_labels is None - assert produced_object.state_id == LabelAnalysisRequestState.CREATED.db_id - assert produced_object.result is None - response_json = response.json() - expected_response_json = { - "base_commit": base_commit_parent_parent.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } - assert response_json == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": produced_object.id}, - apply_async_kwargs={}, - ) get_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.get( - get_url, - format="json", + "view_label_analysis", + kwargs={"external_id": "c4e11164-411a-48f1-b791-91a9f009ea48"}, ) + response = client.get(get_url, format="json") assert response.status_code == 200 - assert response.json() == expected_response_json - - -def test_simple_label_analysis_call_flow_with_fallback_on_base_error_too_long( - db, mocker -): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - repository = RepositoryFactory.create(active=True) - commit = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_root = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=base_commit_root) - current = base_commit_root - attempted_commit_list = [base_commit_root.commitid] - for i in range(12): - current = CommitFactory.create( - parent_commit_id=current.commitid, repository=repository - ) - attempted_commit_list.append(current.commitid) - base_commit = current - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - # reverse and get 10 first elements, thats how far we look - attempted_commit_list = ",".join(list(reversed(attempted_commit_list))[:10]) - expected_response_json = { - "base_commit": [ - f"No possible commits have static analysis sent. Attempted too many commits: {attempted_commit_list}" - ] - } - assert response_json == expected_response_json - assert not mocked_task_service.called + assert response.json() == EMPTY_RESPONSE -def test_simple_label_analysis_call_flow_with_fallback_on_base_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_parent_parent = CommitFactory.create(repository=commit.repository) - base_commit_parent = CommitFactory.create( - parent_commit_id=base_commit_parent_parent.commitid, - repository=commit.repository, - ) - base_commit = CommitFactory.create( - parent_commit_id=base_commit_parent.commitid, repository=commit.repository - ) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - attempted_commit_list = ",".join( - [ - base_commit.commitid, - base_commit_parent.commitid, - base_commit_parent_parent.commitid, - ] - ) - expected_response_json = { - "base_commit": [ - f"No possible commits have static analysis sent. Attempted commits: {attempted_commit_list}" - ] - } - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_call_flow_with_fallback_on_head_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - repository = RepositoryFactory.create(active=True) - head_commit_parent = CommitFactory.create(repository=repository) - head_commit = CommitFactory.create( - parent_commit_id=head_commit_parent.commitid, repository=repository - ) - base_commit = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - StaticAnalysisSuiteFactory.create(commit=head_commit_parent) - token = RepositoryTokenFactory.create( - repository=head_commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": head_commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=head_commit).count() == 0 - assert ( - LabelAnalysisRequest.objects.filter(head_commit=head_commit_parent).count() == 0 - ) - response_json = response.json() - expected_response_json = {"head_commit": ["No static analysis found"]} - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_only_get(db, mocker): +def test_simple_label_analysis_put_labels(db): commit = CommitFactory.create(repository__active=True) base_commit = CommitFactory.create(repository=commit.repository) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.FINISHED.db_id, - result={"some": ["result"]}, - ) - larq_processing_error = LabelAnalysisProcessingError( - label_analysis_request=label_analysis, - error_code="Missing FileSnapshot", - error_params={"message": "Something is wrong"}, - ) - label_analysis.save() - larq_processing_error.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": {"some": ["result"]}, - "state": "finished", - "external_id": str(produced_object.external_id), - "errors": [ - { - "error_code": "Missing FileSnapshot", - "error_params": {"message": "Something is wrong"}, - } - ], - } - get_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 200 - assert response.json() == expected_response_json - -def test_simple_label_analysis_get_does_not_exist(db, mocker): - token = RepositoryTokenFactory.create( - repository__active=True, token_type="static_analysis" - ) client = APIClient() client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - get_url = reverse("view_label_analysis", kwargs={"external_id": uuid4()}) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 404 - assert response.json() == {"detail": "No such Label Analysis exists"} - -def test_simple_label_analysis_put_labels(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.CREATED.db_id, - result=None, - ) - label_analysis.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - assert produced_object.requested_labels is None - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": ["label_1", "label_2", "label_3"], - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } patch_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} + "view_label_analysis", + kwargs={"external_id": "c4e11164-411a-48f1-b791-91a9f009ea48"}, ) response = client.patch( patch_url, @@ -411,47 +63,4 @@ def test_simple_label_analysis_put_labels(db, mocker): }, ) assert response.status_code == 200 - assert response.json() == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": label_analysis.id}, - apply_async_kwargs={}, - ) - - -def test_simple_label_analysis_put_labels_wrong_base_return_404(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.CREATED.db_id, - result=None, - ) - label_analysis.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - assert produced_object.requested_labels is None - patch_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.patch( - patch_url, - format="json", - data={ - "requested_labels": ["label_1", "label_2", "label_3"], - "base_commit": "not_base_commit", - "head_commit": commit.commitid, - }, - ) - assert response.status_code == 404 - mocked_task_service.assert_not_called() + assert response.json() == EMPTY_RESPONSE diff --git a/apps/codecov-api/labelanalysis/urls.py b/apps/codecov-api/labelanalysis/urls.py index 2ebda9b54e..4deff7507f 100644 --- a/apps/codecov-api/labelanalysis/urls.py +++ b/apps/codecov-api/labelanalysis/urls.py @@ -1,19 +1,16 @@ from django.urls import path -from labelanalysis.views import ( - LabelAnalysisRequestCreateView, - LabelAnalysisRequestDetailView, -) +from labelanalysis.views import LabelAnalysisRequestView urlpatterns = [ path( "labels-analysis", - LabelAnalysisRequestCreateView.as_view(), + LabelAnalysisRequestView.as_view(), name="create_label_analysis", ), path( "labels-analysis/", - LabelAnalysisRequestDetailView.as_view(), + LabelAnalysisRequestView.as_view(), name="view_label_analysis", ), ] diff --git a/apps/codecov-api/labelanalysis/views.py b/apps/codecov-api/labelanalysis/views.py index aae8393121..88dbf38c89 100644 --- a/apps/codecov-api/labelanalysis/views.py +++ b/apps/codecov-api/labelanalysis/views.py @@ -1,58 +1,37 @@ -from rest_framework.exceptions import NotFound -from rest_framework.generics import CreateAPIView, RetrieveAPIView, UpdateAPIView +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView from codecov_auth.authentication.repo_auth import RepositoryTokenAuthentication from codecov_auth.permissions import SpecificScopePermission -from labelanalysis.models import LabelAnalysisRequest, LabelAnalysisRequestState -from labelanalysis.serializers import LabelAnalysisRequestSerializer -from services.task import TaskService -from shared.celery_config import label_analysis_task_name - -class LabelAnalysisRequestCreateView(CreateAPIView): - serializer_class = LabelAnalysisRequestSerializer +EMPTY_RESPONSE = { + "external_id": None, + "state": "finished", + "errors": [], + "requested_labels": [], + "base_commit": "", + "head_commit": "", + "result": { + "absent_labels": [], + "present_diff_labels": [], + "present_report_labels": [], + "global_level_labels": [], + }, +} + + +class LabelAnalysisRequestView(APIView): authentication_classes = [RepositoryTokenAuthentication] permission_classes = [SpecificScopePermission] # TODO Consider using a different permission scope required_scopes = ["static_analysis"] - def perform_create(self, serializer): - instance = serializer.save(state_id=LabelAnalysisRequestState.CREATED.db_id) - TaskService().schedule_task( - label_analysis_task_name, - kwargs={"request_id": instance.id}, - apply_async_kwargs={}, - ) - return instance + def post(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE, status=status.HTTP_201_CREATED) - -class LabelAnalysisRequestDetailView(RetrieveAPIView, UpdateAPIView): - serializer_class = LabelAnalysisRequestSerializer - authentication_classes = [RepositoryTokenAuthentication] - permission_classes = [SpecificScopePermission] - # TODO Consider using a different permission scope - required_scopes = ["static_analysis"] + def get(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE) def patch(self, request, *args, **kwargs): - # This is called by the CLI to patch the request_labels information after it's collected - # First we let rest_framework validate and update the larq object - response = super().patch(request, *args, **kwargs) - if response.status_code == 200: - # IF the larq update was successful - # we trigger the task again for the same larq to update the result saved - # The result saved is what we use to get metrics - uid = self.kwargs.get("external_id") - larq = LabelAnalysisRequest.objects.get(external_id=uid) - TaskService().schedule_task( - label_analysis_task_name, - kwargs={"request_id": larq.id}, - apply_async_kwargs={}, - ) - return response - - def get_object(self): - uid = self.kwargs.get("external_id") - try: - return LabelAnalysisRequest.objects.get(external_id=uid) - except LabelAnalysisRequest.DoesNotExist: - raise NotFound("No such Label Analysis exists") + return Response(EMPTY_RESPONSE) diff --git a/apps/codecov-api/staticanalysis/serializers.py b/apps/codecov-api/staticanalysis/serializers.py deleted file mode 100644 index dd71bd3cb0..0000000000 --- a/apps/codecov-api/staticanalysis/serializers.py +++ /dev/null @@ -1,157 +0,0 @@ -import logging -import math - -from rest_framework import exceptions, serializers - -from core.models import Commit -from shared.api_archive.archive import ArchiveService, MinioEndpoints -from staticanalysis.models import ( - StaticAnalysisSingleFileSnapshot, - StaticAnalysisSingleFileSnapshotState, - StaticAnalysisSuite, - StaticAnalysisSuiteFilepath, -) - -log = logging.getLogger(__name__) - - -class CommitFromShaSerializerField(serializers.Field): - def to_representation(self, commit): - return commit.commitid - - def to_internal_value(self, commit_sha): - # TODO: Change this query when we change how we fetch URLs - commit = Commit.objects.filter( - repository__in=self.context["request"].auth.get_repositories(), - commitid=commit_sha, - ).first() - if commit is None: - raise exceptions.NotFound("Commit not found.") - return commit - - -def _dict_to_suite_filepath( - analysis_suite, - repository, - archive_service, - existing_file_snapshots_mapping, - file_dict, -): - if file_dict["file_hash"] in existing_file_snapshots_mapping: - db_element = existing_file_snapshots_mapping[file_dict["file_hash"]] - was_created = False - else: - path = MinioEndpoints.static_analysis_single_file.get_path( - version="v4", - repo_hash=archive_service.storage_hash, - location=f"{file_dict['file_hash']}.json", - ) - # Using get or create in the case the object was already - # created somewhere else first, but also because get_or_create - # is internally get_or_create_or_get, so Django handles the conflicts - # that can arise on race conditions on the create step - # We might choose to change it if the number of extra GETs become too much - ( - db_element, - was_created, - ) = StaticAnalysisSingleFileSnapshot.objects.get_or_create( - file_hash=file_dict["file_hash"], - repository=repository, - defaults={ - "state_id": StaticAnalysisSingleFileSnapshotState.CREATED.db_id, - "content_location": path, - }, - ) - if was_created: - log.debug( - "Created new snapshot for repository", - extra={"repoid": repository.repoid, "snapshot_id": db_element.id}, - ) - return StaticAnalysisSuiteFilepath( - filepath=file_dict["filepath"], - file_snapshot=db_element, - analysis_suite=analysis_suite, - ) - - -class StaticAnalysisSuiteFilepathField(serializers.ModelSerializer): - file_hash = serializers.UUIDField() - raw_upload_location = serializers.SerializerMethodField() - state = serializers.SerializerMethodField() - - class Meta: - model = StaticAnalysisSuiteFilepath - fields = [ - "filepath", - "file_hash", - "raw_upload_location", - "state", - ] - - def get_state(self, obj): - return StaticAnalysisSingleFileSnapshotState.enum_from_int( - obj.file_snapshot.state_id - ).name - - def get_raw_upload_location(self, obj): - # TODO: This has a built-in ttl of 10 seconds. - # We have to consider changing it in case customers are doing a few - # thousand uploads on the first time - return self.context["archive_service"].create_presigned_put( - obj.file_snapshot.content_location - ) - - -class FilepathListField(serializers.ListField): - child = StaticAnalysisSuiteFilepathField() - - def to_representation(self, data): - data = data.select_related( - "file_snapshot", - ).all() - return super().to_representation(data) - - -class StaticAnalysisSuiteSerializer(serializers.ModelSerializer): - commit = CommitFromShaSerializerField(required=True) - filepaths = FilepathListField() - - class Meta: - model = StaticAnalysisSuite - fields = ["external_id", "commit", "filepaths"] - read_only_fields = ["raw_upload_location", "external_id"] - - def create(self, validated_data): - file_metadata_array = validated_data.pop("filepaths") - # `validated_data` only contains `commit` after pop - obj = StaticAnalysisSuite.objects.create(**validated_data) - request = self.context["request"] - repository = request.auth.get_repositories()[0] - archive_service = ArchiveService(repository) - # allow 1s per 10 uploads - ttl = max(math.ceil(len(file_metadata_array) / 10) + 5, 10) - self.context["archive_service"] = ArchiveService(repository, ttl=ttl) - all_hashes = [val["file_hash"] for val in file_metadata_array] - existing_values = StaticAnalysisSingleFileSnapshot.objects.filter( - repository=repository, file_hash__in=all_hashes - ) - existing_values_mapping = {val.file_hash: val for val in existing_values} - created_filepaths = [ - _dict_to_suite_filepath( - obj, - repository, - archive_service, - existing_values_mapping, - file_dict, - ) - for file_dict in file_metadata_array - ] - StaticAnalysisSuiteFilepath.objects.bulk_create(created_filepaths) - log.info( - "Created static analysis filepaths", - extra={ - "created_ids": [f.id for f in created_filepaths], - "repoid": repository.repoid, - }, - ) - return obj diff --git a/apps/codecov-api/staticanalysis/tests/test_views.py b/apps/codecov-api/staticanalysis/tests/test_views.py index 24e31d4df7..7055b2f5ff 100644 --- a/apps/codecov-api/staticanalysis/tests/test_views.py +++ b/apps/codecov-api/staticanalysis/tests/test_views.py @@ -3,22 +3,14 @@ from django.urls import reverse from rest_framework.test import APIClient -from services.task import TaskService -from shared.celery_config import static_analysis_task_name from shared.django_apps.core.tests.factories import ( CommitFactory, RepositoryTokenFactory, ) -from staticanalysis.models import StaticAnalysisSuite -from staticanalysis.tests.factories import StaticAnalysisSuiteFactory +from staticanalysis.views import EMPTY_RESPONSE -def test_simple_static_analysis_call_no_uploads_yet(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - mocked_presigned_put = mocker.patch( - "shared.storage.MinioStorageService.create_presigned_put", - return_value="banana.txt", - ) +def test_simple_static_analysis_call_no_uploads_yet(db): commit = CommitFactory.create(repository__active=True) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" @@ -45,57 +37,20 @@ def test_simple_static_analysis_call_no_uploads_yet(db, mocker): format="json", ) assert response.status_code == 201 - assert StaticAnalysisSuite.objects.filter(commit=commit).count() == 1 - produced_object = StaticAnalysisSuite.objects.filter(commit=commit).get() - response_json = response.json() - assert "filepaths" in response_json - # Popping and sorting because the order doesn't matter, as long as all are there - assert sorted(response_json.pop("filepaths"), key=lambda x: x["filepath"]) == [ - { - "filepath": "banana.cpp", - "file_hash": str(second_uuid), - "raw_upload_location": "banana.txt", - "state": "CREATED", - }, - { - "filepath": "path/to/a.py", - "file_hash": str(some_uuid), - "raw_upload_location": "banana.txt", - "state": "CREATED", - }, - ] - # Now asserting the remaining of the response - assert response_json == { - "external_id": str(produced_object.external_id), - "commit": commit.commitid, - } - mocked_task_service.assert_called_with( - static_analysis_task_name, - kwargs={"suite_id": produced_object.id}, - apply_async_kwargs={"countdown": 10}, - ) - mocked_presigned_put.assert_called_with( - "archive", - mocker.ANY, - 10, - ) + assert response.json() == EMPTY_RESPONSE -def test_static_analysis_finish(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") +def test_static_analysis_finish(db): commit = CommitFactory.create(repository__active=True) - suite = StaticAnalysisSuiteFactory(commit=commit) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) client = APIClient() client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) response = client.post( - reverse("staticanalyses-finish", kwargs={"external_id": suite.external_id}) - ) - assert response.status_code == 204 - mocked_task_service.assert_called_with( - static_analysis_task_name, - kwargs={"suite_id": suite.id}, - apply_async_kwargs={}, + reverse( + "staticanalyses-finish", + kwargs={"external_id": "c4e11164-411a-48f1-b791-91a9f009ea48"}, + ) ) + assert response.status_code == 201 diff --git a/apps/codecov-api/staticanalysis/tests/unit/__init__.py b/apps/codecov-api/staticanalysis/tests/unit/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apps/codecov-api/staticanalysis/tests/unit/test_serializers.py b/apps/codecov-api/staticanalysis/tests/unit/test_serializers.py deleted file mode 100644 index bb09b513bc..0000000000 --- a/apps/codecov-api/staticanalysis/tests/unit/test_serializers.py +++ /dev/null @@ -1,331 +0,0 @@ -import re -from uuid import UUID, uuid4 - -import pytest -from rest_framework.exceptions import NotFound, ValidationError - -from shared.api_archive.archive import ArchiveService -from shared.django_apps.core.tests.factories import CommitFactory, RepositoryFactory -from staticanalysis.models import ( - StaticAnalysisSingleFileSnapshotState, - StaticAnalysisSuite, - StaticAnalysisSuiteFilepath, -) -from staticanalysis.serializers import ( - CommitFromShaSerializerField, - StaticAnalysisSuiteFilepathField, - StaticAnalysisSuiteSerializer, -) -from staticanalysis.tests.factories import ( - StaticAnalysisSingleFileSnapshotFactory, - StaticAnalysisSuiteFilepathFactory, -) - -expected_location_regex = re.compile( - "v4/repos/[A-F0-9]{32}/static_analysis/files/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}.json" -) - - -def test_commit_from_sha_serializer_field_to_internal_value(mocker, db): - commit = CommitFactory.create() - # notice this second commit has a different repo by default - second_commit = CommitFactory.create() - commit.save() - second_commit.save() - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - # silly workaround to not have to manually bind serializers - mocker.patch.object( - CommitFromShaSerializerField, "context", {"request": fake_request} - ) - serializer_field = CommitFromShaSerializerField() - with pytest.raises(NotFound): - assert serializer_field.to_internal_value("abcde" * 8) - with pytest.raises(NotFound): - assert serializer_field.to_internal_value(second_commit.commitid) - assert serializer_field.to_internal_value(commit.commitid) == commit - - -def test_filepath_field(db, mocker): - sasfs = StaticAnalysisSingleFileSnapshotFactory.create( - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id - ) - sasfs.save() - fp = StaticAnalysisSuiteFilepathFactory.create( - filepath="ohoooo", file_snapshot=sasfs - ) - fp.analysis_suite.save() - fp.save() - fake_archive_service = mocker.MagicMock( - create_presigned_put=mocker.MagicMock(return_value="some_url_stuff") - ) - serializer_field = StaticAnalysisSuiteFilepathField( - context={ - "archive_service": fake_archive_service, - } - ) - assert dict(serializer_field.to_representation(fp)) == { - "file_hash": sasfs.file_hash, - "filepath": "ohoooo", - "raw_upload_location": "some_url_stuff", - "state": "VALID", - } - - -class TestStaticAnalysisSuiteSerializer(object): - def test_to_internal_value_missing_filepaths(self, mocker, db): - commit = CommitFactory.create() - commit.save() - input_data = {"commit": commit.commitid} - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - with pytest.raises(ValidationError) as exc: - serializer.to_internal_value(input_data) - assert exc.value.detail == {"filepaths": ["This field is required."]} - - def test_to_internal_value_complete(self, mocker, db): - commit = CommitFactory.create() - commit.save() - input_data = { - "commit": commit.commitid, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": "c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2", - }, - { - "filepath": "anothersomething", - "file_hash": "3998813e-60db-4686-be84-1a0efa7d9b9f", - }, - ], - } - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - res = serializer.to_internal_value(input_data) - assert res == { - "commit": commit, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - }, - { - "filepath": "anothersomething", - "file_hash": UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - }, - ], - } - - def test_create_no_data_previously_exists(self, mocker, db): - first_repository = RepositoryFactory.create() - commit = CommitFactory.create(repository=first_repository) - commit.save() - validated_data = { - "commit": commit, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - }, - { - "filepath": "anothersomething", - "file_hash": UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - }, - ], - } - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - res = serializer.create(validated_data) - assert isinstance(res, StaticAnalysisSuite) - assert res.commit == commit - assert res.filepaths.count() == 2 - first_filepath, second_filepath = sorted( - res.filepaths.all(), key=lambda x: x.filepath - ) - assert isinstance(first_filepath, StaticAnalysisSuiteFilepath) - assert first_filepath.filepath == "anothersomething" - assert first_filepath.file_snapshot is not None - archive_hash = ArchiveService.get_archive_hash(commit.repository) - assert first_filepath.file_snapshot.repository == commit.repository - assert first_filepath.file_snapshot.file_hash == UUID( - "3998813e-60db-4686-be84-1a0efa7d9b9f" - ) - assert archive_hash in first_filepath.file_snapshot.content_location - assert ( - expected_location_regex.match(first_filepath.file_snapshot.content_location) - is not None - ) - assert ( - first_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - - def test_create_some_data_previously_exists(self, mocker, db): - first_repository = RepositoryFactory.create() - second_repository = RepositoryFactory.create() - commit = CommitFactory.create(repository=first_repository) - first_repo_first_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - repository=first_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="first_repo_first_snapshot", - ) - second_repo_first_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - repository=second_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="second_repo_first_snapshot", - ) - second_repo_second_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - repository=second_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="second_repo_second_snapshot", - ) - first_repo_separate_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=uuid4(), - repository=first_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="first_repo_separate_snapshot", - ) - second_repo_separate_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=uuid4(), - repository=second_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="second_repo_separate_snapshot", - ) - first_repo_exists_but_not_valid_yet = ( - StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("31803149-8bd7-4c2b-9a80-71f259360c72"), - repository=first_repository, - state_id=StaticAnalysisSingleFileSnapshotState.CREATED.db_id, - content_location="first_repo_exists_but_not_valid_yet", - ) - ) - first_repo_first_snapshot.save() - second_repo_first_snapshot.save() - second_repo_second_snapshot.save() - first_repo_separate_snapshot.save() - second_repo_separate_snapshot.save() - first_repo_exists_but_not_valid_yet.save() - - validated_data = { - "commit": commit, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - }, - { - "filepath": "anothersomething", - "file_hash": UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - }, - { - "filepath": "oooaaa.rb", - "file_hash": UUID("60228df6-4d11-44d4-a048-ec2fa1ea2c32"), - }, - { - "filepath": "awert.qt", - "file_hash": UUID("31803149-8bd7-4c2b-9a80-71f259360c72"), - }, - ], - } - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - res = serializer.create(validated_data) - assert isinstance(res, StaticAnalysisSuite) - assert res.commit == commit - assert res.filepaths.count() == 4 - first_filepath, second_filepath, third_filepath, fourth_filepath = sorted( - res.filepaths.all(), key=lambda x: x.filepath - ) - archive_hash = ArchiveService.get_archive_hash(first_repository) - # first one - assert isinstance(first_filepath, StaticAnalysisSuiteFilepath) - assert first_filepath.filepath == "anothersomething" - assert first_filepath.file_snapshot is not None - assert first_filepath.file_snapshot.repository == first_repository - assert first_filepath.file_snapshot.file_hash == UUID( - "3998813e-60db-4686-be84-1a0efa7d9b9f" - ) - assert archive_hash in first_filepath.file_snapshot.content_location - assert ( - expected_location_regex.match(first_filepath.file_snapshot.content_location) - is not None - ) - assert ( - first_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - # second one - assert isinstance(second_filepath, StaticAnalysisSuiteFilepath) - assert second_filepath.filepath == "awert.qt" - assert second_filepath.file_snapshot == first_repo_exists_but_not_valid_yet - assert second_filepath.file_snapshot.repository == first_repository - assert second_filepath.file_snapshot.file_hash == UUID( - "31803149-8bd7-4c2b-9a80-71f259360c72" - ) - # content location was already there, so nothing is created - # asserting the old value is still there - assert ( - second_filepath.file_snapshot.content_location - == "first_repo_exists_but_not_valid_yet" - ) - assert ( - second_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - # third one - assert isinstance(third_filepath, StaticAnalysisSuiteFilepath) - assert third_filepath.filepath == "oooaaa.rb" - assert third_filepath.file_snapshot is not None - assert third_filepath.file_snapshot.repository == first_repository - assert third_filepath.file_snapshot.file_hash == UUID( - "60228df6-4d11-44d4-a048-ec2fa1ea2c32" - ) - assert archive_hash in third_filepath.file_snapshot.content_location - assert ( - expected_location_regex.match(third_filepath.file_snapshot.content_location) - is not None - ) - assert ( - third_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - # fourth one - assert isinstance(fourth_filepath, StaticAnalysisSuiteFilepath) - assert fourth_filepath.filepath == "path/to/a.py" - assert fourth_filepath.file_snapshot == first_repo_first_snapshot - assert fourth_filepath.file_snapshot.repository == first_repository - assert fourth_filepath.file_snapshot.file_hash == UUID( - "c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2" - ) - # content location was already there, so nothing is created - # asserting the old value is still there - assert ( - fourth_filepath.file_snapshot.content_location - == "first_repo_first_snapshot" - ) - assert ( - fourth_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.VALID.db_id - ) diff --git a/apps/codecov-api/staticanalysis/urls.py b/apps/codecov-api/staticanalysis/urls.py index 6a2a5510b8..87bf090921 100644 --- a/apps/codecov-api/staticanalysis/urls.py +++ b/apps/codecov-api/staticanalysis/urls.py @@ -1,7 +1,16 @@ -from staticanalysis.views import StaticAnalysisSuiteViewSet -from utils.routers import OptionalTrailingSlashRouter +from django.urls import path -router = OptionalTrailingSlashRouter() -router.register("analyses", StaticAnalysisSuiteViewSet, basename="staticanalyses") +from staticanalysis.views import StaticAnalysisSuiteView -urlpatterns = router.urls +urlpatterns = [ + path( + "staticanalyses", + StaticAnalysisSuiteView.as_view(), + name="staticanalyses-list", + ), + path( + "staticanalyses//finish", + StaticAnalysisSuiteView.as_view(), + name="staticanalyses-finish", + ), +] diff --git a/apps/codecov-api/staticanalysis/views.py b/apps/codecov-api/staticanalysis/views.py index a62377f166..b557d333ab 100644 --- a/apps/codecov-api/staticanalysis/views.py +++ b/apps/codecov-api/staticanalysis/views.py @@ -1,46 +1,21 @@ -import logging - -from django.http import HttpResponse -from rest_framework import mixins, viewsets -from rest_framework.decorators import action +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView from codecov_auth.authentication.repo_auth import RepositoryTokenAuthentication from codecov_auth.permissions import SpecificScopePermission -from services.task import TaskService -from shared.celery_config import static_analysis_task_name -from staticanalysis.models import StaticAnalysisSuite -from staticanalysis.serializers import StaticAnalysisSuiteSerializer -log = logging.getLogger(__name__) +EMPTY_RESPONSE = { + "external_id": "0000", + "filepaths": [], + "commit": "", +} -class StaticAnalysisSuiteViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): - serializer_class = StaticAnalysisSuiteSerializer +class StaticAnalysisSuiteView(APIView): authentication_classes = [RepositoryTokenAuthentication] permission_classes = [SpecificScopePermission] required_scopes = ["static_analysis"] - lookup_field = "external_id" - - def get_queryset(self): - repository = self.request.auth.get_repositories()[0] - return StaticAnalysisSuite.objects.filter(commit__repository=repository) - - def perform_create(self, serializer): - instance = serializer.save() - # TODO: remove this once the CLI is calling the `finish` endpoint - TaskService().schedule_task( - static_analysis_task_name, - kwargs={"suite_id": instance.id}, - apply_async_kwargs={"countdown": 10}, - ) - return instance - @action(detail=True, methods=["post"]) - def finish(self, request, *args, **kwargs): - suite = self.get_object() - TaskService().schedule_task( - static_analysis_task_name, - kwargs={"suite_id": suite.pk}, - apply_async_kwargs={}, - ) - return HttpResponse(status=204) + def post(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE, status=status.HTTP_201_CREATED)