diff --git a/apps/codecov-api/labelanalysis/serializers.py b/apps/codecov-api/labelanalysis/serializers.py deleted file mode 100644 index a75237a1c5..0000000000 --- a/apps/codecov-api/labelanalysis/serializers.py +++ /dev/null @@ -1,89 +0,0 @@ -from rest_framework import exceptions, serializers - -from core.models import Commit -from labelanalysis.models import ( - LabelAnalysisProcessingError, - LabelAnalysisRequest, - LabelAnalysisRequestState, -) - - -class CommitFromShaSerializerField(serializers.Field): - def __init__(self, *args, **kwargs): - self.accepts_fallback = kwargs.pop("accepts_fallback", False) - super().__init__(*args, **kwargs) - - def to_representation(self, commit): - return commit.commitid - - def to_internal_value(self, commit_sha): - commit = Commit.objects.filter( - repository__in=self.context["request"].auth.get_repositories(), - commitid=commit_sha, - ).first() - if commit is None: - raise exceptions.NotFound(f"Commit {commit_sha[:7]} not found.") - if commit.staticanalysissuite_set.exists(): - return commit - if not self.accepts_fallback: - raise serializers.ValidationError("No static analysis found") - attempted_commits = [] - for _ in range(10): - attempted_commits.append(commit.commitid) - commit = commit.parent_commit - if commit is None: - raise serializers.ValidationError( - f"No possible commits have static analysis sent. Attempted commits: {','.join(attempted_commits)}" - ) - if commit.staticanalysissuite_set.exists(): - return commit - raise serializers.ValidationError( - f"No possible commits have static analysis sent. Attempted too many commits: {','.join(attempted_commits)}" - ) - - -class LabelAnalysisProcessingErrorSerializer(serializers.ModelSerializer): - class Meta: - model = LabelAnalysisProcessingError - fields = ("error_code", "error_params") - read_only_fields = ("error_code", "error_params") - - -class ProcessingErrorList(serializers.ListField): - child = LabelAnalysisProcessingErrorSerializer() - - def to_representation(self, data): - data = data.select_related( - "label_analysis_request", - ).all() - return super().to_representation(data) - - -class LabelAnalysisRequestSerializer(serializers.ModelSerializer): - base_commit = CommitFromShaSerializerField(required=True, accepts_fallback=True) - head_commit = CommitFromShaSerializerField(required=True, accepts_fallback=False) - state = serializers.SerializerMethodField() - errors = ProcessingErrorList(required=False) - - def validate(self, data): - if data["base_commit"] == data["head_commit"]: - raise serializers.ValidationError( - {"base_commit": "Base and head must be different commits"} - ) - return data - - class Meta: - model = LabelAnalysisRequest - fields = ( - "base_commit", - "head_commit", - "requested_labels", - "result", - "state", - "external_id", - "errors", - ) - read_only_fields = ("result", "external_id", "errors") - - def get_state(self, obj): - return LabelAnalysisRequestState.enum_from_int(obj.state_id).name.lower() diff --git a/apps/codecov-api/labelanalysis/tests/integration/test_views.py b/apps/codecov-api/labelanalysis/tests/integration/test_views.py index b81c9a0ea9..ad733b9ac5 100644 --- a/apps/codecov-api/labelanalysis/tests/integration/test_views.py +++ b/apps/codecov-api/labelanalysis/tests/integration/test_views.py @@ -1,30 +1,16 @@ -from uuid import uuid4 - from django.urls import reverse from rest_framework.test import APIClient -from labelanalysis.models import ( - LabelAnalysisProcessingError, - LabelAnalysisRequest, - LabelAnalysisRequestState, -) -from labelanalysis.tests.factories import LabelAnalysisRequestFactory -from services.task import TaskService -from shared.celery_config import label_analysis_task_name +from labelanalysis.views import EMPTY_RESPONSE from shared.django_apps.core.tests.factories import ( CommitFactory, - RepositoryFactory, RepositoryTokenFactory, ) -from staticanalysis.tests.factories import StaticAnalysisSuiteFactory -def test_simple_label_analysis_call_flow(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") +def test_simple_label_analysis_call_flow(db): commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) @@ -42,364 +28,30 @@ def test_simple_label_analysis_call_flow(db, mocker): format="json", ) assert response.status_code == 201 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object - assert produced_object.base_commit == base_commit - assert produced_object.head_commit == commit - assert produced_object.requested_labels is None - assert produced_object.state_id == LabelAnalysisRequestState.CREATED.db_id - assert produced_object.result is None - response_json = response.json() - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } - assert response_json == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": produced_object.id}, - apply_async_kwargs={}, - ) - get_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 200 - assert response.json() == expected_response_json - + assert response.json() == EMPTY_RESPONSE -def test_simple_label_analysis_call_flow_same_commit_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - expected_response_json = { - "base_commit": ["Base and head must be different commits"] - } - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_call_flow_with_fallback_on_base(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_parent_parent = CommitFactory.create(repository=commit.repository) - base_commit_parent = CommitFactory.create( - parent_commit_id=base_commit_parent_parent.commitid, - repository=commit.repository, - ) - base_commit = CommitFactory.create( - parent_commit_id=base_commit_parent.commitid, repository=commit.repository - ) - StaticAnalysisSuiteFactory.create(commit=base_commit_parent_parent) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 201 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object - assert produced_object.base_commit == base_commit_parent_parent - assert produced_object.head_commit == commit - assert produced_object.requested_labels is None - assert produced_object.state_id == LabelAnalysisRequestState.CREATED.db_id - assert produced_object.result is None - response_json = response.json() - expected_response_json = { - "base_commit": base_commit_parent_parent.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } - assert response_json == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": produced_object.id}, - apply_async_kwargs={}, - ) get_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.get( - get_url, - format="json", + "view_label_analysis", + kwargs={"external_id": "c4e11164-411a-48f1-b791-91a9f009ea48"}, ) + response = client.get(get_url, format="json") assert response.status_code == 200 - assert response.json() == expected_response_json - - -def test_simple_label_analysis_call_flow_with_fallback_on_base_error_too_long( - db, mocker -): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - repository = RepositoryFactory.create(active=True) - commit = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_root = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=base_commit_root) - current = base_commit_root - attempted_commit_list = [base_commit_root.commitid] - for i in range(12): - current = CommitFactory.create( - parent_commit_id=current.commitid, repository=repository - ) - attempted_commit_list.append(current.commitid) - base_commit = current - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - # reverse and get 10 first elements, thats how far we look - attempted_commit_list = ",".join(list(reversed(attempted_commit_list))[:10]) - expected_response_json = { - "base_commit": [ - f"No possible commits have static analysis sent. Attempted too many commits: {attempted_commit_list}" - ] - } - assert response_json == expected_response_json - assert not mocked_task_service.called + assert response.json() == EMPTY_RESPONSE -def test_simple_label_analysis_call_flow_with_fallback_on_base_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit_parent_parent = CommitFactory.create(repository=commit.repository) - base_commit_parent = CommitFactory.create( - parent_commit_id=base_commit_parent_parent.commitid, - repository=commit.repository, - ) - base_commit = CommitFactory.create( - parent_commit_id=base_commit_parent.commitid, repository=commit.repository - ) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 0 - response_json = response.json() - attempted_commit_list = ",".join( - [ - base_commit.commitid, - base_commit_parent.commitid, - base_commit_parent_parent.commitid, - ] - ) - expected_response_json = { - "base_commit": [ - f"No possible commits have static analysis sent. Attempted commits: {attempted_commit_list}" - ] - } - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_call_flow_with_fallback_on_head_error(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - repository = RepositoryFactory.create(active=True) - head_commit_parent = CommitFactory.create(repository=repository) - head_commit = CommitFactory.create( - parent_commit_id=head_commit_parent.commitid, repository=repository - ) - base_commit = CommitFactory.create(repository=repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - StaticAnalysisSuiteFactory.create(commit=head_commit_parent) - token = RepositoryTokenFactory.create( - repository=head_commit.repository, token_type="static_analysis" - ) - client = APIClient() - url = reverse("create_label_analysis") - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - payload = { - "base_commit": base_commit.commitid, - "head_commit": head_commit.commitid, - "requested_labels": None, - } - response = client.post( - url, - payload, - format="json", - ) - assert response.status_code == 400 - assert LabelAnalysisRequest.objects.filter(head_commit=head_commit).count() == 0 - assert ( - LabelAnalysisRequest.objects.filter(head_commit=head_commit_parent).count() == 0 - ) - response_json = response.json() - expected_response_json = {"head_commit": ["No static analysis found"]} - assert response_json == expected_response_json - assert not mocked_task_service.called - - -def test_simple_label_analysis_only_get(db, mocker): +def test_simple_label_analysis_put_labels(db): commit = CommitFactory.create(repository__active=True) base_commit = CommitFactory.create(repository=commit.repository) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.FINISHED.db_id, - result={"some": ["result"]}, - ) - larq_processing_error = LabelAnalysisProcessingError( - label_analysis_request=label_analysis, - error_code="Missing FileSnapshot", - error_params={"message": "Something is wrong"}, - ) - label_analysis.save() - larq_processing_error.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": None, - "result": {"some": ["result"]}, - "state": "finished", - "external_id": str(produced_object.external_id), - "errors": [ - { - "error_code": "Missing FileSnapshot", - "error_params": {"message": "Something is wrong"}, - } - ], - } - get_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 200 - assert response.json() == expected_response_json - -def test_simple_label_analysis_get_does_not_exist(db, mocker): - token = RepositoryTokenFactory.create( - repository__active=True, token_type="static_analysis" - ) client = APIClient() client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - get_url = reverse("view_label_analysis", kwargs={"external_id": uuid4()}) - response = client.get( - get_url, - format="json", - ) - assert response.status_code == 404 - assert response.json() == {"detail": "No such Label Analysis exists"} - -def test_simple_label_analysis_put_labels(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.CREATED.db_id, - result=None, - ) - label_analysis.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - assert produced_object.requested_labels is None - expected_response_json = { - "base_commit": base_commit.commitid, - "head_commit": commit.commitid, - "requested_labels": ["label_1", "label_2", "label_3"], - "result": None, - "state": "created", - "external_id": str(produced_object.external_id), - "errors": [], - } patch_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} + "view_label_analysis", + kwargs={"external_id": "c4e11164-411a-48f1-b791-91a9f009ea48"}, ) response = client.patch( patch_url, @@ -411,47 +63,4 @@ def test_simple_label_analysis_put_labels(db, mocker): }, ) assert response.status_code == 200 - assert response.json() == expected_response_json - mocked_task_service.assert_called_with( - label_analysis_task_name, - kwargs={"request_id": label_analysis.id}, - apply_async_kwargs={}, - ) - - -def test_simple_label_analysis_put_labels_wrong_base_return_404(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - commit = CommitFactory.create(repository__active=True) - StaticAnalysisSuiteFactory.create(commit=commit) - base_commit = CommitFactory.create(repository=commit.repository) - StaticAnalysisSuiteFactory.create(commit=base_commit) - token = RepositoryTokenFactory.create( - repository=commit.repository, token_type="static_analysis" - ) - label_analysis = LabelAnalysisRequestFactory.create( - head_commit=commit, - base_commit=base_commit, - state_id=LabelAnalysisRequestState.CREATED.db_id, - result=None, - ) - label_analysis.save() - client = APIClient() - client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) - assert LabelAnalysisRequest.objects.filter(head_commit=commit).count() == 1 - produced_object = LabelAnalysisRequest.objects.get(head_commit=commit) - assert produced_object == label_analysis - assert produced_object.requested_labels is None - patch_url = reverse( - "view_label_analysis", kwargs={"external_id": produced_object.external_id} - ) - response = client.patch( - patch_url, - format="json", - data={ - "requested_labels": ["label_1", "label_2", "label_3"], - "base_commit": "not_base_commit", - "head_commit": commit.commitid, - }, - ) - assert response.status_code == 404 - mocked_task_service.assert_not_called() + assert response.json() == EMPTY_RESPONSE diff --git a/apps/codecov-api/labelanalysis/urls.py b/apps/codecov-api/labelanalysis/urls.py index 2ebda9b54e..4deff7507f 100644 --- a/apps/codecov-api/labelanalysis/urls.py +++ b/apps/codecov-api/labelanalysis/urls.py @@ -1,19 +1,16 @@ from django.urls import path -from labelanalysis.views import ( - LabelAnalysisRequestCreateView, - LabelAnalysisRequestDetailView, -) +from labelanalysis.views import LabelAnalysisRequestView urlpatterns = [ path( "labels-analysis", - LabelAnalysisRequestCreateView.as_view(), + LabelAnalysisRequestView.as_view(), name="create_label_analysis", ), path( "labels-analysis/", - LabelAnalysisRequestDetailView.as_view(), + LabelAnalysisRequestView.as_view(), name="view_label_analysis", ), ] diff --git a/apps/codecov-api/labelanalysis/views.py b/apps/codecov-api/labelanalysis/views.py index aae8393121..88dbf38c89 100644 --- a/apps/codecov-api/labelanalysis/views.py +++ b/apps/codecov-api/labelanalysis/views.py @@ -1,58 +1,37 @@ -from rest_framework.exceptions import NotFound -from rest_framework.generics import CreateAPIView, RetrieveAPIView, UpdateAPIView +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView from codecov_auth.authentication.repo_auth import RepositoryTokenAuthentication from codecov_auth.permissions import SpecificScopePermission -from labelanalysis.models import LabelAnalysisRequest, LabelAnalysisRequestState -from labelanalysis.serializers import LabelAnalysisRequestSerializer -from services.task import TaskService -from shared.celery_config import label_analysis_task_name - -class LabelAnalysisRequestCreateView(CreateAPIView): - serializer_class = LabelAnalysisRequestSerializer +EMPTY_RESPONSE = { + "external_id": None, + "state": "finished", + "errors": [], + "requested_labels": [], + "base_commit": "", + "head_commit": "", + "result": { + "absent_labels": [], + "present_diff_labels": [], + "present_report_labels": [], + "global_level_labels": [], + }, +} + + +class LabelAnalysisRequestView(APIView): authentication_classes = [RepositoryTokenAuthentication] permission_classes = [SpecificScopePermission] # TODO Consider using a different permission scope required_scopes = ["static_analysis"] - def perform_create(self, serializer): - instance = serializer.save(state_id=LabelAnalysisRequestState.CREATED.db_id) - TaskService().schedule_task( - label_analysis_task_name, - kwargs={"request_id": instance.id}, - apply_async_kwargs={}, - ) - return instance + def post(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE, status=status.HTTP_201_CREATED) - -class LabelAnalysisRequestDetailView(RetrieveAPIView, UpdateAPIView): - serializer_class = LabelAnalysisRequestSerializer - authentication_classes = [RepositoryTokenAuthentication] - permission_classes = [SpecificScopePermission] - # TODO Consider using a different permission scope - required_scopes = ["static_analysis"] + def get(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE) def patch(self, request, *args, **kwargs): - # This is called by the CLI to patch the request_labels information after it's collected - # First we let rest_framework validate and update the larq object - response = super().patch(request, *args, **kwargs) - if response.status_code == 200: - # IF the larq update was successful - # we trigger the task again for the same larq to update the result saved - # The result saved is what we use to get metrics - uid = self.kwargs.get("external_id") - larq = LabelAnalysisRequest.objects.get(external_id=uid) - TaskService().schedule_task( - label_analysis_task_name, - kwargs={"request_id": larq.id}, - apply_async_kwargs={}, - ) - return response - - def get_object(self): - uid = self.kwargs.get("external_id") - try: - return LabelAnalysisRequest.objects.get(external_id=uid) - except LabelAnalysisRequest.DoesNotExist: - raise NotFound("No such Label Analysis exists") + return Response(EMPTY_RESPONSE) diff --git a/apps/codecov-api/staticanalysis/serializers.py b/apps/codecov-api/staticanalysis/serializers.py deleted file mode 100644 index dd71bd3cb0..0000000000 --- a/apps/codecov-api/staticanalysis/serializers.py +++ /dev/null @@ -1,157 +0,0 @@ -import logging -import math - -from rest_framework import exceptions, serializers - -from core.models import Commit -from shared.api_archive.archive import ArchiveService, MinioEndpoints -from staticanalysis.models import ( - StaticAnalysisSingleFileSnapshot, - StaticAnalysisSingleFileSnapshotState, - StaticAnalysisSuite, - StaticAnalysisSuiteFilepath, -) - -log = logging.getLogger(__name__) - - -class CommitFromShaSerializerField(serializers.Field): - def to_representation(self, commit): - return commit.commitid - - def to_internal_value(self, commit_sha): - # TODO: Change this query when we change how we fetch URLs - commit = Commit.objects.filter( - repository__in=self.context["request"].auth.get_repositories(), - commitid=commit_sha, - ).first() - if commit is None: - raise exceptions.NotFound("Commit not found.") - return commit - - -def _dict_to_suite_filepath( - analysis_suite, - repository, - archive_service, - existing_file_snapshots_mapping, - file_dict, -): - if file_dict["file_hash"] in existing_file_snapshots_mapping: - db_element = existing_file_snapshots_mapping[file_dict["file_hash"]] - was_created = False - else: - path = MinioEndpoints.static_analysis_single_file.get_path( - version="v4", - repo_hash=archive_service.storage_hash, - location=f"{file_dict['file_hash']}.json", - ) - # Using get or create in the case the object was already - # created somewhere else first, but also because get_or_create - # is internally get_or_create_or_get, so Django handles the conflicts - # that can arise on race conditions on the create step - # We might choose to change it if the number of extra GETs become too much - ( - db_element, - was_created, - ) = StaticAnalysisSingleFileSnapshot.objects.get_or_create( - file_hash=file_dict["file_hash"], - repository=repository, - defaults={ - "state_id": StaticAnalysisSingleFileSnapshotState.CREATED.db_id, - "content_location": path, - }, - ) - if was_created: - log.debug( - "Created new snapshot for repository", - extra={"repoid": repository.repoid, "snapshot_id": db_element.id}, - ) - return StaticAnalysisSuiteFilepath( - filepath=file_dict["filepath"], - file_snapshot=db_element, - analysis_suite=analysis_suite, - ) - - -class StaticAnalysisSuiteFilepathField(serializers.ModelSerializer): - file_hash = serializers.UUIDField() - raw_upload_location = serializers.SerializerMethodField() - state = serializers.SerializerMethodField() - - class Meta: - model = StaticAnalysisSuiteFilepath - fields = [ - "filepath", - "file_hash", - "raw_upload_location", - "state", - ] - - def get_state(self, obj): - return StaticAnalysisSingleFileSnapshotState.enum_from_int( - obj.file_snapshot.state_id - ).name - - def get_raw_upload_location(self, obj): - # TODO: This has a built-in ttl of 10 seconds. - # We have to consider changing it in case customers are doing a few - # thousand uploads on the first time - return self.context["archive_service"].create_presigned_put( - obj.file_snapshot.content_location - ) - - -class FilepathListField(serializers.ListField): - child = StaticAnalysisSuiteFilepathField() - - def to_representation(self, data): - data = data.select_related( - "file_snapshot", - ).all() - return super().to_representation(data) - - -class StaticAnalysisSuiteSerializer(serializers.ModelSerializer): - commit = CommitFromShaSerializerField(required=True) - filepaths = FilepathListField() - - class Meta: - model = StaticAnalysisSuite - fields = ["external_id", "commit", "filepaths"] - read_only_fields = ["raw_upload_location", "external_id"] - - def create(self, validated_data): - file_metadata_array = validated_data.pop("filepaths") - # `validated_data` only contains `commit` after pop - obj = StaticAnalysisSuite.objects.create(**validated_data) - request = self.context["request"] - repository = request.auth.get_repositories()[0] - archive_service = ArchiveService(repository) - # allow 1s per 10 uploads - ttl = max(math.ceil(len(file_metadata_array) / 10) + 5, 10) - self.context["archive_service"] = ArchiveService(repository, ttl=ttl) - all_hashes = [val["file_hash"] for val in file_metadata_array] - existing_values = StaticAnalysisSingleFileSnapshot.objects.filter( - repository=repository, file_hash__in=all_hashes - ) - existing_values_mapping = {val.file_hash: val for val in existing_values} - created_filepaths = [ - _dict_to_suite_filepath( - obj, - repository, - archive_service, - existing_values_mapping, - file_dict, - ) - for file_dict in file_metadata_array - ] - StaticAnalysisSuiteFilepath.objects.bulk_create(created_filepaths) - log.info( - "Created static analysis filepaths", - extra={ - "created_ids": [f.id for f in created_filepaths], - "repoid": repository.repoid, - }, - ) - return obj diff --git a/apps/codecov-api/staticanalysis/tests/test_views.py b/apps/codecov-api/staticanalysis/tests/test_views.py index 24e31d4df7..7055b2f5ff 100644 --- a/apps/codecov-api/staticanalysis/tests/test_views.py +++ b/apps/codecov-api/staticanalysis/tests/test_views.py @@ -3,22 +3,14 @@ from django.urls import reverse from rest_framework.test import APIClient -from services.task import TaskService -from shared.celery_config import static_analysis_task_name from shared.django_apps.core.tests.factories import ( CommitFactory, RepositoryTokenFactory, ) -from staticanalysis.models import StaticAnalysisSuite -from staticanalysis.tests.factories import StaticAnalysisSuiteFactory +from staticanalysis.views import EMPTY_RESPONSE -def test_simple_static_analysis_call_no_uploads_yet(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") - mocked_presigned_put = mocker.patch( - "shared.storage.MinioStorageService.create_presigned_put", - return_value="banana.txt", - ) +def test_simple_static_analysis_call_no_uploads_yet(db): commit = CommitFactory.create(repository__active=True) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" @@ -45,57 +37,20 @@ def test_simple_static_analysis_call_no_uploads_yet(db, mocker): format="json", ) assert response.status_code == 201 - assert StaticAnalysisSuite.objects.filter(commit=commit).count() == 1 - produced_object = StaticAnalysisSuite.objects.filter(commit=commit).get() - response_json = response.json() - assert "filepaths" in response_json - # Popping and sorting because the order doesn't matter, as long as all are there - assert sorted(response_json.pop("filepaths"), key=lambda x: x["filepath"]) == [ - { - "filepath": "banana.cpp", - "file_hash": str(second_uuid), - "raw_upload_location": "banana.txt", - "state": "CREATED", - }, - { - "filepath": "path/to/a.py", - "file_hash": str(some_uuid), - "raw_upload_location": "banana.txt", - "state": "CREATED", - }, - ] - # Now asserting the remaining of the response - assert response_json == { - "external_id": str(produced_object.external_id), - "commit": commit.commitid, - } - mocked_task_service.assert_called_with( - static_analysis_task_name, - kwargs={"suite_id": produced_object.id}, - apply_async_kwargs={"countdown": 10}, - ) - mocked_presigned_put.assert_called_with( - "archive", - mocker.ANY, - 10, - ) + assert response.json() == EMPTY_RESPONSE -def test_static_analysis_finish(db, mocker): - mocked_task_service = mocker.patch.object(TaskService, "schedule_task") +def test_static_analysis_finish(db): commit = CommitFactory.create(repository__active=True) - suite = StaticAnalysisSuiteFactory(commit=commit) token = RepositoryTokenFactory.create( repository=commit.repository, token_type="static_analysis" ) client = APIClient() client.credentials(HTTP_AUTHORIZATION="repotoken " + token.key) response = client.post( - reverse("staticanalyses-finish", kwargs={"external_id": suite.external_id}) - ) - assert response.status_code == 204 - mocked_task_service.assert_called_with( - static_analysis_task_name, - kwargs={"suite_id": suite.id}, - apply_async_kwargs={}, + reverse( + "staticanalyses-finish", + kwargs={"external_id": "c4e11164-411a-48f1-b791-91a9f009ea48"}, + ) ) + assert response.status_code == 201 diff --git a/apps/codecov-api/staticanalysis/tests/unit/__init__.py b/apps/codecov-api/staticanalysis/tests/unit/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/apps/codecov-api/staticanalysis/tests/unit/test_serializers.py b/apps/codecov-api/staticanalysis/tests/unit/test_serializers.py deleted file mode 100644 index bb09b513bc..0000000000 --- a/apps/codecov-api/staticanalysis/tests/unit/test_serializers.py +++ /dev/null @@ -1,331 +0,0 @@ -import re -from uuid import UUID, uuid4 - -import pytest -from rest_framework.exceptions import NotFound, ValidationError - -from shared.api_archive.archive import ArchiveService -from shared.django_apps.core.tests.factories import CommitFactory, RepositoryFactory -from staticanalysis.models import ( - StaticAnalysisSingleFileSnapshotState, - StaticAnalysisSuite, - StaticAnalysisSuiteFilepath, -) -from staticanalysis.serializers import ( - CommitFromShaSerializerField, - StaticAnalysisSuiteFilepathField, - StaticAnalysisSuiteSerializer, -) -from staticanalysis.tests.factories import ( - StaticAnalysisSingleFileSnapshotFactory, - StaticAnalysisSuiteFilepathFactory, -) - -expected_location_regex = re.compile( - "v4/repos/[A-F0-9]{32}/static_analysis/files/[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}.json" -) - - -def test_commit_from_sha_serializer_field_to_internal_value(mocker, db): - commit = CommitFactory.create() - # notice this second commit has a different repo by default - second_commit = CommitFactory.create() - commit.save() - second_commit.save() - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - # silly workaround to not have to manually bind serializers - mocker.patch.object( - CommitFromShaSerializerField, "context", {"request": fake_request} - ) - serializer_field = CommitFromShaSerializerField() - with pytest.raises(NotFound): - assert serializer_field.to_internal_value("abcde" * 8) - with pytest.raises(NotFound): - assert serializer_field.to_internal_value(second_commit.commitid) - assert serializer_field.to_internal_value(commit.commitid) == commit - - -def test_filepath_field(db, mocker): - sasfs = StaticAnalysisSingleFileSnapshotFactory.create( - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id - ) - sasfs.save() - fp = StaticAnalysisSuiteFilepathFactory.create( - filepath="ohoooo", file_snapshot=sasfs - ) - fp.analysis_suite.save() - fp.save() - fake_archive_service = mocker.MagicMock( - create_presigned_put=mocker.MagicMock(return_value="some_url_stuff") - ) - serializer_field = StaticAnalysisSuiteFilepathField( - context={ - "archive_service": fake_archive_service, - } - ) - assert dict(serializer_field.to_representation(fp)) == { - "file_hash": sasfs.file_hash, - "filepath": "ohoooo", - "raw_upload_location": "some_url_stuff", - "state": "VALID", - } - - -class TestStaticAnalysisSuiteSerializer(object): - def test_to_internal_value_missing_filepaths(self, mocker, db): - commit = CommitFactory.create() - commit.save() - input_data = {"commit": commit.commitid} - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - with pytest.raises(ValidationError) as exc: - serializer.to_internal_value(input_data) - assert exc.value.detail == {"filepaths": ["This field is required."]} - - def test_to_internal_value_complete(self, mocker, db): - commit = CommitFactory.create() - commit.save() - input_data = { - "commit": commit.commitid, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": "c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2", - }, - { - "filepath": "anothersomething", - "file_hash": "3998813e-60db-4686-be84-1a0efa7d9b9f", - }, - ], - } - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - res = serializer.to_internal_value(input_data) - assert res == { - "commit": commit, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - }, - { - "filepath": "anothersomething", - "file_hash": UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - }, - ], - } - - def test_create_no_data_previously_exists(self, mocker, db): - first_repository = RepositoryFactory.create() - commit = CommitFactory.create(repository=first_repository) - commit.save() - validated_data = { - "commit": commit, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - }, - { - "filepath": "anothersomething", - "file_hash": UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - }, - ], - } - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - res = serializer.create(validated_data) - assert isinstance(res, StaticAnalysisSuite) - assert res.commit == commit - assert res.filepaths.count() == 2 - first_filepath, second_filepath = sorted( - res.filepaths.all(), key=lambda x: x.filepath - ) - assert isinstance(first_filepath, StaticAnalysisSuiteFilepath) - assert first_filepath.filepath == "anothersomething" - assert first_filepath.file_snapshot is not None - archive_hash = ArchiveService.get_archive_hash(commit.repository) - assert first_filepath.file_snapshot.repository == commit.repository - assert first_filepath.file_snapshot.file_hash == UUID( - "3998813e-60db-4686-be84-1a0efa7d9b9f" - ) - assert archive_hash in first_filepath.file_snapshot.content_location - assert ( - expected_location_regex.match(first_filepath.file_snapshot.content_location) - is not None - ) - assert ( - first_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - - def test_create_some_data_previously_exists(self, mocker, db): - first_repository = RepositoryFactory.create() - second_repository = RepositoryFactory.create() - commit = CommitFactory.create(repository=first_repository) - first_repo_first_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - repository=first_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="first_repo_first_snapshot", - ) - second_repo_first_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - repository=second_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="second_repo_first_snapshot", - ) - second_repo_second_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - repository=second_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="second_repo_second_snapshot", - ) - first_repo_separate_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=uuid4(), - repository=first_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="first_repo_separate_snapshot", - ) - second_repo_separate_snapshot = StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=uuid4(), - repository=second_repository, - state_id=StaticAnalysisSingleFileSnapshotState.VALID.db_id, - content_location="second_repo_separate_snapshot", - ) - first_repo_exists_but_not_valid_yet = ( - StaticAnalysisSingleFileSnapshotFactory.create( - file_hash=UUID("31803149-8bd7-4c2b-9a80-71f259360c72"), - repository=first_repository, - state_id=StaticAnalysisSingleFileSnapshotState.CREATED.db_id, - content_location="first_repo_exists_but_not_valid_yet", - ) - ) - first_repo_first_snapshot.save() - second_repo_first_snapshot.save() - second_repo_second_snapshot.save() - first_repo_separate_snapshot.save() - second_repo_separate_snapshot.save() - first_repo_exists_but_not_valid_yet.save() - - validated_data = { - "commit": commit, - "filepaths": [ - { - "filepath": "path/to/a.py", - "file_hash": UUID("c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2"), - }, - { - "filepath": "anothersomething", - "file_hash": UUID("3998813e-60db-4686-be84-1a0efa7d9b9f"), - }, - { - "filepath": "oooaaa.rb", - "file_hash": UUID("60228df6-4d11-44d4-a048-ec2fa1ea2c32"), - }, - { - "filepath": "awert.qt", - "file_hash": UUID("31803149-8bd7-4c2b-9a80-71f259360c72"), - }, - ], - } - fake_request = mocker.MagicMock( - auth=mocker.MagicMock( - get_repositories=mocker.MagicMock(return_value=[commit.repository]) - ) - ) - serializer = StaticAnalysisSuiteSerializer(context={"request": fake_request}) - res = serializer.create(validated_data) - assert isinstance(res, StaticAnalysisSuite) - assert res.commit == commit - assert res.filepaths.count() == 4 - first_filepath, second_filepath, third_filepath, fourth_filepath = sorted( - res.filepaths.all(), key=lambda x: x.filepath - ) - archive_hash = ArchiveService.get_archive_hash(first_repository) - # first one - assert isinstance(first_filepath, StaticAnalysisSuiteFilepath) - assert first_filepath.filepath == "anothersomething" - assert first_filepath.file_snapshot is not None - assert first_filepath.file_snapshot.repository == first_repository - assert first_filepath.file_snapshot.file_hash == UUID( - "3998813e-60db-4686-be84-1a0efa7d9b9f" - ) - assert archive_hash in first_filepath.file_snapshot.content_location - assert ( - expected_location_regex.match(first_filepath.file_snapshot.content_location) - is not None - ) - assert ( - first_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - # second one - assert isinstance(second_filepath, StaticAnalysisSuiteFilepath) - assert second_filepath.filepath == "awert.qt" - assert second_filepath.file_snapshot == first_repo_exists_but_not_valid_yet - assert second_filepath.file_snapshot.repository == first_repository - assert second_filepath.file_snapshot.file_hash == UUID( - "31803149-8bd7-4c2b-9a80-71f259360c72" - ) - # content location was already there, so nothing is created - # asserting the old value is still there - assert ( - second_filepath.file_snapshot.content_location - == "first_repo_exists_but_not_valid_yet" - ) - assert ( - second_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - # third one - assert isinstance(third_filepath, StaticAnalysisSuiteFilepath) - assert third_filepath.filepath == "oooaaa.rb" - assert third_filepath.file_snapshot is not None - assert third_filepath.file_snapshot.repository == first_repository - assert third_filepath.file_snapshot.file_hash == UUID( - "60228df6-4d11-44d4-a048-ec2fa1ea2c32" - ) - assert archive_hash in third_filepath.file_snapshot.content_location - assert ( - expected_location_regex.match(third_filepath.file_snapshot.content_location) - is not None - ) - assert ( - third_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.CREATED.db_id - ) - # fourth one - assert isinstance(fourth_filepath, StaticAnalysisSuiteFilepath) - assert fourth_filepath.filepath == "path/to/a.py" - assert fourth_filepath.file_snapshot == first_repo_first_snapshot - assert fourth_filepath.file_snapshot.repository == first_repository - assert fourth_filepath.file_snapshot.file_hash == UUID( - "c8c23bea-c383-4abf-8a7e-6b9cadbeb5b2" - ) - # content location was already there, so nothing is created - # asserting the old value is still there - assert ( - fourth_filepath.file_snapshot.content_location - == "first_repo_first_snapshot" - ) - assert ( - fourth_filepath.file_snapshot.state_id - == StaticAnalysisSingleFileSnapshotState.VALID.db_id - ) diff --git a/apps/codecov-api/staticanalysis/urls.py b/apps/codecov-api/staticanalysis/urls.py index 6a2a5510b8..87bf090921 100644 --- a/apps/codecov-api/staticanalysis/urls.py +++ b/apps/codecov-api/staticanalysis/urls.py @@ -1,7 +1,16 @@ -from staticanalysis.views import StaticAnalysisSuiteViewSet -from utils.routers import OptionalTrailingSlashRouter +from django.urls import path -router = OptionalTrailingSlashRouter() -router.register("analyses", StaticAnalysisSuiteViewSet, basename="staticanalyses") +from staticanalysis.views import StaticAnalysisSuiteView -urlpatterns = router.urls +urlpatterns = [ + path( + "staticanalyses", + StaticAnalysisSuiteView.as_view(), + name="staticanalyses-list", + ), + path( + "staticanalyses//finish", + StaticAnalysisSuiteView.as_view(), + name="staticanalyses-finish", + ), +] diff --git a/apps/codecov-api/staticanalysis/views.py b/apps/codecov-api/staticanalysis/views.py index a62377f166..b557d333ab 100644 --- a/apps/codecov-api/staticanalysis/views.py +++ b/apps/codecov-api/staticanalysis/views.py @@ -1,46 +1,21 @@ -import logging - -from django.http import HttpResponse -from rest_framework import mixins, viewsets -from rest_framework.decorators import action +from rest_framework import status +from rest_framework.response import Response +from rest_framework.views import APIView from codecov_auth.authentication.repo_auth import RepositoryTokenAuthentication from codecov_auth.permissions import SpecificScopePermission -from services.task import TaskService -from shared.celery_config import static_analysis_task_name -from staticanalysis.models import StaticAnalysisSuite -from staticanalysis.serializers import StaticAnalysisSuiteSerializer -log = logging.getLogger(__name__) +EMPTY_RESPONSE = { + "external_id": "0000", + "filepaths": [], + "commit": "", +} -class StaticAnalysisSuiteViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): - serializer_class = StaticAnalysisSuiteSerializer +class StaticAnalysisSuiteView(APIView): authentication_classes = [RepositoryTokenAuthentication] permission_classes = [SpecificScopePermission] required_scopes = ["static_analysis"] - lookup_field = "external_id" - - def get_queryset(self): - repository = self.request.auth.get_repositories()[0] - return StaticAnalysisSuite.objects.filter(commit__repository=repository) - - def perform_create(self, serializer): - instance = serializer.save() - # TODO: remove this once the CLI is calling the `finish` endpoint - TaskService().schedule_task( - static_analysis_task_name, - kwargs={"suite_id": instance.id}, - apply_async_kwargs={"countdown": 10}, - ) - return instance - @action(detail=True, methods=["post"]) - def finish(self, request, *args, **kwargs): - suite = self.get_object() - TaskService().schedule_task( - static_analysis_task_name, - kwargs={"suite_id": suite.pk}, - apply_async_kwargs={}, - ) - return HttpResponse(status=204) + def post(self, request, *args, **kwargs): + return Response(EMPTY_RESPONSE, status=status.HTTP_201_CREATED)