From 8e652d223ab69aa46e5ce6e7643e239fca6e4d4e Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:38:15 -0400 Subject: [PATCH 01/31] rebase - serializers --- coldfront/plugins/api/serializers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/coldfront/plugins/api/serializers.py b/coldfront/plugins/api/serializers.py index d58e892368..96eb057e56 100644 --- a/coldfront/plugins/api/serializers.py +++ b/coldfront/plugins/api/serializers.py @@ -151,6 +151,7 @@ class Meta: fields = ( 'id', 'allocation', + 'resource', 'justification', 'status', 'created', From 96cdbbaffcd14a58d8acb9683a05f427648c0269 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Fri, 28 Jun 2024 10:55:06 -0400 Subject: [PATCH 02/31] add created_before/created_after filtering to allocation and project views --- coldfront/plugins/api/serializers.py | 17 ++++++++++++++++- coldfront/plugins/api/views.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/coldfront/plugins/api/serializers.py b/coldfront/plugins/api/serializers.py index 96eb057e56..cb0380a2ae 100644 --- a/coldfront/plugins/api/serializers.py +++ b/coldfront/plugins/api/serializers.py @@ -94,8 +94,15 @@ class Meta: 'usage', 'pct_full', 'cost', + 'created', ) + def get_type(self, obj): + resource = obj.get_parent_resource + if resource: + return resource.resource_type.name + return None + class AllocationRequestSerializer(serializers.ModelSerializer): project = serializers.SlugRelatedField(slug_field='title', read_only=True) @@ -206,7 +213,15 @@ class ProjectSerializer(serializers.ModelSerializer): class Meta: model = Project - fields = ('id', 'title', 'pi', 'status', 'project_users', 'allocations') + fields = ( + 'id', + 'title', + 'pi', + 'status', + 'project_users', + 'allocations', + 'created', + ) def get_project_users(self, obj): request = self.context.get('request', None) diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index 5622a246ae..57ed9da407 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -24,8 +24,23 @@ class ResourceViewSet(viewsets.ReadOnlyModelViewSet): queryset = Resource.objects.all() +class AllocationFilter(filters.FilterSet): + '''Filters for AllocationViewSet. + created_before is the date the request was created before. + created_after is the date the request was created after. + ''' + created = filters.DateFromToRangeFilter() + + class Meta: + model = Allocation + fields = [ + 'created', + ] + + class AllocationViewSet(viewsets.ReadOnlyModelViewSet): serializer_class = serializers.AllocationSerializer + filterset_class = AllocationFilter # permission_classes = (permissions.IsAuthenticatedOrReadOnly,) def get_queryset(self): @@ -246,6 +261,17 @@ def get_queryset(self): return requests +class ProjectFilter(filters.FilterSet): + '''Filters for ProjectViewSet. + ''' + created = filters.DateFromToRangeFilter() + + class Meta: + model = Project + fields = [ + 'created', + ] + class ProjectViewSet(viewsets.ReadOnlyModelViewSet): ''' Query parameters: @@ -253,8 +279,10 @@ class ProjectViewSet(viewsets.ReadOnlyModelViewSet): Show related allocation data. - project_users (default false) Show related user data. + - created_before/created_after (structure date as 'YYYY-MM-DD') ''' serializer_class = serializers.ProjectSerializer + filterset_class = ProjectFilter def get_queryset(self): projects = Project.objects.prefetch_related('status') From 243bf03306d4ba4c28f4103f8700937ff294d4c5 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 2 Jul 2024 15:04:53 -0400 Subject: [PATCH 03/31] update api config --- coldfront/config/base.py | 14 -------------- coldfront/config/plugins/api.py | 8 +++++--- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/coldfront/config/base.py b/coldfront/config/base.py index 3c70aa1d52..0b53e04350 100644 --- a/coldfront/config/base.py +++ b/coldfront/config/base.py @@ -51,7 +51,6 @@ 'django_tables2', 'table', 'rest_framework_datatables', - 'rest_framework', 'easy_pdf', ] @@ -148,19 +147,6 @@ }, ] -REST_FRAMEWORK = { - 'DEFAULT_RENDERER_CLASSES': ( - 'rest_framework.renderers.JSONRenderer', - 'rest_framework.renderers.BrowsableAPIRenderer', - 'rest_framework_datatables.renderers.DatatablesRenderer', - ), - 'DEFAULT_FILTER_BACKENDS': ( - 'rest_framework_datatables.filters.DatatablesFilterBackend', - ), - 'DEFAULT_PAGINATION_CLASS': 'rest_framework_datatables.pagination.DatatablesPageNumberPagination', - 'PAGE_SIZE': 500, -} - # Add local site templates files if set SITE_TEMPLATES = ENV.str('SITE_TEMPLATES', default='') if len(SITE_TEMPLATES) > 0: diff --git a/coldfront/config/plugins/api.py b/coldfront/config/plugins/api.py index cee06d0c5b..daf342b8ae 100644 --- a/coldfront/config/plugins/api.py +++ b/coldfront/config/plugins/api.py @@ -1,9 +1,11 @@ from coldfront.config.base import INSTALLED_APPS INSTALLED_APPS += [ - 'django_filters', - 'coldfront.plugins.api' - ] + 'django_filters', + 'knox', + 'rest_framework', + 'coldfront.plugins.api', +] REST_FRAMEWORK = { 'DEFAULT_AUTHENTICATION_CLASSES': ( From aa213da7b7af7fa34743c462da820d67a8ea1c3e Mon Sep 17 00:00:00 2001 From: claire-peters Date: Mon, 1 Jul 2024 14:23:31 -0400 Subject: [PATCH 04/31] add custom admin renderer --- coldfront/plugins/api/views.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index 57ed9da407..fdecc26e0a 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -7,6 +7,7 @@ from ifxuser.models import Organization from rest_framework import viewsets from rest_framework.permissions import IsAuthenticated, IsAdminUser +from rest_framework.renderers import AdminRenderer, JSONRenderer from simple_history.utils import get_history_model_for_model from coldfront.core.utils.common import import_from_settings @@ -19,6 +20,27 @@ 'PENDING_ALLOCATION_STATUSES', ['New', 'In Progress', 'On Hold', 'Pending Activation'] ) +class CustomAdminRenderer(AdminRenderer): + def render(self, data, accepted_media_type=None, renderer_context=None): + # Get the count of objects + count = len(data['results']) if 'results' in data else len(data) + + # Create the count HTML + count_html = f'
Total Objects: {count}
' + + # Render the original content + original_content = super().render(data, accepted_media_type, renderer_context) + + # Ensure original_content is a string + if isinstance(original_content, bytes): + original_content = original_content.decode('utf-8') + + # Insert the count HTML after the docstring and before the results table + parts = original_content.split(' Date: Tue, 2 Jul 2024 15:22:53 -0400 Subject: [PATCH 05/31] remove basicauthentication, add knox dependencies --- coldfront/config/plugins/api.py | 1 - requirements.txt | 3 +++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/coldfront/config/plugins/api.py b/coldfront/config/plugins/api.py index daf342b8ae..9cbbca56dd 100644 --- a/coldfront/config/plugins/api.py +++ b/coldfront/config/plugins/api.py @@ -11,7 +11,6 @@ 'DEFAULT_AUTHENTICATION_CLASSES': ( 'rest_framework.authentication.TokenAuthentication', 'rest_framework.authentication.SessionAuthentication', - 'rest_framework.authentication.BasicAuthentication', ), 'DEFAULT_PERMISSION_CLASSES': [ 'rest_framework.permissions.IsAuthenticated' diff --git a/requirements.txt b/requirements.txt index c271e65f71..45734dbc34 100644 --- a/requirements.txt +++ b/requirements.txt @@ -62,3 +62,6 @@ XlsxWriter python-dateutil django-extensions natural-keys +knox +jinja2 +boto3 From 81217f0a8861b0a780c682bffa8f17e50d036806 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Tue, 2 Jul 2024 15:57:30 -0400 Subject: [PATCH 06/31] change allocationchangerequest api response content --- coldfront/plugins/api/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coldfront/plugins/api/serializers.py b/coldfront/plugins/api/serializers.py index cb0380a2ae..e2f50149a6 100644 --- a/coldfront/plugins/api/serializers.py +++ b/coldfront/plugins/api/serializers.py @@ -146,7 +146,8 @@ def get_fulfilled_by(self, obj): class AllocationChangeRequestSerializer(serializers.ModelSerializer): - allocation = AllocationSerializer(read_only=True) + project = serializers.ReadOnlyField(source='allocation.project.title') + resource = serializers.ReadOnlyField(source='allocation.get_resources_as_string') status = serializers.SlugRelatedField(slug_field='name', read_only=True) created_by = serializers.SerializerMethodField(read_only=True) fulfilled_date = serializers.DateTimeField(read_only=True) @@ -158,6 +159,7 @@ class Meta: fields = ( 'id', 'allocation', + 'project', 'resource', 'justification', 'status', From a13f3762b9504c299b397876384aee01dd5edf30 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 4 Sep 2024 13:05:09 -0400 Subject: [PATCH 07/31] remove knox for now --- coldfront/config/plugins/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/coldfront/config/plugins/api.py b/coldfront/config/plugins/api.py index 9cbbca56dd..061510749b 100644 --- a/coldfront/config/plugins/api.py +++ b/coldfront/config/plugins/api.py @@ -2,7 +2,6 @@ INSTALLED_APPS += [ 'django_filters', - 'knox', 'rest_framework', 'coldfront.plugins.api', ] From 094a343fa3144169104f09c1e066368af971244a Mon Sep 17 00:00:00 2001 From: claire-peters Date: Wed, 11 Sep 2024 19:15:41 -0400 Subject: [PATCH 08/31] update allocation request api view --- coldfront/plugins/api/serializers.py | 14 +++++++++++--- coldfront/plugins/api/views.py | 18 +++++++++++------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/coldfront/plugins/api/serializers.py b/coldfront/plugins/api/serializers.py index e2f50149a6..66b32b8bad 100644 --- a/coldfront/plugins/api/serializers.py +++ b/coldfront/plugins/api/serializers.py @@ -106,10 +106,15 @@ def get_type(self, obj): class AllocationRequestSerializer(serializers.ModelSerializer): project = serializers.SlugRelatedField(slug_field='title', read_only=True) - resource = serializers.ReadOnlyField(source='get_resources_as_string', read_only=True) + pi = serializers.ReadOnlyField(source='project.pi.full_name') + resource = serializers.ReadOnlyField(source='get_resources_as_string', allow_null=True) + tier = serializers.ReadOnlyField(source='get_parent_resource.parent_resource.name', allow_null=True) status = serializers.SlugRelatedField(slug_field='name', read_only=True) - fulfilled_date = serializers.DateTimeField(read_only=True) + requested_size = serializers.ReadOnlyField(source='quantity') + current_size = serializers.ReadOnlyField(source='size') + created = serializers.DateTimeField(format="%Y-%m-%d %H:%M", read_only=True) created_by = serializers.SerializerMethodField(read_only=True) + fulfilled_date = serializers.DateTimeField(format="%Y-%m-%d %H:%M", read_only=True) fulfilled_by = serializers.SerializerMethodField(read_only=True) time_to_fulfillment = serializers.DurationField(read_only=True) @@ -118,10 +123,13 @@ class Meta: fields = ( 'id', 'project', + 'pi', 'resource', + 'tier', 'path', 'status', - 'size', + 'requested_size', + 'current_size', 'created', 'created_by', 'fulfilled_date', diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index fdecc26e0a..1f50bb52ae 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -8,6 +8,8 @@ from rest_framework import viewsets from rest_framework.permissions import IsAuthenticated, IsAdminUser from rest_framework.renderers import AdminRenderer, JSONRenderer +from rest_framework import filters as drf_filters + from simple_history.utils import get_history_model_for_model from coldfront.core.utils.common import import_from_settings @@ -94,10 +96,10 @@ class AllocationRequestFilter(filters.FilterSet): created_before is the date the request was created before. created_after is the date the request was created after. ''' - created = filters.DateFromToRangeFilter() - fulfilled = filters.BooleanFilter(method='filter_fulfilled') - fulfilled_date = filters.DateFromToRangeFilter() - time_to_fulfillment = filters.NumericRangeFilter(method='filter_time_to_fulfillment') + created = filters.DateFromToRangeFilter(label='Created Range') + fulfilled = filters.BooleanFilter(label='Fulfilled', method='filter_fulfilled') + fulfilled_date = filters.DateFromToRangeFilter(label='Fulfilled Date Range') + time_to_fulfillment = filters.NumericRangeFilter(label='Time-to-fulfillment Range', method='filter_time_to_fulfillment') class Meta: model = Allocation @@ -151,10 +153,12 @@ class AllocationRequestViewSet(viewsets.ReadOnlyModelViewSet): Set to the maximum/minimum number of days between request creation and time_to_fulfillment. ''' serializer_class = serializers.AllocationRequestSerializer - filter_backends = (filters.DjangoFilterBackend,) + filter_backends = (filters.DjangoFilterBackend, drf_filters.OrderingFilter) filterset_class = AllocationRequestFilter permission_classes = [IsAuthenticated, IsAdminUser] - renderer_classes = [CustomAdminRenderer, JSONRenderer] + renderer_classes = [AdminRenderer, JSONRenderer] + ordering_fields = ['id', 'project', 'pi', 'status', 'requested_size', 'created', 'fulfilled_date', 'time_to_fulfillment'] + ordering = ['created'] def get_queryset(self): HistoricalAllocation = get_history_model_for_model(Allocation) @@ -171,7 +175,7 @@ def get_queryset(self): # Annotate allocations with the status_id of their earliest historical record allocations = Allocation.objects.annotate( earliest_status_name=Subquery(earliest_history) - ).filter(earliest_status_name='New').order_by('created') + ).filter(earliest_status_name='New') allocations = allocations.annotate( fulfilled_date=Subquery(fulfilled_date) From 9ef1d0977338293e3a644b65b71272a955c05a0c Mon Sep 17 00:00:00 2001 From: claire-peters Date: Wed, 11 Sep 2024 19:44:02 -0400 Subject: [PATCH 09/31] update allocation change request values --- coldfront/plugins/api/serializers.py | 9 +++++++-- coldfront/plugins/api/views.py | 13 +++++++------ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/coldfront/plugins/api/serializers.py b/coldfront/plugins/api/serializers.py index 66b32b8bad..ba705f097b 100644 --- a/coldfront/plugins/api/serializers.py +++ b/coldfront/plugins/api/serializers.py @@ -107,7 +107,7 @@ def get_type(self, obj): class AllocationRequestSerializer(serializers.ModelSerializer): project = serializers.SlugRelatedField(slug_field='title', read_only=True) pi = serializers.ReadOnlyField(source='project.pi.full_name') - resource = serializers.ReadOnlyField(source='get_resources_as_string', allow_null=True) + resource = serializers.ReadOnlyField(source='get_parent_resource.name', allow_null=True) tier = serializers.ReadOnlyField(source='get_parent_resource.parent_resource.name', allow_null=True) status = serializers.SlugRelatedField(slug_field='name', read_only=True) requested_size = serializers.ReadOnlyField(source='quantity') @@ -155,10 +155,13 @@ def get_fulfilled_by(self, obj): class AllocationChangeRequestSerializer(serializers.ModelSerializer): project = serializers.ReadOnlyField(source='allocation.project.title') + pi = serializers.ReadOnlyField(source='allocation.project.pi.full_name') resource = serializers.ReadOnlyField(source='allocation.get_resources_as_string') + tier = serializers.ReadOnlyField(source='allocation.get_parent_resource.parent_resource.name', allow_null=True) status = serializers.SlugRelatedField(slug_field='name', read_only=True) + created = serializers.DateTimeField(format="%Y-%m-%d %H:%M", read_only=True) created_by = serializers.SerializerMethodField(read_only=True) - fulfilled_date = serializers.DateTimeField(read_only=True) + fulfilled_date = serializers.DateTimeField(format="%Y-%m-%d %H:%M", read_only=True) fulfilled_by = serializers.SerializerMethodField(read_only=True) time_to_fulfillment = serializers.DurationField(read_only=True) @@ -168,7 +171,9 @@ class Meta: 'id', 'allocation', 'project', + 'pi', 'resource', + 'tier', 'justification', 'status', 'created', diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index 1f50bb52ae..9cced92f94 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -156,7 +156,7 @@ class AllocationRequestViewSet(viewsets.ReadOnlyModelViewSet): filter_backends = (filters.DjangoFilterBackend, drf_filters.OrderingFilter) filterset_class = AllocationRequestFilter permission_classes = [IsAuthenticated, IsAdminUser] - renderer_classes = [AdminRenderer, JSONRenderer] + renderer_classes = [CustomAdminRenderer, JSONRenderer] ordering_fields = ['id', 'project', 'pi', 'status', 'requested_size', 'created', 'fulfilled_date', 'time_to_fulfillment'] ordering = ['created'] @@ -196,9 +196,9 @@ class AllocationChangeRequestFilter(filters.FilterSet): created_after is the date the request was created after. ''' created = filters.DateFromToRangeFilter() - fulfilled = filters.BooleanFilter(method='filter_fulfilled') - fulfilled_date = filters.DateFromToRangeFilter() - time_to_fulfillment = filters.NumericRangeFilter(method='filter_time_to_fulfillment') + fulfilled = filters.BooleanFilter(label='Fulfilled', method='filter_fulfilled') + fulfilled_date = filters.DateFromToRangeFilter(label='Fulfilled Date Range') + time_to_fulfillment = filters.NumericRangeFilter(label='Time-to-fulfillment Range', method='filter_time_to_fulfillment') class Meta: model = AllocationChangeRequest @@ -247,9 +247,11 @@ class AllocationChangeRequestViewSet(viewsets.ReadOnlyModelViewSet): Set to the maximum/minimum number of days between request creation and time_to_fulfillment. ''' serializer_class = serializers.AllocationChangeRequestSerializer - filter_backends = (filters.DjangoFilterBackend,) + filter_backends = (filters.DjangoFilterBackend, drf_filters.OrderingFilter) filterset_class = AllocationChangeRequestFilter renderer_classes = [CustomAdminRenderer, JSONRenderer] + ordering_fields = ['id', 'allocation', 'project', 'pi', 'status', 'created', 'fulfilled_date', 'time_to_fulfillment'] + ordering = ['created'] def get_queryset(self): requests = AllocationChangeRequest.objects.prefetch_related( @@ -284,7 +286,6 @@ def get_queryset(self): output_field=fields.DurationField() ) ) - requests = requests.order_by('created') return requests From b797d08ed50c48284e3f4b82532f7ffa4a696837 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Thu, 12 Sep 2024 15:21:45 -0700 Subject: [PATCH 10/31] update documentation --- coldfront/plugins/api/views.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index 9cced92f94..174aee5113 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -130,10 +130,13 @@ def filter_time_to_fulfillment(self, queryset, name, value): class AllocationRequestViewSet(viewsets.ReadOnlyModelViewSet): '''Report view on allocations requested through Coldfront. + Data: - id: allocation id - project: project name - - resource: resource name + - pi: full name of project PI + - resource: name of allocation's resource + - tier: storage tier of allocation's resource - path: path to the allocation on the resource - status: current status of the allocation - size: current size of the allocation @@ -141,8 +144,9 @@ class AllocationRequestViewSet(viewsets.ReadOnlyModelViewSet): - created_by: user who submitted the allocation request - fulfilled_date: date the allocation's status was first set to "Active" - fulfilled_by: user who first set the allocation status to "Active" - - time_to_fulfillment: time between request creation and time_to_fulfillment - displayed as "DAY_INTEGER HH:MM:SS" + - time_to_fulfillment: time from request creation to time_to_fulfillment displayed as "DAY_INTEGER HH:MM:SS" + + Filters and ordering can be added either by manually defining in the url or by clicking on the "filters" button in the top right corner. Filters: - created_before/created_after (structure date as 'YYYY-MM-DD') @@ -230,7 +234,12 @@ def filter_time_to_fulfillment(self, queryset, name, value): class AllocationChangeRequestViewSet(viewsets.ReadOnlyModelViewSet): ''' Data: - - allocation: allocation object details + - id: allocationchangerequest id + - allocation: allocation id + - project: project title + - pi: full name of project PI + - resource: allocation's resource + - tier: storage tier of allocation's resource - justification: justification provided at time of filing - status: request status - created: date created @@ -238,7 +247,9 @@ class AllocationChangeRequestViewSet(viewsets.ReadOnlyModelViewSet): - fulfilled_date: date the allocationchangerequests's status was first set to "Approved" - fulfilled_by: user who last modified an approved object. - Query parameters: + Filters and ordering can be added either by manually defining in the url or by clicking on the "filters" button in the top right corner. + + Filters: - created_before/created_after (structure date as 'YYYY-MM-DD') - fulfilled (boolean) Set to true to return all approved requests, false to return all pending and denied requests. From b6b84f78c673064f152cf698c21eef1e3cee5418 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Thu, 12 Sep 2024 20:28:40 -0700 Subject: [PATCH 11/31] add csv export button --- coldfront/plugins/api/views.py | 115 +++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 12 deletions(-) diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index 174aee5113..a176622921 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -1,8 +1,12 @@ +import csv from datetime import timedelta from django.contrib.auth import get_user_model -from django.db.models import OuterRef, Subquery, Q, F, ExpressionWrapper, fields + +from django.db.models import OuterRef, Subquery, Q, F, ExpressionWrapper, Case, When, Value, fields, DurationField from django.db.models.functions import Cast +from django.http import HttpResponse +from django.utils.http import urlencode from django_filters import rest_framework as filters from ifxuser.models import Organization from rest_framework import viewsets @@ -24,24 +28,42 @@ class CustomAdminRenderer(AdminRenderer): def render(self, data, accepted_media_type=None, renderer_context=None): + # Get the count of objects count = len(data['results']) if 'results' in data else len(data) - + # Create the count HTML count_html = f'
Total Objects: {count}
' - + # Render the original content original_content = super().render(data, accepted_media_type, renderer_context) - + # Ensure original_content is a string if isinstance(original_content, bytes): original_content = original_content.decode('utf-8') - + + + # Get the request object + request = renderer_context.get('request') + + # Generate the CSV export URL + query_params = request.GET.copy() + params_present = request.build_absolute_uri(request.path) != request.build_absolute_uri() + connector = '&' if params_present else '?' + export_url = f"{request.build_absolute_uri()}{connector}export=csv" + + # Create the button HTML + button_html = f''' + + ''' + # Insert the count HTML after the docstring and before the results table parts = original_content.split(' Date: Fri, 13 Sep 2024 18:28:26 -0400 Subject: [PATCH 12/31] update filters --- coldfront/plugins/api/views.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index a176622921..7f8acad41c 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -121,6 +121,9 @@ class AllocationRequestFilter(filters.FilterSet): created = filters.DateFromToRangeFilter(label='Created Range') fulfilled = filters.BooleanFilter(label='Fulfilled', method='filter_fulfilled') fulfilled_date = filters.DateFromToRangeFilter(label='Fulfilled Date Range') + requested_size = filters.NumericRangeFilter(label='Requested Size', field_name='requested_size') + pi = filters.CharFilter(label='PI', field_name='project__pi__full_name', lookup_expr='icontains') + project = filters.CharFilter(label='Project', field_name='project__title', lookup_expr='icontains') time_to_fulfillment = filters.NumericRangeFilter(label='Time-to-fulfillment Range', method='filter_time_to_fulfillment') class Meta: @@ -129,6 +132,8 @@ class Meta: 'created', 'fulfilled', 'fulfilled_date', + 'pi', + 'requested_size', 'time_to_fulfillment', ] @@ -262,6 +267,8 @@ class AllocationChangeRequestFilter(filters.FilterSet): created = filters.DateFromToRangeFilter() fulfilled = filters.BooleanFilter(label='Fulfilled', method='filter_fulfilled') fulfilled_date = filters.DateFromToRangeFilter(label='Fulfilled Date Range') + pi = filters.CharFilter(label='PI', field_name='allocation__project__pi__full_name', lookup_expr='icontains') + project = filters.CharFilter(label='Project', field_name='allocation__project__title', lookup_expr='icontains') time_to_fulfillment = filters.NumericRangeFilter(label='Time-to-fulfillment Range', method='filter_time_to_fulfillment') class Meta: @@ -270,6 +277,8 @@ class Meta: 'created', 'fulfilled', 'fulfilled_date', + 'pi', + 'project', 'time_to_fulfillment', ] From 616d0ce2a3c432e2063d934a9c98d4dc4a94eac7 Mon Sep 17 00:00:00 2001 From: claire-peters Date: Fri, 13 Sep 2024 19:24:50 -0400 Subject: [PATCH 13/31] use filter class for ordering --- coldfront/plugins/api/views.py | 35 +++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/coldfront/plugins/api/views.py b/coldfront/plugins/api/views.py index 7f8acad41c..7902dec64f 100644 --- a/coldfront/plugins/api/views.py +++ b/coldfront/plugins/api/views.py @@ -124,7 +124,20 @@ class AllocationRequestFilter(filters.FilterSet): requested_size = filters.NumericRangeFilter(label='Requested Size', field_name='requested_size') pi = filters.CharFilter(label='PI', field_name='project__pi__full_name', lookup_expr='icontains') project = filters.CharFilter(label='Project', field_name='project__title', lookup_expr='icontains') + status = filters.CharFilter(label='Status', field_name='status__name', lookup_expr='icontains') time_to_fulfillment = filters.NumericRangeFilter(label='Time-to-fulfillment Range', method='filter_time_to_fulfillment') + o = filters.OrderingFilter( + fields=( + ('id', 'id'), + ('created', 'created'), + ('project__title', 'project'), + ('project__pi__full_name', 'pi'), + ('status__name', 'status'), + ('quantity', 'requested_size'), + ('fulfilled_date', 'fulfilled_date'), + ('time_to_fulfillment', 'time_to_fulfillment'), + ) + ) class Meta: model = Allocation @@ -184,13 +197,11 @@ class AllocationRequestViewSet(viewsets.ReadOnlyModelViewSet): Set to the maximum/minimum number of days between request creation and time_to_fulfillment. ''' serializer_class = serializers.AllocationRequestSerializer - filter_backends = (filters.DjangoFilterBackend, drf_filters.OrderingFilter) + filter_backends = (filters.DjangoFilterBackend, ) filterset_class = AllocationRequestFilter permission_classes = [IsAuthenticated, IsAdminUser] renderer_classes = [CustomAdminRenderer, JSONRenderer] csv_filename = 'allocation_requests.csv' - ordering_fields = ['id', 'project', 'pi', 'status', 'requested_size', 'created', 'fulfilled_date', 'time_to_fulfillment'] - ordering = ['created'] def get_queryset(self): HistoricalAllocation = get_history_model_for_model(Allocation) @@ -270,10 +281,22 @@ class AllocationChangeRequestFilter(filters.FilterSet): pi = filters.CharFilter(label='PI', field_name='allocation__project__pi__full_name', lookup_expr='icontains') project = filters.CharFilter(label='Project', field_name='allocation__project__title', lookup_expr='icontains') time_to_fulfillment = filters.NumericRangeFilter(label='Time-to-fulfillment Range', method='filter_time_to_fulfillment') - + o = filters.OrderingFilter( + fields=( + ('created', 'created'), + ('id', 'id'), + ('allocation__id', 'allocation'), + ('allocation__project__title', 'project'), + ('allocation__project__pi__full_name', 'pi'), + ('status__name', 'status'), + ('fulfilled_date', 'fulfilled_date'), + ('time_to_fulfillment', 'time_to_fulfillment'), + ) + ) class Meta: model = AllocationChangeRequest fields = [ + 'id', 'created', 'fulfilled', 'fulfilled_date', @@ -327,12 +350,10 @@ class AllocationChangeRequestViewSet(viewsets.ReadOnlyModelViewSet): Set to the maximum/minimum number of days between request creation and time_to_fulfillment. ''' serializer_class = serializers.AllocationChangeRequestSerializer - filter_backends = (filters.DjangoFilterBackend, drf_filters.OrderingFilter) + filter_backends = (filters.DjangoFilterBackend, ) filterset_class = AllocationChangeRequestFilter renderer_classes = [CustomAdminRenderer, JSONRenderer] csv_filename = 'allocation_change_requests.csv' - ordering_fields = ['id', 'allocation', 'project', 'pi', 'status', 'created', 'fulfilled_date', 'time_to_fulfillment'] - ordering = ['created'] def get_queryset(self): requests = AllocationChangeRequest.objects.prefetch_related( From 37d4ff3c570d2954f3b175c162ef331737acbf72 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 21 May 2024 15:37:50 -0700 Subject: [PATCH 14/31] replace action imports with signals --- coldfront/core/project/signals.py | 7 +++ coldfront/core/project/views.py | 92 ++++++++++++++----------------- coldfront/plugins/ldap/utils.py | 24 ++++++-- 3 files changed, 67 insertions(+), 56 deletions(-) diff --git a/coldfront/core/project/signals.py b/coldfront/core/project/signals.py index e0563ba5d9..fbb1255ce3 100644 --- a/coldfront/core/project/signals.py +++ b/coldfront/core/project/signals.py @@ -4,6 +4,13 @@ project_create = django.dispatch.Signal() #providing_args=["project_title"] project_post_create = django.dispatch.Signal() + #providing_args=["project_obj"] + +project_add_projectuser = django.dispatch.Signal() + #providing_args=["user_name", "group_name"] + +project_preremove_projectuser = django.dispatch.Signal() + #providing_args=["user_name", "group_name"] project_activate_user = django.dispatch.Signal() #providing_args=["project_user_pk"] diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index c95b996cde..3e179bb151 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -29,7 +29,11 @@ allocation_remove_user, allocation_activate_user, ) -from coldfront.core.project.signals import project_create, project_post_create +from coldfront.core.project.signals import ( + project_make_projectuser, + project_create, + project_post_create +) from coldfront.core.grant.models import Grant from coldfront.core.project.forms import ( ProjectReviewForm, @@ -611,8 +615,6 @@ def post(self, request, *args, **kwargs): ) added_users_count = 0 - if 'coldfront.plugins.ldap' in settings.INSTALLED_APPS: - ldap_conn = LDAPConn() if formset.is_valid() and allocation_form.is_valid(): projuserstatus_active = ProjectUserStatusChoice.objects.get(name='Active') @@ -649,28 +651,19 @@ def post(self, request, *args, **kwargs): role_choice = user_form_data.get('role') - if 'coldfront.plugins.ldap' in settings.INSTALLED_APPS: - try: - ldap_conn.add_user_to_group( - user_obj.username, project_obj.title, - ) - logger.info( - "P678: Coldfront user %s added AD User for %s to AD Group %s", - self.request.user, - user_obj.username, - project_obj.title, - ) - except Exception as e: - error = f"Could not add user {user_obj} to AD Group for {project_obj.title}: {e}\nPlease contact Coldfront administration for further assistance." - logger.error( - "P685: user %s could not add AD user of %s to AD Group of %s: %s", - self.request.user, user_obj, project_obj.title, e - ) - errors.append(error) - continue - success_msg = f"User {user_obj} added by {request.user} to AD Group for {project_obj.title}" - logger.info(success_msg) - successes.append(success_msg) + try: + project_make_projectuser.send( + sender=self.__class__, + user_name=user_obj.username, group_name=project_obj.title + ) + except Exception as e: + error = f"Could not add user {user_obj} to AD Group for {project_obj.title}: {e}\nPlease contact Coldfront administration for further assistance." + logger.exception('P646: %s', e) + errors.append(error) + continue + success_msg = f"User {user_obj} added by {request.user} to AD Group for {project_obj.title}" + logger.info(success_msg) + successes.append(success_msg) # Is the user already in the project? project_obj.projectuser_set.update_or_create( @@ -798,7 +791,6 @@ def post(self, request, *args, **kwargs): ingroup = lambda u: u['username'] in users_main_group users_no_removal, users_to_remove = sort_by(users_to_remove, ingroup, how="condition") - formset = formset_factory(ProjectRemoveUserForm, max_num=len(users_to_remove)) formset = formset(request.POST, initial=users_to_remove, prefix='userform') @@ -822,30 +814,30 @@ def post(self, request, *args, **kwargs): project_user_obj = project_obj.projectuser_set.get(user=user_obj) - if 'coldfront.plugins.ldap' in settings.INSTALLED_APPS: - try: - ldap_conn.remove_member_from_group( - user_obj.username, project_obj.title, - ) - logger.info( - "P835: Coldfront user %s removed AD User for %s from AD Group for %s", - self.request.user, - user_obj.username, - project_obj.title, - ) - except Exception as e: - messages.error( - request, - f"could not remove user {user_obj}: {e}" - ) - logger.error( - "P846: Coldfront user %s could NOT remove AD User for %s from AD Group for %s: %s", - self.request.user, - user_obj.username, - project_obj.title, - e - ) - continue + try: + project_preremove_projectuser.send( + sender=self.__class__, + user_name=user_obj.username, group_name=project_obj.title + ) + logger.info( + "P802: Coldfront user %s removed AD User for %s from AD Group for %s", + self.request.user, + user_obj.username, + project_obj.title, + ) + except Exception as e: + messages.error( + request, + f"could not remove user {user_obj}: {e}" + ) + logger.error( + "P802: Coldfront user %s could NOT remove AD User for %s from AD Group for %s: %s", + self.request.user, + user_obj.username, + project_obj.title, + e + ) + continue project_user_obj.status = projectuser_status_removed project_user_obj.save() diff --git a/coldfront/plugins/ldap/utils.py b/coldfront/plugins/ldap/utils.py index 0e7eeb9ae2..b1c8b7bd5b 100644 --- a/coldfront/plugins/ldap/utils.py +++ b/coldfront/plugins/ldap/utils.py @@ -14,7 +14,12 @@ from ldap3.extend.microsoft.addMembersToGroups import ad_add_members_to_groups from ldap3.extend.microsoft.removeMembersFromGroups import ad_remove_members_from_groups -from coldfront.core.project.signals import project_create, project_post_create +from coldfront.core.project.signals import ( + project_preremove_projectuser, + project_make_projectuser, + project_create, + project_post_create, +) from coldfront.core.utils.common import ( import_from_settings, uniques_and_intersection ) @@ -179,11 +184,6 @@ def return_group_by_name(self, groupname, return_as='dict', attributes=ALL_ATTRI raise ValueError("no groups returned") return group[0] - def add_user_to_group(self, user_name, group_name): - group = self.return_group_by_name(group_name) - user = self.return_user_by_name(user_name) - self.add_member_to_group(user, group) - def add_group_to_group(self, group_name, parent_group_name): group = self.return_group_by_name(group_name) parent_group = self.return_group_by_name(parent_group_name) @@ -714,3 +714,15 @@ def update_new_project(sender, **kwargs): role=ProjectUserRoleChoice.objects.get(name=role_name), status=ProjectUserStatusChoice.objects.get(name='Active'), ) + +@receiver(project_make_projectuser) +def add_user_to_group(sender, **kwargs): + ldap_conn = LDAPConn() + group = ldap_conn.return_group_by_name(kwargs['group_name']) + user = ldap_conn.return_user_by_name(kwargs['user_name']) + ldap_conn.add_member_to_group(user, group) + +@receiver(project_preremove_projectuser) +def remove_member_from_group(sender, **kwargs): + ldap_conn = LDAPConn() + ldap_conn.remove_member_from_group(kwargs['user_name'], kwargs['group_name']) From 209acc05d3b690aa7eb23c0aa8acf191b4e99fab Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 21 May 2024 16:12:18 -0700 Subject: [PATCH 15/31] use signals for info grab in project views --- coldfront/core/project/signals.py | 4 ++++ coldfront/core/project/views.py | 27 ++++++++------------------- coldfront/plugins/ldap/utils.py | 12 +++++++++++- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/coldfront/core/project/signals.py b/coldfront/core/project/signals.py index fbb1255ce3..04e5f2899e 100644 --- a/coldfront/core/project/signals.py +++ b/coldfront/core/project/signals.py @@ -12,6 +12,10 @@ project_preremove_projectuser = django.dispatch.Signal() #providing_args=["user_name", "group_name"] +project_filter_users_to_remove = django.dispatch.Signal() + #providing_args=["project_user_list"] + # return tuple of (no_removal, can_remove) + project_activate_user = django.dispatch.Signal() #providing_args=["project_user_pk"] project_remove_user = django.dispatch.Signal() diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index 3e179bb151..cb2452fd00 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -30,6 +30,8 @@ allocation_activate_user, ) from coldfront.core.project.signals import ( + project_filter_users_to_remove, + project_preremove_projectuser, project_make_projectuser, project_create, project_post_create @@ -72,9 +74,6 @@ if 'django_q' in settings.INSTALLED_APPS: from django_q.tasks import Task -if 'coldfront.plugins.ldap' in settings.INSTALLED_APPS: - from coldfront.plugins.ldap.utils import LDAPConn - ALLOCATION_ENABLE_ALLOCATION_RENEWAL = import_from_settings( 'ALLOCATION_ENABLE_ALLOCATION_RENEWAL', True) ALLOCATION_DEFAULT_ALLOCATION_LENGTH = import_from_settings( @@ -754,14 +753,9 @@ def get(self, request, *args, **kwargs): users_no_removal = None # if ldap is activated, prevent selection of users with project corresponding to primary group - if 'coldfront.plugins.ldap' in settings.INSTALLED_APPS: - - usernames = [u['username'] for u in users_to_remove] - ldap_conn = LDAPConn() - users_main_group = ldap_conn.users_in_primary_group( - usernames, project_obj.title) - ingroup = lambda u: u['username'] in users_main_group - users_no_removal, users_to_remove = sort_by(users_to_remove, ingroup, how="condition") + users_no_removal, users_to_remove = project_filter_users_to_remove.send( + sender=self.__class__, users=users_to_remove, project=project_obj + ) context = {} @@ -782,14 +776,9 @@ def post(self, request, *args, **kwargs): users_to_remove = self.get_users_to_remove(project_obj) # if ldap is activated, prevent selection of users with project corresponding to primary group - if 'coldfront.plugins.ldap' in settings.INSTALLED_APPS: - - usernames = [u['username'] for u in users_to_remove] - ldap_conn = LDAPConn() - users_main_group = ldap_conn.users_in_primary_group( - usernames, project_obj.title) - ingroup = lambda u: u['username'] in users_main_group - users_no_removal, users_to_remove = sort_by(users_to_remove, ingroup, how="condition") + users_no_removal, users_to_remove = project_filter_users_to_remove.send( + sender=self.__class__, users=users_to_remove, project=project_obj + ) formset = formset_factory(ProjectRemoveUserForm, max_num=len(users_to_remove)) formset = formset(request.POST, initial=users_to_remove, prefix='userform') diff --git a/coldfront/plugins/ldap/utils.py b/coldfront/plugins/ldap/utils.py index b1c8b7bd5b..84e3afd461 100644 --- a/coldfront/plugins/ldap/utils.py +++ b/coldfront/plugins/ldap/utils.py @@ -5,7 +5,6 @@ import operator from functools import reduce -from coldfront.core import field_of_science from django.db.models import Q from django.dispatch import receiver from django.utils import timezone @@ -15,6 +14,7 @@ from ldap3.extend.microsoft.removeMembersFromGroups import ad_remove_members_from_groups from coldfront.core.project.signals import ( + project_filter_users_to_remove, project_preremove_projectuser, project_make_projectuser, project_create, @@ -715,6 +715,16 @@ def update_new_project(sender, **kwargs): status=ProjectUserStatusChoice.objects.get(name='Active'), ) +@receiver(project_filter_users_to_remove) +def filter_project_users_to_remove(sender, **kwargs): + users_to_remove = kwargs['users_to_remove'] + usernames = [u['username'] for u in users_to_remove] + ldap_conn = LDAPConn() + users_main_group = ldap_conn.users_in_primary_group(usernames, kwargs['project'].title) + ingroup = lambda u: u['username'] in users_main_group + users_no_removal, users_to_remove = sort_by(users_to_remove, ingroup, how="condition") + return (users_no_removal, users_to_remove) + @receiver(project_make_projectuser) def add_user_to_group(sender, **kwargs): ldap_conn = LDAPConn() From 3ecbb682fff4e286342d1328877aafa666a8dbc3 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 21 May 2024 16:35:29 -0700 Subject: [PATCH 16/31] fix signal name --- coldfront/core/project/signals.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/coldfront/core/project/signals.py b/coldfront/core/project/signals.py index 04e5f2899e..c7982c2720 100644 --- a/coldfront/core/project/signals.py +++ b/coldfront/core/project/signals.py @@ -1,12 +1,11 @@ import django.dispatch - project_create = django.dispatch.Signal() #providing_args=["project_title"] project_post_create = django.dispatch.Signal() #providing_args=["project_obj"] -project_add_projectuser = django.dispatch.Signal() +project_make_projectuser = django.dispatch.Signal() #providing_args=["user_name", "group_name"] project_preremove_projectuser = django.dispatch.Signal() @@ -15,8 +14,3 @@ project_filter_users_to_remove = django.dispatch.Signal() #providing_args=["project_user_list"] # return tuple of (no_removal, can_remove) - -project_activate_user = django.dispatch.Signal() - #providing_args=["project_user_pk"] -project_remove_user = django.dispatch.Signal() - #providing_args=["project_user_pk"] From ada5fba699cc4a73c04746ba25ced2a4ce9117d5 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:56:33 -0400 Subject: [PATCH 17/31] add holylfs06 --- .../core/resource/management/commands/add_resource_defaults.py | 1 + 1 file changed, 1 insertion(+) diff --git a/coldfront/core/resource/management/commands/add_resource_defaults.py b/coldfront/core/resource/management/commands/add_resource_defaults.py index ee0cf09afb..a492978689 100644 --- a/coldfront/core/resource/management/commands/add_resource_defaults.py +++ b/coldfront/core/resource/management/commands/add_resource_defaults.py @@ -83,6 +83,7 @@ def handle(self, *args, **options): ('Tier 3', 'Attic Storage - Tape', True, storage_tier, None, 20, True, True), ('holylfs04/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1, True, True), ('holylfs05/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1, True, True), + ('holylfs06/tier0', 'Holyoke data center lustre storage', True, storage, 'Tier 0', 1, True, True), ('nesetape/tier3', 'Cold storage for past projects', True, storage, 'Tier 3', 20, True, True), ('holy-isilon/tier1', 'Tier1 storage with snapshots and disaster recovery copy', True, storage, 'Tier 1', 1, True, True), ('bos-isilon/tier1', 'Tier1 storage for on-campus storage mounting', True, storage, 'Tier 1', 1, True, True), From cb3a391bfa5e826238801fb44fa94ec38f996f97 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 4 Sep 2024 15:58:40 -0400 Subject: [PATCH 18/31] tier 2 gets billed --- .../commands/add_resource_defaults.py | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/coldfront/core/resource/management/commands/add_resource_defaults.py b/coldfront/core/resource/management/commands/add_resource_defaults.py index a492978689..d159b97037 100644 --- a/coldfront/core/resource/management/commands/add_resource_defaults.py +++ b/coldfront/core/resource/management/commands/add_resource_defaults.py @@ -88,23 +88,23 @@ def handle(self, *args, **options): ('holy-isilon/tier1', 'Tier1 storage with snapshots and disaster recovery copy', True, storage, 'Tier 1', 1, True, True), ('bos-isilon/tier1', 'Tier1 storage for on-campus storage mounting', True, storage, 'Tier 1', 1, True, True), ('holystore01/tier0', 'Luster storage under Tier0', True, storage, 'Tier 0', 1, True, True), - ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs11-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs12-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs13-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs14-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs15-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), - ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, False, True), + ('b-nfs02-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs03-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs04-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs05-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs06-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs07-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs08-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('b-nfs09-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs11-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs12-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs13-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs14-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs15-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs16-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs17-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs18-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), + ('h-nfs19-p/tier2', 'Tier2 CEPH storage', True, storage, 'Tier 2', 1, True, True), ('boslfs02', 'complimentary lab storage', True, storage, 'Tier 0', 1, False, False), ('holylabs', 'complimentary lab storage', True, storage, 'Tier 0', 1, False, False), ): From e87ffa61e0a5d9e922578c03136880e413ae268d Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:06:16 -0400 Subject: [PATCH 19/31] fix variable name --- coldfront/core/project/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index cb2452fd00..b223dc1061 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -754,7 +754,7 @@ def get(self, request, *args, **kwargs): # if ldap is activated, prevent selection of users with project corresponding to primary group users_no_removal, users_to_remove = project_filter_users_to_remove.send( - sender=self.__class__, users=users_to_remove, project=project_obj + sender=self.__class__, users_to_remove=users_to_remove, project=project_obj ) context = {} From 8103e2b9ccddc973be86f318a25a8c8eb1e82843 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 4 Sep 2024 16:56:36 -0400 Subject: [PATCH 20/31] update project_filter_users_to_remove signal processing --- coldfront/core/project/views.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index b223dc1061..f075189948 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -750,12 +750,14 @@ def get(self, request, *args, **kwargs): pk = self.kwargs.get('pk') project_obj = get_object_or_404(Project, pk=pk) users_to_remove = self.get_users_to_remove(project_obj) - users_no_removal = None # if ldap is activated, prevent selection of users with project corresponding to primary group - users_no_removal, users_to_remove = project_filter_users_to_remove.send( + signal_response = project_filter_users_to_remove.send( sender=self.__class__, users_to_remove=users_to_remove, project=project_obj ) + user_categories = signal_response[0][1] + users_no_removal = user_categories[0] + users_to_remove = user_categories[1] context = {} @@ -776,9 +778,12 @@ def post(self, request, *args, **kwargs): users_to_remove = self.get_users_to_remove(project_obj) # if ldap is activated, prevent selection of users with project corresponding to primary group - users_no_removal, users_to_remove = project_filter_users_to_remove.send( - sender=self.__class__, users=users_to_remove, project=project_obj + signal_response = project_filter_users_to_remove.send( + sender=self.__class__, users_to_remove=users_to_remove, project=project_obj ) + user_categories = signal_response[0][1] + users_no_removal = user_categories[0] + users_to_remove = user_categories[1] formset = formset_factory(ProjectRemoveUserForm, max_num=len(users_to_remove)) formset = formset(request.POST, initial=users_to_remove, prefix='userform') From 873125c6f24c387942f64b89fcd023033e4ff561 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Mon, 9 Sep 2024 14:27:58 -0400 Subject: [PATCH 21/31] work on testing --- coldfront/core/project/tests/test_views.py | 44 ++++++++++++++++++++-- coldfront/core/project/views.py | 23 +++++++---- coldfront/core/test_helpers/utils.py | 2 +- coldfront/plugins/system_monitor/utils.py | 2 +- 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/coldfront/core/project/tests/test_views.py b/coldfront/core/project/tests/test_views.py index 04e0519a31..32c617d750 100644 --- a/coldfront/core/project/tests/test_views.py +++ b/coldfront/core/project/tests/test_views.py @@ -1,6 +1,6 @@ import logging -from django.test import TestCase, tag +from django.test import TestCase, tag, override_settings from django.urls import reverse from coldfront.core.test_helpers import utils @@ -12,7 +12,11 @@ ProjectStatusChoiceFactory, ProjectAttributeTypeFactory, ) -from coldfront.core.project.models import Project, ProjectUserStatusChoice +from coldfront.core.project.models import ( + Project, ProjectUser, + ProjectUserRoleChoice, + ProjectUserStatusChoice +) logging.disable(logging.CRITICAL) @@ -161,7 +165,6 @@ def test_project_attribute_create_post_required_values(self): def test_project_attribute_create_value_type_match(self): """ProjectAttributeCreate correctly flags value-type mismatch""" - self.client.force_login(self.admin_user, backend='django.contrib.auth.backends.ModelBackend') # test that value must be numeric if proj_attr_type is string @@ -351,7 +354,7 @@ def test_projectnotecreateview_access(self): self.project_access_tstbase(self.url) -class ProjectAddUsersSearchView(ProjectViewTestBase): +class ProjectAddUsersSearchViewTest(ProjectViewTestBase): """Tests for ProjectAddUsersSearchView""" def setUp(self): """set up users and project for testing""" @@ -365,6 +368,39 @@ def test_projectadduserssearchview_access(self): utils.test_user_cannot_access(self, self.proj_datamanager, self.url)# data manager cannot access utils.test_user_cannot_access(self, self.proj_allocation_user, self.url)# user cannot access +class ProjectAddUsersViewTest(ProjectViewTestBase): + """Tests for ProjectAddUsersView""" + def setUp(self): + """set up users and project for testing""" + self.url = reverse('project-add-users', kwargs={'pk': self.project.pk}) + + @override_settings(PLUGIN_LDAP=True) + def test_projectaddusers_ldapsignalfail_messages(self): + """Test the messages displayed when the add user signal fails""" + self.client.force_login(self.pi_user) + + def test_add_users_form_validation(self): + """Test that the formset and allocation form are validated correctly""" + self.client.force_login(self.proj_accessmanager) + # Prepare form data for adding a user + form_data = { + 'q': 'search_user', + 'search_by': 'username', + 'userform-TOTAL_FORMS': '1', + 'userform-INITIAL_FORMS': '0', + 'userform-MIN_NUM_FORMS': '0', + 'userform-MAX_NUM_FORMS': '1', + 'userform-0-selected': 'on', + 'userform-0-role': ProjectUserRoleChoice.objects.get(name='User').pk, + 'userform-0-username': self.nonproj_allocation_user.username, + 'allocationform-allocation': [self.proj_allocation.pk] + } + response = self.client.post(self.url, data=form_data) + self.assertEqual(response.url, reverse('project-detail', kwargs={'pk': self.project.pk})) + self.assertEqual(response.status_code, 302) + # Check that user was added + self.assertTrue(ProjectUser.objects.filter(project=self.project, user=self.nonproj_allocation_user).exists()) + class ProjectUserDetailViewTest(ProjectViewTestBase): """Tests for ProjectUserDetailView""" diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index f075189948..7c7ed6dba5 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -749,15 +749,20 @@ def get_users_to_remove(self, project_obj): def get(self, request, *args, **kwargs): pk = self.kwargs.get('pk') project_obj = get_object_or_404(Project, pk=pk) - users_to_remove = self.get_users_to_remove(project_obj) + users_list = self.get_users_to_remove(project_obj) # if ldap is activated, prevent selection of users with project corresponding to primary group signal_response = project_filter_users_to_remove.send( - sender=self.__class__, users_to_remove=users_to_remove, project=project_obj + sender=self.__class__, users_to_remove=users_list, project=project_obj ) - user_categories = signal_response[0][1] - users_no_removal = user_categories[0] - users_to_remove = user_categories[1] + print('signal_response', signal_response) + if signal_response: + user_categories = signal_response[0][1] + users_no_removal = user_categories[0] + users_to_remove = user_categories[1] + else: + users_no_removal = users_list + users_to_remove = [] context = {} @@ -781,9 +786,11 @@ def post(self, request, *args, **kwargs): signal_response = project_filter_users_to_remove.send( sender=self.__class__, users_to_remove=users_to_remove, project=project_obj ) - user_categories = signal_response[0][1] - users_no_removal = user_categories[0] - users_to_remove = user_categories[1] + if signal_response: + user_categories = signal_response[0][1] + users_to_remove = user_categories[1] + else: + users_to_remove = users_to_remove formset = formset_factory(ProjectRemoveUserForm, max_num=len(users_to_remove)) formset = formset(request.POST, initial=users_to_remove, prefix='userform') diff --git a/coldfront/core/test_helpers/utils.py b/coldfront/core/test_helpers/utils.py index fe1c858c0d..8ce58c7337 100644 --- a/coldfront/core/test_helpers/utils.py +++ b/coldfront/core/test_helpers/utils.py @@ -1,5 +1,5 @@ """utility functions for unit and integration testing""" -from bs4 import BeautifulSoup +from beautifulsoup4 import BeautifulSoup def page_contains_for_user(test_case, user, url, text): """Check that page contains text for user""" diff --git a/coldfront/plugins/system_monitor/utils.py b/coldfront/plugins/system_monitor/utils.py index f07003b980..c4d7a50c94 100644 --- a/coldfront/plugins/system_monitor/utils.py +++ b/coldfront/plugins/system_monitor/utils.py @@ -1,7 +1,7 @@ import re import requests -from bs4 import BeautifulSoup +from beautifulsoup4 import BeautifulSoup from coldfront.core.utils.common import import_from_settings From f6481820ca32c81ce742031f5d71469e4291ced3 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Fri, 13 Sep 2024 20:35:44 -0700 Subject: [PATCH 22/31] fix addusers test --- coldfront/core/project/tests/test_views.py | 5 ++--- coldfront/core/project/views.py | 7 ++----- coldfront/core/test_helpers/utils.py | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/coldfront/core/project/tests/test_views.py b/coldfront/core/project/tests/test_views.py index 32c617d750..bd094bbed3 100644 --- a/coldfront/core/project/tests/test_views.py +++ b/coldfront/core/project/tests/test_views.py @@ -384,15 +384,14 @@ def test_add_users_form_validation(self): self.client.force_login(self.proj_accessmanager) # Prepare form data for adding a user form_data = { - 'q': 'search_user', - 'search_by': 'username', + 'q': self.nonproj_allocation_user.username, + 'search_by': 'username_only', 'userform-TOTAL_FORMS': '1', 'userform-INITIAL_FORMS': '0', 'userform-MIN_NUM_FORMS': '0', 'userform-MAX_NUM_FORMS': '1', 'userform-0-selected': 'on', 'userform-0-role': ProjectUserRoleChoice.objects.get(name='User').pk, - 'userform-0-username': self.nonproj_allocation_user.username, 'allocationform-allocation': [self.proj_allocation.pk] } response = self.client.post(self.url, data=form_data) diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index 7c7ed6dba5..9d54d2ca06 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -612,7 +612,6 @@ def post(self, request, *args, **kwargs): allocation_form = ProjectAddUsersToAllocationForm( request.user, project_obj.pk, request.POST, prefix='allocationform' ) - added_users_count = 0 if formset.is_valid() and allocation_form.is_valid(): @@ -672,7 +671,6 @@ def post(self, request, *args, **kwargs): 'status': projuserstatus_active, } ) - added_users_count += 1 for allocation in Allocation.objects.filter( pk__in=allocation_form_data @@ -755,14 +753,13 @@ def get(self, request, *args, **kwargs): signal_response = project_filter_users_to_remove.send( sender=self.__class__, users_to_remove=users_list, project=project_obj ) - print('signal_response', signal_response) if signal_response: user_categories = signal_response[0][1] users_no_removal = user_categories[0] users_to_remove = user_categories[1] else: - users_no_removal = users_list - users_to_remove = [] + users_no_removal = [] + users_to_remove = users_list context = {} diff --git a/coldfront/core/test_helpers/utils.py b/coldfront/core/test_helpers/utils.py index 8ce58c7337..fe1c858c0d 100644 --- a/coldfront/core/test_helpers/utils.py +++ b/coldfront/core/test_helpers/utils.py @@ -1,5 +1,5 @@ """utility functions for unit and integration testing""" -from beautifulsoup4 import BeautifulSoup +from bs4 import BeautifulSoup def page_contains_for_user(test_case, user, url, text): """Check that page contains text for user""" From 0e6c6c00b0761845364956d38137289777b4312d Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:10:02 -0700 Subject: [PATCH 23/31] add removeuser signal test --- coldfront/core/project/tests/test_views.py | 45 ++++++++++++++++------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/coldfront/core/project/tests/test_views.py b/coldfront/core/project/tests/test_views.py index bd094bbed3..fa65911c15 100644 --- a/coldfront/core/project/tests/test_views.py +++ b/coldfront/core/project/tests/test_views.py @@ -2,6 +2,7 @@ from django.test import TestCase, tag, override_settings from django.urls import reverse +from unittest.mock import patch from coldfront.core.test_helpers import utils from coldfront.core.test_helpers.factories import ( @@ -309,6 +310,16 @@ def test_pi_user_cannot_be_removed(self): users_to_remove = context['formset'].initial self.assertNotIn(self.pi_user.username, [u['username'] for u in users_to_remove]) + @patch('coldfront.core.project.signals.project_preremove_projectuser.send') + def test_projectremove_users_signal_fail(self, mock_signal): + """Test that the add users form fails when the signal sent to LDAP fails""" + self.client.force_login(self.proj_accessmanager) + mock_signal.side_effect = Exception("LDAP error occurred") + # Prepare form data for adding a user + response = self.client.post(self.url, data=self.form_data, follow=True) + self.assertContains(response, 'LDAP error occurred') + self.assertContains(response, 'Could not remove user') + class ProjectUpdateViewTest(ProjectViewTestBase): """Tests for ProjectUpdateView""" @@ -373,17 +384,7 @@ class ProjectAddUsersViewTest(ProjectViewTestBase): def setUp(self): """set up users and project for testing""" self.url = reverse('project-add-users', kwargs={'pk': self.project.pk}) - - @override_settings(PLUGIN_LDAP=True) - def test_projectaddusers_ldapsignalfail_messages(self): - """Test the messages displayed when the add user signal fails""" - self.client.force_login(self.pi_user) - - def test_add_users_form_validation(self): - """Test that the formset and allocation form are validated correctly""" - self.client.force_login(self.proj_accessmanager) - # Prepare form data for adding a user - form_data = { + self.form_data = { 'q': self.nonproj_allocation_user.username, 'search_by': 'username_only', 'userform-TOTAL_FORMS': '1', @@ -394,12 +395,32 @@ def test_add_users_form_validation(self): 'userform-0-role': ProjectUserRoleChoice.objects.get(name='User').pk, 'allocationform-allocation': [self.proj_allocation.pk] } - response = self.client.post(self.url, data=form_data) + + @override_settings(PLUGIN_LDAP=True) + def test_projectaddusers_ldapsignalfail_messages(self): + """Test the messages displayed when the add user signal fails""" + self.client.force_login(self.pi_user) + + def test_projectaddusers_form_validation(self): + """Test that the formset and allocation form are validated correctly""" + self.client.force_login(self.proj_accessmanager) + # Prepare form data for adding a user + response = self.client.post(self.url, data=self.form_data) self.assertEqual(response.url, reverse('project-detail', kwargs={'pk': self.project.pk})) self.assertEqual(response.status_code, 302) # Check that user was added self.assertTrue(ProjectUser.objects.filter(project=self.project, user=self.nonproj_allocation_user).exists()) + @patch('coldfront.core.project.signals.project_make_projectuser.send') + def test_projectaddusers_signal_fail(self, mock_signal): + """Test that the add users form fails when the signal sent to LDAP fails""" + self.client.force_login(self.proj_accessmanager) + mock_signal.side_effect = Exception("LDAP error occurred") + # Prepare form data for adding a user + response = self.client.post(self.url, data=self.form_data, follow=True) + self.assertContains(response, 'LDAP error occurred') + self.assertContains(response, 'Added 0 users') + class ProjectUserDetailViewTest(ProjectViewTestBase): """Tests for ProjectUserDetailView""" From 835037623e37341c5b719bf4acbf0da10ec5f88c Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:59:06 -0700 Subject: [PATCH 24/31] remove test --- coldfront/core/project/tests/test_views.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/coldfront/core/project/tests/test_views.py b/coldfront/core/project/tests/test_views.py index fa65911c15..9f3b29325a 100644 --- a/coldfront/core/project/tests/test_views.py +++ b/coldfront/core/project/tests/test_views.py @@ -310,16 +310,6 @@ def test_pi_user_cannot_be_removed(self): users_to_remove = context['formset'].initial self.assertNotIn(self.pi_user.username, [u['username'] for u in users_to_remove]) - @patch('coldfront.core.project.signals.project_preremove_projectuser.send') - def test_projectremove_users_signal_fail(self, mock_signal): - """Test that the add users form fails when the signal sent to LDAP fails""" - self.client.force_login(self.proj_accessmanager) - mock_signal.side_effect = Exception("LDAP error occurred") - # Prepare form data for adding a user - response = self.client.post(self.url, data=self.form_data, follow=True) - self.assertContains(response, 'LDAP error occurred') - self.assertContains(response, 'Could not remove user') - class ProjectUpdateViewTest(ProjectViewTestBase): """Tests for ProjectUpdateView""" From 6bbbe6b6cc915a59d89ea4bbf9826460b9037918 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:41:51 -0700 Subject: [PATCH 25/31] update test --- coldfront/core/project/tests/test_views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/coldfront/core/project/tests/test_views.py b/coldfront/core/project/tests/test_views.py index 9f3b29325a..b8873dacd6 100644 --- a/coldfront/core/project/tests/test_views.py +++ b/coldfront/core/project/tests/test_views.py @@ -391,9 +391,11 @@ def test_projectaddusers_ldapsignalfail_messages(self): """Test the messages displayed when the add user signal fails""" self.client.force_login(self.pi_user) - def test_projectaddusers_form_validation(self): + @patch('coldfront.core.project.signals.project_make_projectuser.send') + def test_projectaddusers_form_validation(self, mock_signal): """Test that the formset and allocation form are validated correctly""" self.client.force_login(self.proj_accessmanager) + mock_signal.return_value = None # Prepare form data for adding a user response = self.client.post(self.url, data=self.form_data) self.assertEqual(response.url, reverse('project-detail', kwargs={'pk': self.project.pk})) From d6838990d2e9cca9b91961c57dc39b762b37df1f Mon Sep 17 00:00:00 2001 From: claire-peters Date: Wed, 18 Sep 2024 01:31:20 -0400 Subject: [PATCH 26/31] update logging --- coldfront/core/project/views.py | 2 +- coldfront/plugins/ldap/utils.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index 9d54d2ca06..0b742687cb 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -828,7 +828,7 @@ def post(self, request, *args, **kwargs): request, f"could not remove user {user_obj}: {e}" ) - logger.error( + logger.exception( "P802: Coldfront user %s could NOT remove AD User for %s from AD Group for %s: %s", self.request.user, user_obj.username, diff --git a/coldfront/plugins/ldap/utils.py b/coldfront/plugins/ldap/utils.py index 84e3afd461..2afa305e2b 100644 --- a/coldfront/plugins/ldap/utils.py +++ b/coldfront/plugins/ldap/utils.py @@ -198,8 +198,9 @@ def add_member_to_group(self, member, group): except Exception as e: logger.exception("Error encountered while adding user to group: %s", e) raise LDAPUserAdditionError("Error adding user to group.") - if not self.member_in_group(member_dn, group_dn): + if not self.member_in_group(member_dn, group_dn) or not result: raise LDAPUserAdditionError("Member not successfully added to group.") + logger.info('user %s added to AD group %s', member_dn, group_dn) return result def remove_member_from_group(self, user_name, group_name): @@ -219,8 +220,9 @@ def remove_member_from_group(self, user_name, group_name): except Exception as e: logger.exception("Error encountered while removing user from group: %s", e) raise LDAPUserRemovalError("Error removing user from group.") - if self.member_in_group(user_dn, group_dn): + if self.member_in_group(user_dn, group_dn) or not result: raise LDAPUserRemovalError("Member not successfully removed from group.") + logger.info('user %s removed from AD group %s', user_dn, group_dn) return result def users_in_primary_group(self, usernames, groupname): From 3bc65a520bfbaeb77fdba3f44f3460135154b650 Mon Sep 17 00:00:00 2001 From: Aaron Kitzmiller Date: Mon, 23 Sep 2024 16:47:56 -0400 Subject: [PATCH 27/31] Update ifxbilling / fiine.client; improves synchronization (including organization changes), adds organization to facility codes for things like SEAS-specific codes in OAR Updated ifxuser (mostly updates for nice like demographic information) --- fiine.client | 2 +- ifxbilling | 2 +- ifxuser | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fiine.client b/fiine.client index c1eac23df0..624c1013ee 160000 --- a/fiine.client +++ b/fiine.client @@ -1 +1 @@ -Subproject commit c1eac23df0f218e512ea8dc207b78baef33deba4 +Subproject commit 624c1013ee3de8005a0de5e632447213d1c02b3f diff --git a/ifxbilling b/ifxbilling index 177760142b..66cb9d6942 160000 --- a/ifxbilling +++ b/ifxbilling @@ -1 +1 @@ -Subproject commit 177760142bf90f2fab5f313fae3c55c8a4328a72 +Subproject commit 66cb9d6942585eac2f2380496c4d8dbe4e11b638 diff --git a/ifxuser b/ifxuser index 30bf561df8..b6e01339ce 160000 --- a/ifxuser +++ b/ifxuser @@ -1 +1 @@ -Subproject commit 30bf561df8e58d83512d2979434c34b6f5a0d3de +Subproject commit b6e01339ce852c7eebbcfdfa624157968efc4425 From a0ab22769d477ed8b01b2bd5915907e1b0abe693 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:34:54 -0400 Subject: [PATCH 28/31] give department approvers user permissions for child projects --- coldfront/core/project/models.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/coldfront/core/project/models.py b/coldfront/core/project/models.py index b3c4f12909..fb8fb7dad2 100644 --- a/coldfront/core/project/models.py +++ b/coldfront/core/project/models.py @@ -9,6 +9,7 @@ from model_utils.models import TimeStampedModel from simple_history.models import HistoricalRecords +from coldfront.core.department.models import DepartmentProject from coldfront.core.field_of_science.models import FieldOfScience from coldfront.core.utils.common import import_from_settings @@ -214,6 +215,14 @@ def user_permissions(self, user): if self.pi.id == user.id: permissions.append(ProjectPermission.PI) + + # if the user is an approver in a department connected to the project, + # give them user permissions + department = DepartmentProject.objects.get(project=self).department + for parent_department in department.parents.filter(org_tree='Research Computing Storage Billing'): + if user in parent_department.useraffiliation_set.filter(role='approver'): + permissions.append(ProjectPermission.USER) + return permissions def has_perm(self, user, perm): From 9059cddb66b0a31fef9ee87d80a7b9cd8639b6d9 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 16 Oct 2024 15:57:25 -0400 Subject: [PATCH 29/31] fix circular import --- coldfront/core/project/models.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/coldfront/core/project/models.py b/coldfront/core/project/models.py index fb8fb7dad2..3e578ca145 100644 --- a/coldfront/core/project/models.py +++ b/coldfront/core/project/models.py @@ -9,7 +9,7 @@ from model_utils.models import TimeStampedModel from simple_history.models import HistoricalRecords -from coldfront.core.department.models import DepartmentProject +from coldfront.core.department.models import Department from coldfront.core.field_of_science.models import FieldOfScience from coldfront.core.utils.common import import_from_settings @@ -218,9 +218,12 @@ def user_permissions(self, user): # if the user is an approver in a department connected to the project, # give them user permissions - department = DepartmentProject.objects.get(project=self).department - for parent_department in department.parents.filter(org_tree='Research Computing Storage Billing'): - if user in parent_department.useraffiliation_set.filter(role='approver'): + departments = Department.objects.filter( + org_tree='Research Computing Storage Billing' + ) + proj_departments = [d for d in departments if self in d.get_projects()] + for department in proj_departments: + if user in department.useraffiliation_set.filter(role='approver'): permissions.append(ProjectPermission.USER) return permissions From e61f48bb5f5fe3f8101e753672b58f90489eab31 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Wed, 16 Oct 2024 16:02:46 -0400 Subject: [PATCH 30/31] fix circular import --- coldfront/core/project/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/coldfront/core/project/models.py b/coldfront/core/project/models.py index 3e578ca145..8cb7656629 100644 --- a/coldfront/core/project/models.py +++ b/coldfront/core/project/models.py @@ -9,7 +9,7 @@ from model_utils.models import TimeStampedModel from simple_history.models import HistoricalRecords -from coldfront.core.department.models import Department +from ifxuser.models import Organization from coldfront.core.field_of_science.models import FieldOfScience from coldfront.core.utils.common import import_from_settings @@ -218,7 +218,7 @@ def user_permissions(self, user): # if the user is an approver in a department connected to the project, # give them user permissions - departments = Department.objects.filter( + departments = Organization.objects.filter( org_tree='Research Computing Storage Billing' ) proj_departments = [d for d in departments if self in d.get_projects()] From f83f3c9ad513677aaca18f82e087bb3415e144d4 Mon Sep 17 00:00:00 2001 From: geistling <34081638+geistling@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:42:05 -0800 Subject: [PATCH 31/31] update project user removal query option signal --- coldfront/core/project/views.py | 22 ++++++++++------------ coldfront/plugins/ldap/utils.py | 7 ++++--- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/coldfront/core/project/views.py b/coldfront/core/project/views.py index 0b742687cb..1ccf29cd1c 100644 --- a/coldfront/core/project/views.py +++ b/coldfront/core/project/views.py @@ -754,13 +754,12 @@ def get(self, request, *args, **kwargs): sender=self.__class__, users_to_remove=users_list, project=project_obj ) if signal_response: - user_categories = signal_response[0][1] - users_no_removal = user_categories[0] - users_to_remove = user_categories[1] + users_to_remove = signal_response[0][1] else: - users_no_removal = [] users_to_remove = users_list + users_no_removal = [u for u in users_list if u not in users_to_remove] + context = {} if users_to_remove: @@ -784,8 +783,7 @@ def post(self, request, *args, **kwargs): sender=self.__class__, users_to_remove=users_to_remove, project=project_obj ) if signal_response: - user_categories = signal_response[0][1] - users_to_remove = user_categories[1] + users_to_remove = signal_response[0][1] else: users_to_remove = users_to_remove @@ -793,6 +791,7 @@ def post(self, request, *args, **kwargs): formset = formset(request.POST, initial=users_to_remove, prefix='userform') remove_users_count = 0 + failed_user_removals = [] if formset.is_valid(): projectuser_status_removed = ProjectUserStatusChoice.objects.get( @@ -818,18 +817,15 @@ def post(self, request, *args, **kwargs): user_name=user_obj.username, group_name=project_obj.title ) logger.info( - "P802: Coldfront user %s removed AD User for %s from AD Group for %s", + "P815: Coldfront user %s removed AD User for %s from AD Group for %s", self.request.user, user_obj.username, project_obj.title, ) except Exception as e: - messages.error( - request, - f"could not remove user {user_obj}: {e}" - ) + failed_user_removals += [f"could not remove user {user_obj}: {e}"] logger.exception( - "P802: Coldfront user %s could NOT remove AD User for %s from AD Group for %s: %s", + "P815: Coldfront user %s could NOT remove AD User for %s from AD Group for %s: %s", self.request.user, user_obj.username, project_obj.title, @@ -856,6 +852,8 @@ def post(self, request, *args, **kwargs): ) remove_users_count += 1 user_pl = 'user' if remove_users_count == 1 else 'users' + for fail in failed_user_removals: + messages.error(request, fail) messages.success( request, f'Removed {remove_users_count} {user_pl} from project.' ) diff --git a/coldfront/plugins/ldap/utils.py b/coldfront/plugins/ldap/utils.py index 2afa305e2b..812c0f9c96 100644 --- a/coldfront/plugins/ldap/utils.py +++ b/coldfront/plugins/ldap/utils.py @@ -723,9 +723,10 @@ def filter_project_users_to_remove(sender, **kwargs): usernames = [u['username'] for u in users_to_remove] ldap_conn = LDAPConn() users_main_group = ldap_conn.users_in_primary_group(usernames, kwargs['project'].title) - ingroup = lambda u: u['username'] in users_main_group - users_no_removal, users_to_remove = sort_by(users_to_remove, ingroup, how="condition") - return (users_no_removal, users_to_remove) + users_to_remove = [ + u for u in users_to_remove if u['username'] not in users_main_group + ] + return users_to_remove @receiver(project_make_projectuser) def add_user_to_group(sender, **kwargs):