From 7ca259c7c25682f5de8d7a36d382f0539c47482e Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Tue, 27 Jun 2023 13:20:07 +0200 Subject: [PATCH 01/22] django changes: urlresolvers module was moved to django.urls --- src/jane/documents/admin.py | 2 +- src/jane/documents/models.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jane/documents/admin.py b/src/jane/documents/admin.py index d819a87..ae7ffa6 100644 --- a/src/jane/documents/admin.py +++ b/src/jane/documents/admin.py @@ -4,8 +4,8 @@ from django.conf.urls import url from django.contrib.gis import admin -from django.core.urlresolvers import reverse from django.http import HttpResponse +from django.urls import reverse from jane.documents import models diff --git a/src/jane/documents/models.py b/src/jane/documents/models.py index 42c2fd7..3688bbf 100644 --- a/src/jane/documents/models.py +++ b/src/jane/documents/models.py @@ -22,13 +22,13 @@ from django.contrib.gis.db import models from django.contrib.gis.measure import Distance from django.contrib.postgres.fields import jsonb -from django.core.urlresolvers import reverse from django.db import connection from django.db.models.aggregates import Count from django.db.models.expressions import OrderBy, RawSQL from django.http import Http404 from django.shortcuts import get_object_or_404 from django.template.defaultfilters import filesizeformat +from django.urls import reverse from djangoplugins.fields import PluginField, ManyPluginField from obspy.core.utcdatetime import UTCDateTime from rest_framework import status From b84914e406e0feccc4748566e0be9f3052abb163 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Tue, 27 Jun 2023 13:20:07 +0200 Subject: [PATCH 02/22] django changes: ForeignKey on_delete is now mandatory argument So fill in the default value, see https://docs.djangoproject.com/en/dev/releases/1.9/#foreignkey-and-onetoonefield-on-delete-argument --- src/jane/documents/migrations/0001_initial.py | 14 ++++++------- src/jane/documents/models.py | 21 ++++++++++++------- src/jane/waveforms/migrations/0001_initial.py | 12 +++++------ src/jane/waveforms/models.py | 18 ++++++++++------ 4 files changed, 39 insertions(+), 26 deletions(-) diff --git a/src/jane/documents/migrations/0001_initial.py b/src/jane/documents/migrations/0001_initial.py index 912d093..12d6ea2 100644 --- a/src/jane/documents/migrations/0001_initial.py +++ b/src/jane/documents/migrations/0001_initial.py @@ -27,7 +27,7 @@ class Migration(migrations.Migration): ('sha1', models.CharField(unique=True, db_index=True, max_length=40, editable=False)), ('created_at', models.DateTimeField(auto_now_add=True)), ('modified_at', models.DateTimeField(auto_now=True)), - ('created_by', models.ForeignKey(related_name='documents_created', to=settings.AUTH_USER_MODEL)), + ('created_by', models.ForeignKey(related_name='documents_created', on_delete=models.CASCADE, to=settings.AUTH_USER_MODEL)), ], options={ 'verbose_name': 'Document', @@ -41,7 +41,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(verbose_name='ID', auto_created=True, primary_key=True, serialize=False)), ('json', jsonb.JSONField(verbose_name='JSON')), ('geometry', django.contrib.gis.db.models.fields.GeometryCollectionField(srid=4326, geography=True, blank=True, null=True)), - ('document', models.ForeignKey(related_name='indices', to='documents.Document')), + ('document', models.ForeignKey(related_name='indices', to='documents.Document', on_delete=models.CASCADE)), ], options={ 'verbose_name': 'Index', @@ -58,9 +58,9 @@ class Migration(migrations.Migration): ('data', models.BinaryField()), ('created_at', models.DateTimeField(auto_now_add=True)), ('modified_at', models.DateTimeField(auto_now=True)), - ('created_by', models.ForeignKey(related_name='attachments_created', to=settings.AUTH_USER_MODEL)), - ('index', models.ForeignKey(related_name='attachments', to='documents.DocumentIndex')), - ('modified_by', models.ForeignKey(related_name='attachments_modified', to=settings.AUTH_USER_MODEL)), + ('created_by', models.ForeignKey(related_name='attachments_created', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), + ('index', models.ForeignKey(related_name='attachments', to='documents.DocumentIndex', on_delete=models.CASCADE)), + ('modified_by', models.ForeignKey(related_name='attachments_modified', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE)), ], options={ 'verbose_name': 'Attachment', @@ -87,12 +87,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='document', name='document_type', - field=models.ForeignKey(related_name='documents', to='documents.DocumentType'), + field=models.ForeignKey(related_name='documents', to='documents.DocumentType', on_delete=models.CASCADE), ), migrations.AddField( model_name='document', name='modified_by', - field=models.ForeignKey(related_name='documents_modified', to=settings.AUTH_USER_MODEL), + field=models.ForeignKey(related_name='documents_modified', to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE), ), migrations.AlterUniqueTogether( name='document', diff --git a/src/jane/documents/models.py b/src/jane/documents/models.py index 3688bbf..9be82f8 100644 --- a/src/jane/documents/models.py +++ b/src/jane/documents/models.py @@ -171,7 +171,8 @@ class Document(models.Model): any type that is described by indices. """ # The type of the document. Depends on the available Jane plug-ins. - document_type = models.ForeignKey(DocumentType, related_name='documents') + document_type = models.ForeignKey(DocumentType, related_name='documents', + on_delete=models.CASCADE) # The name of that particular document. Oftentimes the filename. Unique # together with the document type to enable a nice REST API. name = models.CharField(max_length=255, db_index=True) @@ -189,9 +190,11 @@ class Document(models.Model): modified_at = models.DateTimeField(auto_now=True, editable=False) # Users responsible for the aforementioned actions. created_by = models.ForeignKey(settings.AUTH_USER_MODEL, - related_name='documents_created') + related_name='documents_created', + on_delete=models.CASCADE) modified_by = models.ForeignKey(settings.AUTH_USER_MODEL, - related_name='documents_modified') + related_name='documents_modified', + on_delete=models.CASCADE) objects = DocumentManager() class Meta: @@ -504,7 +507,8 @@ class DocumentIndex(models.Model): """ Indexed values for a specific document. """ - document = models.ForeignKey(Document, related_name='indices') + document = models.ForeignKey(Document, related_name='indices', + on_delete=models.CASCADE) json = jsonb.JSONField(verbose_name="JSON") geometry = models.GeometryCollectionField(blank=True, null=True, geography=True) @@ -653,7 +657,8 @@ class DocumentIndexAttachment(models.Model): """ Attachments for one Document. """ - index = models.ForeignKey(DocumentIndex, related_name='attachments') + index = models.ForeignKey(DocumentIndex, related_name='attachments', + on_delete=models.CASCADE) category = models.CharField(max_length=50, db_index=True) content_type = models.CharField(max_length=255) data = models.BinaryField() @@ -663,9 +668,11 @@ class DocumentIndexAttachment(models.Model): modified_at = models.DateTimeField(auto_now=True, editable=False) # Users responsible for the aforementioned actions. created_by = models.ForeignKey(settings.AUTH_USER_MODEL, - related_name='attachments_created') + related_name='attachments_created', + on_delete=models.CASCADE) modified_by = models.ForeignKey(settings.AUTH_USER_MODEL, - related_name='attachments_modified') + related_name='attachments_modified', + on_delete=models.CASCADE) objects = DocumentIndexAttachmentManager() class Meta: diff --git a/src/jane/waveforms/migrations/0001_initial.py b/src/jane/waveforms/migrations/0001_initial.py index 172bb34..68f4d64 100644 --- a/src/jane/waveforms/migrations/0001_initial.py +++ b/src/jane/waveforms/migrations/0001_initial.py @@ -69,8 +69,8 @@ class Migration(migrations.Migration): ('file_regex', models.CharField(max_length=255, blank=True)), ('created_at', models.DateTimeField(auto_now_add=True)), ('modified_at', models.DateTimeField(auto_now=True)), - ('created_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, editable=False, related_name='mappings_created')), - ('modified_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, editable=False, related_name='mappings_modified')), + ('created_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, editable=False, related_name='mappings_created')), + ('modified_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, editable=False, related_name='mappings_modified')), ], options={ 'ordering': ['-timerange'], @@ -97,20 +97,20 @@ class Migration(migrations.Migration): ('comment', models.TextField(null=True, blank=True)), ('created_at', models.DateTimeField(auto_now_add=True)), ('modified_at', models.DateTimeField(auto_now=True)), - ('created_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, editable=False, related_name='restrictions_created')), - ('modified_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, editable=False, related_name='restrictions_modified')), + ('created_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, editable=False, related_name='restrictions_created')), + ('modified_by', models.ForeignKey(null=True, to=settings.AUTH_USER_MODEL, on_delete=models.CASCADE, editable=False, related_name='restrictions_modified')), ('users', models.ManyToManyField(db_index=True, to=settings.AUTH_USER_MODEL)), ], ), migrations.AddField( model_name='file', name='path', - field=models.ForeignKey(related_name='files', to='waveforms.Path'), + field=models.ForeignKey(related_name='files', to='waveforms.Path', on_delete=models.CASCADE), ), migrations.AddField( model_name='continuoustrace', name='file', - field=models.ForeignKey(related_name='traces', to='waveforms.File'), + field=models.ForeignKey(related_name='traces', to='waveforms.File', on_delete=models.CASCADE), ), migrations.AlterUniqueTogether( name='file', diff --git a/src/jane/waveforms/models.py b/src/jane/waveforms/models.py index 7fedcfa..bc63470 100644 --- a/src/jane/waveforms/models.py +++ b/src/jane/waveforms/models.py @@ -40,7 +40,8 @@ def save(self, *args, **kwargs): class File(models.Model): - path = models.ForeignKey(Path, related_name='files') + path = models.ForeignKey( + Path, related_name='files', on_delete=models.CASCADE) name = models.CharField(max_length=255, db_index=True) size = models.IntegerField() gaps = models.IntegerField(default=0, db_index=True) @@ -71,7 +72,8 @@ def save(self, *args, **kwargs): class ContinuousTrace(models.Model): - file = models.ForeignKey(File, related_name='traces') + file = models.ForeignKey( + File, related_name='traces', on_delete=models.CASCADE) pos = models.IntegerField(default=0) network = models.CharField(max_length=2, db_index=True, blank=True) station = models.CharField(max_length=5, db_index=True, blank=True) @@ -198,10 +200,12 @@ class Mapping(models.Model): max_length=255, blank=False, default=r"^.*$") created_at = models.DateTimeField(auto_now_add=True, editable=False) created_by = models.ForeignKey(User, null=True, editable=False, - related_name='mappings_created') + related_name='mappings_created', + on_delete=models.CASCADE) modified_at = models.DateTimeField(auto_now=True, editable=False) modified_by = models.ForeignKey(User, null=True, editable=False, - related_name='mappings_modified') + related_name='mappings_modified', + on_delete=models.CASCADE) def __str__(self): return "%s.%s.%s.%s | %s ==> %s.%s.%s.%s" % ( @@ -241,10 +245,12 @@ class Restriction(models.Model): comment = models.TextField(blank=True, null=True) created_at = models.DateTimeField(auto_now_add=True, editable=False) created_by = models.ForeignKey(User, null=True, editable=False, - related_name='restrictions_created') + related_name='restrictions_created', + on_delete=models.CASCADE) modified_at = models.DateTimeField(auto_now=True, editable=False) modified_by = models.ForeignKey(User, null=True, editable=False, - related_name='restrictions_modified') + related_name='restrictions_modified', + on_delete=models.CASCADE) def save(self, *args, **kwargs): # ensure uppercase and no whitespaces around network/station ids From 3801465ddbabc69e368c16e5833f92bf1ee23838 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Tue, 27 Jun 2023 13:20:07 +0200 Subject: [PATCH 03/22] django update: GeoManager was removed likely have to adjust code for this removal more.. --- src/jane/documents/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jane/documents/models.py b/src/jane/documents/models.py index 9be82f8..7580f65 100644 --- a/src/jane/documents/models.py +++ b/src/jane/documents/models.py @@ -231,7 +231,7 @@ def save(self, *args, **kwargs): signals.index_document(sender=None, instance=self, created=None) -class DocumentIndexManager(models.GeoManager): +class DocumentIndexManager(models.Manager): """ Custom queryset manager for the document indices. """ From 158531627d89fc0907250a1714d43469a0052c46 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Tue, 27 Jun 2023 13:20:07 +0200 Subject: [PATCH 04/22] django update: include() helper in urls module changed syntax --- src/jane/urls.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/jane/urls.py b/src/jane/urls.py index 591c27c..1823439 100644 --- a/src/jane/urls.py +++ b/src/jane/urls.py @@ -10,7 +10,8 @@ urlpatterns = [ - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', + include(admin.site.urls[:2], namespace=admin.site.urls[2])), url(r'', include('jane.waveforms.urls')), url(r'', include('jane.documents.urls')), url(r'', include('jane.jane.urls')), From 46898e428e4be1567c8289fe499460b14941b098 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Tue, 27 Jun 2023 13:20:07 +0200 Subject: [PATCH 05/22] django rest framework change: some decorators got combined in a new one and removed --- src/jane/waveforms/views.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jane/waveforms/views.py b/src/jane/waveforms/views.py index 0fcf174..804be1a 100644 --- a/src/jane/waveforms/views.py +++ b/src/jane/waveforms/views.py @@ -3,7 +3,7 @@ from rest_framework import viewsets, renderers, filters from rest_framework.response import Response -from rest_framework.decorators import detail_route +from rest_framework.decorators import action from jane.waveforms import models, serializer @@ -48,12 +48,12 @@ def get_queryset(self): filter_backends = (filters.OrderingFilter,) ordering_fields = '__all__' - @detail_route(renderer_classes=[PNGRenderer]) + @action(detail=True, renderer_classes=[PNGRenderer]) def plot(self, request, *args, **kwargs): obj = self.get_object() return Response(obj.preview_image) - @detail_route(renderer_classes=[BinaryRenderer]) + @action(detail=True, renderer_classes=[BinaryRenderer]) def file(self, request, *args, **kwargs): file_obj = self.get_object().file filename = os.path.join(file_obj.path.name, file_obj.name) From d432bebb61075f4be662f1ba59d209a4c9382f1f Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Tue, 27 Jun 2023 13:20:07 +0200 Subject: [PATCH 06/22] django rest framework change: minor change in API --- src/jane/documents/urls.py | 6 +++--- src/jane/waveforms/urls.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jane/documents/urls.py b/src/jane/documents/urls.py index b1e6c70..4092119 100644 --- a/src/jane/documents/urls.py +++ b/src/jane/documents/urls.py @@ -37,15 +37,15 @@ router = OptionalTrailingSlashSimpleRouter(trailing_slash=False) router.register(prefix='documents/(?P[a-zA-Z0-9]+)', viewset=views.DocumentsView, - base_name="rest_documents") + basename="rest_documents") router.register(prefix='document_indices/(?P[a-zA-Z0-9]+)', viewset=views.DocumentIndicesView, - base_name="rest_document_indices") + basename="rest_document_indices") router.register( prefix=('document_indices/(?P[a-zA-Z0-9]+)/(?P[0-9]+)' '/attachments'), viewset=views.DocumentIndexAttachmentsView, - base_name="rest_document_index_attachments") + basename="rest_document_index_attachments") urlpatterns.append(url(r'^rest/', include(router.urls))) diff --git a/src/jane/waveforms/urls.py b/src/jane/waveforms/urls.py index fe8c256..01ddcbb 100644 --- a/src/jane/waveforms/urls.py +++ b/src/jane/waveforms/urls.py @@ -6,7 +6,7 @@ router = OptionalTrailingSlashSimpleRouter(trailing_slash=False) -router.register(r'waveforms', views.WaveformView, base_name='rest_waveforms') +router.register(r'waveforms', views.WaveformView, basename='rest_waveforms') urlpatterns = [ url(r'^rest/', include(router.urls)), From 3cb3995254bfaade80b774a9ecc2b05143303277 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Tue, 27 Jun 2023 17:25:30 +0200 Subject: [PATCH 07/22] django update: try to adjust to middleware changes see https://docs.djangoproject.com/en/4.2/topics/http/middleware/#upgrading-pre-django-1-10-style-middleware --- src/jane/jane/middleware.py | 45 +++++++++++++++++++++---------------- src/jane/settings.py | 2 +- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/src/jane/jane/middleware.py b/src/jane/jane/middleware.py index 2c91e89..0e01f77 100644 --- a/src/jane/jane/middleware.py +++ b/src/jane/jane/middleware.py @@ -6,27 +6,34 @@ class AutoLogoutMiddleware(object): - def process_request(self, request): + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + set_last_touch = True + # can't log out if not logged in if not request.user.is_authenticated(): - return - + # do nothing + set_last_touch = False # check if auto logout is activated - try: - if not settings.AUTO_LOGOUT_MINUTES: - return - except: - return - - try: - delta = time.time() - request.session['last_touch'] - except (KeyError, TypeError): - pass + elif not settings.AUTO_LOGOUT_MINUTES: + # do nothing + set_last_touch = False else: - seconds = settings.AUTO_LOGOUT_MINUTES * 60 - if delta > seconds: - del request.session['last_touch'] - auth.logout(request) - return + try: + delta = time.time() - request.session['last_touch'] + except (KeyError, TypeError): + pass + else: + seconds = settings.AUTO_LOGOUT_MINUTES * 60 + if delta > seconds: + del request.session['last_touch'] + auth.logout(request) + set_last_touch = False + + if set_last_touch: + request.session['last_touch'] = time.time() - request.session['last_touch'] = time.time() + response = self.get_response(request) + return response diff --git a/src/jane/settings.py b/src/jane/settings.py index e5b9d13..e337219 100644 --- a/src/jane/settings.py +++ b/src/jane/settings.py @@ -77,7 +77,7 @@ }] -MIDDLEWARE_CLASSES = [ +MIDDLEWARE = [ 'django.middleware.gzip.GZipMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', From ca800881c761dfb12be344e16c687223d6919e5a Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 29 Jun 2023 10:27:59 +0200 Subject: [PATCH 08/22] django changes: validators need to be specified as objects ..not as strings anymore. Not sure if it works in the migration like that or if it might be safer to duplicate the code and put it in the migration.. --- src/jane/waveforms/migrations/0001_initial.py | 4 +++- src/jane/waveforms/models.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/jane/waveforms/migrations/0001_initial.py b/src/jane/waveforms/migrations/0001_initial.py index 68f4d64..fc37a6d 100644 --- a/src/jane/waveforms/migrations/0001_initial.py +++ b/src/jane/waveforms/migrations/0001_initial.py @@ -6,6 +6,8 @@ import django.contrib.postgres.fields from django.conf import settings +from jane.waveforms.models import validate_name + class Migration(migrations.Migration): @@ -79,7 +81,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Path', fields=[ - ('name', models.CharField(max_length=255, validators=['validate_name'], serialize=False, primary_key=True)), + ('name', models.CharField(max_length=255, validators=[validate_name], serialize=False, primary_key=True)), ('ctime', models.DateTimeField()), ('mtime', models.DateTimeField()), ('created_at', models.DateTimeField(auto_now_add=True)), diff --git a/src/jane/waveforms/models.py b/src/jane/waveforms/models.py index bc63470..faf003a 100644 --- a/src/jane/waveforms/models.py +++ b/src/jane/waveforms/models.py @@ -13,9 +13,16 @@ User = settings.AUTH_USER_MODEL +def validate_name(value): + if not os.path.isdir(value): + raise ValidationError(u'%s is not a valid path' % value) + if not os.path.isabs(value): + raise ValidationError(u'%s is not a absolute path' % value) + + class Path(models.Model): name = models.CharField(max_length=255, primary_key=True, - validators=['validate_name']) + validators=[validate_name]) ctime = models.DateTimeField() mtime = models.DateTimeField() created_at = models.DateTimeField(auto_now_add=True, editable=False) @@ -26,12 +33,6 @@ def __str__(self): class Meta: ordering = ['name'] - def validate_name(self, value): - if not os.path.isdir(value): - raise ValidationError(u'%s is not a valid path' % value) - if not os.path.isabs(value): - raise ValidationError(u'%s is not a absolute path' % value) - def save(self, *args, **kwargs): stats = os.stat(self.name) self.mtime = to_datetime(stats.st_mtime) From 2d7b970a35aac772a5fa67876cc8bb0727afeb5d Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 29 Jun 2023 10:45:10 +0200 Subject: [PATCH 09/22] django update: specify DEFAULT_AUTO_FIELD in settings to avoid django warnings. new in django 3.2 (?) https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field --- src/jane/settings.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/jane/settings.py b/src/jane/settings.py index e337219..9d962f1 100644 --- a/src/jane/settings.py +++ b/src/jane/settings.py @@ -19,6 +19,15 @@ PROJECT_DIR = os.path.abspath(os.path.dirname(__file__)) +# new in django 3.2 (?), creates warnings if not set anywhere +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field +# this is old behavior of having int32 as automatic primary key fields. Other +# option could be "BigAutoField" which is int64 but that would need a migration +# (at least if changed on an existing project) +# also see https://dev.to/weplayinternet/upgrading-to-django-3-2-and-fixing-defaultautofield-warnings-518n +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' + + # Hosts/domain names that are valid for this site; required if DEBUG is False # See https://docs.djangoproject.com/en/1.5/ref/settings/#allowed-hosts ALLOWED_HOSTS = ['*'] From 1db003d30b096a832e9fa54c5bbaeb47e78fe353 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 29 Jun 2023 11:06:41 +0200 Subject: [PATCH 10/22] django update: JSONField from contrib is deprecated https://docs.djangoproject.com/en/3.1/releases/3.1/#jsonfield-for-all-supported-database-backends --- src/jane/documents/migrations/0001_initial.py | 4 ++-- src/jane/documents/models.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jane/documents/migrations/0001_initial.py b/src/jane/documents/migrations/0001_initial.py index 12d6ea2..68a4c20 100644 --- a/src/jane/documents/migrations/0001_initial.py +++ b/src/jane/documents/migrations/0001_initial.py @@ -3,8 +3,8 @@ from django.conf import settings import django.contrib.gis.db.models.fields -from django.contrib.postgres.fields import jsonb from django.db import models, migrations +from django.db.models import JSONField import djangoplugins.fields @@ -39,7 +39,7 @@ class Migration(migrations.Migration): name='DocumentIndex', fields=[ ('id', models.AutoField(verbose_name='ID', auto_created=True, primary_key=True, serialize=False)), - ('json', jsonb.JSONField(verbose_name='JSON')), + ('json', JSONField(verbose_name='JSON')), ('geometry', django.contrib.gis.db.models.fields.GeometryCollectionField(srid=4326, geography=True, blank=True, null=True)), ('document', models.ForeignKey(related_name='indices', to='documents.Document', on_delete=models.CASCADE)), ], diff --git a/src/jane/documents/models.py b/src/jane/documents/models.py index 7580f65..82e9f62 100644 --- a/src/jane/documents/models.py +++ b/src/jane/documents/models.py @@ -21,8 +21,8 @@ from django.contrib.auth.models import AnonymousUser from django.contrib.gis.db import models from django.contrib.gis.measure import Distance -from django.contrib.postgres.fields import jsonb from django.db import connection +from django.db.models import JSONField from django.db.models.aggregates import Count from django.db.models.expressions import OrderBy, RawSQL from django.http import Http404 @@ -509,7 +509,7 @@ class DocumentIndex(models.Model): """ document = models.ForeignKey(Document, related_name='indices', on_delete=models.CASCADE) - json = jsonb.JSONField(verbose_name="JSON") + json = JSONField(verbose_name="JSON") geometry = models.GeometryCollectionField(blank=True, null=True, geography=True) From c4b59f6101dbde5ff18d1a275da542d009e93393 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 29 Jun 2023 12:45:11 +0200 Subject: [PATCH 11/22] django changes: many-to-many direct assignment not allowed anymore --- src/jane/documents/plugins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/jane/documents/plugins.py b/src/jane/documents/plugins.py index ce3d582..529d076 100644 --- a/src/jane/documents/plugins.py +++ b/src/jane/documents/plugins.py @@ -171,9 +171,9 @@ def initialize_plugins(): indexer=indexer) resource_type.save() - resource_type.validators = validators - resource_type.retrieve_permissions = retrieve_permissions - resource_type.upload_permissions = upload_permissions + resource_type.validators.set(validators) + resource_type.retrieve_permissions.set(retrieve_permissions) + resource_type.upload_permissions.set(upload_permissions) resource_type.save() # Permissions. From b7c5a322e3b2ee052387c318be62bf873c15fd39 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 29 Jun 2023 13:22:04 +0200 Subject: [PATCH 12/22] django changes: user.is_authenticated() method removed use it as a property instead --- src/jane/fdsnws/views/dataselect_1.py | 2 +- src/jane/fdsnws/views/station_1.py | 2 +- src/jane/jane/decorators.py | 2 +- src/jane/jane/middleware.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/jane/fdsnws/views/dataselect_1.py b/src/jane/fdsnws/views/dataselect_1.py index 1ffcb48..fe4524c 100644 --- a/src/jane/fdsnws/views/dataselect_1.py +++ b/src/jane/fdsnws/views/dataselect_1.py @@ -156,7 +156,7 @@ def query(request): return _error(request, msg) longestonly = bool(longestonly) # user - if request.user.is_authenticated(): + if request.user.is_authenticated: user = request.user else: user = None diff --git a/src/jane/fdsnws/views/station_1.py b/src/jane/fdsnws/views/station_1.py index 995c340..67371a6 100644 --- a/src/jane/fdsnws/views/station_1.py +++ b/src/jane/fdsnws/views/station_1.py @@ -250,7 +250,7 @@ def query(request): url = request.build_absolute_uri(request.get_full_path()) # user - if request.user.is_authenticated(): + if request.user.is_authenticated: user = request.user else: user = None diff --git a/src/jane/jane/decorators.py b/src/jane/jane/decorators.py index 1549547..77432d5 100644 --- a/src/jane/jane/decorators.py +++ b/src/jane/jane/decorators.py @@ -81,7 +81,7 @@ def your_view: def view_decorator(func): def wrapper(request, *args, **kwargs): return view_or_basicauth(func, request, - lambda u: u.is_authenticated(), + lambda u: u.is_authenticated, realm, *args, **kwargs) return wrapper return view_decorator diff --git a/src/jane/jane/middleware.py b/src/jane/jane/middleware.py index 0e01f77..5c47acf 100644 --- a/src/jane/jane/middleware.py +++ b/src/jane/jane/middleware.py @@ -13,7 +13,7 @@ def __call__(self, request): set_last_touch = True # can't log out if not logged in - if not request.user.is_authenticated(): + if not request.user.is_authenticated: # do nothing set_last_touch = False # check if auto logout is activated From 24efb4565e08427e08a8a16b031fa2fa7d6c0b96 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 29 Jun 2023 13:26:59 +0200 Subject: [PATCH 13/22] django changes: {% load staticfiles %} was removed {% load staticfiles %} and {% load admin_static %} were deprecated in django 2.1 and later removed, in favor of {% load static %}, which works the same. https://docs.djangoproject.com/en/2.2/releases/2.1/#miscellaneous-1 --- src/jane/jane/templates/jane/base.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jane/jane/templates/jane/base.html b/src/jane/jane/templates/jane/base.html index a6845df..539ee80 100644 --- a/src/jane/jane/templates/jane/base.html +++ b/src/jane/jane/templates/jane/base.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} From 986b07e1ac261bde1adfd13b00be899ca21ffa82 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 29 Jun 2023 14:45:25 +0200 Subject: [PATCH 14/22] mkdocs update: adjust config yaml to some config syntax changes also fall back to one of the two builtin themes, cosmo is not shipped in Debian packages that we use for mkdocs now --- docs/mkdocs.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index 84a0b6c..cf21609 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -1,9 +1,10 @@ site_name: Jane repo_url: https://github.com/krischer/jane/ use_directory_urls: false -theme: cosmo +theme: + name: mkdocs copyright: Jane Developers, 2014-2016 -pages: +nav: - 'Home': 'index.md' - 'Introduction': 'introduction.md' - 'Details': From 74d0347999fcf4b538772af28cfe9a8931c43097 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 11 Apr 2024 16:21:20 +0200 Subject: [PATCH 15/22] add logging to file and and option to set a custom logfile in local_settings --- src/jane/local_settings.py.example | 2 ++ src/jane/settings.py | 22 +++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/jane/local_settings.py.example b/src/jane/local_settings.py.example index aa0d0ee..eef124c 100644 --- a/src/jane/local_settings.py.example +++ b/src/jane/local_settings.py.example @@ -93,6 +93,8 @@ JANE_ACCENT_COLOR = "#D9230F" JANE_FDSN_STATIONXML_SENDER = "Jane" JANE_FDSN_STATIONXML_SOURCE = "Jane" +# optionally set a custom logfile destination, overriding the default value from settings.py +#LOGFILE = '/home/sysop/logs/jane-rotational.log' # Change the settings for the test database here! # for pytest-django, this did not work for me: diff --git a/src/jane/settings.py b/src/jane/settings.py index 9d962f1..b13fd01 100644 --- a/src/jane/settings.py +++ b/src/jane/settings.py @@ -144,6 +144,12 @@ 'class': 'logging.StreamHandler', 'formatter': 'simple', }, + 'file': { + 'level': 'DEBUG', + 'class': 'logging.FileHandler', + 'filename': '/tmp/jane.log', + 'formatter': 'verbose', + }, 'mail_admins': { 'level': 'ERROR', 'filters': ['require_debug_false'], @@ -152,18 +158,23 @@ }, }, 'loggers': { + 'jane': { + 'handlers': ['file'], + 'propagate': True, + 'level': 'DEBUG', + }, 'django': { - 'handlers': ['console'], + 'handlers': ['console', 'file'], 'propagate': True, 'level': 'WARN', }, 'django.db.backends': { - 'handers': ['console'], + 'handers': ['console', 'file'], 'level': 'WARN', 'propagate': True, }, 'django.request': { - 'handlers': ['mail_admins'], + 'handlers': ['mail_admins', 'file'], 'level': 'ERROR', 'propagate': False, }, @@ -296,6 +307,11 @@ def show_toolbar(request): "local_settings.py and edit its content before running this " "service.") exit() +else: + try: + LOGGING['handlers']['file']['filename'] = LOGFILE + except: + pass # speed up tests From 4be05efc1d10b5ed8c5b7e73c31bcff629dbaa9e Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Wed, 8 Nov 2023 13:08:29 +0100 Subject: [PATCH 16/22] GIS: remove Stamen layers, make OpenTopoMap default Stamen layers not freely available anymore --- src/jane/static/web_gis/src/directives/map.js | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/jane/static/web_gis/src/directives/map.js b/src/jane/static/web_gis/src/directives/map.js index db71ea2..86a6094 100644 --- a/src/jane/static/web_gis/src/directives/map.js +++ b/src/jane/static/web_gis/src/directives/map.js @@ -43,26 +43,26 @@ app.directive('openlayers3', function($q, $log, bing_key, $modal) { crossOrigin: null }) }), - "Stamen Toner-Lite": new ol.layer.Tile({ - visible: false, - source: new ol.source.Stamen({layer: 'toner-lite'}) - }), - "Stamen Toner": new ol.layer.Tile({ - visible: false, - source: new ol.source.Stamen({layer: 'toner'}) - }), - "Stamen Watercolor": new ol.layer.Tile({ - visible: false, - source: new ol.source.Stamen({layer: 'watercolor'}) - }), - "MapQuest (Street)": new ol.layer.Tile({ - visible: false, - source: new ol.source.MapQuest({layer: 'osm'}) - }), - "MapQuest (Satellite)": new ol.layer.Tile({ - visible: false, - source: new ol.source.MapQuest({layer: 'sat'}) - }), + //"Stamen Toner-Lite": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.Stamen({layer: 'toner-lite'}) + //}), + //"Stamen Toner": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.Stamen({layer: 'toner'}) + //}), + //"Stamen Watercolor": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.Stamen({layer: 'watercolor'}) + //}), + //"MapQuest (Street)": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.MapQuest({layer: 'osm'}) + //}), + //"MapQuest (Satellite)": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.MapQuest({layer: 'sat'}) + //}), "Open Street Map": new ol.layer.Tile({ visible: false, source: new ol.source.OSM() @@ -73,35 +73,35 @@ app.directive('openlayers3', function($q, $log, bing_key, $modal) { url: '//{a-c}.tile.openstreetmap.fr/hot/{z}/{x}/{y}.png', crossOrigin: null }) - }), - "OSM EU TOPO (4umaps.eu)": new ol.layer.Tile({ - visible: false, - source: new ol.source.OSM({ - url: '//4umaps.eu/{z}/{x}/{y}.png', - crossOrigin: null - }) - }), - "Bing (Road)": new ol.layer.Tile({ - visible: false, - source: new ol.source.BingMaps({ - key: bing_key, - imagerySet: 'Road' - }) - }), - "Bing (Aerial)": new ol.layer.Tile({ - visible: false, - source: new ol.source.BingMaps({ - key: bing_key, - imagerySet: 'Aerial' - }) - }), - "Bing (Aerial with Labels)": new ol.layer.Tile({ - visible: false, - source: new ol.source.BingMaps({ - key: bing_key, - imagerySet: 'AerialWithLabels' - }) }) + //"OSM EU TOPO (4umaps.eu)": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.OSM({ + // url: '//4umaps.eu/{z}/{x}/{y}.png', + // crossOrigin: null + // }) + //}), + //"Bing (Road)": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.BingMaps({ + // key: bing_key, + // imagerySet: 'Road' + // }) + //}), + //"Bing (Aerial)": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.BingMaps({ + // key: bing_key, + // imagerySet: 'Aerial' + // }) + //}), + //"Bing (Aerial with Labels)": new ol.layer.Tile({ + // visible: false, + // source: new ol.source.BingMaps({ + // key: bing_key, + // imagerySet: 'AerialWithLabels' + // }) + //}) }; $scope.change_base_layer = function(layer_name) { From d71a90513d6342b3e828f2b08c0cc17146c16c93 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 11 Apr 2024 17:04:01 +0200 Subject: [PATCH 17/22] webgis: reduce default opacity after opentopomap being the default which is quite heavily colored and obscures events/stations --- src/jane/static/web_gis/src/baynetapp.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jane/static/web_gis/src/baynetapp.js b/src/jane/static/web_gis/src/baynetapp.js index 55ef142..8b925bb 100644 --- a/src/jane/static/web_gis/src/baynetapp.js +++ b/src/jane/static/web_gis/src/baynetapp.js @@ -213,7 +213,7 @@ module.controller("BayNetController", function($scope, $log, stations, station_c }; $scope.rotation = 0; - $scope.base_layer_opacity = 100.0; + $scope.base_layer_opacity = 60.0; $scope.show_bavaria_outline = false; From e5af7987f22e54c5da28b4ab0b63ec408986b32c Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Fri, 28 Jun 2024 15:21:57 +0200 Subject: [PATCH 18/22] geopy/postgis: adjust to API changes --- src/jane/quakeml/plugins.py | 2 +- src/jane/static/web_gis/src/baynetapp.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jane/quakeml/plugins.py b/src/jane/quakeml/plugins.py index 2180780..f18b98b 100644 --- a/src/jane/quakeml/plugins.py +++ b/src/jane/quakeml/plugins.py @@ -217,7 +217,7 @@ def index(self, document): (horizontal_uncertainty_min, horizontal_uncertainty_max_azimuth + 90)): azimuth = azimuth % 180 - distance = geopy.distance.VincentyDistance( + distance = geopy.distance.geodesic( kilometers=distance / 1e3) end1 = distance.destination( point=start, bearing=azimuth) diff --git a/src/jane/static/web_gis/src/baynetapp.js b/src/jane/static/web_gis/src/baynetapp.js index 8b925bb..c361a33 100644 --- a/src/jane/static/web_gis/src/baynetapp.js +++ b/src/jane/static/web_gis/src/baynetapp.js @@ -82,7 +82,7 @@ module.factory('events', function($http, $log, jane_server) { "coordinates": [j.longitude, j.latitude]}, { "type": "MultiLineString", - "coordinates": i.geometry.coordinates[1]} + "coordinates": i.geometry.geometries[1].coordinates} ]} }; return geojson; From 17b349611782d9dda76580a7fc8ca5254d0b68c8 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Fri, 28 Jun 2024 15:27:43 +0200 Subject: [PATCH 19/22] webgis html: remove a superfluous closing tag --- src/jane/static/web_gis/index.html | 1 - 1 file changed, 1 deletion(-) diff --git a/src/jane/static/web_gis/index.html b/src/jane/static/web_gis/index.html index 7f63b37..9a4b7e4 100644 --- a/src/jane/static/web_gis/index.html +++ b/src/jane/static/web_gis/index.html @@ -92,7 +92,6 @@

Jane WebGIS

margin-top: -2px; margin-left: 50px"> -
From 462b6bb7e31a7913d96d720419ff74d8398ee61d Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Fri, 25 Oct 2024 13:38:47 +0200 Subject: [PATCH 20/22] add new migration needed due to newer postgresql version on Debian bookworm?! migration was needed on clean system right after cloning the jane repo and applying the existing migrations --- .../migrations/0003_auto_20241025_1136.py | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 src/jane/documents/migrations/0003_auto_20241025_1136.py diff --git a/src/jane/documents/migrations/0003_auto_20241025_1136.py b/src/jane/documents/migrations/0003_auto_20241025_1136.py new file mode 100644 index 0000000..a7404f6 --- /dev/null +++ b/src/jane/documents/migrations/0003_auto_20241025_1136.py @@ -0,0 +1,41 @@ +# Generated by Django 3.2.19 on 2024-10-25 11:36 + +from django.db import migrations +import django.db.models.deletion +import djangoplugins.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('djangoplugins', '0002_add_id_to_plugins_uniq'), + ('documents', '0002_auto_20161018_0646'), + ] + + operations = [ + migrations.AlterField( + model_name='documenttype', + name='definition', + field=djangoplugins.fields.PluginField(limit_choices_to={'point__pythonpath': 'jane.documents.plugins.DocumentPluginPoint'}, on_delete=django.db.models.deletion.CASCADE, related_name='definition', to='djangoplugins.plugin'), + ), + migrations.AlterField( + model_name='documenttype', + name='indexer', + field=djangoplugins.fields.PluginField(limit_choices_to={'point__pythonpath': 'jane.documents.plugins.IndexerPluginPoint'}, on_delete=django.db.models.deletion.CASCADE, related_name='indexer', to='djangoplugins.plugin'), + ), + migrations.AlterField( + model_name='documenttype', + name='retrieve_permissions', + field=djangoplugins.fields.ManyPluginField(blank=True, limit_choices_to={'point__pythonpath': 'jane.documents.plugins.RetrievePermissionPluginPoint'}, related_name='retrieve_permissions', to='djangoplugins.Plugin'), + ), + migrations.AlterField( + model_name='documenttype', + name='upload_permissions', + field=djangoplugins.fields.ManyPluginField(blank=True, limit_choices_to={'point__pythonpath': 'jane.documents.plugins.UploadPermissionPluginPoint'}, related_name='upload_permissions', to='djangoplugins.Plugin'), + ), + migrations.AlterField( + model_name='documenttype', + name='validators', + field=djangoplugins.fields.ManyPluginField(blank=True, limit_choices_to={'point__pythonpath': 'jane.documents.plugins.ValidatorPluginPoint'}, related_name='validators', to='djangoplugins.Plugin'), + ), + ] From d82ec5f4510d28622256f4f4b1ba85274be835aa Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Fri, 7 Feb 2025 13:56:08 +0100 Subject: [PATCH 21/22] local settings example: add note on max upload size in config --- src/jane/local_settings.py.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/jane/local_settings.py.example b/src/jane/local_settings.py.example index eef124c..9bc3b94 100644 --- a/src/jane/local_settings.py.example +++ b/src/jane/local_settings.py.example @@ -33,6 +33,12 @@ DEFAULT_FROM_EMAIL = 'noreply@localhost' AUTO_LOGOUT_MINUTES = 60 +# size of http request body is limited to 2.5MB by django by default +# if large files need to be uploaded (e.g. station metadata grouped in one +# file) this needs to be set higher here or upload will fail. +#DATA_UPLOAD_MAX_MEMORY_SIZE = 94371840 # 90 MB + + # see https://docs.djangoproject.com/en/dev/topics/cache/#setting-up-the-cache CACHE = { 'default': { From 7e111cca5ff498439da5894f5c2f483d9ffaa1c6 Mon Sep 17 00:00:00 2001 From: Tobias Megies Date: Thu, 27 Nov 2025 11:08:28 +0100 Subject: [PATCH 22/22] FDSNWS: fix a bug in event contributor view The view errored out if any agency was indexed as None. Need to force a str() cast for potential None values --- src/jane/fdsnws/views/event_1.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jane/fdsnws/views/event_1.py b/src/jane/fdsnws/views/event_1.py index 1b8b6a2..7af6d1a 100644 --- a/src/jane/fdsnws/views/event_1.py +++ b/src/jane/fdsnws/views/event_1.py @@ -263,7 +263,7 @@ def contributors(request): # @UnusedVariable xml = E.Contributors( E.total(str(len(values))), - *[E.Contributor(_i) for _i in values] + *[E.Contributor(str(_i)) for _i in values] ) with io.BytesIO() as fh: