Minio Migrations

.example.env

@@ -45,6 +45,10 @@ SMTP_HOST_USER=
 SMTP_PORT=
 
 # Minio
+# Make sure the access key could read, write, and delete object
 MINIO_ROOT_USER=minisass_admin
 MINIO_ROOT_PASSWORD=secure_minio_secret
-BUCKET=demo
+MINIO_ACCESS_KEY=xJBv0IxgtGViYqH6n6Sg
+MINIO_SECRET_KEY=jw9Dxc2tAuUz5G1dXgn5lTHJvaInnJv5ML0pshki
+BUCKET=minisass
+MINIO_ENDPOINT=http://minio:9000

deployment/docker/requirements.txt

@@ -63,4 +63,6 @@ sentry-sdk==1.14.0
 geopandas==0.14.1
 pandas==2.1.3
 numpy==1.26.2
-factory-boy==3.3.0
+factory-boy==3.3.0
+django-storages==1.14.2
+boto3==1.34.16

django_project/minisass/migrations/0007_alter_mobileapp_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-01-10 02:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('minisass', '0006_mobileapp'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='mobileapp',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

django_project/minisass/settings.py

@@ -237,3 +237,21 @@
 )
 
 AUTHENTICATION_BACKENDS = ['minisass_authentication.backends.EmailBackend']
+
+DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
+
+MINIO_ACCESS_KEY = os.getenv("MINIO_ACCESS_KEY")
+MINIO_SECRET_KEY = os.getenv("MINIO_SECRET_KEY")
+MINIO_BUCKET_NAME = os.getenv("MINIO_BUCKET")
+MINIO_ENDPOINT = os.getenv("MINIO_ENDPOINT")
+
+AWS_ACCESS_KEY_ID = MINIO_ACCESS_KEY
+AWS_SECRET_ACCESS_KEY = MINIO_SECRET_KEY
+AWS_STORAGE_BUCKET_NAME = MINIO_BUCKET_NAME
+AWS_S3_ENDPOINT_URL = MINIO_ENDPOINT
+AWS_DEFAULT_ACL = None
+AWS_QUERYSTRING_AUTH = True
+AWS_S3_FILE_OVERWRITE = False
+
+# Set storage to use MINIO
+DEFAULT_FILE_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"

django_project/minisass/utils.py

@@ -1,4 +1,6 @@
 import os
+import boto3
+from django.conf import settings
 
 # Absolute filesystem path to the Django project directory:
 DJANGO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -12,5 +14,22 @@ def absolute_path(*args):
 def delete_file_field(file_field):
     """Delete actual file from file_field."""
     if file_field:
-        if os.path.isfile(file_field.path):
-            os.remove(file_field.path)
+        try:
+            if os.path.isfile(file_field.path):
+                os.remove(file_field.path)
+        except Exception:
+            pass
+
+        try:
+            s3 = boto3.client(
+                's3',
+                endpoint_url=settings.AWS_S3_ENDPOINT_URL,
+                aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
+                aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY
+            )
+            s3.delete_object(
+                Bucket=settings.AWS_STORAGE_BUCKET_NAME,
+                Key=file_field.name,
+            )
+        except Exception:
+            pass

django_project/minisass_authentication/migrations/0011_alter_passwordhistory_options.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.7 on 2024-01-10 02:31
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('minisass_authentication', '0010_passwordhistory'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='passwordhistory',
+            options={'verbose_name_plural': 'Password Histories'},
+        ),
+    ]

django_project/minisass_authentication/migrations/0012_alter_passwordhistory_id.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.7 on 2024-01-10 02:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('minisass_authentication', '0011_alter_passwordhistory_options'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='passwordhistory',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

django_project/monitor/admin.py

@@ -36,9 +36,10 @@ class ObservationsAdmin(admin.ModelAdmin):
         'minisass_ml_score',
         'ml_model_version',
         'ml_model_type',
-        'flag'
+        'flag',
+        'is_validated'
     )
-    list_filter = ('flag',)
+    list_filter = ('flag', 'is_validated')
     search_fields = ('site__site_name', 'site__river_name')
     actions = [make_verified, make_unverified]
     inlines = (ObservationPestImageInline,)

django_project/monitor/management/__init__.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+"""
+GeoSight is UNICEF's geospatial web-based business intelligence platform.
+
+Contact : geosight-no-reply@unicef.org
+
+.. note:: This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+"""
+__author__ = 'zakki@kartoza.com'
+__date__ = '11/01/24'
+__copyright__ = ('Copyright 2023, Unicef')

django_project/monitor/management/commands/__init__.py

@@ -0,0 +1,15 @@
+# coding=utf-8
+"""
+GeoSight is UNICEF's geospatial web-based business intelligence platform.
+
+Contact : geosight-no-reply@unicef.org
+
+.. note:: This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation; either version 3 of the License, or
+    (at your option) any later version.
+
+"""
+__author__ = 'zakki@kartoza.com'
+__date__ = '11/01/24'
+__copyright__ = ('Copyright 2023, Unicef')

django_project/monitor/management/commands/migrate_image_data.py

@@ -0,0 +1,75 @@
+# import os.path
+#
+# import boto3
+# from django.core.management.base import BaseCommand
+# from django.conf import settings
+# from monitor.models import Observations, ObservationPestImage, SiteImage, site_image_path, observation_pest_image_path
+#
+#
+# class Command(BaseCommand):
+#     help = 'Migrate images data from current minIO to https://minio.do.kartoza.com'
+#
+#     def add_arguments(self, parser):
+#         parser.add_argument('--copy-image', type=str, help='Copy image to  new location')
+#         parser.add_argument('--update-path', type=str, help='Update image path to new location')
+    #     parser.add_argument('-b', '--bucket', type=str, help='MinIO Bucket')
+    #     parser.add_argument('-u', '--url', type=str, help='MinIO endpoint URL')
+    #     parser.add_argument('-k', '--access-key', type=str, help='MinIO access key')
+    #     parser.add_argument('-s', '--secret-key', type=str, help='MinIO secret key')
+
+    # def handle(self, *args, **kwargs):
+        # true_words = ['1', 'yes', 'no']
+        #
+        # # setup minIO client
+        # copy_image = kwargs.get('copy_image')
+        # copy_image = 'false' if not copy_image else copy_image
+        # if copy_image.lower() in true_words:
+        #     s3 = boto3.client(
+        #         's3',
+        #         endpoint_url=settings.MINIO_ENDPOINT,
+        #         aws_access_key_id=settings.MINIO_ACCESS_KEY,
+        #         aws_secret_access_key=settings.MINIO_SECRET_KEY
+        #     )
+        #
+        #
+        #
+        # update_path = kwargs.get('update_path')
+        # update_path = 'false' if not update_path else update_path
+        # if update_path.lower() in true_words:
+        #     # for image in SiteImage.objects.all():
+        #     #     old_path = image.image.name.replace('demo/', 'old_minisass/')
+        #     #     old_path = os.path.join('minisass', old_path)
+        #     #     new_path = site_image_path(image, os.path.basename(old_path))
+        #     #     print(new_path)
+        #         # print(image.image.name)
+        #
+        #         # image.save()
+        #
+        #     # for image in ObservationPestImage.objects.all():
+        #     for image in ObservationPestImage.objects.filter(id=16):
+        #         old_path = image.image.name.replace('demo/', 'old_minisass/')
+        #         # image.image.name = os.path.join('minisass', old_path)
+        #         print(old_path)
+        #         # image.save()
+
+        # for image in ObservationPestImage.objects.all():
+        #     image.image.name = 'demo/1416/2519/demoiselle-4857850_640.jpg'
+        #     image.save()
+            # if image.observation.user.userprofile.is_expert:
+            #     image.valid = True
+            # print(image.image.name)
+            # old_path = image.image.name.replace('demo/', 'old_minisass/')
+            # old_path = os.path.join('minisass', old_path)
+            # print(old_path)
+            # print('---')
+            # image.image.name = old_path
+
+            # print(old_path, len(old_path))
+            # image.save()
+
+        # for image in SiteImage.objects.all():
+        #     old_path = image.image.name.replace('demo/', 'old_minisass/')
+        #     old_path = os.path.join('minisass', old_path)
+        #     image.image.name = old_path
+        #     print(len(old_path))
+            # image.save()

django_project/monitor/migrations/0006_alter_observations_options_alter_sites_options_and_more.py

@@ -0,0 +1,50 @@
+# Generated by Django 4.2.7 on 2024-01-11 00:05
+
+from django.db import migrations, models
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('monitor', '0005_merge_20231211_1322'),
+    ]
+
+    operations = [
+        migrations.AlterModelOptions(
+            name='observations',
+            options={'verbose_name_plural': 'Observations'},
+        ),
+        migrations.AlterModelOptions(
+            name='sites',
+            options={'verbose_name_plural': 'Sites'},
+        ),
+        migrations.AddField(
+            model_name='observationpestimage',
+            name='valid',
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AddField(
+            model_name='observations',
+            name='is_validated',
+            field=models.BooleanField(default=False, help_text='Flag whether observation correctness has been validated'),
+        ),
+        migrations.AlterField(
+            model_name='observationpestimage',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='observations',
+            name='flag',
+            field=models.CharField(choices=[('dirty', 'Dirty'), ('clean', 'Clean')], default='dirty', help_text='Flag whether observation comes from expert or novice', max_length=5),
+        ),
+        migrations.AlterField(
+            model_name='pest',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='siteimage',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]

django_project/monitor/migrations/0007_alter_observationpestimage_image_and_more.py

@@ -0,0 +1,49 @@
+# Generated by Django 4.2.7 on 2024-01-11 09:13
+
+import os
+from django.db import migrations, models
+import monitor.models
+
+
+def update_observation_validity(apps, schema_editor):
+    Observation = apps.get_model('monitor', 'Observations')
+    ObservationPestImage = apps.get_model('monitor', 'ObservationPestImage')
+    SiteImage = apps.get_model('monitor', 'SiteImage')
+
+    for observation in Observation.objects.all():
+        if observation.user.userprofile.is_expert:
+            observation.is_validated = True
+            observation.save()
+
+    for image in ObservationPestImage.objects.all():
+        if image.observation.user.userprofile.is_expert:
+            image.valid = True
+        old_path = image.image.name.replace('demo/', 'old_minisass/')
+        image.image.name = old_path
+        image.save()
+
+    for image in SiteImage.objects.all():
+        old_path = image.image.name.replace('demo/', 'old_minisass/')
+        image.image.name = old_path
+        image.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('monitor', '0006_alter_observations_options_alter_sites_options_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='observationpestimage',
+            name='image',
+            field=models.ImageField(max_length=250, upload_to=monitor.models.observation_pest_image_path),
+        ),
+        migrations.AlterField(
+            model_name='siteimage',
+            name='image',
+            field=models.ImageField(max_length=250, upload_to=monitor.models.site_image_path),
+        ),
+        migrations.RunPython(update_observation_validity, lambda a, b: None),
+    ]