fix: update .gitignore and enhance error handling in raster commands

Author: jkgeo

.gitignore

@@ -186,6 +186,7 @@ local_files/
 # Local settings - use env vars instead
 local_settings_dev.py
 local_settings_local.py
+local*settings.py
 
 # IDE
 .vscode

glam_api/glam/management/commands/get_new_rasters.py

@@ -1,8 +1,12 @@
 from django.core.management.base import BaseCommand, CommandError
 from django.conf import settings
+from botocore.exceptions import ClientError
 
 from glam.tasks import download_new, upload_files_from_directory
 from glam.ingest import add_product_rasters_from_storage
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 class Command(BaseCommand):
@@ -20,4 +24,19 @@ def handle(self, *args, **options):
         )
 
         self.stdout.write(f"Adding new Rasters to DB")
-        add_product_rasters_from_storage()
+        try:
+            add_product_rasters_from_storage()
+        except ClientError as e:
+            error_code = e.response.get("Error", {}).get("Code", "Unknown")
+            self.stdout.write(
+                self.style.ERROR(
+                    f"AWS Error ({error_code}): Failed to access S3 bucket. "
+                    f"Check your AWS credentials and permissions. Details: {str(e)}"
+                )
+            )
+            logger.error(f"S3 access error: {str(e)}")
+        except Exception as e:
+            self.stdout.write(
+                self.style.ERROR(f"Unexpected error while adding rasters: {str(e)}")
+            )
+            logger.error(f"Unexpected error: {str(e)}")

glam_api/glam/tasks.py

@@ -33,6 +33,11 @@ def upload_files_from_directory(directory, bucket, prefix=""):
     import os
     import tqdm
 
+    # Check if directory exists, if not create it
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+        return
+
     # Create S3 client
     s3_client = boto3.client("s3")
 
@@ -127,3 +132,66 @@ def download_new():
     logging.info(f"Total downloads: {len(downloads)}")
     logging.info(f"{downloads}")
     return downloads
+
+
+def download_new_by_product(product_id):
+
+    try:
+        product = Product.objects.get(product_id=product_id)
+
+        downloads = []
+
+        vi = None
+
+        try:
+            valid_product = Product.objects.get(product_id=product_id)
+            latest = (
+                ProductRaster.objects.filter(product=valid_product)
+                .order_by("-date")
+                .first()
+            )
+            logging.info(f"latest date for {product_id}: {latest.date.isoformat()}")
+
+        except Product.DoesNotExist:
+            return
+
+        parts = product_id.split("-")
+        if parts[-1] in ["ndvi", "ndwi"]:
+            vi = parts[-1].upper()
+            product = Downloader(parts[0].upper())
+        elif parts[-1] == "swi":
+            product = Downloader(parts[-1])
+        elif parts[-1] == "precip":
+            product = Downloader(parts[0])
+        elif parts[-1] == "esi":
+            product = Downloader(f"{parts[-1]}/{parts[-2].upper()}")
+
+        if valid_product.composite:
+            start_date = latest.date + timedelta(days=valid_product.composite_period)
+        else:
+            start_date = latest.date + timedelta(days=1)
+
+        end_date = latest.date + timedelta(days=30)
+
+        if not os.path.exists(settings.PRODUCT_DATASET_LOCAL_PATH):
+            os.makedirs(settings.PRODUCT_DATASET_LOCAL_PATH)
+
+        if vi:
+            out = product.download_vi_composites(
+                start_date.isoformat(),
+                end_date.isoformat(),
+                settings.PRODUCT_DATASET_LOCAL_PATH,
+                vi=vi,
+            )
+        else:
+            out = product.download_composites(
+                start_date.isoformat(),
+                end_date.isoformat(),
+                settings.PRODUCT_DATASET_LOCAL_PATH,
+            )
+        downloads.append(out)
+        logging.info(f"{product_id} files downloaded: {out}")
+    except Exception as e:
+        logging.error(f"Failed to download {product_id}: {e}")
+
+    return downloads

glam_api/glam/utils.py

@@ -69,9 +69,15 @@ def get_product_id_from_filename(filename):
         return "chirps-precip"
     elif "swi" in filename:
         return "copernicus-swi"
-    elif any(text in filename for text in ["dfppm_4wk", "dfppm-4wk"]):
+    elif any(
+        text in filename
+        for text in ["dfppm_4wk", "dfppm-4wk", "dfppm_aqua_4wk", "dfppm-aqua-4wk"]
+    ):
         return "servir-4wk-esi"
-    elif any(text in filename for text in ["dfppm_12wk", "dfppm-12wk"]):
+    elif any(
+        text in filename
+        for text in ["dfppm_12wk", "dfppm-12wk", "dfppm_aqua_12wk", "dfppm-aqua-12wk"]
+    ):
         return "servir-12wk-esi"
     elif any(text in filename for text in ["vnp09h1.ndvi", "vnp09h1-ndvi"]):
         return "vnp09h1-ndvi"