geonetwork datadir checker useless ressources

docker/Dockerfile_celery

@@ -27,4 +27,4 @@ USER celery
 COPY . /geordash
 WORKDIR /geordash
 
-CMD ["sh", "-c", "python3 -m celery -A make_celery worker --loglevel INFO -P solo -B -E -s workdir/celerybeat-schedule -s celerybeat-schedule"]
+CMD ["sh", "-c", "python3 -m celery -A make_celery worker --loglevel INFO -P solo -B -E -s workdir/celerybeat-schedule"]

docker/Dockerfile_flask

@@ -5,6 +5,7 @@ ENV FLASK_APP=geordash
 ENV FLASK_OPTS="-h 0.0.0.0 -p 5002"
 ENV georchestradatadir=/etc/georchestra
 ENV REDISURL=""
+#ENV FLASK_DEBUG=1
 
 # set fixed UID and GID - see github.com/hexops/dockerfile
 ARG UID=10000

docker/docker-compose.yml

@@ -8,10 +8,12 @@ services:
   redis:
     image: "redis:alpine"
     command: redis-server
-
     environment:
       - REDIS_REPLICATION_MODE=master
+      -
   gaia-back:
+    # if you want to switch back to official image
+    #  image: "georchestra/gaia:latest"
     build:
       context: ../
       dockerfile: ./docker/Dockerfile_flask
@@ -65,6 +67,8 @@ services:
 
 
   gaia-celery:
+    # if you want to switch back to official image
+    # image: "georchestra/gaia-celery:latest"
     build:
       context: ../
       dockerfile: ./docker/Dockerfile_celery

geordash/__init__.py

@@ -11,6 +11,7 @@
 from geordash.georchestraconfig import GeorchestraConfig
 from geordash.result_backend.redisbackend import RedisClient
 from geordash.checks.mapstore import MapstoreChecker
+from geordash.checks.gn_datadir import GeonetworkDatadirChecker
 from geordash.decorators import is_superuser
 from config import url as redisurl
 import threading
@@ -86,6 +87,7 @@ def inject_globals():
     app.extensions["conf"] = conf
     app.extensions["owscache"] = OwsCapCache(conf, app)
     app.extensions["msc"] = MapstoreChecker(conf)
+    app.extensions["gndc"] = GeonetworkDatadirChecker(conf)
     app.extensions["rcli"] = RedisClient(redisurl)
     from . import views, api, admin, dashboard
 

geordash/admin.py

@@ -52,6 +52,17 @@ def geonetwork():
     return render_template("admin/geonetwork.html", portals=portals)
 
 
+@admin_bp.route("/geonetwork/datadir")
+@check_role(role="GN_ADMIN")
+def geonetwork_datadir():
+    all_jobs_for_gnconfigs = app.extensions["rcli"].get_taskids_by_taskname_and_args(
+        "geordash.checks.gn_datadir.check_gn_meta", []
+    )
+    return render_template(
+        "admin/geonetwork_datadir.html", previous_configs_jobs=all_jobs_for_gnconfigs
+    )
+
+
 @admin_bp.route("/geoserver")
 @check_role(role="ADMINISTRATOR")
 def geoserver():

geordash/celeryconfig.py.example

@@ -11,6 +11,7 @@ imports = (
     "geordash.checks.csw",
     "geordash.checks.mviewer",
     "geordash.checks.gsd",
+    "geordash.checks.gn_datadir",
 )
 # worker_pool = solo
 worker_log_format = (
@@ -56,7 +57,7 @@ beat_schedule = {
     "check-gs-datadir-every-sunday": {
         "task": "geordash.checks.gsd.gsdatadir",
         "args": [],
-        "schedule": crontab(day_of_week='sunday',minute=30, hour=1),
+        "schedule": crontab(day_of_week="sunday", minute=30, hour=1),
     },
     "check-gn-metadatas-every-night": {
         "task": "geordash.checks.csw.check_catalog",
@@ -68,6 +69,11 @@ beat_schedule = {
     #    "args": [],
     #    "schedule": crontab(minute=0, hour=1),
     # },
+    "check-gn-metadatadir-every-night": {
+        "task": "geordash.checks.gn_datadir.check_gn_meta",
+        "args": [""],
+        "schedule": crontab(minute=55, hour=0),
+    },
 }
 # otherwise scheduled hours is taken as UTC
 timezone = "Europe/Paris"

geordash/checks/gn_datadir.py

@@ -0,0 +1,107 @@
+#!/bin/env python3
+# -*- coding: utf-8 -*-
+# vim: ts=4 sw=4 et
+from celery import shared_task
+from geordash.logwrap import get_logger
+from flask import current_app as app
+from sqlalchemy import create_engine, MetaData, select, Table
+from sqlalchemy.engine import URL
+from sqlalchemy.ext.automap import automap_base
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import sessionmaker
+import glob
+from pathlib import Path
+
+
+def get_folder_size(folder):
+    return sum(file.stat().st_size for file in Path(folder).rglob("*"))
+
+
+class GeonetworkDatadirChecker:
+    def __init__(self, conf):
+        url = URL.create(
+            drivername="postgresql",
+            username=conf.get("jdbc.username", "geonetwork"),
+            host=conf.get("jdbc.host", "geonetwork"),
+            port=conf.get("jdbc.port", "geonetwork"),
+            password=conf.get("jdbc.password", "geonetwork"),
+            database=conf.get("jdbc.database", "geonetwork"),
+        )
+
+        engine = create_engine(
+            url,
+            connect_args={
+                "options": f"-csearch_path={conf.get('jdbc.schema', 'geonetwork')}"
+            },
+        )
+        self.sessionm = sessionmaker(bind=engine)
+        self.sessiono = self.sessionm()
+
+        # Perform database reflection to analyze tables and relationships
+        m = MetaData(schema=conf.get("jdbc.schema", "geonetwork"))
+        Table("metadata", m, autoload_with=engine)
+        Base = automap_base(metadata=m)
+        Base.prepare()
+        self.Metadata = Base.classes.metadata
+
+    def session(self):
+        try:
+            self.sessiono.execute(select(1))
+        except OperationalError:
+            print("Reconnecting to the database...")
+            self.sessiono = self.sessionm()
+        return self.sessiono
+
+    def get_meta_list(self):
+        return self.session().query(self.Metadata).all()
+
+
+@shared_task(bind=True)
+def check_gn_meta(self):
+    get_logger("CheckGNDatadir").debug("Start gn datadir checker")
+    metadatabase = app.extensions["gndc"]
+    gnmetadatas = metadatabase.get_meta_list()
+    geonetwork_dir_path = app.extensions["conf"].get("geonetwork.dir", "geonetwork")
+    geonetwork_datadir_path = (
+        app.extensions["conf"]
+        .get("geonetwork.data.dir", "geonetwork")
+        .replace("${geonetwork.dir}", geonetwork_dir_path)
+    )
+    # self.gnmetadatas.sort(key=lambda x: x.id)
+    meta = dict()
+    meta["searching_path"] = geonetwork_datadir_path
+    meta["problems"] = list()
+    total_could_be_deleted = 0
+    for foldermeta in glob.glob(geonetwork_datadir_path + "*/*"):
+        idmeta = foldermeta.split("/")[-1]
+        subpath = foldermeta.split("/")[-2]
+        get_logger("CheckGNDatadir").debug(foldermeta)
+        existing_index = 0
+
+        for index, item in enumerate(gnmetadatas):
+            if item.id == int(idmeta):
+                existing_index = index
+                break
+        if existing_index:
+            continue
+        else:
+            # append useless folder
+            meta["problems"].append(
+                {
+                    "url": subpath + "/" + idmeta,
+                    "problem": get_folder_size(foldermeta),
+                }
+            )
+            total_could_be_deleted += get_folder_size(foldermeta)
+    get_logger("CheckGNDatadir").debug("finish gn datadir checker")
+
+    if len(meta["problems"]) > 0:
+        meta["problems"].append(
+            {
+                "type": "UnusedFileResTotal",
+                "size": total_could_be_deleted,
+                "total": get_folder_size(geonetwork_datadir_path),
+            }
+        )
+
+    return meta

geordash/dashboard.py

@@ -6,6 +6,7 @@
 from flask import request, render_template, abort, url_for
 from flask import current_app as app
 import requests
+from functools import wraps
 
 from geordash.decorators import is_superuser, check_role
 from geordash.checks.mapstore import get_resources_using_ows, get_res
@@ -23,6 +24,16 @@
 )
 
 
+def debug_only(f):
+    @wraps(f)
+    def wrapped(**kwargs):
+        if not app.debug:
+            abort(404)
+        return f(**kwargs)
+
+    return wrapped
+
+
 def get_rescontent_from_resid(restype, resid):
     r = mapstore_get(request, f"rest/geostore/data/{resid}", False)
     res = dict()
@@ -67,6 +78,12 @@ def home():
     return render_template("home.html")
 
 
+@dash_bp.route("/debug")
+@debug_only
+def debug():
+    return app.extensions["conf"].tostr()
+
+
 @dash_bp.route("/my-metadata")
 @check_role(role="GN_EDITOR")
 def my_metadata():

geordash/georchestraconfig.py

@@ -50,6 +50,12 @@ def __init__(self):
             except:
                 # safe default value
                 self.sections["urls"]["localgs"] = "geoserver"
+
+        with open(f"{self.datadirpath}/geonetwork/geonetwork.properties") as lines:
+            lines = chain(("[section]",), lines)  # This line does the trick.
+            parser.read_file(lines)
+        self.sections["geonetwork"] = parser["section"]
+
         # read current commit from .git/HEAD which might lead to the branch tip
         prefix = getcwd() + "/.git/"
         self.sections["gaia"] = {"commit": None}
@@ -67,6 +73,24 @@ def __init__(self):
             # failed to read .git/HEAD or .git/refs/heads/* ?
             pass
 
+    def tostr(self):
+        str = ""
+        for key in self.sections:
+            str += key + ":\r\n<br>"
+            for key2 in self.sections[key]:
+                str += " \t&emsp;" + key2 + " : "
+                if self.sections[key][key2] == self.get(key2, section=key):
+                    str += " \t&emsp;" + self.sections[key][key2] + "\r\n<br> "
+                else:
+                    str += (
+                        " \t&emsp;"
+                        + self.sections[key][key2]
+                        + " = "
+                        + self.get(key2, section=key)
+                        + "\r\n<br> "
+                    )
+        return str
+
     def get(self, key, section="default"):
         if section not in self.sections:
             return None