Add Kibana log link to the requests table dashboard

helm/servicex/values.yaml

@@ -289,7 +289,8 @@ logging:
     port: 5959
     protocol: TCP
     monitor: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/c2cc1f30-4a5b-11ed-afcf-d91dad577662?embed=true&_g=(filters%3A!()%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A0)%2Ctime%3A(from%3Anow-24h%2Fh%2Cto%3Anow))&show-time-filter=true"
-    logs: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662?embed=true&_g=(filters%3A!(('%24state'%3A(store%3AglobalState)%2Cmeta%3A(alias%3A!n%2Cdisabled%3A!f%2Cindex%3A'923eaa00-45b9-11ed-afcf-d91dad577662'%2Ckey%3Ainstance%2Cnegate%3A!f%2Cparams%3A(query%3Aservicex)%2Ctype%3Aphrase)%2Cquery%3A(term%3A(instance%3A{{ .Release.Name }}))))%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A0)%2Ctime%3A(from%3Anow-24h%2Fh%2Cto%3Anow))&show-query-input=true&show-time-filter=true&hide-filter-bar=true"
+    # This URL format is tied to a parser in servicex_app/servicex_app/web/kibana_url_filter.py
+    logs: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),time:(from:now-24h%2Fh,to:now))&_a=(filters:!((query:(match_phrase:(instance:{{.Release.Name}})))),index:'923eaa00-45b9-11ed-afcf-d91dad577662')"
 minio:
   image:
     repository: bitnamilegacy/minio

servicex_app/servicex_app/__init__.py

@@ -379,6 +379,12 @@ def inject_modules():
 
             import humanize
 
-            return dict(datetime=datetime, humanize=humanize)
+            from servicex_app.web import create_kibana_link
+
+            return dict(
+                datetime=datetime,
+                humanize=humanize,
+                create_kibana_link=create_kibana_link,
+            )
 
     return app

servicex_app/servicex_app/templates/requests_table.html

@@ -21,6 +21,9 @@
     {% for req in pagination.items %}
     <tr>
       <th scope="row" style="word-break: break-all">
+        <a href="{{ create_kibana_link(req.request_id, "ERROR") }}" target="_blank">
+        <img src="{{ url_for('static', filename='logs.svg') }}" height="25" alt="Get Logs" />
+        </a>
         <a href="/transformation-request/{{ req.id }}">{{ req.title or "Untitled" }}</a>
       </th>
 

servicex_app/servicex_app/templates/transformation_request.html

@@ -61,6 +61,10 @@ <h4 class="mb-3 col-auto">Transformation Request</h4>
     <dd class="col-sm-9">{{ req.failure_description }} s</dd>
     {% endif %}
 
+      <dt class="col-sm-3"><a href="{{ create_kibana_link(req.request_id, "ERROR") }}" target="_blank">
+          <img src="{{ url_for('static', filename='logs.svg') }}" height="25" alt="Get Logs" />
+          See Logs</a></dt>
+
   </dl>
 </div>
 {% endblock %}

servicex_app/servicex_app/web/__init__.py

@@ -0,0 +1,7 @@
+from flask import current_app
+from .kibana_url_filter import filter_kibana_url
+
+
+def create_kibana_link(transform_id=None, log_level="INFO"):
+    log_url = current_app.config["LOGS_URL"]
+    return filter_kibana_url(log_url, transform_id, log_level)

servicex_app/servicex_app/web/kibana_url_filter.py

@@ -0,0 +1,37 @@
+"""
+Kibana URL Filter Modifier
+
+This module provides functionality to add requestID filters to Kibana dashboard URLs.
+"""
+
+import re
+import urllib.parse
+
+from urllib.parse import urlunparse
+
+
+def filter_kibana_url(url: str, request_id: str, log_level: str) -> str:
+    """
+    Add a filter to a Kibana dashboard URL to show only results for a given request ID
+    with an ERROR level or higher.
+    """
+    decoded_url = urllib.parse.urlparse(url)
+
+    view_match = re.search(r"/view/([^?]+)", decoded_url.fragment)
+    instance_match = re.search(r"instance:([^)]+)", decoded_url.fragment)
+    index_match = re.search(r"index:'([^']+)'", decoded_url.fragment)
+
+    view = view_match.group(1) if view_match else None
+    instance = instance_match.group(1) if instance_match else None
+    index = index_match.group(1) if index_match else None
+
+    # If we are unable to parse the fragment, return the original URL
+    if view is None or instance is None or index is None:
+        return url
+
+    _a = f"(filters:!((query:(match_phrase:(instance:{instance}))),(query:(match_phrase:(requestId:'{request_id}'))),(query:(match_phrase:(level:{log_level})))),index:'{index}')"  # NOQA  E502
+
+    new_fragment = f"/view/{view}?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),time:(from:now-24h/h,to:now))"  # NOQA  E502
+    new_fragment += f"&_a={urllib.parse.quote(_a)}"
+    decoded_url = decoded_url._replace(fragment=new_fragment)
+    return urlunparse(decoded_url)

servicex_app/servicex_app_test/web/test_kibana_url_filter.py

@@ -0,0 +1,55 @@
+from servicex_app.web.kibana_url_filter import filter_kibana_url
+
+
+class TestAddRequestIdFilter:
+    """Tests for add_request_id_filter using a real Kibana dashboard URL."""
+
+    EXAMPLE_URL = (
+        "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards"
+        "?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662#"
+        "?embed=true"
+        "&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),"
+        "time:(from:now-24h/h,to:now))"
+        "&_a=(filters:!((query:(match_phrase:(instance:servicex-unit-test))))"
+        ",index:'923eaa00-45b9-11ed-afcf-d91dad577662')"
+    )
+
+    def test_preserves_base_url(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert result.startswith(
+            "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards"
+        )
+
+    def test_preserves_auth_query(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert "?auth_provider_hint=anonymous1" in result
+
+    def test_preserves_view_path(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert "#/view/bb682100-5558-11ed-afcf-d91dad577662" in result
+
+    def test_includes_embed_true(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert "embed=true" in result
+
+    def test_preserves_instance_in_query(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert "servicex-unit-test" in result
+
+    def test_includes_request_id_in_query(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert "abc-123" in result
+
+    def test_includes_log_level_in_query(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="DEBUG")
+        assert "level%3ADEBUG" in result
+
+    def test_includes_app_state_filter(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert "_a=" in result
+        assert "requestId" in result
+
+    def test_preserves_time_range(self):
+        result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO")
+        assert "now-24h/h" in result
+        assert "to:now" in result

servicex_app/servicex_app_test/web/web_test_base.py

@@ -90,6 +90,7 @@ def _app_config():
             "DID_RUCIO_FINDER_TAG": "develop",
             "DID_CERNOPENDATA_FINDER_TAG": "develop",
             "APP_IMAGE_TAG": "develop",
+            "LOGS_URL": "http://kibana.example.com",
         }
 
     @staticmethod