diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index c1b0cfba8..42275b637 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -289,7 +289,8 @@ logging: port: 5959 protocol: TCP monitor: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/c2cc1f30-4a5b-11ed-afcf-d91dad577662?embed=true&_g=(filters%3A!()%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A0)%2Ctime%3A(from%3Anow-24h%2Fh%2Cto%3Anow))&show-time-filter=true" - logs: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662?embed=true&_g=(filters%3A!(('%24state'%3A(store%3AglobalState)%2Cmeta%3A(alias%3A!n%2Cdisabled%3A!f%2Cindex%3A'923eaa00-45b9-11ed-afcf-d91dad577662'%2Ckey%3Ainstance%2Cnegate%3A!f%2Cparams%3A(query%3Aservicex)%2Ctype%3Aphrase)%2Cquery%3A(term%3A(instance%3A{{ .Release.Name }}))))%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A0)%2Ctime%3A(from%3Anow-24h%2Fh%2Cto%3Anow))&show-query-input=true&show-time-filter=true&hide-filter-bar=true" + # This URL format is tied to a parser in servicex_app/servicex_app/web/kibana_url_filter.py + logs: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),time:(from:now-24h%2Fh,to:now))&_a=(filters:!((query:(match_phrase:(instance:{{.Release.Name}})))),index:'923eaa00-45b9-11ed-afcf-d91dad577662')" minio: image: repository: bitnamilegacy/minio diff --git a/servicex_app/servicex_app/__init__.py b/servicex_app/servicex_app/__init__.py index aa6afe378..5a49a647c 100644 --- a/servicex_app/servicex_app/__init__.py +++ b/servicex_app/servicex_app/__init__.py @@ -379,6 +379,12 @@ def inject_modules(): import humanize - return dict(datetime=datetime, humanize=humanize) + from servicex_app.web import create_kibana_link + + return dict( + datetime=datetime, + humanize=humanize, + create_kibana_link=create_kibana_link, + ) return app diff --git a/servicex_app/servicex_app/static/logs.svg b/servicex_app/servicex_app/static/logs.svg new file mode 100644 index 000000000..ae9ebe3e8 --- /dev/null +++ b/servicex_app/servicex_app/static/logs.svg @@ -0,0 +1,4 @@ + + + +Created by Muhammad Arifinfrom Noun Project diff --git a/servicex_app/servicex_app/templates/requests_table.html b/servicex_app/servicex_app/templates/requests_table.html index f3199903b..e24821ac3 100644 --- a/servicex_app/servicex_app/templates/requests_table.html +++ b/servicex_app/servicex_app/templates/requests_table.html @@ -21,6 +21,9 @@ {% for req in pagination.items %} + + Get Logs + {{ req.title or "Untitled" }} diff --git a/servicex_app/servicex_app/templates/transformation_request.html b/servicex_app/servicex_app/templates/transformation_request.html index 0b5a2284a..3b96f5d48 100644 --- a/servicex_app/servicex_app/templates/transformation_request.html +++ b/servicex_app/servicex_app/templates/transformation_request.html @@ -61,6 +61,10 @@

Transformation Request

{{ req.failure_description }} s
{% endif %} +
+ Get Logs + See Logs
+ {% endblock %} diff --git a/servicex_app/servicex_app/web/__init__.py b/servicex_app/servicex_app/web/__init__.py index e69de29bb..2d368d33d 100644 --- a/servicex_app/servicex_app/web/__init__.py +++ b/servicex_app/servicex_app/web/__init__.py @@ -0,0 +1,7 @@ +from flask import current_app +from .kibana_url_filter import filter_kibana_url + + +def create_kibana_link(transform_id=None, log_level="INFO"): + log_url = current_app.config["LOGS_URL"] + return filter_kibana_url(log_url, transform_id, log_level) diff --git a/servicex_app/servicex_app/web/kibana_url_filter.py b/servicex_app/servicex_app/web/kibana_url_filter.py new file mode 100644 index 000000000..25608d332 --- /dev/null +++ b/servicex_app/servicex_app/web/kibana_url_filter.py @@ -0,0 +1,37 @@ +""" +Kibana URL Filter Modifier + +This module provides functionality to add requestID filters to Kibana dashboard URLs. +""" + +import re +import urllib.parse + +from urllib.parse import urlunparse + + +def filter_kibana_url(url: str, request_id: str, log_level: str) -> str: + """ + Add a filter to a Kibana dashboard URL to show only results for a given request ID + with an ERROR level or higher. + """ + decoded_url = urllib.parse.urlparse(url) + + view_match = re.search(r"/view/([^?]+)", decoded_url.fragment) + instance_match = re.search(r"instance:([^)]+)", decoded_url.fragment) + index_match = re.search(r"index:'([^']+)'", decoded_url.fragment) + + view = view_match.group(1) if view_match else None + instance = instance_match.group(1) if instance_match else None + index = index_match.group(1) if index_match else None + + # If we are unable to parse the fragment, return the original URL + if view is None or instance is None or index is None: + return url + + _a = f"(filters:!((query:(match_phrase:(instance:{instance}))),(query:(match_phrase:(requestId:'{request_id}'))),(query:(match_phrase:(level:{log_level})))),index:'{index}')" # NOQA E502 + + new_fragment = f"/view/{view}?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),time:(from:now-24h/h,to:now))" # NOQA E502 + new_fragment += f"&_a={urllib.parse.quote(_a)}" + decoded_url = decoded_url._replace(fragment=new_fragment) + return urlunparse(decoded_url) diff --git a/servicex_app/servicex_app_test/web/test_kibana_url_filter.py b/servicex_app/servicex_app_test/web/test_kibana_url_filter.py new file mode 100644 index 000000000..b87914c8a --- /dev/null +++ b/servicex_app/servicex_app_test/web/test_kibana_url_filter.py @@ -0,0 +1,55 @@ +from servicex_app.web.kibana_url_filter import filter_kibana_url + + +class TestAddRequestIdFilter: + """Tests for add_request_id_filter using a real Kibana dashboard URL.""" + + EXAMPLE_URL = ( + "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards" + "?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662#" + "?embed=true" + "&_g=(filters:!(),refreshInterval:(pause:!t,value:1000)," + "time:(from:now-24h/h,to:now))" + "&_a=(filters:!((query:(match_phrase:(instance:servicex-unit-test))))" + ",index:'923eaa00-45b9-11ed-afcf-d91dad577662')" + ) + + def test_preserves_base_url(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert result.startswith( + "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards" + ) + + def test_preserves_auth_query(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "?auth_provider_hint=anonymous1" in result + + def test_preserves_view_path(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "#/view/bb682100-5558-11ed-afcf-d91dad577662" in result + + def test_includes_embed_true(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "embed=true" in result + + def test_preserves_instance_in_query(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "servicex-unit-test" in result + + def test_includes_request_id_in_query(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "abc-123" in result + + def test_includes_log_level_in_query(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="DEBUG") + assert "level%3ADEBUG" in result + + def test_includes_app_state_filter(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "_a=" in result + assert "requestId" in result + + def test_preserves_time_range(self): + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "now-24h/h" in result + assert "to:now" in result diff --git a/servicex_app/servicex_app_test/web/web_test_base.py b/servicex_app/servicex_app_test/web/web_test_base.py index d803c59cb..16d51e111 100644 --- a/servicex_app/servicex_app_test/web/web_test_base.py +++ b/servicex_app/servicex_app_test/web/web_test_base.py @@ -90,6 +90,7 @@ def _app_config(): "DID_RUCIO_FINDER_TAG": "develop", "DID_CERNOPENDATA_FINDER_TAG": "develop", "APP_IMAGE_TAG": "develop", + "LOGS_URL": "http://kibana.example.com", } @staticmethod