From 34f25a6f25cb61cc326a22492f286ae7cf970846 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Thu, 5 Feb 2026 15:04:01 -0600 Subject: [PATCH 1/3] Add Kibana log link to the requests table dashboard Adds a clickable log icon next to each transformation request that links directly to Kibana with pre-filtered query parameters for the request ID and log level. Co-Authored-By: Claude Opus 4.6 --- servicex_app/servicex_app/__init__.py | 8 +- servicex_app/servicex_app/static/logs.svg | 4 + .../templates/requests_table.html | 3 + servicex_app/servicex_app/web/__init__.py | 39 +++ .../servicex_app/web/kibana_url_filter.py | 259 ++++++++++++++++++ .../servicex_app_test/web/test_kibana_link.py | 78 ++++++ 6 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 servicex_app/servicex_app/static/logs.svg create mode 100644 servicex_app/servicex_app/web/kibana_url_filter.py create mode 100644 servicex_app/servicex_app_test/web/test_kibana_link.py diff --git a/servicex_app/servicex_app/__init__.py b/servicex_app/servicex_app/__init__.py index aa6afe378..bc456c959 100644 --- a/servicex_app/servicex_app/__init__.py +++ b/servicex_app/servicex_app/__init__.py @@ -379,6 +379,12 @@ def inject_modules(): import humanize - return dict(datetime=datetime, humanize=humanize) + from servicex_app.web import create_kibana_link_parameters + + return dict( + datetime=datetime, + humanize=humanize, + create_kibana_link=create_kibana_link_parameters, + ) return app diff --git a/servicex_app/servicex_app/static/logs.svg b/servicex_app/servicex_app/static/logs.svg new file mode 100644 index 000000000..ae9ebe3e8 --- /dev/null +++ b/servicex_app/servicex_app/static/logs.svg @@ -0,0 +1,4 @@ + + + +Created by Muhammad Arifinfrom Noun Project diff --git a/servicex_app/servicex_app/templates/requests_table.html b/servicex_app/servicex_app/templates/requests_table.html index f3199903b..aaf247d95 100644 --- a/servicex_app/servicex_app/templates/requests_table.html +++ b/servicex_app/servicex_app/templates/requests_table.html @@ -21,6 +21,9 @@ {% for req in pagination.items %} + + Get Logs + {{ req.title or "Untitled" }} diff --git a/servicex_app/servicex_app/web/__init__.py b/servicex_app/servicex_app/web/__init__.py index e69de29bb..f86bebeb5 100644 --- a/servicex_app/servicex_app/web/__init__.py +++ b/servicex_app/servicex_app/web/__init__.py @@ -0,0 +1,39 @@ +from enum import Enum +from flask import current_app +import re + + +class LogLevel(str, Enum): + r""" + Level of the log messages: INFO & ERROR + This controls the log level of the logs displayed in the dashboard + """ + + info = "INFO" + error = "ERROR" + + +def add_query(key, value): + """ + Creates query string from the key and value pairs for use in constructing + the kibana link + """ + query_string = "(query:(match_phrase:({0}:'{1}')))".format(key, value) + return query_string + + +def create_kibana_link_parameters( + transform_id=None, + log_level=LogLevel.info, +): + """ + Construct n url for the kibana dashboard based on the input parameters + """ + assert log_level and log_level in LogLevel, "Log level must be one of INFO or ERROR" + log_url = current_app.config["LOGS_URL"] + a_parameter = ( + f"&_a=(filters:!({add_query('requestId', transform_id)}," + f"{add_query('level', str(log_level.value).lower())}))" + ) + kibana_link = re.sub(r"&_g=\(\)", a_parameter, log_url) + return kibana_link diff --git a/servicex_app/servicex_app/web/kibana_url_filter.py b/servicex_app/servicex_app/web/kibana_url_filter.py new file mode 100644 index 000000000..2e9ca1c58 --- /dev/null +++ b/servicex_app/servicex_app/web/kibana_url_filter.py @@ -0,0 +1,259 @@ +""" +Kibana URL Filter Modifier + +This module provides functionality to add requestID filters to Kibana dashboard URLs. +It handles Kibana's rison-encoded state format and properly constructs filter objects. +""" + +import urllib.parse +import re + + +def add_request_id_filter(kibana_url, request_id): + """ + Add a requestID filter to a Kibana dashboard URL. + + This function parses a Kibana dashboard URL, extracts the global state (_g parameter) + and the index pattern from existing filters, adds a new filter for the specified + requestID, and returns the modified URL with both the original filters and the new + requestID filter combined with AND logic. + + Args: + kibana_url (str): Original Kibana dashboard URL containing the _g parameter + request_id (str): Request ID value to filter by (e.g., '31f32a39-c998-4784-b8f1-3a0ef6f0bd8b') + + Returns: + str: Modified URL with the requestID filter added to the filters array + + Raises: + ValueError: If no index pattern can be extracted from existing filters in the URL + + Example: + >>> original_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?..." + >>> request_id = "31f32a39-c998-4784-b8f1-3a0ef6f0bd8b" + >>> new_url = add_request_id_filter(original_url, request_id) + >>> print(new_url) + + Notes: + - The function preserves all existing filters and URL parameters + - Filters are combined with AND logic (all filters must match) + - The index pattern is extracted from existing filters in the URL + - The _g parameter uses Kibana's rison format where: + * () represents objects + * !() represents arrays + * !n = null, !f = false, !t = true + * Single quotes are used instead of double quotes + """ + # Parse the URL into components + parsed = urllib.parse.urlparse(kibana_url) + + # Split the fragment (hash) part which contains the dashboard path and query + fragment_parts = parsed.fragment.split('?', 1) + dashboard_path = fragment_parts[0] + + if len(fragment_parts) > 1: + fragment_query = fragment_parts[1] + else: + raise ValueError("No query parameters found in URL fragment") + + # Parse the query parameters from the fragment + fragment_params = urllib.parse.parse_qs(fragment_query) + + # Get the _g parameter (which contains the global state) + if '_g' not in fragment_params: + raise ValueError("No _g parameter found in URL") + + # Decode the _g parameter (it's URL encoded) + _g_encoded = fragment_params['_g'][0] + _g_decoded = urllib.parse.unquote(_g_encoded) + + # Extract the index pattern from existing filters + index_pattern = _extract_index_pattern(_g_decoded) + if not index_pattern: + raise ValueError("Could not extract index pattern from existing filters in URL") + + # Create the new filter in rison format (Kibana's custom encoding format) + # This matches the structure of existing filters in the URL + new_filter_rison = ( + f"('$state':(store:globalState)," + f"meta:(alias:!n,disabled:!f,index:'{index_pattern}',key:requestId,negate:!f," + f"params:(query:'{request_id}'),type:phrase)," + f"query:(term:(requestID:'{request_id}')))" + ) + + # Find the filters array in the _g parameter + # Filters are encoded as: filters:!(filter1,filter2,...) + filters_start = _g_decoded.find('filters:!(') + + if filters_start != -1: + # Find the matching closing parenthesis for the filters array + # We need to track nested parentheses to find the correct closing paren + start_pos = filters_start + len('filters:!(') + paren_count = 1 + i = start_pos + + while i < len(_g_decoded) and paren_count > 0: + if _g_decoded[i] == '(': + paren_count += 1 + elif _g_decoded[i] == ')': + paren_count -= 1 + i += 1 + + end_pos = i - 1 # Position of the closing ) + + # Extract existing filters content + existing_filters = _g_decoded[start_pos:end_pos] + + # Add the new filter to the existing filters array + if existing_filters.strip(): + new_filters_content = f"{existing_filters},{new_filter_rison}" + else: + new_filters_content = new_filter_rison + + # Build the modified _g string by replacing the filters content + _g_modified = ( + _g_decoded[:start_pos] + + new_filters_content + + _g_decoded[end_pos:] + ) + else: + # No filters exist, add a new filters array at the beginning + _g_modified = _g_decoded.replace( + "(", + f"(filters:!({new_filter_rison}),", + 1 + ) + + # URL encode the modified _g parameter + _g_new_encoded = urllib.parse.quote(_g_modified, safe='') + + # Update the fragment params with the new _g value + fragment_params['_g'] = [_g_new_encoded] + + # Reconstruct the fragment query string + # We use custom encoding to preserve the exact format + param_parts = [] + for key, values in fragment_params.items(): + for value in values: + param_parts.append(f"{key}={value}") + new_fragment_query = "&".join(param_parts) + + # Rebuild the fragment with the dashboard path and modified query + new_fragment = f"{dashboard_path}?{new_fragment_query}" + + # Reconstruct the full URL with all components + new_url = urllib.parse.urlunparse(( + parsed.scheme, + parsed.netloc, + parsed.path, + parsed.params, + parsed.query, + new_fragment + )) + + return new_url + + +def _extract_index_pattern(decoded_g_param): + """ + Extract the index pattern ID from existing filters in the decoded _g parameter. + + Args: + decoded_g_param (str): Decoded _g parameter string in rison format + + Returns: + str: Index pattern ID if found, None otherwise + """ + # Look for the pattern: index:'...' + # This appears in the meta section of filters + match = re.search(r"index:'([^']+)'", decoded_g_param) + if match: + return match.group(1) + return None + + +#!/usr/bin/env python3 +""" +Kibana Field Name Diagnostic Tool + +This script helps you find the correct field name to use for your requestID filter. +It generates test URLs with different field name formats so you can try them in Kibana. +""" + +import sys +import os + + +def diagnose_field_name(original_url, request_id): + """ + Generate test URLs with different field name formats. + + Args: + original_url: Your original Kibana URL + request_id: The request ID value to filter by + """ + print("="*80) + print("KIBANA FIELD NAME DIAGNOSTIC TOOL") + print("="*80) + print("\nThis tool generates URLs with different field name formats.") + print("Test each URL in Kibana to find which one shows results.\n") + + # Common field name variations + field_names = [ + ('requestID', 'camelCase (default)'), + ('request_id', 'snake_case'), + ('request.id', 'nested field (dot notation)'), + ('requestId', 'camelCase with lowercase d'), + ('RequestID', 'PascalCase'), + ('req_id', 'abbreviated snake_case'), + ('reqId', 'abbreviated camelCase'), + ] + + print(f"Request ID to filter: {request_id}\n") + print("="*80) + print("TEST URLS") + print("="*80) + + for field_name, description in field_names: + try: + url = add_request_id_filter(original_url, request_id, field_name=field_name) + print(f"\n{field_name} ({description}):") + print("-"*80) + print(url) + print() + except Exception as e: + print(f"\n{field_name}: ERROR - {e}\n") + + print("\n" + "="*80) + print("INSTRUCTIONS") + print("="*80) + print(""" +1. Copy each URL above and paste it into your browser +2. The URL that shows results has the correct field name +3. Use that field name in your code: + + new_url = add_request_id_filter( + original_url, + request_id, + field_name='THE_CORRECT_FIELD_NAME' # <- Replace with the working one + ) + +4. If none work, you can manually check the field name: + a. In Kibana, manually add a filter that works + b. Copy the URL from your browser + c. Run: python -c "from kibana_url_filter import decode_kibana_url; print(decode_kibana_url('YOUR_URL'))" + d. Look for "key:YOUR_FIELD_NAME" in the output + """) + + +if __name__ == "__main__": + if len(sys.argv) < 3: + print("Usage: python diagnose_field_name.py ") + print("\nExample:") + print(' python diagnose_field_name.py "https://kibana.example.com/..." "31f32a39-c998-4784-b8f1-3a0ef6f0bd8b"') + sys.exit(1) + + original_url = sys.argv[1] + request_id = sys.argv[2] + + diagnose_field_name(original_url, request_id) diff --git a/servicex_app/servicex_app_test/web/test_kibana_link.py b/servicex_app/servicex_app_test/web/test_kibana_link.py new file mode 100644 index 000000000..6ce092907 --- /dev/null +++ b/servicex_app/servicex_app_test/web/test_kibana_link.py @@ -0,0 +1,78 @@ +import pytest +from flask import Flask + +from servicex_app.web import ( + LogLevel, + add_query, + create_kibana_link_parameters, +) + +SAMPLE_LOGS_URL = "http://kibana.example.com/app/discover#/?_t=foo&_g=()" + + +@pytest.fixture +def app(): + app = Flask(__name__) + app.config["LOGS_URL"] = SAMPLE_LOGS_URL + return app + + +class TestAddQuery: + def test_formats_request_id(self): + result = add_query("requestId", "abc-123") + assert result == "(query:(match_phrase:(requestId:'abc-123')))" + + def test_formats_level(self): + result = add_query("level", "info") + assert result == "(query:(match_phrase:(level:'info')))" + + +class TestCreateKibanaLinkParameters: + def test_default_log_level_is_info(self, app): + with app.app_context(): + result = create_kibana_link_parameters(transform_id="abc-123") + assert "(query:(match_phrase:(level:'info')))" in result + assert "(query:(match_phrase:(requestId:'abc-123')))" in result + + def test_error_log_level(self, app): + with app.app_context(): + result = create_kibana_link_parameters( + transform_id="abc-123", + log_level=LogLevel.error, + ) + assert "(query:(match_phrase:(level:'error')))" in result + assert "(query:(match_phrase:(requestId:'abc-123')))" in result + + def test_no_log_level_omits_level_filter(self, app): + with app.app_context(): + with pytest.raises( + AssertionError, match="Log level must be one of INFO or ERROR" + ): + _ = create_kibana_link_parameters( + transform_id="abc-123", + log_level=None, + ) + + def test_replaces_g_parameter_in_url(self, app): + with app.app_context(): + result = create_kibana_link_parameters(transform_id="abc-123") + assert "&_g=()" not in result + assert result.startswith("http://kibana.example.com/app/discover#/?_t=foo") + + def test_full_url_with_info_level(self, app): + with app.app_context(): + result = create_kibana_link_parameters(transform_id="abc-123") + expected = ( + "http://kibana.example.com/app/discover#/?_t=foo" + "&_a=(filters:!(" + "(query:(match_phrase:(requestId:'abc-123')))," + "(query:(match_phrase:(level:'info')))" + "))" + ) + assert result == expected + + def test_no_substitution_when_pattern_missing(self, app): + app.config["LOGS_URL"] = "http://kibana.example.com/app/discover#/" + with app.app_context(): + result = create_kibana_link_parameters(transform_id="abc-123") + assert result == "http://kibana.example.com/app/discover#/" From 7977a781fffcfc44302fd99ded1f49c9e7c97de1 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Fri, 20 Feb 2026 09:03:10 -0600 Subject: [PATCH 2/3] Update the default Kibana URL to make it easier to programatically manipulate the query and add requestId --- helm/servicex/values.yaml | 3 +- .../templates/requests_table.html | 2 +- servicex_app/servicex_app/web/__init__.py | 38 +-- .../servicex_app/web/kibana_url_filter.py | 262 ++---------------- .../servicex_app_test/web/test_kibana_link.py | 78 ------ .../web/test_kibana_url_filter.py | 55 ++++ 6 files changed, 81 insertions(+), 357 deletions(-) delete mode 100644 servicex_app/servicex_app_test/web/test_kibana_link.py create mode 100644 servicex_app/servicex_app_test/web/test_kibana_url_filter.py diff --git a/helm/servicex/values.yaml b/helm/servicex/values.yaml index c1b0cfba8..42275b637 100644 --- a/helm/servicex/values.yaml +++ b/helm/servicex/values.yaml @@ -289,7 +289,8 @@ logging: port: 5959 protocol: TCP monitor: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/c2cc1f30-4a5b-11ed-afcf-d91dad577662?embed=true&_g=(filters%3A!()%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A0)%2Ctime%3A(from%3Anow-24h%2Fh%2Cto%3Anow))&show-time-filter=true" - logs: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662?embed=true&_g=(filters%3A!(('%24state'%3A(store%3AglobalState)%2Cmeta%3A(alias%3A!n%2Cdisabled%3A!f%2Cindex%3A'923eaa00-45b9-11ed-afcf-d91dad577662'%2Ckey%3Ainstance%2Cnegate%3A!f%2Cparams%3A(query%3Aservicex)%2Ctype%3Aphrase)%2Cquery%3A(term%3A(instance%3A{{ .Release.Name }}))))%2CrefreshInterval%3A(pause%3A!t%2Cvalue%3A0)%2Ctime%3A(from%3Anow-24h%2Fh%2Cto%3Anow))&show-query-input=true&show-time-filter=true&hide-filter-bar=true" + # This URL format is tied to a parser in servicex_app/servicex_app/web/kibana_url_filter.py + logs: "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),time:(from:now-24h%2Fh,to:now))&_a=(filters:!((query:(match_phrase:(instance:{{.Release.Name}})))),index:'923eaa00-45b9-11ed-afcf-d91dad577662')" minio: image: repository: bitnamilegacy/minio diff --git a/servicex_app/servicex_app/templates/requests_table.html b/servicex_app/servicex_app/templates/requests_table.html index aaf247d95..e24821ac3 100644 --- a/servicex_app/servicex_app/templates/requests_table.html +++ b/servicex_app/servicex_app/templates/requests_table.html @@ -21,7 +21,7 @@ {% for req in pagination.items %} - + Get Logs {{ req.title or "Untitled" }} diff --git a/servicex_app/servicex_app/web/__init__.py b/servicex_app/servicex_app/web/__init__.py index f86bebeb5..48c54ba0f 100644 --- a/servicex_app/servicex_app/web/__init__.py +++ b/servicex_app/servicex_app/web/__init__.py @@ -1,39 +1,7 @@ -from enum import Enum from flask import current_app -import re +from .kibana_url_filter import add_request_id_filter -class LogLevel(str, Enum): - r""" - Level of the log messages: INFO & ERROR - This controls the log level of the logs displayed in the dashboard - """ - - info = "INFO" - error = "ERROR" - - -def add_query(key, value): - """ - Creates query string from the key and value pairs for use in constructing - the kibana link - """ - query_string = "(query:(match_phrase:({0}:'{1}')))".format(key, value) - return query_string - - -def create_kibana_link_parameters( - transform_id=None, - log_level=LogLevel.info, -): - """ - Construct n url for the kibana dashboard based on the input parameters - """ - assert log_level and log_level in LogLevel, "Log level must be one of INFO or ERROR" +def create_kibana_link(transform_id=None, log_level="INFO"): log_url = current_app.config["LOGS_URL"] - a_parameter = ( - f"&_a=(filters:!({add_query('requestId', transform_id)}," - f"{add_query('level', str(log_level.value).lower())}))" - ) - kibana_link = re.sub(r"&_g=\(\)", a_parameter, log_url) - return kibana_link + return add_request_id_filter(log_url, transform_id, log_level) diff --git a/servicex_app/servicex_app/web/kibana_url_filter.py b/servicex_app/servicex_app/web/kibana_url_filter.py index 2e9ca1c58..dfb55c398 100644 --- a/servicex_app/servicex_app/web/kibana_url_filter.py +++ b/servicex_app/servicex_app/web/kibana_url_filter.py @@ -2,258 +2,36 @@ Kibana URL Filter Modifier This module provides functionality to add requestID filters to Kibana dashboard URLs. -It handles Kibana's rison-encoded state format and properly constructs filter objects. """ -import urllib.parse import re +import urllib.parse - -def add_request_id_filter(kibana_url, request_id): - """ - Add a requestID filter to a Kibana dashboard URL. - - This function parses a Kibana dashboard URL, extracts the global state (_g parameter) - and the index pattern from existing filters, adds a new filter for the specified - requestID, and returns the modified URL with both the original filters and the new - requestID filter combined with AND logic. - - Args: - kibana_url (str): Original Kibana dashboard URL containing the _g parameter - request_id (str): Request ID value to filter by (e.g., '31f32a39-c998-4784-b8f1-3a0ef6f0bd8b') - - Returns: - str: Modified URL with the requestID filter added to the filters array - - Raises: - ValueError: If no index pattern can be extracted from existing filters in the URL - - Example: - >>> original_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?..." - >>> request_id = "31f32a39-c998-4784-b8f1-3a0ef6f0bd8b" - >>> new_url = add_request_id_filter(original_url, request_id) - >>> print(new_url) - - Notes: - - The function preserves all existing filters and URL parameters - - Filters are combined with AND logic (all filters must match) - - The index pattern is extracted from existing filters in the URL - - The _g parameter uses Kibana's rison format where: - * () represents objects - * !() represents arrays - * !n = null, !f = false, !t = true - * Single quotes are used instead of double quotes - """ - # Parse the URL into components - parsed = urllib.parse.urlparse(kibana_url) - - # Split the fragment (hash) part which contains the dashboard path and query - fragment_parts = parsed.fragment.split('?', 1) - dashboard_path = fragment_parts[0] - - if len(fragment_parts) > 1: - fragment_query = fragment_parts[1] - else: - raise ValueError("No query parameters found in URL fragment") - - # Parse the query parameters from the fragment - fragment_params = urllib.parse.parse_qs(fragment_query) - - # Get the _g parameter (which contains the global state) - if '_g' not in fragment_params: - raise ValueError("No _g parameter found in URL") - - # Decode the _g parameter (it's URL encoded) - _g_encoded = fragment_params['_g'][0] - _g_decoded = urllib.parse.unquote(_g_encoded) - - # Extract the index pattern from existing filters - index_pattern = _extract_index_pattern(_g_decoded) - if not index_pattern: - raise ValueError("Could not extract index pattern from existing filters in URL") - - # Create the new filter in rison format (Kibana's custom encoding format) - # This matches the structure of existing filters in the URL - new_filter_rison = ( - f"('$state':(store:globalState)," - f"meta:(alias:!n,disabled:!f,index:'{index_pattern}',key:requestId,negate:!f," - f"params:(query:'{request_id}'),type:phrase)," - f"query:(term:(requestID:'{request_id}')))" - ) - - # Find the filters array in the _g parameter - # Filters are encoded as: filters:!(filter1,filter2,...) - filters_start = _g_decoded.find('filters:!(') - - if filters_start != -1: - # Find the matching closing parenthesis for the filters array - # We need to track nested parentheses to find the correct closing paren - start_pos = filters_start + len('filters:!(') - paren_count = 1 - i = start_pos - - while i < len(_g_decoded) and paren_count > 0: - if _g_decoded[i] == '(': - paren_count += 1 - elif _g_decoded[i] == ')': - paren_count -= 1 - i += 1 - - end_pos = i - 1 # Position of the closing ) - - # Extract existing filters content - existing_filters = _g_decoded[start_pos:end_pos] - - # Add the new filter to the existing filters array - if existing_filters.strip(): - new_filters_content = f"{existing_filters},{new_filter_rison}" - else: - new_filters_content = new_filter_rison - - # Build the modified _g string by replacing the filters content - _g_modified = ( - _g_decoded[:start_pos] + - new_filters_content + - _g_decoded[end_pos:] - ) - else: - # No filters exist, add a new filters array at the beginning - _g_modified = _g_decoded.replace( - "(", - f"(filters:!({new_filter_rison}),", - 1 - ) - - # URL encode the modified _g parameter - _g_new_encoded = urllib.parse.quote(_g_modified, safe='') - - # Update the fragment params with the new _g value - fragment_params['_g'] = [_g_new_encoded] - - # Reconstruct the fragment query string - # We use custom encoding to preserve the exact format - param_parts = [] - for key, values in fragment_params.items(): - for value in values: - param_parts.append(f"{key}={value}") - new_fragment_query = "&".join(param_parts) - - # Rebuild the fragment with the dashboard path and modified query - new_fragment = f"{dashboard_path}?{new_fragment_query}" - - # Reconstruct the full URL with all components - new_url = urllib.parse.urlunparse(( - parsed.scheme, - parsed.netloc, - parsed.path, - parsed.params, - parsed.query, - new_fragment - )) - - return new_url - - -def _extract_index_pattern(decoded_g_param): - """ - Extract the index pattern ID from existing filters in the decoded _g parameter. - - Args: - decoded_g_param (str): Decoded _g parameter string in rison format - - Returns: - str: Index pattern ID if found, None otherwise - """ - # Look for the pattern: index:'...' - # This appears in the meta section of filters - match = re.search(r"index:'([^']+)'", decoded_g_param) - if match: - return match.group(1) - return None - - -#!/usr/bin/env python3 -""" -Kibana Field Name Diagnostic Tool - -This script helps you find the correct field name to use for your requestID filter. -It generates test URLs with different field name formats so you can try them in Kibana. -""" - -import sys -import os +from urllib.parse import urlunparse -def diagnose_field_name(original_url, request_id): +def add_request_id_filter(url: str, request_id: str, log_level: str) -> str: """ - Generate test URLs with different field name formats. - - Args: - original_url: Your original Kibana URL - request_id: The request ID value to filter by + Add a filter to a Kibana dashboard URL to show only results for a given request ID + with an ERROR level or higher. """ - print("="*80) - print("KIBANA FIELD NAME DIAGNOSTIC TOOL") - print("="*80) - print("\nThis tool generates URLs with different field name formats.") - print("Test each URL in Kibana to find which one shows results.\n") - - # Common field name variations - field_names = [ - ('requestID', 'camelCase (default)'), - ('request_id', 'snake_case'), - ('request.id', 'nested field (dot notation)'), - ('requestId', 'camelCase with lowercase d'), - ('RequestID', 'PascalCase'), - ('req_id', 'abbreviated snake_case'), - ('reqId', 'abbreviated camelCase'), - ] - - print(f"Request ID to filter: {request_id}\n") - print("="*80) - print("TEST URLS") - print("="*80) - - for field_name, description in field_names: - try: - url = add_request_id_filter(original_url, request_id, field_name=field_name) - print(f"\n{field_name} ({description}):") - print("-"*80) - print(url) - print() - except Exception as e: - print(f"\n{field_name}: ERROR - {e}\n") - - print("\n" + "="*80) - print("INSTRUCTIONS") - print("="*80) - print(""" -1. Copy each URL above and paste it into your browser -2. The URL that shows results has the correct field name -3. Use that field name in your code: - - new_url = add_request_id_filter( - original_url, - request_id, - field_name='THE_CORRECT_FIELD_NAME' # <- Replace with the working one - ) + decoded_url = urllib.parse.urlparse(url) -4. If none work, you can manually check the field name: - a. In Kibana, manually add a filter that works - b. Copy the URL from your browser - c. Run: python -c "from kibana_url_filter import decode_kibana_url; print(decode_kibana_url('YOUR_URL'))" - d. Look for "key:YOUR_FIELD_NAME" in the output - """) + view_match = re.search(r"/view/([^?]+)", decoded_url.fragment) + instance_match = re.search(r"instance:([^)]+)", decoded_url.fragment) + index_match = re.search(r"index:'([^']+)'", decoded_url.fragment) + view = view_match.group(1) if view_match else None + instance = instance_match.group(1) if instance_match else None + index = index_match.group(1) if index_match else None -if __name__ == "__main__": - if len(sys.argv) < 3: - print("Usage: python diagnose_field_name.py ") - print("\nExample:") - print(' python diagnose_field_name.py "https://kibana.example.com/..." "31f32a39-c998-4784-b8f1-3a0ef6f0bd8b"') - sys.exit(1) + # If we are unable to parse the fragment, return the original URL + if view is None or instance is None or index is None: + return url - original_url = sys.argv[1] - request_id = sys.argv[2] + _a = f"(filters:!((query:(match_phrase:(instance:{instance}))),(query:(match_phrase:(requestId:'{request_id}'))),(query:(match_phrase:(level:{log_level})))),index:'{index}')" # NOQA E502 - diagnose_field_name(original_url, request_id) + new_fragment = f"/view/{view}?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),time:(from:now-24h/h,to:now))" # NOQA E502 + new_fragment += f"&_a={urllib.parse.quote(_a)}" + decoded_url = decoded_url._replace(fragment=new_fragment) + return urlunparse(decoded_url) diff --git a/servicex_app/servicex_app_test/web/test_kibana_link.py b/servicex_app/servicex_app_test/web/test_kibana_link.py deleted file mode 100644 index 6ce092907..000000000 --- a/servicex_app/servicex_app_test/web/test_kibana_link.py +++ /dev/null @@ -1,78 +0,0 @@ -import pytest -from flask import Flask - -from servicex_app.web import ( - LogLevel, - add_query, - create_kibana_link_parameters, -) - -SAMPLE_LOGS_URL = "http://kibana.example.com/app/discover#/?_t=foo&_g=()" - - -@pytest.fixture -def app(): - app = Flask(__name__) - app.config["LOGS_URL"] = SAMPLE_LOGS_URL - return app - - -class TestAddQuery: - def test_formats_request_id(self): - result = add_query("requestId", "abc-123") - assert result == "(query:(match_phrase:(requestId:'abc-123')))" - - def test_formats_level(self): - result = add_query("level", "info") - assert result == "(query:(match_phrase:(level:'info')))" - - -class TestCreateKibanaLinkParameters: - def test_default_log_level_is_info(self, app): - with app.app_context(): - result = create_kibana_link_parameters(transform_id="abc-123") - assert "(query:(match_phrase:(level:'info')))" in result - assert "(query:(match_phrase:(requestId:'abc-123')))" in result - - def test_error_log_level(self, app): - with app.app_context(): - result = create_kibana_link_parameters( - transform_id="abc-123", - log_level=LogLevel.error, - ) - assert "(query:(match_phrase:(level:'error')))" in result - assert "(query:(match_phrase:(requestId:'abc-123')))" in result - - def test_no_log_level_omits_level_filter(self, app): - with app.app_context(): - with pytest.raises( - AssertionError, match="Log level must be one of INFO or ERROR" - ): - _ = create_kibana_link_parameters( - transform_id="abc-123", - log_level=None, - ) - - def test_replaces_g_parameter_in_url(self, app): - with app.app_context(): - result = create_kibana_link_parameters(transform_id="abc-123") - assert "&_g=()" not in result - assert result.startswith("http://kibana.example.com/app/discover#/?_t=foo") - - def test_full_url_with_info_level(self, app): - with app.app_context(): - result = create_kibana_link_parameters(transform_id="abc-123") - expected = ( - "http://kibana.example.com/app/discover#/?_t=foo" - "&_a=(filters:!(" - "(query:(match_phrase:(requestId:'abc-123')))," - "(query:(match_phrase:(level:'info')))" - "))" - ) - assert result == expected - - def test_no_substitution_when_pattern_missing(self, app): - app.config["LOGS_URL"] = "http://kibana.example.com/app/discover#/" - with app.app_context(): - result = create_kibana_link_parameters(transform_id="abc-123") - assert result == "http://kibana.example.com/app/discover#/" diff --git a/servicex_app/servicex_app_test/web/test_kibana_url_filter.py b/servicex_app/servicex_app_test/web/test_kibana_url_filter.py new file mode 100644 index 000000000..2f9a081d5 --- /dev/null +++ b/servicex_app/servicex_app_test/web/test_kibana_url_filter.py @@ -0,0 +1,55 @@ +from servicex_app.web.kibana_url_filter import add_request_id_filter + + +class TestAddRequestIdFilter: + """Tests for add_request_id_filter using a real Kibana dashboard URL.""" + + EXAMPLE_URL = ( + "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards" + "?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662#" + "?embed=true" + "&_g=(filters:!(),refreshInterval:(pause:!t,value:1000)," + "time:(from:now-24h/h,to:now))" + "&_a=(filters:!((query:(match_phrase:(instance:servicex-unit-test))))" + ",index:'923eaa00-45b9-11ed-afcf-d91dad577662')" + ) + + def test_preserves_base_url(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert result.startswith( + "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards" + ) + + def test_preserves_auth_query(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "?auth_provider_hint=anonymous1" in result + + def test_preserves_view_path(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "#/view/bb682100-5558-11ed-afcf-d91dad577662" in result + + def test_includes_embed_true(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "embed=true" in result + + def test_preserves_instance_in_query(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "servicex-unit-test" in result + + def test_includes_request_id_in_query(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "abc-123" in result + + def test_includes_log_level_in_query(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="DEBUG") + assert "level%3ADEBUG" in result + + def test_includes_app_state_filter(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "_a=" in result + assert "requestId" in result + + def test_preserves_time_range(self): + result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + assert "now-24h/h" in result + assert "to:now" in result From eb80c5fd4537fdde14e3d37bb33ca388cf1273af Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Fri, 20 Feb 2026 17:36:19 -0600 Subject: [PATCH 3/3] Add log link to the transformation detail web page --- servicex_app/servicex_app/__init__.py | 4 ++-- .../templates/transformation_request.html | 4 ++++ servicex_app/servicex_app/web/__init__.py | 4 ++-- .../servicex_app/web/kibana_url_filter.py | 2 +- .../web/test_kibana_url_filter.py | 20 +++++++++---------- .../servicex_app_test/web/web_test_base.py | 1 + 6 files changed, 20 insertions(+), 15 deletions(-) diff --git a/servicex_app/servicex_app/__init__.py b/servicex_app/servicex_app/__init__.py index bc456c959..5a49a647c 100644 --- a/servicex_app/servicex_app/__init__.py +++ b/servicex_app/servicex_app/__init__.py @@ -379,12 +379,12 @@ def inject_modules(): import humanize - from servicex_app.web import create_kibana_link_parameters + from servicex_app.web import create_kibana_link return dict( datetime=datetime, humanize=humanize, - create_kibana_link=create_kibana_link_parameters, + create_kibana_link=create_kibana_link, ) return app diff --git a/servicex_app/servicex_app/templates/transformation_request.html b/servicex_app/servicex_app/templates/transformation_request.html index 0b5a2284a..3b96f5d48 100644 --- a/servicex_app/servicex_app/templates/transformation_request.html +++ b/servicex_app/servicex_app/templates/transformation_request.html @@ -61,6 +61,10 @@

Transformation Request

{{ req.failure_description }} s
{% endif %} +
+ Get Logs + See Logs
+ {% endblock %} diff --git a/servicex_app/servicex_app/web/__init__.py b/servicex_app/servicex_app/web/__init__.py index 48c54ba0f..2d368d33d 100644 --- a/servicex_app/servicex_app/web/__init__.py +++ b/servicex_app/servicex_app/web/__init__.py @@ -1,7 +1,7 @@ from flask import current_app -from .kibana_url_filter import add_request_id_filter +from .kibana_url_filter import filter_kibana_url def create_kibana_link(transform_id=None, log_level="INFO"): log_url = current_app.config["LOGS_URL"] - return add_request_id_filter(log_url, transform_id, log_level) + return filter_kibana_url(log_url, transform_id, log_level) diff --git a/servicex_app/servicex_app/web/kibana_url_filter.py b/servicex_app/servicex_app/web/kibana_url_filter.py index dfb55c398..25608d332 100644 --- a/servicex_app/servicex_app/web/kibana_url_filter.py +++ b/servicex_app/servicex_app/web/kibana_url_filter.py @@ -10,7 +10,7 @@ from urllib.parse import urlunparse -def add_request_id_filter(url: str, request_id: str, log_level: str) -> str: +def filter_kibana_url(url: str, request_id: str, log_level: str) -> str: """ Add a filter to a Kibana dashboard URL to show only results for a given request ID with an ERROR level or higher. diff --git a/servicex_app/servicex_app_test/web/test_kibana_url_filter.py b/servicex_app/servicex_app_test/web/test_kibana_url_filter.py index 2f9a081d5..b87914c8a 100644 --- a/servicex_app/servicex_app_test/web/test_kibana_url_filter.py +++ b/servicex_app/servicex_app_test/web/test_kibana_url_filter.py @@ -1,4 +1,4 @@ -from servicex_app.web.kibana_url_filter import add_request_id_filter +from servicex_app.web.kibana_url_filter import filter_kibana_url class TestAddRequestIdFilter: @@ -15,41 +15,41 @@ class TestAddRequestIdFilter: ) def test_preserves_base_url(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert result.startswith( "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards" ) def test_preserves_auth_query(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert "?auth_provider_hint=anonymous1" in result def test_preserves_view_path(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert "#/view/bb682100-5558-11ed-afcf-d91dad577662" in result def test_includes_embed_true(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert "embed=true" in result def test_preserves_instance_in_query(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert "servicex-unit-test" in result def test_includes_request_id_in_query(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert "abc-123" in result def test_includes_log_level_in_query(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="DEBUG") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="DEBUG") assert "level%3ADEBUG" in result def test_includes_app_state_filter(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert "_a=" in result assert "requestId" in result def test_preserves_time_range(self): - result = add_request_id_filter(self.EXAMPLE_URL, "abc-123", log_level="INFO") + result = filter_kibana_url(self.EXAMPLE_URL, "abc-123", log_level="INFO") assert "now-24h/h" in result assert "to:now" in result diff --git a/servicex_app/servicex_app_test/web/web_test_base.py b/servicex_app/servicex_app_test/web/web_test_base.py index d803c59cb..16d51e111 100644 --- a/servicex_app/servicex_app_test/web/web_test_base.py +++ b/servicex_app/servicex_app_test/web/web_test_base.py @@ -90,6 +90,7 @@ def _app_config(): "DID_RUCIO_FINDER_TAG": "develop", "DID_CERNOPENDATA_FINDER_TAG": "develop", "APP_IMAGE_TAG": "develop", + "LOGS_URL": "http://kibana.example.com", } @staticmethod