Skip to content

Wrapper for URL's with commas #99

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pooleycodes merged 2 commits into from
Apr 8, 2026
+16 −3
Merged

Wrapper for URL's with commas #99

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2ca58a6
added a wrapper for url's that have commas
pooleycodes Mar 31, 2026
eb1be1d
lint
pooleycodes Mar 31, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions request-processor/src/application/core/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,13 @@
logger = get_logger(__name__)


def _quote_url_if_comma(url):
"""Wrap url in double quotes if it contains a comma, to prevent CSV parsing errors."""
if url and "," in url:
return f'"{url}"'
return url


def get_request(url, verify_ssl=True):
# log["ssl-verify"] = verify_ssl
log = {"status": "", "message": ""}
Expand Down Expand Up @@ -315,14 +322,16 @@ def validate_endpoint(url, config_dir, plugin, start_date=None):
]
)

safe_url = _quote_url_if_comma(url)

endpoint_exists = False
existing_entry = None

try:
with open(endpoint_csv_path, "r", encoding="utf-8") as f:
reader = csv.DictReader(f)
for row in reader:
if row.get("endpoint-url", "").strip() == url.strip():
if row.get("endpoint-url", "").strip() == safe_url.strip():
endpoint_exists = True
existing_entry = {
"endpoint": row.get("endpoint", ""),
Expand Down Expand Up @@ -350,7 +359,7 @@ def validate_endpoint(url, config_dir, plugin, start_date=None):

endpoint_key, new_endpoint_row = append_endpoint(
endpoint_csv_path=endpoint_csv_path,
endpoint_url=url,
endpoint_url=safe_url,
entry_date=entry_date,
start_date=start_date,
end_date="",
Expand Down Expand Up @@ -387,6 +396,10 @@ def validate_source(
if not documentation_url:
logger.warning("No documentation URL provided")

safe_documentation_url = (
_quote_url_if_comma(documentation_url) if documentation_url else ""
)

if not start_date:
start_date = datetime.now().strftime("%Y-%m-%d")
entry_date = datetime.now().isoformat()
Expand All @@ -397,7 +410,7 @@ def validate_source(
organisation=organisation,
endpoint_key=endpoint_key,
attribution="",
documentation_url=documentation_url or "",
documentation_url=safe_documentation_url,
licence=licence or "",
pipelines=dataset,
entry_date=entry_date,
Expand Down
Loading