From 97dbfd102708b8e9d59ea710d4df5c54e0e9e891 Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Thu, 4 Dec 2025 20:38:42 +0000 Subject: [PATCH 1/2] feat: block all crawlers by default the default for all configurations should be to block crawlers in the robot file. FIXES: APER-4252 --- playbooks/roles/nginx/defaults/main.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/nginx/defaults/main.yml b/playbooks/roles/nginx/defaults/main.yml index 0be8b8d93..8b5f969de 100644 --- a/playbooks/roles/nginx/defaults/main.yml +++ b/playbooks/roles/nginx/defaults/main.yml @@ -190,7 +190,10 @@ nginx_ecommerce_gunicorn_hosts: nginx_credentails_gunicorn_hosts: - 127.0.0.1 -NGINX_ROBOT_RULES: [ ] +# Block all crawlers by default +NGINX_ROBOT_RULES: + - agent: "*" + disallow: "/" NGINX_EDXAPP_EMBARGO_CIDRS: [] NGINX_P3P_MESSAGE: 'CP="Open edX does not have a P3P policy."' From 9727a4a9fb0543cea5d17d9154295daf743dc02c Mon Sep 17 00:00:00 2001 From: Deborah Kaplan Date: Thu, 4 Dec 2025 21:05:04 +0000 Subject: [PATCH 2/2] feat: updating changelog updating changelog --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4078cb81e..d2b1c34a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. Add any new changes to the top (right below this line). +- 2025-12-05 + - Set `NGINX_ROBOT_RULES` to a default of block everything. + - 2025-09-26 - Moved `EDXAPP_FEATURES_DEFAULT` and `EDXAPP_FEATURES_EXTRA` into top-level settings.