Adds robots.txt and sitemap views with indexing control

Implements `robots.txt` and `sitemap.xml` views to improve SEO.

Introduces a setting (`INDEXABLE`) to control whether the sitemap is available,
returning a 404 if indexing is disabled.

Adds tests for `robots.txt` and `sitemap.xml` to verify correct behavior based on `INDEXABLE` setting.

Author: anlu85

core_directory/context_processors.py

@@ -1,4 +1,19 @@
+"""
+Context processor for SEO-related settings.
+
+Provides the INDEXABLE setting to Django templates for use in meta tags and robots.txt.
+"""
+
 from django.conf import settings
 
 def seo_settings(request):
+    """
+    Add the INDEXABLE setting to the template context.
+
+    Args:
+        request: The current HttpRequest object.
+
+    Returns:
+        dict: A dictionary with the INDEXABLE setting.
+    """
     return {'INDEXABLE': settings.INDEXABLE}

core_directory/sitemaps.py

@@ -3,13 +3,15 @@
 Classes:
     ProjectSitemap: Generates sitemap entries for all Project objects.
     CorePackageSitemap: Generates sitemap entries for all CorePackage objects.
-    StaticViewSitemap: Generates sitemap entries for static views such as 'landing', 'core-package-list', and 'vendor-list'.
-Each sitemap class specifies the change frequency and priority for its entries, and provides methods to retrieve the items to be included in the sitemap.
+    StaticViewSitemap: Generates sitemap entries for static views such as 'landing',
+    'core-package-list', and 'vendor-list'.
+Each sitemap class specifies the change frequency and priority for its entries,
+and provides methods to retrieve the items to be included in the sitemap.
 """
 
 from django.contrib.sitemaps import Sitemap
-from .models import CorePackage, Vendor
 from django.urls import reverse
+from .models import CorePackage, Vendor
 
 class VendorSitemap(Sitemap):
     """
@@ -24,7 +26,7 @@ class VendorSitemap(Sitemap):
     priority = 0.6
 
     def items(self):
-        return Vendor.objects.all()
+        return Vendor.objects.all().order_by('sanitized_name', 'pk')
 
 class CorePackageSitemap(Sitemap):
     """
@@ -44,7 +46,7 @@ class CorePackageSitemap(Sitemap):
     changefreq = "daily"
     priority = 0.8
     def items(self):
-        return CorePackage.objects.all()
+        return CorePackage.objects.all().order_by('vlnv_name')
 
 class StaticViewSitemap(Sitemap):
     """

core_directory/tests/system/__init__.py

@@ -0,0 +1,22 @@
+import pytest
+from django.urls import reverse
+from django.conf import settings
+
+@pytest.mark.django_db
+def test_robots_txt_indexable_true(settings, client):
+    settings.INDEXABLE = True
+    url = reverse("robots_txt")
+    response = client.get(url)
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("text/plain")
+    assert "Disallow:" in response.content.decode()
+    assert "Disallow: /" not in response.content.decode()
+
+@pytest.mark.django_db
+def test_robots_txt_indexable_false(settings, client):
+    settings.INDEXABLE = False
+    url = reverse("robots_txt")
+    response = client.get(url)
+    assert response.status_code == 200
+    assert response["Content-Type"].startswith("text/plain")
+    assert "Disallow: /" in response.content.decode()

core_directory/tests/system/test_robots_txt.py

@@ -0,0 +1,47 @@
+# tests/test_sitemaps.py
+
+import pytest
+from django.urls import reverse, NoReverseMatch
+from django.conf import settings
+from django.test import override_settings
+
+from core_directory.models import Vendor, Library, Project, CorePackage
+
+@pytest.mark.django_db
+@override_settings(INDEXABLE=True)
+def test_sitemap_with_data(client):
+    # Create test data
+    vendor = Vendor.objects.create(name="Acme")
+    library = Library.objects.create(vendor=vendor, name="Lib1")
+    project = Project.objects.create(vendor=vendor, library=library, name="Core1", description="desc")
+    core_package = CorePackage.objects.create(
+        project=project,
+        vlnv_name="acme:lib1:core1:1.0.0",
+        version="1.0.0",
+        core_url="https://example.com/core",
+        description="desc"
+    )
+
+    # The sitemap should be available
+    url = reverse("sitemap")
+    response = client.get(url)
+    assert response.status_code == 200
+    content = response.content.decode()
+
+    # Check for expected URLs in the sitemap
+    assert "<urlset" in content
+    # Vendor page
+    assert vendor.get_absolute_url() in content
+    # CorePackage page
+    assert core_package.get_absolute_url() in content
+    # Static views
+    assert reverse("landing") in content
+    assert reverse("core-package-list") in content
+    assert reverse("vendor-list") in content
+
+@pytest.mark.django_db
+@override_settings(INDEXABLE=False)
+def test_sitemap_not_available_when_indexing_disabled(client):
+    url = reverse("sitemap")
+    response = client.get(url)
+    assert response.status_code == 404

core_directory/tests/test_sitemap.py

@@ -12,7 +12,6 @@
     ("api_docs_landing", {}, 200, False, "get"),
     ("schema", {}, 200, False, "get"),
     ("swagger_ui", {}, 200, False, "get"),
-    ("redoc_ui", {}, 200, False, "get"),
 ])
 def test_url_resolves_and_returns(client, mocker, url_name, kwargs, expected_status, needs_object, method):
     """

core_directory/tests/test_urls.py

@@ -1,6 +1,9 @@
 import pytest
 from django.urls import reverse, resolve
+from django.test import override_settings
 
+@pytest.mark.django_db
+@override_settings(INDEXABLE=False)
 @pytest.mark.parametrize("url_name, kwargs, expected_status", [
     ("landing", {}, 200),
     ("core-detail", {"pk": 1}, 200),
@@ -13,6 +16,8 @@
     }, 200),
     ("vendor-list", {}, 200),
     ("vendor-detail", {"sanitized_name": "acme"}, 200),
+    ("robots_txt", {}, 200),
+    ("sitemap", {}, 200),
 ])
 def test_project_urls(client, url_name, kwargs, expected_status):
     """

project/tests/tests_project_urls.py project/tests/test_project_urls.pyproject/tests/tests_project_urls.py renamed to project/tests/test_project_urls.py

@@ -19,6 +19,7 @@
         2. Add a URL to urlpatterns:  path('blog/', include('blog.urls'))
 """
 from django.conf import settings
+from django.http import HttpResponseNotFound
 from django.contrib import admin
 from django.contrib.sitemaps.views import sitemap
 from django.urls import include, path
@@ -41,6 +42,14 @@
     'vendors': VendorSitemap,
 }
 
+def guarded_sitemap_view(request, *args, **kwargs):
+    """
+    Return the sitemap if INDEXABLE is True, else return 404.
+    """
+    if not settings.INDEXABLE:
+        return HttpResponseNotFound("Sitemap is disabled.")
+    return sitemap(request, *args, **kwargs)
+
 urlpatterns = [
 
     path('', landing, name='landing'),
@@ -64,10 +73,6 @@
     path('admin/', admin.site.urls),
     path('api/', include('core_directory.urls')),
 
-    path('robots.txt', robots_txt, name='robots_txt')
+    path('robots.txt', robots_txt, name='robots_txt'),
+    path('sitemap.xml', guarded_sitemap_view, {'sitemaps': sitemaps}, name='sitemap'),
 ]
-
-if settings.INDEXABLE:
-    urlpatterns += [
-        path('sitemap.xml', sitemap, {'sitemaps': sitemaps}, name='django.contrib.sitemaps.views.sitemap'),
-    ]