From e25d2381eae3e541f63dc9131784a45a47307b96 Mon Sep 17 00:00:00 2001 From: marcperrinoptel Date: Thu, 16 Feb 2023 11:09:42 -0500 Subject: [PATCH 1/6] Fix issue with static files on Windows by mimicking Django's work on url before calling finders.find (which really does need an OS-standardized path as input) --- src/whitenoise/middleware.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 3f5a8091..920c94b4 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -1,8 +1,9 @@ from __future__ import annotations import os -from posixpath import basename +from posixpath import basename, normpath from urllib.parse import urlparse +from urllib.request import url2pathname from django.conf import settings from django.contrib.staticfiles import finders @@ -155,7 +156,10 @@ def add_files_from_finders(self): def candidate_paths_for_url(self, url): if self.use_finders and url.startswith(self.static_prefix): - path = finders.find(url[len(self.static_prefix) :]) + relative_url = url[len(self.static_prefix) :] + path = url2pathname(relative_url) + normalized_path = normpath(path).lstrip('/') + path = finders.find(normalized_path) if path: yield path paths = super().candidate_paths_for_url(url) From af62aeb638512adc129f953c12266a77b82e11a2 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 16 Feb 2023 16:12:54 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/whitenoise/middleware.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py index 920c94b4..57cfc71b 100644 --- a/src/whitenoise/middleware.py +++ b/src/whitenoise/middleware.py @@ -1,7 +1,8 @@ from __future__ import annotations import os -from posixpath import basename, normpath +from posixpath import basename +from posixpath import normpath from urllib.parse import urlparse from urllib.request import url2pathname @@ -158,7 +159,7 @@ def candidate_paths_for_url(self, url): if self.use_finders and url.startswith(self.static_prefix): relative_url = url[len(self.static_prefix) :] path = url2pathname(relative_url) - normalized_path = normpath(path).lstrip('/') + normalized_path = normpath(path).lstrip("/") path = finders.find(normalized_path) if path: yield path From e1d2b07cef18b1c4fe2b078880be7a2c4cc8a01a Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 27 Aug 2024 01:54:24 -0700 Subject: [PATCH 3/6] Merge into servestatic --- src/servestatic/middleware.py | 8 +- src/whitenoise/middleware.py | 207 ---------------------------------- 2 files changed, 6 insertions(+), 209 deletions(-) delete mode 100644 src/whitenoise/middleware.py diff --git a/src/servestatic/middleware.py b/src/servestatic/middleware.py index f693106a..b269c0ee 100644 --- a/src/servestatic/middleware.py +++ b/src/servestatic/middleware.py @@ -4,9 +4,10 @@ import concurrent.futures import contextlib import os -from posixpath import basename +from posixpath import basename, normpath from typing import AsyncIterable from urllib.parse import urlparse +from urllib.request import url2pathname import django from aiofiles.base import AiofilesContextManager @@ -252,7 +253,10 @@ def add_files_from_finders(self): def candidate_paths_for_url(self, url): if self.use_finders and url.startswith(self.static_prefix): - path = finders.find(url[len(self.static_prefix) :]) + relative_url = url[len(self.static_prefix) :] + path = url2pathname(relative_url) + normalized_path = normpath(path).lstrip("/") + path = finders.find(normalized_path) if path: yield path paths = super().candidate_paths_for_url(url) diff --git a/src/whitenoise/middleware.py b/src/whitenoise/middleware.py deleted file mode 100644 index 57cfc71b..00000000 --- a/src/whitenoise/middleware.py +++ /dev/null @@ -1,207 +0,0 @@ -from __future__ import annotations - -import os -from posixpath import basename -from posixpath import normpath -from urllib.parse import urlparse -from urllib.request import url2pathname - -from django.conf import settings -from django.contrib.staticfiles import finders -from django.contrib.staticfiles.storage import staticfiles_storage -from django.http import FileResponse -from django.urls import get_script_prefix - -from .base import WhiteNoise -from .string_utils import ensure_leading_trailing_slash - -__all__ = ["WhiteNoiseMiddleware"] - - -class WhiteNoiseFileResponse(FileResponse): - """ - Wrap Django's FileResponse to prevent setting any default headers. For the - most part these just duplicate work already done by WhiteNoise but in some - cases (e.g. the content-disposition header introduced in Django 3.0) they - are actively harmful. - """ - - def set_headers(self, *args, **kwargs): - pass - - -class WhiteNoiseMiddleware(WhiteNoise): - """ - Wrap WhiteNoise to allow it to function as Django middleware, rather - than WSGI middleware. - """ - - def __init__(self, get_response=None, settings=settings): - self.get_response = get_response - - try: - autorefresh: bool = settings.WHITENOISE_AUTOREFRESH - except AttributeError: - autorefresh = settings.DEBUG - try: - max_age = settings.WHITENOISE_MAX_AGE - except AttributeError: - if settings.DEBUG: - max_age = 0 - else: - max_age = 60 - try: - allow_all_origins = settings.WHITENOISE_ALLOW_ALL_ORIGINS - except AttributeError: - allow_all_origins = True - try: - charset = settings.WHITENOISE_CHARSET - except AttributeError: - charset = "utf-8" - try: - mimetypes = settings.WHITENOISE_MIMETYPES - except AttributeError: - mimetypes = None - try: - add_headers_function = settings.WHITENOISE_ADD_HEADERS_FUNCTION - except AttributeError: - add_headers_function = None - try: - index_file = settings.WHITENOISE_INDEX_FILE - except AttributeError: - index_file = None - try: - immutable_file_test = settings.WHITENOISE_IMMUTABLE_FILE_TEST - except AttributeError: - immutable_file_test = None - - super().__init__( - application=None, - autorefresh=autorefresh, - max_age=max_age, - allow_all_origins=allow_all_origins, - charset=charset, - mimetypes=mimetypes, - add_headers_function=add_headers_function, - index_file=index_file, - immutable_file_test=immutable_file_test, - ) - - try: - self.use_finders = settings.WHITENOISE_USE_FINDERS - except AttributeError: - self.use_finders = settings.DEBUG - - try: - self.static_prefix = settings.WHITENOISE_STATIC_PREFIX - except AttributeError: - self.static_prefix = urlparse(settings.STATIC_URL or "").path - script_prefix = get_script_prefix().rstrip("/") - if script_prefix: - if self.static_prefix.startswith(script_prefix): - self.static_prefix = self.static_prefix[len(script_prefix) :] - self.static_prefix = ensure_leading_trailing_slash(self.static_prefix) - - self.static_root = settings.STATIC_ROOT - if self.static_root: - self.add_files(self.static_root, prefix=self.static_prefix) - - try: - root = settings.WHITENOISE_ROOT - except AttributeError: - root = None - if root: - self.add_files(root) - - if self.use_finders and not self.autorefresh: - self.add_files_from_finders() - - def __call__(self, request): - if self.autorefresh: - static_file = self.find_file(request.path_info) - else: - static_file = self.files.get(request.path_info) - if static_file is not None: - return self.serve(static_file, request) - return self.get_response(request) - - @staticmethod - def serve(static_file, request): - response = static_file.get_response(request.method, request.META) - status = int(response.status) - http_response = WhiteNoiseFileResponse(response.file or (), status=status) - # Remove default content-type - del http_response["content-type"] - for key, value in response.headers: - http_response[key] = value - return http_response - - def add_files_from_finders(self): - files = {} - for finder in finders.get_finders(): - for path, storage in finder.list(None): - prefix = (getattr(storage, "prefix", None) or "").strip("/") - url = "".join( - ( - self.static_prefix, - prefix, - "/" if prefix else "", - path.replace("\\", "/"), - ) - ) - # Use setdefault as only first matching file should be used - files.setdefault(url, storage.path(path)) - stat_cache = {path: os.stat(path) for path in files.values()} - for url, path in files.items(): - self.add_file_to_dictionary(url, path, stat_cache=stat_cache) - - def candidate_paths_for_url(self, url): - if self.use_finders and url.startswith(self.static_prefix): - relative_url = url[len(self.static_prefix) :] - path = url2pathname(relative_url) - normalized_path = normpath(path).lstrip("/") - path = finders.find(normalized_path) - if path: - yield path - paths = super().candidate_paths_for_url(url) - for path in paths: - yield path - - def immutable_file_test(self, path, url): - """ - Determine whether given URL represents an immutable file (i.e. a - file with a hash of its contents as part of its name) which can - therefore be cached forever - """ - if not url.startswith(self.static_prefix): - return False - name = url[len(self.static_prefix) :] - name_without_hash = self.get_name_without_hash(name) - if name == name_without_hash: - return False - static_url = self.get_static_url(name_without_hash) - # If the static_url function maps the name without hash - # back to the original name, then we know we've got a - # versioned filename - if static_url and basename(static_url) == basename(url): - return True - return False - - def get_name_without_hash(self, filename): - """ - Removes the version hash from a filename e.g, transforms - 'css/application.f3ea4bcc2.css' into 'css/application.css' - - Note: this is specific to the naming scheme used by Django's - CachedStaticFilesStorage. You may have to override this if - you are using a different static files versioning system - """ - name_with_hash, ext = os.path.splitext(filename) - name = os.path.splitext(name_with_hash)[0] - return name + ext - - def get_static_url(self, name): - try: - return staticfiles_storage.url(name) - except ValueError: - return None From 743b29c8a7d38e378e2ba3ba3b285ba19f0efee3 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 27 Aug 2024 02:17:45 -0700 Subject: [PATCH 4/6] Add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5356597c..9db356ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,6 +37,7 @@ Using the following categories, list your changes in this order: ### Fixed - Fix Django "StreamingHttpResponse must consume synchronous iterators" warning +- Fix bug where non-standard file paths could fail to be followed on Windows ([Upstream PR](https://github.com/evansd/whitenoise/pull/474)) ## [1.0.0] - 2024-05-08 From 78e0ace4b11eee9caf067557e76fed61784008ca Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 27 Aug 2024 02:20:56 -0700 Subject: [PATCH 5/6] Modify changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9db356ae..3ff63b82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,8 +36,8 @@ Using the following categories, list your changes in this order: ### Fixed -- Fix Django "StreamingHttpResponse must consume synchronous iterators" warning -- Fix bug where non-standard file paths could fail to be followed on Windows ([Upstream PR](https://github.com/evansd/whitenoise/pull/474)) +- Fix Django `StreamingHttpResponse must consume synchronous iterators` warning +- Fix bug where file paths could fail to be followed on Windows ([Upstream PR](https://github.com/evansd/whitenoise/pull/474)) ## [1.0.0] - 2024-05-08 From 8c7f2d6a38d486b14ce09f8a0b666c076de8f7b8 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 27 Aug 2024 02:24:54 -0700 Subject: [PATCH 6/6] Clarify that the fix is django only --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ff63b82..f12d0801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -37,7 +37,7 @@ Using the following categories, list your changes in this order: ### Fixed - Fix Django `StreamingHttpResponse must consume synchronous iterators` warning -- Fix bug where file paths could fail to be followed on Windows ([Upstream PR](https://github.com/evansd/whitenoise/pull/474)) +- Fix Django bug where file paths could fail to be followed on Windows ([Upstream PR](https://github.com/evansd/whitenoise/pull/474)) ## [1.0.0] - 2024-05-08