From e35d7eb1109d52b1797e3a67baf18b075b2443f8 Mon Sep 17 00:00:00 2001 From: Kshitij Sobti Date: Mon, 17 Apr 2023 16:53:18 +0530 Subject: [PATCH 001/282] chore: Switch from edx-sphinx-theme to sphinx-book-theme The edx-sphinx theme is being deprecated, and replaced with sphinx-book-theme. This removes references to the deprecated theme and replaces them with the new standard theme for the platform. --- CHANGELOG.rst | 6 ++++-- docs/conf.py | 47 +++++++++++++++++++++++++++++++++++--------- requirements/doc.in | 2 +- requirements/doc.txt | 29 ++++++++++++++++++++------- 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index fcbcbdf2e..9c2000774 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,8 +14,10 @@ Change Log Unreleased ~~~~~~~~~~ -* - +* Switch from ``edx-sphinx-theme`` to ``sphinx-book-theme`` since the former is + deprecated. See https://github.com/openedx/edx-sphinx-theme/issues/184 for + more details. + [0.1.0] - 2021-08-08 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/conf.py b/docs/conf.py index 5c9e7aceb..763a5ffb6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,9 +14,9 @@ import os import re import sys +from datetime import datetime from subprocess import check_call -import edx_theme from django import setup as django_setup @@ -59,7 +59,6 @@ def get_version(*file_paths): # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'edx_theme', 'sphinx.ext.autodoc', 'sphinxcontrib_django', 'sphinx.ext.doctest', @@ -91,8 +90,8 @@ def get_version(*file_paths): # General information about the project. project = 'openedx-learning' -copyright = edx_theme.COPYRIGHT # pylint: disable=redefined-builtin -author = edx_theme.AUTHOR +copyright = f'{datetime.now().year}, edX Inc.' # pylint: disable=redefined-builtin +author = 'edX Inc.' project_title = 'Open edX Learning Core' documentation_title = f"{project_title}" @@ -163,16 +162,46 @@ def get_version(*file_paths): # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'edx_theme' +html_theme = 'sphinx_book_theme' # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. # -# html_theme_options = {} +html_theme_options = { + "repository_url": "https://github.com/openedx/openedx-learning", + "repository_branch": "main", + "path_to_docs": "docs/", + "home_page_in_toc": True, + "use_repository_button": True, + "use_issues_button": True, + "use_edit_page_button": True, + # Please don't change unless you know what you're doing. + "extra_footer": """ + + Creative Commons License + +
+ These works by + The Axim Collaborative + are licensed under a + Creative Commons Attribution-ShareAlike 4.0 International License. + """ +} # Add any paths that contain custom themes here, relative to this directory. -html_theme_path = [edx_theme.get_html_theme_path()] +# html_theme_path = [] # The name for this set of Sphinx documents. # " v documentation" by default. @@ -186,13 +215,13 @@ def get_version(*file_paths): # The name of an image file (relative to this directory) to place at the top # of the sidebar. # -# html_logo = None +html_logo = "https://logos.openedx.org/open-edx-logo-color.png" # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. # -# html_favicon = None +html_favicon = "https://logos.openedx.org/open-edx-favicon.ico" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, diff --git a/requirements/doc.in b/requirements/doc.in index f340ba2c9..4869da915 100644 --- a/requirements/doc.in +++ b/requirements/doc.in @@ -4,7 +4,7 @@ -r test.txt # Core and testing dependencies for this package doc8 # reStructuredText style checker -edx_sphinx_theme # edX theme for Sphinx output +sphinx-book-theme # Common theme for all Open edX projects readme_renderer # Validates README.rst for usage on PyPI Sphinx # Documentation builder sphinxcontrib-django # Django-awareness for Sphinx diff --git a/requirements/doc.txt b/requirements/doc.txt index 338ada5fa..4418ed34a 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -4,6 +4,8 @@ # # make upgrade # +accessible-pygments==0.0.4 + # via pydata-sphinx-theme alabaster==0.7.12 # via sphinx asgiref==3.5.2 @@ -15,11 +17,15 @@ attrs==22.1.0 # -r requirements/test.txt # pytest babel==2.10.3 - # via sphinx + # via + # pydata-sphinx-theme + # sphinx backports-zoneinfo==0.2.1 # via # -r requirements/test.txt # django +beautifulsoup4==4.12.2 + # via pydata-sphinx-theme bleach==5.0.1 # via readme-renderer certifi==2022.6.15 @@ -50,11 +56,10 @@ doc8==1.0.0 docutils==0.19 # via # doc8 + # pydata-sphinx-theme # readme-renderer # restructuredtext-lint # sphinx -edx-sphinx-theme==3.0.0 - # via -r requirements/doc.in idna==3.3 # via requests imagesize==1.4.1 @@ -82,6 +87,7 @@ markupsafe==2.1.1 packaging==21.3 # via # -r requirements/test.txt + # pydata-sphinx-theme # pytest # sphinx pbr==5.10.0 @@ -96,9 +102,13 @@ py==1.11.0 # via # -r requirements/test.txt # pytest +pydata-sphinx-theme==0.13.3 + # via sphinx-book-theme pygments==2.13.0 # via + # accessible-pygments # doc8 + # pydata-sphinx-theme # readme-renderer # sphinx pyparsing==3.0.9 @@ -134,15 +144,18 @@ requests==2.28.1 restructuredtext-lint==1.4.0 # via doc8 six==1.16.0 - # via - # bleach - # edx-sphinx-theme + # via bleach snowballstemmer==2.2.0 # via sphinx +soupsieve==2.4.1 + # via beautifulsoup4 sphinx==5.1.1 # via # -r requirements/doc.in - # edx-sphinx-theme + # pydata-sphinx-theme + # sphinx-book-theme +sphinx-book-theme==1.0.1 + # via -r requirements/doc.in sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 @@ -176,6 +189,8 @@ tomli==2.0.1 # coverage # doc8 # pytest +typing-extensions==4.5.0 + # via pydata-sphinx-theme urllib3==1.26.12 # via requests webencodings==0.5.1 From a838a91644d05ef0becfdafa0b11e496f90654bf Mon Sep 17 00:00:00 2001 From: Giovanni Cimolin da Silva Date: Wed, 19 Apr 2023 16:44:02 +0100 Subject: [PATCH 002/282] docs: ADR for tagging service. (#40) --- docs/decisions/0004-content-tagging.rst | 45 +++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 docs/decisions/0004-content-tagging.rst diff --git a/docs/decisions/0004-content-tagging.rst b/docs/decisions/0004-content-tagging.rst new file mode 100644 index 000000000..dff13aec1 --- /dev/null +++ b/docs/decisions/0004-content-tagging.rst @@ -0,0 +1,45 @@ +4. Content Tagging +================== + +Context +------- + +Tagging content is central to enable content re-use, facilitate the implementation of flexible content structures different from the current implementation and allow adaptive learning in the Open edX platform. + +Content tagging should be classified as "kernel" component following `OEP-57's `_ guidelines, as a baked-in feature of the platform. + +Decision +-------- + +Implement the new tagging service as a pluggable django app, and installed alongside learning-core as a dependency in the edx-platform. + +Tagging data models will follow the guidelines for this repository, and focus on extensibility and flexibility. + +Since some use cases for content tagging are not considered "kernel" (like providing data for a marketing site), a generic mechanism to differentiate those uses cases will be built, and proper Python and REST APIs will be provided, to different taxonomies/tags for the same content. + + +Rejected Alternatives +--------------------- + +Implementing tagging as Discovery service plugin +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Discovery is already used as the source of truth place for course/program/block metadata in the previous implementations, so keeping it here means keeping the platform metadata management "consistent" (specially since it needs to work with all kinds of contents - from atomic learning units all the way up to courses). + +Concerns: + +#. The course discovery repository is not well documented and has code that is only used by edX/2U. +#. Some implementations are tied to specific closed-source services (example: `openedx/taxonomy-connector `_ uses a 3rd party closed-source service to tag video blocks automatically). +#. The data flow from the discovery service is confusing, and evolved as the needs for that repo changed. `Reference `_. +#. Many people prefer/need to run the platform without the discovery service. + +Implement tagging as a new IDA +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A new IDA dedicated for tagging would represent a break from the current codebases and force implementation using REST APIs. +This option adds extra complexity without a good compelling reason for it. + +Concerns: + +#. One more app to host: maintenance cost increases over time. +#. Extra response time between services when handling synchronous operations related to tagging. From f112ea4fb7bf47929a018bb7a189996aeb1fe861 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 12 Feb 2023 19:11:31 -0500 Subject: [PATCH 003/282] feat: add support for static assets This alters the data model in the components app in a number of ways in order to support static assets, along with some refactoring: * Content has been renamed to RawContent to make it more clear when we are talking about "content" as a general concept and the actual data model that holds the raw bytes. * RawContent now uses FileField instead of BinaryField, giving us more cheaply scalable storage in exchange for higher latency. This is offset by the new TextContent model that will be used to store the text versions of RawContent that needs low-latency access (like XBlock OLX). * A primitive media_server app now exists to view static assets during development. It is NOT safe to use on a running site yet. --- .gitignore | 2 + olx_importer/apps.py | 1 + .../management/commands/load_components.py | 106 +++++--- .../contrib/media_server/__init__.py | 0 openedx_learning/contrib/media_server/apps.py | 22 ++ .../contrib/media_server/readme.rst | 16 ++ openedx_learning/contrib/media_server/urls.py | 16 ++ .../contrib/media_server/views.py | 41 +++ openedx_learning/contrib/readme.rst | 9 + openedx_learning/core/components/admin.py | 140 +++++----- openedx_learning/core/components/api.py | 43 ++++ .../components/migrations/0001_initial.py | 187 +++++++++----- openedx_learning/core/components/models.py | 241 +++++++++++++----- openedx_learning/core/components/readme.rst | 3 +- .../publishing/migrations/0001_initial.py | 31 ++- openedx_learning/lib/admin_utils.py | 28 ++ openedx_learning/lib/fields.py | 13 + openedx_learning/lib/validators.py | 12 + openedx_learning/rest_api/urls.py | 1 - projects/dev.py | 103 ++++---- projects/urls.py | 8 +- requirements/base.in | 11 +- requirements/base.txt | 23 +- requirements/ci.txt | 20 +- requirements/dev.txt | 63 ++--- requirements/doc.txt | 34 +-- requirements/quality.txt | 54 ++-- requirements/test.txt | 35 +-- test_settings.py | 47 ++-- 29 files changed, 823 insertions(+), 487 deletions(-) create mode 100644 openedx_learning/contrib/media_server/__init__.py create mode 100644 openedx_learning/contrib/media_server/apps.py create mode 100644 openedx_learning/contrib/media_server/readme.rst create mode 100644 openedx_learning/contrib/media_server/urls.py create mode 100644 openedx_learning/contrib/media_server/views.py create mode 100644 openedx_learning/core/components/api.py create mode 100644 openedx_learning/lib/admin_utils.py create mode 100644 openedx_learning/lib/validators.py diff --git a/.gitignore b/.gitignore index f642f3b20..9cabbe5db 100644 --- a/.gitignore +++ b/.gitignore @@ -65,3 +65,5 @@ dev.db .vscode +# Media files (for uploads) +media/ diff --git a/olx_importer/apps.py b/olx_importer/apps.py index 416dfe97e..68fb21b89 100644 --- a/olx_importer/apps.py +++ b/olx_importer/apps.py @@ -9,5 +9,6 @@ class OLXImporterConfig(AppConfig): """ Configuration for the OLX Importer Django application. """ + name = "olx_importer" verbose_name = "OLX Importer" diff --git a/olx_importer/management/commands/load_components.py b/olx_importer/management/commands/load_components.py index 1e06df716..08fc47cc6 100644 --- a/olx_importer/management/commands/load_components.py +++ b/olx_importer/management/commands/load_components.py @@ -11,9 +11,9 @@ Open Question: If the data model is extensible, how do we know whether a change has really happened between what's currently stored/published for a particular -item and the new value we want to set? For Content that's easy, because we have -actual hashes of the data. But it's not clear how that would work for something -like an ComponentVersion. We'd have to have some kind of mechanism where every +item and the new value we want to set? For RawContent that's easy, because we +have actual hashes of the data. But it's not clear how that would work for +something like an ComponentVersion. We'd have to have some kind of mechanism where every pp that wants to attach data gets to answer the question of "has anything changed?" in order to decide if we really make a new ComponentVersion or not. """ @@ -25,22 +25,28 @@ import re import xml.etree.ElementTree as ET +from django.core.files.base import ContentFile from django.core.management.base import BaseCommand from django.db import transaction from openedx_learning.core.publishing.models import LearningPackage, PublishLogEntry from openedx_learning.core.components.models import ( - Content, Component, ComponentVersion, ComponentVersionContent, - ComponentPublishLogEntry, PublishedComponent, + Component, + ComponentVersion, + ComponentVersionRawContent, + ComponentPublishLogEntry, + PublishedComponent, + RawContent, + TextContent, ) from openedx_learning.lib.fields import create_hash_digest -SUPPORTED_TYPES = ['problem', 'video', 'html'] +SUPPORTED_TYPES = ["problem", "video", "html"] logger = logging.getLogger(__name__) class Command(BaseCommand): - help = 'Load sample Component data from course export' + help = "Load sample Component data from course export" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -61,11 +67,10 @@ def init_known_types(self): # officially "text/javascript" mimetypes.add_type("text/javascript", ".js") mimetypes.add_type("text/javascript", ".mjs") - def add_arguments(self, parser): - parser.add_argument('course_data_path', type=pathlib.Path) - parser.add_argument('learning_package_identifier', type=str) + parser.add_argument("course_data_path", type=pathlib.Path) + parser.add_argument("learning_package_identifier", type=str) def handle(self, course_data_path, learning_package_identifier, **options): self.course_data_path = course_data_path @@ -73,8 +78,8 @@ def handle(self, course_data_path, learning_package_identifier, **options): self.load_course_data(learning_package_identifier) def get_course_title(self): - course_type_dir = self.course_data_path / 'course' - course_xml_file = next(course_type_dir.glob('*.xml')) + course_type_dir = self.course_data_path / "course" + course_xml_file = next(course_type_dir.glob("*.xml")) course_root = ET.parse(course_xml_file).getroot() return course_root.attrib.get("display_name", "Unknown Course") @@ -87,9 +92,9 @@ def load_course_data(self, learning_package_identifier): learning_package, _created = LearningPackage.objects.get_or_create( identifier=learning_package_identifier, defaults={ - 'title': title, - 'created': now, - 'updated': now, + "title": title, + "created": now, + "updated": now, }, ) self.learning_package = learning_package @@ -105,11 +110,13 @@ def load_course_data(self, learning_package_identifier): self.import_block_type(block_type, now, publish_log_entry) def create_content(self, static_local_path, now, component_version): - identifier = pathlib.Path('static') / static_local_path + identifier = pathlib.Path("static") / static_local_path real_path = self.course_data_path / identifier mime_type, _encoding = mimetypes.guess_type(identifier) if mime_type is None: - logger.error(f" no mimetype found for {real_path}, defaulting to application/binary") + logger.error( + f" no mimetype found for {real_path}, defaulting to application/binary" + ) mime_type = "application/binary" try: @@ -120,20 +127,26 @@ def create_content(self, static_local_path, now, component_version): hash_digest = create_hash_digest(data_bytes) - content, _created = Content.objects.get_or_create( + raw_content, created = RawContent.objects.get_or_create( learning_package=self.learning_package, mime_type=mime_type, hash_digest=hash_digest, - defaults = dict( - data=data_bytes, + defaults=dict( size=len(data_bytes), created=now, - ) + ), ) - ComponentVersionContent.objects.get_or_create( + if created: + raw_content.file.save( + f"{raw_content.learning_package.uuid}/{hash_digest}", + ContentFile(data_bytes), + ) + + ComponentVersionRawContent.objects.get_or_create( component_version=component_version, - content=content, + raw_content=raw_content, identifier=identifier, + learner_downloadable=True, ) def import_block_type(self, block_type, now, publish_log_entry): @@ -146,35 +159,49 @@ def import_block_type(self, block_type, now, publish_log_entry): static_files_regex = re.compile(r"""['"]\/static\/(.+?)["'\?]""") block_data_path = self.course_data_path / block_type - for xml_file_path in block_data_path.glob('*.xml'): + for xml_file_path in block_data_path.glob("*.xml"): components_found += 1 identifier = xml_file_path.stem # Find or create the Component itself component, _created = Component.objects.get_or_create( learning_package=self.learning_package, - namespace='xblock.v1', + namespace="xblock.v1", type=block_type, identifier=identifier, - defaults = { - 'created': now, - } + defaults={ + "created": now, + }, ) - # Create the Content entry for the raw data... + # Create the RawContent entry for the raw data... data_bytes = xml_file_path.read_bytes() hash_digest = create_hash_digest(data_bytes) - data_str = codecs.decode(data_bytes, 'utf-8') - content, _created = Content.objects.get_or_create( + + raw_content, created = RawContent.objects.get_or_create( learning_package=self.learning_package, - mime_type=f'application/vnd.openedx.xblock.v1.{block_type}+xml', + mime_type=f"application/vnd.openedx.xblock.v1.{block_type}+xml", hash_digest=hash_digest, - defaults = dict( - data=data_bytes, + defaults=dict( + # text=data_str, size=len(data_bytes), created=now, - ) + ), ) + if created: + raw_content.file.save( + f"{raw_content.learning_package.uuid}/{hash_digest}", + ContentFile(data_bytes), + ) + + # Decode as utf-8-sig in order to strip any BOM from the data. + data_str = codecs.decode(data_bytes, "utf-8-sig") + TextContent.objects.create( + raw_content=raw_content, + text=data_str, + length=len(data_str), + ) + # TODO: Get associated file contents, both with the static regex, as # well as with XBlock-specific code that examines attributes in # video and HTML tag definitions. @@ -185,7 +212,7 @@ def import_block_type(self, block_type, now, publish_log_entry): logger.error(f"Parse error for {xml_file_path}: {err}") continue - display_name = block_root.attrib.get('display_name', "") + display_name = block_root.attrib.get("display_name", "") # Create the ComponentVersion component_version = ComponentVersion.objects.create( @@ -195,10 +222,11 @@ def import_block_type(self, block_type, now, publish_log_entry): created=now, created_by=None, ) - ComponentVersionContent.objects.create( + ComponentVersionRawContent.objects.create( component_version=component_version, - content=content, - identifier='source.xml', + raw_content=raw_content, + identifier="source.xml", + learner_downloadable=False, ) static_files_found = static_files_regex.findall(data_str) for static_local_path in static_files_found: diff --git a/openedx_learning/contrib/media_server/__init__.py b/openedx_learning/contrib/media_server/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_learning/contrib/media_server/apps.py b/openedx_learning/contrib/media_server/apps.py new file mode 100644 index 000000000..2612be027 --- /dev/null +++ b/openedx_learning/contrib/media_server/apps.py @@ -0,0 +1,22 @@ +from django.apps import AppConfig +from django.core.exceptions import ImproperlyConfigured +from django.conf import settings + + +class MediaServerConfig(AppConfig): + """ + Configuration for the Media Server application. + """ + + name = "openedx_learning.contrib.media_server" + verbose_name = "Learning Core: Media Server" + default_auto_field = "django.db.models.BigAutoField" + + def ready(self): + if not settings.DEBUG: + # Until we get proper security and support for running this app + # under a separate domain, just don't allow it to be run in + # production environments. + raise ImproperlyConfigured( + "The media_server app should only be run in DEBUG mode!" + ) diff --git a/openedx_learning/contrib/media_server/readme.rst b/openedx_learning/contrib/media_server/readme.rst new file mode 100644 index 000000000..aef395661 --- /dev/null +++ b/openedx_learning/contrib/media_server/readme.rst @@ -0,0 +1,16 @@ +Media Server App +================ + +The ``media_server`` app exists to serve media files that are ultimately backed by the RawContent model, *for development purposes and for sites with light-to-moderate traffic*. It also provides an API that can be used by CDNs for high traffic sites. + +Motivation +---------- + +The ``components`` app stores large binary file data by calculating the hash and creating a django-storages backed file named after that hash. This is efficient from a storage point of view, because we don't store redundant copies for every version of a Component. There are at least two drawbacks: + +* We have unintelligibly named files that are confusing for clients. +* Intra-file links between media files break. For instance, if we have a piece of HTML that makes a reference to a VTT file, that filename will have changed. + +This app tries to bridge that gap by serving URLs that preserve the original file names and give the illusion that there is a seprate set of media files for every version of a Component, but does a lookup behind the scenes to serve the correct hash-based-file. + +The big caveat on this is that Django is not really optimized to do this sort of asset serving. The most scalable approach is to have a CDN-backed solution where ``media_server`` serves the locations of files that are converted by worker code to serving the actual assets. (More details to follow when that part gets built out.) diff --git a/openedx_learning/contrib/media_server/urls.py b/openedx_learning/contrib/media_server/urls.py new file mode 100644 index 000000000..554966ace --- /dev/null +++ b/openedx_learning/contrib/media_server/urls.py @@ -0,0 +1,16 @@ +from django.urls import path + +from .views import component_asset + +urlpatterns = [ + path( + ( + "component_asset/" + "/" + "/" + "/" + "" + ), + component_asset, + ) +] diff --git a/openedx_learning/contrib/media_server/views.py b/openedx_learning/contrib/media_server/views.py new file mode 100644 index 000000000..86e92d131 --- /dev/null +++ b/openedx_learning/contrib/media_server/views.py @@ -0,0 +1,41 @@ +from pathlib import Path + +from django.http import FileResponse +from django.http import Http404 +from django.core.exceptions import ObjectDoesNotExist, PermissionDenied + +from openedx_learning.core.components.api import get_component_version_content + + +def component_asset( + request, learning_package_identifier, component_identifier, version_num, asset_path +): + """ + Serve the ComponentVersion asset data. + + This function maps from a logical URL with Component and verison data like: + media_server/component_asset/course101/finalexam-problem14/1/static/images/fig3.png + To the actual data file as stored in file/object storage, which looks like: + media/055499fd-f670-451a-9727-501ea9dfbf5b/a9528d66739a297aa0cd17106b0bc0f7515b8e78 + + TODO: + * ETag support + * Range queries + * Serving from a different domain than the rest of the service + """ + try: + cvc = get_component_version_content( + learning_package_identifier, component_identifier, version_num, asset_path + ) + except ObjectDoesNotExist: + raise Http404("File not found") + + if not cvc.learner_downloadable and not ( + request.user and request.user.is_superuser + ): + raise PermissionDenied("This file is not publicly downloadable.") + + response = FileResponse(cvc.raw_content.file, filename=Path(asset_path).name) + response["Content-Type"] = cvc.raw_content.mime_type + + return response diff --git a/openedx_learning/contrib/readme.rst b/openedx_learning/contrib/readme.rst index e69de29bb..908ed6217 100644 --- a/openedx_learning/contrib/readme.rst +++ b/openedx_learning/contrib/readme.rst @@ -0,0 +1,9 @@ +Contrib Package +=============== + +The ``contrib`` package holds Django apps that *could* be implemented in separate repos, but are bundled here because it's more convenient to do so. + +Guidelines +---------- + +Nothing from ``lib`` or ``core`` should *ever* import from ``contrib``. diff --git a/openedx_learning/core/components/admin.py b/openedx_learning/core/components/admin.py index 36e4a7999..10712d6a9 100644 --- a/openedx_learning/core/components/admin.py +++ b/openedx_learning/core/components/admin.py @@ -1,5 +1,3 @@ -import base64 - from django.contrib import admin from django.db.models.aggregates import Count, Sum from django.template.defaultfilters import filesizeformat @@ -9,20 +7,11 @@ from .models import ( Component, ComponentVersion, - Content, + ComponentVersionRawContent, PublishedComponent, + RawContent, ) - - -class ReadOnlyModelAdmin(admin.ModelAdmin): - def has_add_permission(self, request): - return False - - def has_change_permission(self, request, obj=None): - return False - - def has_delete_permission(self, request, obj=None): - return False +from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin class ComponentVersionInline(admin.TabularInline): @@ -70,8 +59,8 @@ def get_queryset(self, request): "component_version", "component_publish_log_entry__publish_log_entry", ) - .annotate(size=Sum("component_version__contents__size")) - .annotate(content_count=Count("component_version__contents")) + .annotate(size=Sum("component_version__raw_contents__size")) + .annotate(content_count=Count("component_version__raw_contents")) ) readonly_fields = ["component", "component_version", "component_publish_log_entry"] @@ -111,12 +100,16 @@ def identifier(self, pc): """ return format_html( '{}', - reverse("admin:components_componentversion_change", args=(pc.component_version_id,)), + reverse( + "admin:components_componentversion_change", + args=(pc.component_version_id,), + ), pc.component.identifier, ) def content_count(self, pc): return pc.content_count + content_count.short_description = "#" def size(self, pc): @@ -135,24 +128,36 @@ def title(self, pc): return pc.component_version.title -class ContentInline(admin.TabularInline): - model = ComponentVersion.contents.through - fields = ["format_identifier", "format_size", "rendered_data"] - readonly_fields = ["content", "format_identifier", "format_size", "rendered_data"] +class RawContentInline(admin.TabularInline): + model = ComponentVersion.raw_contents.through + fields = [ + "format_identifier", + "format_size", + "learner_downloadable", + "rendered_data", + ] + readonly_fields = [ + "raw_content", + "format_identifier", + "format_size", + "rendered_data", + ] extra = 0 - def rendered_data(self, cv_obj): - return content_preview(cv_obj.content, 100_000) + def rendered_data(self, cvc_obj): + return content_preview(cvc_obj) + + def format_size(self, cvc_obj): + return filesizeformat(cvc_obj.raw_content.size) - def format_size(self, cv_obj): - return filesizeformat(cv_obj.content.size) format_size.short_description = "Size" - def format_identifier(self, cv_obj): + def format_identifier(self, cvc_obj): return format_html( '{}', - reverse("admin:components_content_change", args=(cv_obj.content_id,)), - cv_obj.identifier, + link_for_cvc(cvc_obj), + # reverse("admin:components_content_change", args=(cvc_obj.content_id,)), + cvc_obj.identifier, ) format_identifier.short_description = "Identifier" @@ -166,7 +171,7 @@ class ComponentVersionAdmin(ReadOnlyModelAdmin): "title", "version_num", "created", - "contents", + "raw_contents", ] fields = [ "component", @@ -175,48 +180,46 @@ class ComponentVersionAdmin(ReadOnlyModelAdmin): "version_num", "created", ] - inlines = [ContentInline] + inlines = [RawContentInline] -@admin.register(Content) -class ContentAdmin(ReadOnlyModelAdmin): +@admin.register(RawContent) +class RawContentAdmin(ReadOnlyModelAdmin): list_display = [ "hash_digest", "learning_package", "mime_type", - "format_size", + "size", "created", ] fields = [ "learning_package", "hash_digest", "mime_type", - "format_size", + "size", "created", - "rendered_data", + "text_preview", ] readonly_fields = [ "learning_package", "hash_digest", "mime_type", - "format_size", + "size", "created", - "rendered_data", + "text_preview", ] list_filter = ("mime_type", "learning_package") - search_fields = ("hash_digest", "size") + search_fields = ("hash_digest",) - def format_size(self, content_obj): - return filesizeformat(content_obj.size) - format_size.short_description = "Size" - - def rendered_data(self, content_obj): - return content_preview(content_obj, 10_000_000) + def text_preview(self, raw_content_obj): + if hasattr(raw_content_obj, "text_content"): + return format_text_for_admin_display(raw_content_obj.text_content.text) + return "" def is_displayable_text(mime_type): - # Our usual text files, includiing things like text/markdown, text/html - media_type, media_subtype = mime_type.split('/') + # Our usual text files, including things like text/markdown, text/html + media_type, media_subtype = mime_type.split("/") if media_type == "text": return True @@ -228,7 +231,7 @@ def is_displayable_text(mime_type): # Other application/* types that we know we can display. if media_subtype in ["json", "x-subrip"]: return True - + # Other formats that are really specific types of JSON if media_subtype.endswith("+json"): return True @@ -236,21 +239,40 @@ def is_displayable_text(mime_type): return False -def content_preview(content_obj, size_limit): - if content_obj.size > size_limit: - return f"Too large to preview." +def link_for_cvc(cvc_obj: ComponentVersionRawContent): + return "/media_server/component_asset/{}/{}/{}/{}".format( + cvc_obj.raw_content.learning_package.identifier, + cvc_obj.component_version.component.identifier, + cvc_obj.component_version.version_num, + cvc_obj.identifier, + ) + + +def format_text_for_admin_display(text): + return format_html( + '
\n{}\n
', + text, + ) - # image before text check, since SVGs can be either, but we probably want to - # see the image version in the admin. - if content_obj.mime_type.startswith("image/"): - b64_str = base64.b64encode(content_obj.data).decode("ascii") - encoded_img_src = f"data:{content_obj.mime_type};base64,{b64_str}" - return format_html('', encoded_img_src) - if is_displayable_text(content_obj.mime_type): +def content_preview(cvc_obj: ComponentVersionRawContent): + raw_content_obj = cvc_obj.raw_content + + if raw_content_obj.mime_type.startswith("image/"): return format_html( - '
\n{}\n
', - content_obj.data.decode("utf-8"), + '', + # TODO: configure with settings value: + "/media_server/component_asset/{}/{}/{}/{}".format( + cvc_obj.raw_content.learning_package.identifier, + cvc_obj.component_version.component.identifier, + cvc_obj.component_version.version_num, + cvc_obj.identifier, + ), + ) + + if hasattr(raw_content_obj, "text_content"): + return format_text_for_admin_display( + raw_content_obj.text_content.text, ) return format_html("This content type cannot be displayed.") diff --git a/openedx_learning/core/components/api.py b/openedx_learning/core/components/api.py new file mode 100644 index 000000000..41d92f462 --- /dev/null +++ b/openedx_learning/core/components/api.py @@ -0,0 +1,43 @@ +""" +Public API for querying and manipulating Components. + +This API is still under construction and should not be considered "stable" until +this repo hits a 1.0 release. +""" +from django.db.models import Q +from pathlib import Path + +from .models import ComponentVersionRawContent + + +def get_component_version_content( + learning_package_identifier: str, + component_identifier: str, + version_num: int, + identifier: Path, +) -> ComponentVersionRawContent: + """ + Look up ComponentVersionRawContent by human readable identifiers. + + Notes: + + 1. This function is returning a model, which we generally frown upon. + 2. I'd like to experiment with different lookup methods + (see https://github.com/openedx/openedx-learning/issues/34) + + Can raise a django.core.exceptions.ObjectDoesNotExist error if there is no + matching ComponentVersionRawContent. + """ + return ComponentVersionRawContent.objects.select_related( + "raw_content", + "component_version", + "component_version__component", + "component_version__component__learning_package", + ).get( + Q( + component_version__component__learning_package__identifier=learning_package_identifier + ) + & Q(component_version__component__identifier=component_identifier) + & Q(component_version__version_num=version_num) + & Q(identifier=identifier) + ) diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py index 273a54ffb..f4a2c284b 100644 --- a/openedx_learning/core/components/migrations/0001_initial.py +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -1,14 +1,14 @@ -# Generated by Django 4.1 on 2023-02-10 18:58 +# Generated by Django 4.1.6 on 2023-04-14 00:12 from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.validators import uuid class Migration(migrations.Migration): - initial = True dependencies = [ @@ -41,7 +41,14 @@ class Migration(migrations.Migration): ("namespace", models.CharField(max_length=100)), ("type", models.CharField(blank=True, max_length=100)), ("identifier", models.CharField(max_length=255)), - ("created", models.DateTimeField()), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ( "learning_package", models.ForeignKey( @@ -83,7 +90,14 @@ class Migration(migrations.Migration): validators=[django.core.validators.MinValueValidator(1)] ), ), - ("created", models.DateTimeField()), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ( "component", models.ForeignKey( @@ -91,6 +105,14 @@ class Migration(migrations.Migration): to="components.component", ), ), + ( + "created_by", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ "verbose_name": "Component Version", @@ -98,7 +120,7 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name="Content", + name="RawContent", fields=[ ( "id", @@ -114,11 +136,18 @@ class Migration(migrations.Migration): ( "size", models.PositiveBigIntegerField( - validators=[django.core.validators.MaxValueValidator(10000000)] + validators=[django.core.validators.MaxValueValidator(50000000)] ), ), - ("created", models.DateTimeField()), - ("data", models.BinaryField(max_length=10000000)), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), + ("file", models.FileField(null=True, upload_to="")), ( "learning_package", models.ForeignKey( @@ -127,9 +156,48 @@ class Migration(migrations.Migration): ), ), ], + options={ + "verbose_name": "Raw Content", + "verbose_name_plural": "Raw Contents", + }, + ), + migrations.CreateModel( + name="PublishedComponent", + fields=[ + ( + "component", + models.OneToOneField( + on_delete=django.db.models.deletion.RESTRICT, + primary_key=True, + serialize=False, + to="components.component", + ), + ), + ], + options={ + "verbose_name": "Published Component", + "verbose_name_plural": "Published Components", + }, ), migrations.CreateModel( - name="ComponentVersionContent", + name="TextContent", + fields=[ + ( + "raw_content", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="text_content", + serialize=False, + to="components.rawcontent", + ), + ), + ("text", models.TextField(blank=True, max_length=100000)), + ("length", models.PositiveIntegerField()), + ], + ), + migrations.CreateModel( + name="ComponentVersionRawContent", fields=[ ( "id", @@ -140,7 +208,17 @@ class Migration(migrations.Migration): verbose_name="ID", ), ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), ("identifier", models.CharField(max_length=255)), + ("learner_downloadable", models.BooleanField(default=False)), ( "component_version", models.ForeignKey( @@ -149,30 +227,21 @@ class Migration(migrations.Migration): ), ), ( - "content", + "raw_content", models.ForeignKey( on_delete=django.db.models.deletion.RESTRICT, - to="components.content", + to="components.rawcontent", ), ), ], ), migrations.AddField( model_name="componentversion", - name="contents", + name="raw_contents", field=models.ManyToManyField( related_name="component_versions", - through="components.ComponentVersionContent", - to="components.content", - ), - ), - migrations.AddField( - model_name="componentversion", - name="created_by", - field=models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, + through="components.ComponentVersionRawContent", + to="components.rawcontent", ), ), migrations.CreateModel( @@ -211,84 +280,66 @@ class Migration(migrations.Migration): ), ], ), - migrations.CreateModel( - name="PublishedComponent", - fields=[ - ( - "component", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="components.component", - ), - ), - ( - "component_publish_log_entry", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="components.componentpublishlogentry", - ), - ), - ( - "component_version", - models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="components.componentversion", - ), - ), - ], - options={ - "verbose_name": "Published Component", - "verbose_name_plural": "Published Components", - }, - ), migrations.AddIndex( - model_name="content", + model_name="rawcontent", index=models.Index( fields=["learning_package", "mime_type"], name="content_idx_lp_mime_type", ), ), migrations.AddIndex( - model_name="content", + model_name="rawcontent", index=models.Index( fields=["learning_package", "-size"], name="content_idx_lp_rsize" ), ), migrations.AddIndex( - model_name="content", + model_name="rawcontent", index=models.Index( fields=["learning_package", "-created"], name="content_idx_lp_rcreated" ), ), migrations.AddConstraint( - model_name="content", + model_name="rawcontent", constraint=models.UniqueConstraint( fields=("learning_package", "mime_type", "hash_digest"), name="content_uniq_lc_mime_type_hash_digest", ), ), + migrations.AddField( + model_name="publishedcomponent", + name="component_publish_log_entry", + field=models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + to="components.componentpublishlogentry", + ), + ), + migrations.AddField( + model_name="publishedcomponent", + name="component_version", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to="components.componentversion", + ), + ), migrations.AddIndex( - model_name="componentversioncontent", + model_name="componentversionrawcontent", index=models.Index( - fields=["content", "component_version"], - name="componentversioncontent_c_cv", + fields=["raw_content", "component_version"], name="cvrawcontent_c_cv" ), ), migrations.AddIndex( - model_name="componentversioncontent", + model_name="componentversionrawcontent", index=models.Index( - fields=["component_version", "content"], - name="componentversioncontent_cv_d", + fields=["component_version", "raw_content"], name="cvrawcontent_cv_d" ), ), migrations.AddConstraint( - model_name="componentversioncontent", + model_name="componentversionrawcontent", constraint=models.UniqueConstraint( fields=("component_version", "identifier"), - name="componentversioncontent_uniq_cv_id", + name="cvrawcontent_uniq_cv_id", ), ), migrations.AddIndex( diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index fcd284d32..b1d683394 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -1,5 +1,5 @@ """ -The model hierarchy is Component -> ComponentVersion -> Content. +The model hierarchy is Component -> ComponentVersion -> RawContent. A Component is an entity like a Problem or Video. It has enough information to identify the Component and determine what the handler should be (e.g. XBlock @@ -11,13 +11,13 @@ publish status is tracked in PublishedComponent, with historical publish data in ComponentPublishLogEntry. -Content is a simple model holding unversioned, raw data, along with some simple -metadata like size and MIME type. +RawContent is a simple model holding unversioned, raw data, along with some +simple metadata like size and MIME type. -Multiple pieces of Content may be associated with a ComponentVersion, through -the ComponentVersionContent model. ComponentVersionContent allows to specify a -Component-local identifier. We're using this like a file path by convention, but -it's possible we might want to have special identifiers later. +Multiple pieces of RawContent may be associated with a ComponentVersion, through +the ComponentVersionRawContent model. ComponentVersionRawContent allows to +specify a Component-local identifier. We're using this like a file path by +convention, but it's possible we might want to have special identifiers later. """ from django.db import models from django.conf import settings @@ -34,22 +34,40 @@ class Component(models.Model): """ - This represents any content that has ever existed in a LearningPackage. + This represents any Component that has ever existed in a LearningPackage. + + What is a Component + ------------------- + + A Component is an entity like a Problem or Video. It has enough information + to identify itself and determine what the handler should be (e.g. XBlock + Problem), but little beyond that. A Component will have many ComponentVersions over time, and most metadata is - associated with the ComponentVersion model. Make a foreign key to this model - when you need a stable reference that will exist for as long as the - LearningPackage itself exists. It is possible for an Component to have no - published ComponentVersion, either because it was never published or because - it's been "deleted" (made unavailable). + associated with the ComponentVersion model and the RawContent that + ComponentVersions are associated with. A Component belongs to one and only one LearningPackage. - The UUID should be treated as immutable. The identifier field *is* mutable, - but changing it will affect all ComponentVersions. If you are referencing - this model from within the same process, use a foreign key to the id. If you - are referencing this Component from an external system, use the UUID. Do NOT - use the identifier if you can help it, since this can be changed. + How to use this model + --------------------- + + Make a foreign key to the Component model when you need a stable reference + that will exist for as long as the LearningPackage itself exists. It is + possible for an Component to have no published ComponentVersion, either + because it was never published or because it's been "deleted" (made + unavailable) at some point, but the Component will continue to exist. + + The UUID should be treated as immutable. + + The identifier field *is* mutable, but changing it will affect all + ComponentVersions. + + If you are referencing this model from within the same process, use a + foreign key to the id. If you are referencing this Component from an + external system/service, use the UUID. The identifier is the part that is + most likely to be human-readable, and may be exported/copied, but try not to + rely on it, since this value may change. Note: When we actually implement the ability to change identifiers, we should make a history table and a modified attribute on this model. @@ -82,7 +100,7 @@ class Meta: # a given LearningPackage. Note that this means it is possible to # have two Components that have the exact same identifier. An XBlock # would be modeled as namespace="xblock.v1" with the type as the - # block_type, so the identifier would only be the block_id (the + # block_type, so the identifier would only be the block_id (the # very last part of the UsageKey). models.UniqueConstraint( fields=[ @@ -102,7 +120,6 @@ class Meta: fields=["learning_package", "identifier"], name="component_idx_lp_identifier", ), - # Global Identifier Index: # * Search by identifier across all Components on the site. This # would be a support-oriented tool from Django Admin. @@ -110,7 +127,6 @@ class Meta: fields=["identifier"], name="component_idx_identifier", ), - # LearningPackage (reverse) Created Index: # * Search for most recently *created* Components for a given # LearningPackage, since they're the most likely to be actively @@ -134,11 +150,11 @@ class ComponentVersion(models.Model): A particular version of a Component. This holds the title (because that's versioned information) and the contents - via a M:M relationship with Content via ComponentVersionContent. + via a M:M relationship with RawContent via ComponentVersionRawContent. * Each ComponentVersion belongs to one and only one Component. * ComponentVersions have a version_num that should increment by one with - each new version. + each new version. """ uuid = immutable_uuid_field() @@ -165,16 +181,16 @@ class ComponentVersion(models.Model): # removed. Open edX in general doesn't let you remove users, but we should # try to model it so that this is possible eventually. created_by = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.SET_NULL, - null=True, + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, ) # The contents hold the actual interesting data associated with this # ComponentVersion. - contents = models.ManyToManyField( - "Content", - through="ComponentVersionContent", + raw_contents = models.ManyToManyField( + "RawContent", + through="ComponentVersionRawContent", related_name="component_versions", ) @@ -206,11 +222,12 @@ class Meta: fields=["component", "-created"], name="cv_idx_component_rcreated", ), - # Title Index: # * Search by title. models.Index( - fields=["title",], + fields=[ + "title", + ], name="cv_idx_title", ), ] @@ -268,38 +285,47 @@ class Meta: verbose_name_plural = "Published Components" -class Content(models.Model): +class RawContent(models.Model): """ This is the most basic piece of raw content data, with no version metadata. - Content stores data in an immutable Binary BLOB `data` field. This data is - not auto-normalized in any way, meaning that pieces of content that are - semantically equivalent (e.g. differently spaced/sorted JSON) will result in + RawContent stores data using the "file" field. This data is not + auto-normalized in any way, meaning that pieces of content that are + semantically equivalent (e.g. differently spaced/sorted JSON) may result in new entries. This model is intentionally ignorant of what these things mean, because it expects supplemental data models to build on top of it. - Two Content instances _can_ have the same hash_digest if they are of + Two RawContent instances _can_ have the same hash_digest if they are of different MIME types. For instance, an empty text file and an empty SRT file will both hash the same way, but be considered different entities. - The other fields on Content are for data that is intrinsic to the file data - itself (e.g. the size). Any smart parsing of the contents into more - structured metadata should happen in other models that hang off of Content. + The other fields on RawContent are for data that is intrinsic to the file + data itself (e.g. the size). Any smart parsing of the contents into more + structured metadata should happen in other models that hang off of + RawContent. + + RawContent models are not versioned in any way. The concept of versioning + only exists at a higher level. - Content models are not versioned in any way. The concept of versioning only - exists at a higher level. + RawContent is optimized for cheap storage, not low latency. It stores + content in a FileField. If you need faster text access across multiple rows, + add a TextContent entry that corresponds to the relevant RawContent. - Since this model uses a BinaryField to hold its data, we have to be careful - about scalability issues. For instance, video files should not be stored - here directly. There is a 10 MB limit set for the moment, to accomodate - things like PDF files and images, but the itention is for the vast majority - of rows to be much smaller than that. + If you need to transform this RawContent into more structured data for your + application, create a model with a OneToOneField(primary_key=True) + relationship to RawContent. Just remember that *you should always create the + RawContent entry* first, to ensure content is always exportable, even if + your app goes away in the future. """ - # Cap item size at 10 MB for now. - MAX_SIZE = 10_000_000 + # 50 MB is our current limit, based on the current Open edX Studio file + # upload size limit. + MAX_FILE_SIZE = 50_000_000 learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + + # This hash value may be calculated using create_hash_digest from the + # openedx.lib.fields module. hash_digest = hash_field() # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type @@ -310,16 +336,28 @@ class Content(models.Model): # that becomes necessary. mime_type = models.CharField(max_length=255, blank=False, null=False) + # This is the size of the raw data file in bytes. This can be different than + # the character length, since UTF-8 encoding can use anywhere between 1-4 + # bytes to represent any given character. size = models.PositiveBigIntegerField( - validators=[MaxValueValidator(MAX_SIZE)], + validators=[MaxValueValidator(MAX_FILE_SIZE)], ) - # This should be manually set so that multiple Content rows being set in the - # same transaction are created with the same timestamp. The timestamp should - # be UTC. + # This should be manually set so that multiple RawContent rows being set in + # the same transaction are created with the same timestamp. The timestamp + # should be UTC. created = manual_date_time_field() - data = models.BinaryField(null=False, max_length=MAX_SIZE) + # All content for the LearningPackage should be stored in files. See model + # docstring for more details on how to store this data in supplementary data + # models that offer better latency guarantees. + file = models.FileField( + null=True, + storage=settings.OPENEDX_LEARNING.get( + "STORAGE", + settings.DEFAULT_FILE_STORAGE, + ), + ) class Meta: constraints = [ @@ -336,7 +374,7 @@ class Meta: ] indexes = [ # LearningPackage MIME type Index: - # * Break down Content counts by type/subtype within a + # * Break down Content counts by type/subtype with in a # LearningPackage. # * Find all the Content in a LearningPackage that matches a # certain MIME type (e.g. "image/png", "application/pdf". @@ -356,31 +394,102 @@ class Meta: models.Index( fields=["learning_package", "-created"], name="content_idx_lp_rcreated", - ) + ), ] + verbose_name = "Raw Content" + verbose_name_plural = "Raw Contents" + + +class TextContent(models.Model): + """ + TextContent supplements RawContent to give an in-table text copy. + + This model exists so that we can have lower-latency access to this data, + particularly if we're pulling back multiple rows at once. + Apps are encouraged to create their own data models that further extend this + one with a more intelligent, parsed data model. For example, individual + XBlocks might parse the OLX in this model into separate data models for + VideoBlock, ProblemBlock, etc. -class ComponentVersionContent(models.Model): + The reason this is built directly into the Learning Core data model is + because we want to be able to easily access and browse this data even if the + app-extended models get deleted (e.g. if they are deprecated and removed). """ - Determines the Content for a given ComponentVersion. + + # 100K is our limit for text data, like OLX. This means 100K *characters*, + # not bytes. Since UTF-8 encodes characters using as many as 4 bytes, this + # couled be as much as 400K of data if we had nothing but emojis. + MAX_TEXT_LENGTH = 100_000 + + raw_content = models.OneToOneField( + RawContent, + on_delete=models.CASCADE, + primary_key=True, + related_name="text_content", + ) + text = models.TextField(null=False, blank=True, max_length=MAX_TEXT_LENGTH) + length = models.PositiveIntegerField(null=False) + + +class ComponentVersionRawContent(models.Model): + """ + Determines the RawContent for a given ComponentVersion. An ComponentVersion may be associated with multiple pieces of binary data. For instance, a Video ComponentVersion might be associated with multiple transcripts in different languages. - When Content is associated with an ComponentVersion, it has some local + When RawContent is associated with an ComponentVersion, it has some local identifier that is unique within the the context of that ComponentVersion. This allows the ComponentVersion to do things like store an image file and reference it by a "path" identifier. - Content is immutable and sharable across multiple ComponentVersions and even - across LearningPackages. + RawContent is immutable and sharable across multiple ComponentVersions and + even across LearningPackages. """ + raw_content = models.ForeignKey(RawContent, on_delete=models.RESTRICT) component_version = models.ForeignKey(ComponentVersion, on_delete=models.CASCADE) - content = models.ForeignKey(Content, on_delete=models.RESTRICT) + + uuid = immutable_uuid_field() identifier = identifier_field() + # Is this RawContent downloadable during the learning experience? This is + # NOT about public vs. private permissions on course assets, as that will be + # a policy that can be changed independently of new versions of the content. + # For instance, a course team could decide to flip their course assets from + # private to public for CDN caching reasons, and that should not require + # new ComponentVersions to be created. + # + # What the ``learner_downloadable`` field refers to is whether this asset is + # supposed to *ever* be directly downloadable by browsers during the + # learning experience. This will be True for things like images, PDFs, and + # video transcript files. This field will be False for things like: + # + # * Problem Block OLX will contain the answers to the problem. The XBlock + # runtime and ProblemBlock will use this information to generate HTML and + # grade responses, but the the user's browser is never permitted to + # actually download the raw OLX itself. + # * Many courses include a python_lib.zip file holding custom Python code + # to be used by codejail to assess student answers. This code will also + # potentially reveal answers, and is never intended to be downloadable by + # the student's browser. + # * Some course teams will upload other file formats that their OLX is + # derived from (e.g. specially formatted LaTeX files). These files will + # likewise contain answers and should never be downloadable by the + # student. + # * Other custom metadata may be attached as files in the import, such as + # custom identifiers, author information, etc. + # + # Even if ``learner_downloadble`` is True, the LMS may decide that this + # particular student isn't allowed to see this particular piece of content + # yet–e.g. because they are not enrolled, or because the exam this Component + # is a part of hasn't started yet. That's a matter of LMS permissions and + # policy that is not intrinsic to the content itself, and exists at a layer + # above this. + learner_downloadable = models.BooleanField(default=False) + class Meta: constraints = [ # Uniqueness is only by ComponentVersion and identifier. If for some @@ -388,16 +497,16 @@ class Meta: # content with two different identifiers, that is permitted. models.UniqueConstraint( fields=["component_version", "identifier"], - name="componentversioncontent_uniq_cv_id", + name="cvrawcontent_uniq_cv_id", ), ] indexes = [ models.Index( - fields=["content", "component_version"], - name="componentversioncontent_c_cv", + fields=["raw_content", "component_version"], + name="cvrawcontent_c_cv", ), models.Index( - fields=["component_version", "content"], - name="componentversioncontent_cv_d", + fields=["component_version", "raw_content"], + name="cvrawcontent_cv_d", ), ] diff --git a/openedx_learning/core/components/readme.rst b/openedx_learning/core/components/readme.rst index e2752f63a..70cf5a5f6 100644 --- a/openedx_learning/core/components/readme.rst +++ b/openedx_learning/core/components/readme.rst @@ -18,5 +18,6 @@ Architecture Guidelines * We're keeping nearly unlimited history, so per-version metadata (i.e. the space/time cost of making a new version) must be kept low. * Do not assume that all Components will be XBlocks. -* Encourage other apps to make models that join to (and add their own metadata to) Component, ComponentVersion, Content, etc. But it should be done in such a way that this app is not aware of them. +* Encourage other apps to make models that join to (and add their own metadata to) Component, ComponentVersion, RawContent, TextContent etc. But it should be done in such a way that this app is not aware of them. * Always preserve the most raw version of the data possible, e.g. OLX, even if XBlocks then extend that with more sophisticated data models. At some point those XBlocks will get deprecated/removed, and we will still want to be able to export the raw data. +* Exports should be fast and *not* require the invocation of plugin code. \ No newline at end of file diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index 16f011832..350a1849a 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,13 +1,13 @@ -# Generated by Django 4.1 on 2023-02-10 18:56 +# Generated by Django 4.1.6 on 2023-04-14 00:12 from django.conf import settings from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.validators import uuid class Migration(migrations.Migration): - initial = True dependencies = [ @@ -38,8 +38,22 @@ class Migration(migrations.Migration): ), ("identifier", models.CharField(max_length=255)), ("title", models.CharField(max_length=1000)), - ("created", models.DateTimeField()), - ("updated", models.DateTimeField()), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), + ( + "updated", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ], options={ "verbose_name": "Learning Package", @@ -68,7 +82,14 @@ class Migration(migrations.Migration): ), ), ("message", models.CharField(blank=True, default="", max_length=1000)), - ("published_at", models.DateTimeField()), + ( + "published_at", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), ( "learning_package", models.ForeignKey( diff --git a/openedx_learning/lib/admin_utils.py b/openedx_learning/lib/admin_utils.py new file mode 100644 index 000000000..f7509726f --- /dev/null +++ b/openedx_learning/lib/admin_utils.py @@ -0,0 +1,28 @@ +""" +Convenience utilities for the Django Admin. +""" +from django.contrib import admin + + +class ReadOnlyModelAdmin(admin.ModelAdmin): + """ + ModelAdmin subclass that removes any editing ability. + + The Django Admin is really useful for quickly examining model data. At the + same time, model creation and updates follow specific rules that are meant + to be enforced above the model layer (in api.py files), so making edits in + the Django Admin is potentially dangerous. + + In general, if you're providing Django Admin interfaces for your + openedx-learning related app data models, you should subclass this class + instead of subclassing admin.ModelAdmin directly. + """ + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index b18137c06..4955ad11c 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -6,6 +6,11 @@ * Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a primary key + separate UUID column. https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0038-Data-Modeling.html +* The UUID fields are intended to be globally unique identifiers that other + services can store and rely on staying the same. +* The "identifier" fields can be more human-friendly strings, but these may only + be unique within a given context. These values should be treated as mutable, + even if they rarely change in practice. TODO: * Try making a CaseSensitiveCharField and CaseInsensitiveCharField @@ -22,6 +27,8 @@ from django.db import models +from .validators import validate_utc_datetime + def identifier_field(): """ @@ -86,6 +93,9 @@ def manual_date_time_field(): """ DateTimeField that does not auto-generate values. + The datetimes entered for this field *must be UTC* or it will raise a + ValidationError. + The reason for this convention is that we are often creating many rows of data in the same transaction. They are semantically being created or modified "at the same time", even if each individual row is milliseconds @@ -103,4 +113,7 @@ def manual_date_time_field(): auto_now=False, auto_now_add=False, null=False, + validators=[ + validate_utc_datetime, + ], ) diff --git a/openedx_learning/lib/validators.py b/openedx_learning/lib/validators.py new file mode 100644 index 000000000..035a35540 --- /dev/null +++ b/openedx_learning/lib/validators.py @@ -0,0 +1,12 @@ +from datetime import datetime, timezone + +from django.core.exceptions import ValidationError +from django.utils.translation import gettext_lazy as _ + + +def validate_utc_datetime(dt: datetime): + if dt.tzinfo != timezone.utc: + raise ValidationError( + _("The timezone for %(datetime)s is not UTC."), + params={"datetime": dt}, + ) diff --git a/openedx_learning/rest_api/urls.py b/openedx_learning/rest_api/urls.py index f6e1c2e88..edc533ff9 100644 --- a/openedx_learning/rest_api/urls.py +++ b/openedx_learning/rest_api/urls.py @@ -1,4 +1,3 @@ -from django.contrib import admin from django.urls import include, path urlpatterns = [path("v1/", include("openedx_learning.rest_api.v1.urls"))] diff --git a/projects/dev.py b/projects/dev.py index 13cb86c6c..13c351d02 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -4,91 +4,94 @@ from pathlib import Path # Build paths inside the project like this: BASE_DIR / {dir_name} / -BASE_DIR = Path(__file__).resolve().parents[2] +BASE_DIR = Path(__file__).resolve().parents[1] DEBUG = True DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'dev.db', - 'USER': '', - 'PASSWORD': '', - 'HOST': '', - 'PORT': '', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "dev.db", + "USER": "", + "PASSWORD": "", + "HOST": "", + "PORT": "", } } INSTALLED_APPS = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.messages', - 'django.contrib.sessions', - 'django.contrib.staticfiles', - + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.messages", + "django.contrib.sessions", + "django.contrib.staticfiles", # Admin - 'django.contrib.admin', - 'django.contrib.admindocs', - + "django.contrib.admin", + "django.contrib.admindocs", # Learning Core Apps - 'openedx_learning.core.components.apps.ComponentsConfig', - 'openedx_learning.core.publishing.apps.PublishingConfig', - + "openedx_learning.core.components.apps.ComponentsConfig", + "openedx_learning.core.publishing.apps.PublishingConfig", # Learning Contrib Apps - + "openedx_learning.contrib.media_server.apps.MediaServerConfig", # Apps that don't belong in this repo in the long term, but are here to make # testing/iteration easier until the APIs stabilize. - 'olx_importer.apps.OLXImporterConfig', - + "olx_importer.apps.OLXImporterConfig", # REST API - 'rest_framework', - 'openedx_learning.rest_api.apps.RESTAPIConfig', + "rest_framework", + "openedx_learning.rest_api.apps.RESTAPIConfig", ) MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', - + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", # Admin-specific - 'django.contrib.admindocs.middleware.XViewMiddleware', + "django.contrib.admindocs.middleware.XViewMiddleware", ] TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ] - } + }, }, ] LOCALE_PATHS = [ - BASE_DIR / 'openedx_learning' / 'conf' / 'locale', + BASE_DIR / "conf" / "locale", ] -ROOT_URLCONF = 'projects.urls' +ROOT_URLCONF = "projects.urls" -SECRET_KEY = 'insecure-secret-key' +SECRET_KEY = "insecure-secret-key" -STATIC_URL = '/static/' +STATIC_URL = "/static/" STATICFILES_FINDERS = [ - 'django.contrib.staticfiles.finders.FileSystemFinder', - 'django.contrib.staticfiles.finders.AppDirectoriesFinder', + "django.contrib.staticfiles.finders.FileSystemFinder", + "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] STATICFILES_DIRS = [ -# BASE_DIR / 'projects' / 'static' + # BASE_DIR / 'projects' / 'static' ] -MEDIA_URL = '/media/' +MEDIA_URL = "/media/" +MEDIA_ROOT = BASE_DIR / "media" USE_TZ = True + +# openedx-learning required configuration +OPENEDX_LEARNING = { + # Custom file storage, though this is better done through Django's + # STORAGES setting in Django >= 4.2 + "STORAGE": None, +} diff --git a/projects/urls.py b/projects/urls.py index 92928a097..67364cfcb 100644 --- a/projects/urls.py +++ b/projects/urls.py @@ -2,8 +2,8 @@ from django.urls import include, path urlpatterns = [ - path('admin/doc/', include('django.contrib.admindocs.urls')), - path('admin/', admin.site.urls), - - path('rest_api/', include('openedx_learning.rest_api.urls')) + path("admin/doc/", include("django.contrib.admindocs.urls")), + path("admin/", admin.site.urls), + path("media_server/", include("openedx_learning.contrib.media_server.urls")), + path("rest_api/", include("openedx_learning.rest_api.urls")), ] diff --git a/requirements/base.in b/requirements/base.in index c0122e9d1..6a9095226 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,15 +1,6 @@ # Core requirements for using this application -c constraints.txt -Django # Web application framework +Django<4.0 # Web application framework -# For the Python API layer (eventually) attrs - -# Serialization -pyyaml - -# Django Rest Framework + extras for the openedx_lor project -djangorestframework -markdown -django-filter diff --git a/requirements/base.txt b/requirements/base.txt index ff9722db2..40f87f788 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -8,26 +8,9 @@ asgiref==3.5.2 # via django attrs==22.1.0 # via -r requirements/base.in -backports-zoneinfo==0.2.1 - # via django -django==4.1 - # via - # -r requirements/base.in - # django-filter - # djangorestframework -django-filter==22.1 - # via -r requirements/base.in -djangorestframework==3.13.1 - # via -r requirements/base.in -importlib-metadata==4.12.0 - # via markdown -markdown==3.4.1 - # via -r requirements/base.in -pytz==2022.2.1 - # via djangorestframework -pyyaml==6.0 +django==3.2.15 # via -r requirements/base.in +pytz==2022.1 + # via django sqlparse==0.4.2 # via django -zipp==3.8.1 - # via importlib-metadata diff --git a/requirements/ci.txt b/requirements/ci.txt index 912932dfe..bd0b6ffab 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -6,27 +6,27 @@ # certifi==2022.6.15 # via requests -charset-normalizer==2.1.1 +charset-normalizer==2.1.0 # via requests click==8.1.3 # via import-linter codecov==2.1.12 # via -r requirements/ci.in -coverage==6.4.4 +coverage==6.4.3 # via codecov -distlib==0.3.6 +distlib==0.3.5 # via virtualenv -filelock==3.8.0 +filelock==3.7.1 # via # tox # virtualenv -grimp==1.3 +grimp==1.2.3 # via import-linter idna==3.3 # via requests -import-linter==1.3.0 +import-linter==1.2.7 # via -r requirements/ci.in -networkx==2.8.6 +networkx==2.8.5 # via grimp packaging==21.3 # via tox @@ -46,9 +46,7 @@ toml==0.10.2 # via tox tox==3.25.1 # via -r requirements/ci.in -typing-extensions==4.3.0 - # via import-linter -urllib3==1.26.12 +urllib3==1.26.11 # via requests -virtualenv==20.16.4 +virtualenv==20.16.3 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 71421be7a..33087f213 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ asgiref==3.5.2 # via # -r requirements/quality.txt # django -astroid==2.12.5 +astroid==2.11.7 # via # -r requirements/quality.txt # pylint @@ -17,10 +17,6 @@ attrs==22.1.0 # via # -r requirements/quality.txt # pytest -backports-zoneinfo==0.2.1 - # via - # -r requirements/quality.txt - # django bleach==5.0.1 # via # -r requirements/quality.txt @@ -36,7 +32,7 @@ certifi==2022.6.15 # requests chardet==5.0.0 # via diff-cover -charset-normalizer==2.1.1 +charset-normalizer==2.1.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -65,7 +61,7 @@ commonmark==0.9.1 # via # -r requirements/quality.txt # rich -coverage[toml]==6.4.4 +coverage[toml]==6.4.3 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -77,20 +73,14 @@ dill==0.3.5.1 # via # -r requirements/quality.txt # pylint -distlib==0.3.6 +distlib==0.3.5 # via # -r requirements/ci.txt # virtualenv -django==4.1 +django==3.2.15 # via # -r requirements/quality.txt - # django-filter - # djangorestframework # edx-i18n-tools -django-filter==22.1 - # via -r requirements/quality.txt -djangorestframework==3.13.1 - # via -r requirements/quality.txt docutils==0.19 # via # -r requirements/quality.txt @@ -99,12 +89,12 @@ edx-i18n-tools==0.9.1 # via -r requirements/dev.in edx-lint==5.2.4 # via -r requirements/quality.txt -filelock==3.8.0 +filelock==3.7.1 # via # -r requirements/ci.txt # tox # virtualenv -grimp==1.3 +grimp==1.2.3 # via # -r requirements/ci.txt # import-linter @@ -113,13 +103,12 @@ idna==3.3 # -r requirements/ci.txt # -r requirements/quality.txt # requests -import-linter==1.3.0 +import-linter==1.2.7 # via -r requirements/ci.txt importlib-metadata==4.12.0 # via # -r requirements/quality.txt # keyring - # markdown # twine iniconfig==1.1.1 # via @@ -129,16 +118,12 @@ isort==5.10.1 # via # -r requirements/quality.txt # pylint -jaraco-classes==3.2.2 - # via - # -r requirements/quality.txt - # keyring jinja2==3.1.2 # via # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.9.0 +keyring==23.8.1 # via # -r requirements/quality.txt # twine @@ -146,8 +131,6 @@ lazy-object-proxy==1.7.1 # via # -r requirements/quality.txt # astroid -markdown==3.4.1 - # via -r requirements/quality.txt markupsafe==2.1.1 # via # -r requirements/quality.txt @@ -156,11 +139,7 @@ mccabe==0.7.0 # via # -r requirements/quality.txt # pylint -more-itertools==8.14.0 - # via - # -r requirements/quality.txt - # jaraco-classes -networkx==2.8.6 +networkx==2.8.5 # via # -r requirements/ci.txt # grimp @@ -174,7 +153,7 @@ packaging==21.3 # tox path==16.4.0 # via edx-i18n-tools -pbr==5.10.0 +pbr==5.9.0 # via # -r requirements/quality.txt # stevedore @@ -213,13 +192,13 @@ pycodestyle==2.9.1 # via -r requirements/quality.txt pydocstyle==6.1.1 # via -r requirements/quality.txt -pygments==2.13.0 +pygments==2.12.0 # via # -r requirements/quality.txt # diff-cover # readme-renderer # rich -pylint==2.15.0 +pylint==2.14.5 # via # -r requirements/quality.txt # edx-lint @@ -245,7 +224,7 @@ pyparsing==3.0.9 # -r requirements/pip-tools.txt # -r requirements/quality.txt # packaging -pytest==7.1.3 +pytest==7.1.2 # via # -r requirements/quality.txt # pytest-cov @@ -258,16 +237,16 @@ python-slugify==6.1.2 # via # -r requirements/quality.txt # code-annotations -pytz==2022.2.1 +pytz==2022.1 # via # -r requirements/quality.txt - # djangorestframework + # django pyyaml==6.0 # via # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==37.0 +readme-renderer==36.0 # via # -r requirements/quality.txt # twine @@ -326,7 +305,7 @@ tomli==2.0.1 # pep517 # pylint # pytest -tomlkit==0.11.4 +tomlkit==0.11.1 # via # -r requirements/quality.txt # pylint @@ -340,19 +319,17 @@ twine==4.0.1 # via -r requirements/quality.txt typing-extensions==4.3.0 # via - # -r requirements/ci.txt # -r requirements/quality.txt # astroid - # import-linter # pylint # rich -urllib3==1.26.12 +urllib3==1.26.11 # via # -r requirements/ci.txt # -r requirements/quality.txt # requests # twine -virtualenv==20.16.4 +virtualenv==20.16.3 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index 4418ed34a..342d77336 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -30,7 +30,7 @@ bleach==5.0.1 # via readme-renderer certifi==2022.6.15 # via requests -charset-normalizer==2.1.1 +charset-normalizer==2.1.0 # via requests click==8.1.3 # via @@ -38,18 +38,11 @@ click==8.1.3 # code-annotations code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==6.4.4 +coverage[toml]==6.4.3 # via # -r requirements/test.txt # pytest-cov -django==4.1 - # via - # -r requirements/test.txt - # django-filter - # djangorestframework -django-filter==22.1 - # via -r requirements/test.txt -djangorestframework==3.13.1 +django==3.2.15 # via -r requirements/test.txt doc8==1.0.0 # via -r requirements/doc.in @@ -65,10 +58,7 @@ idna==3.3 imagesize==1.4.1 # via sphinx importlib-metadata==4.12.0 - # via - # -r requirements/test.txt - # markdown - # sphinx + # via sphinx iniconfig==1.1.1 # via # -r requirements/test.txt @@ -78,8 +68,6 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -markdown==3.4.1 - # via -r requirements/test.txt markupsafe==2.1.1 # via # -r requirements/test.txt @@ -90,7 +78,7 @@ packaging==21.3 # pydata-sphinx-theme # pytest # sphinx -pbr==5.10.0 +pbr==5.9.0 # via # -r requirements/test.txt # stevedore @@ -115,7 +103,7 @@ pyparsing==3.0.9 # via # -r requirements/test.txt # packaging -pytest==7.1.3 +pytest==7.1.2 # via # -r requirements/test.txt # pytest-cov @@ -128,16 +116,16 @@ python-slugify==6.1.2 # via # -r requirements/test.txt # code-annotations -pytz==2022.2.1 +pytz==2022.1 # via # -r requirements/test.txt # babel - # djangorestframework + # django pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.0 +readme-renderer==36.0 # via -r requirements/doc.in requests==2.28.1 # via sphinx @@ -196,6 +184,4 @@ urllib3==1.26.12 webencodings==0.5.1 # via bleach zipp==3.8.1 - # via - # -r requirements/test.txt - # importlib-metadata + # via importlib-metadata diff --git a/requirements/quality.txt b/requirements/quality.txt index 615387d02..5b541835a 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,7 +8,7 @@ asgiref==3.5.2 # via # -r requirements/test.txt # django -astroid==2.12.5 +astroid==2.11.7 # via # pylint # pylint-celery @@ -16,15 +16,11 @@ attrs==22.1.0 # via # -r requirements/test.txt # pytest -backports-zoneinfo==0.2.1 - # via - # -r requirements/test.txt - # django bleach==5.0.1 # via readme-renderer certifi==2022.6.15 # via requests -charset-normalizer==2.1.1 +charset-normalizer==2.1.0 # via requests click==8.1.3 # via @@ -40,20 +36,13 @@ code-annotations==1.3.0 # edx-lint commonmark==0.9.1 # via rich -coverage[toml]==6.4.4 +coverage[toml]==6.4.3 # via # -r requirements/test.txt # pytest-cov dill==0.3.5.1 # via pylint -django==4.1 - # via - # -r requirements/test.txt - # django-filter - # djangorestframework -django-filter==22.1 - # via -r requirements/test.txt -djangorestframework==3.13.1 +django==3.2.15 # via -r requirements/test.txt docutils==0.19 # via readme-renderer @@ -63,9 +52,7 @@ idna==3.3 # via requests importlib-metadata==4.12.0 # via - # -r requirements/test.txt # keyring - # markdown # twine iniconfig==1.1.1 # via @@ -75,31 +62,25 @@ isort==5.10.1 # via # -r requirements/quality.in # pylint -jaraco-classes==3.2.2 - # via keyring jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.9.0 +keyring==23.8.1 # via twine lazy-object-proxy==1.7.1 # via astroid -markdown==3.4.1 - # via -r requirements/test.txt markupsafe==2.1.1 # via # -r requirements/test.txt # jinja2 mccabe==0.7.0 # via pylint -more-itertools==8.14.0 - # via jaraco-classes packaging==21.3 # via # -r requirements/test.txt # pytest -pbr==5.10.0 +pbr==5.9.0 # via # -r requirements/test.txt # stevedore @@ -119,11 +100,11 @@ pycodestyle==2.9.1 # via -r requirements/quality.in pydocstyle==6.1.1 # via -r requirements/quality.in -pygments==2.13.0 +pygments==2.12.0 # via # readme-renderer # rich -pylint==2.15.0 +pylint==2.14.5 # via # edx-lint # pylint-celery @@ -141,7 +122,7 @@ pyparsing==3.0.9 # via # -r requirements/test.txt # packaging -pytest==7.1.3 +pytest==7.1.2 # via # -r requirements/test.txt # pytest-cov @@ -154,15 +135,15 @@ python-slugify==6.1.2 # via # -r requirements/test.txt # code-annotations -pytz==2022.2.1 +pytz==2022.1 # via # -r requirements/test.txt - # djangorestframework + # django pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.0 +readme-renderer==36.0 # via twine requests==2.28.1 # via @@ -198,7 +179,7 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.11.4 +tomlkit==0.11.1 # via pylint twine==4.0.1 # via -r requirements/quality.in @@ -207,7 +188,7 @@ typing-extensions==4.3.0 # astroid # pylint # rich -urllib3==1.26.12 +urllib3==1.26.11 # via # requests # twine @@ -216,6 +197,7 @@ webencodings==0.5.1 wrapt==1.14.1 # via astroid zipp==3.8.1 - # via - # -r requirements/test.txt - # importlib-metadata + # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools diff --git a/requirements/test.txt b/requirements/test.txt index a055df200..3146dfa4b 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -12,39 +12,22 @@ attrs==22.1.0 # via # -r requirements/base.txt # pytest -backports-zoneinfo==0.2.1 - # via - # -r requirements/base.txt - # django click==8.1.3 # via code-annotations code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==6.4.4 +coverage[toml]==6.4.3 # via pytest-cov - # via - # -r requirements/base.txt - # django-filter - # djangorestframework -django-filter==22.1 - # via -r requirements/base.txt -djangorestframework==3.13.1 # via -r requirements/base.txt -importlib-metadata==4.12.0 - # via - # -r requirements/base.txt - # markdown iniconfig==1.1.1 # via pytest jinja2==3.1.2 # via code-annotations -markdown==3.4.1 - # via -r requirements/base.txt markupsafe==2.1.1 # via jinja2 packaging==21.3 # via pytest -pbr==5.10.0 +pbr==5.9.0 # via stevedore pluggy==1.0.0 # via pytest @@ -52,7 +35,7 @@ py==1.11.0 # via pytest pyparsing==3.0.9 # via packaging -pytest==7.1.3 +pytest==7.1.2 # via # pytest-cov # pytest-django @@ -62,14 +45,12 @@ pytest-django==4.5.2 # via -r requirements/test.in python-slugify==6.1.2 # via code-annotations -pytz==2022.2.1 +pytz==2022.1 # via # -r requirements/base.txt - # djangorestframework + # django pyyaml==6.0 - # via - # -r requirements/base.txt - # code-annotations + # via code-annotations sqlparse==0.4.2 # via # -r requirements/base.txt @@ -82,7 +63,3 @@ tomli==2.0.1 # via # coverage # pytest -zipp==3.8.1 - # via - # -r requirements/base.txt - # importlib-metadata diff --git a/test_settings.py b/test_settings.py index e41be8f37..d2d0820fc 100644 --- a/test_settings.py +++ b/test_settings.py @@ -16,38 +16,43 @@ def root(*args): DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'default.db', - 'USER': '', - 'PASSWORD': '', - 'HOST': '', - 'PORT': '', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "default.db", + "USER": "", + "PASSWORD": "", + "HOST": "", + "PORT": "", } } INSTALLED_APPS = [ - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.messages', - 'django.contrib.sessions', - 'django.contrib.staticfiles', - + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.messages", + "django.contrib.sessions", + "django.contrib.staticfiles", # Admin -# 'django.contrib.admin', -# 'django.contrib.admindocs', - + # 'django.contrib.admin', + # 'django.contrib.admindocs', # Our own apps - 'openedx_learning.core.publishing.apps.PublishingConfig', - 'openedx_learning.core.components.apps.ComponentsConfig', + "openedx_learning.core.publishing.apps.PublishingConfig", + "openedx_learning.core.components.apps.ComponentsConfig", ] LOCALE_PATHS = [ - root('openedx_learning', 'conf', 'locale'), + root("openedx_learning", "conf", "locale"), ] -ROOT_URLCONF = 'projects.urls' +ROOT_URLCONF = "projects.urls" -SECRET_KEY = 'insecure-secret-key' +SECRET_KEY = "insecure-secret-key" USE_TZ = True + +# openedx-learning required configuration +OPENEDX_LEARNING = { + # Custom file storage, though this is better done through Django's + # STORAGES setting in Django >= 4.2 + "STORAGE": None, +} From 5390dd4f32179ed9324e8d60e62cc2b8bc8f40c8 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Thu, 13 Apr 2023 22:46:23 -0400 Subject: [PATCH 004/282] chore: add DRF, remove codecov, rebuild requirements The codecov dependency had to be replaced because it got pulled from PyPI. The DRF dependency somehow never made it in there before. --- requirements/base.in | 2 +- requirements/base.txt | 22 ++-- requirements/ci.in | 2 +- requirements/ci.txt | 67 ++++++------ requirements/dev.txt | 202 ++++++++++++++++++++----------------- requirements/doc.txt | 58 ++++++----- requirements/pip-tools.txt | 22 ++-- requirements/quality.txt | 124 ++++++++++++----------- requirements/test.txt | 43 ++++---- 9 files changed, 280 insertions(+), 262 deletions(-) diff --git a/requirements/base.in b/requirements/base.in index 6a9095226..f5ebd9403 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -3,4 +3,4 @@ Django<4.0 # Web application framework -attrs +djangorestframework<4.0 # REST API diff --git a/requirements/base.txt b/requirements/base.txt index 40f87f788..7c80551d3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,16 +1,20 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 +asgiref==3.6.0 # via django -attrs==22.1.0 +django==3.2.18 + # via + # -r requirements/base.in + # djangorestframework +djangorestframework==3.14.0 # via -r requirements/base.in -django==3.2.15 - # via -r requirements/base.in -pytz==2022.1 - # via django -sqlparse==0.4.2 +pytz==2023.3 + # via + # django + # djangorestframework +sqlparse==0.4.3 # via django diff --git a/requirements/ci.in b/requirements/ci.in index ebc123521..8776af102 100644 --- a/requirements/ci.in +++ b/requirements/ci.in @@ -1,6 +1,6 @@ # Requirements for running tests on CI -c constraints.txt -codecov # Code coverage reporting +coverage # Code coverage reporting tox # Virtualenv management for tests import-linter # Track our internal dependencies diff --git a/requirements/ci.txt b/requirements/ci.txt index bd0b6ffab..e6ca7b96a 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,52 +1,51 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -certifi==2022.6.15 - # via requests -charset-normalizer==2.1.0 - # via requests +cachetools==5.3.0 + # via tox +chardet==5.1.0 + # via tox click==8.1.3 # via import-linter -codecov==2.1.12 +colorama==0.4.6 + # via tox +coverage==7.2.3 # via -r requirements/ci.in -coverage==6.4.3 - # via codecov -distlib==0.3.5 +distlib==0.3.6 # via virtualenv -filelock==3.7.1 +filelock==3.11.0 # via # tox # virtualenv -grimp==1.2.3 +grimp==2.3 # via import-linter -idna==3.3 - # via requests -import-linter==1.2.7 +import-linter==1.8.0 # via -r requirements/ci.in -networkx==2.8.5 - # via grimp -packaging==21.3 - # via tox -platformdirs==2.5.2 - # via virtualenv +packaging==23.1 + # via + # pyproject-api + # tox +platformdirs==3.2.0 + # via + # tox + # virtualenv pluggy==1.0.0 # via tox -py==1.11.0 - # via tox -pyparsing==3.0.9 - # via packaging -requests==2.28.1 - # via codecov -six==1.16.0 - # via tox -toml==0.10.2 +pyproject-api==1.5.1 # via tox -tox==3.25.1 +tomli==2.0.1 + # via + # import-linter + # pyproject-api + # tox +tox==4.4.12 # via -r requirements/ci.in -urllib3==1.26.11 - # via requests -virtualenv==20.16.3 +typing-extensions==4.5.0 + # via + # grimp + # import-linter +virtualenv==20.21.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 33087f213..2fbb634cf 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,40 +1,41 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 +asgiref==3.6.0 # via # -r requirements/quality.txt # django -astroid==2.11.7 +astroid==2.15.2 # via # -r requirements/quality.txt # pylint # pylint-celery -attrs==22.1.0 - # via - # -r requirements/quality.txt - # pytest -bleach==5.0.1 +bleach==6.0.0 # via # -r requirements/quality.txt # readme-renderer -build==0.8.0 +build==0.10.0 # via # -r requirements/pip-tools.txt # pip-tools -certifi==2022.6.15 +cachetools==5.3.0 # via # -r requirements/ci.txt + # tox +certifi==2022.12.7 + # via # -r requirements/quality.txt # requests -chardet==5.0.0 - # via diff-cover -charset-normalizer==2.1.0 +chardet==5.1.0 # via # -r requirements/ci.txt + # diff-cover + # tox +charset-normalizer==3.1.0 + # via # -r requirements/quality.txt # requests click==8.1.3 @@ -55,83 +56,98 @@ code-annotations==1.3.0 # via # -r requirements/quality.txt # edx-lint -codecov==2.1.12 - # via -r requirements/ci.txt -commonmark==0.9.1 +colorama==0.4.6 # via - # -r requirements/quality.txt - # rich -coverage[toml]==6.4.3 + # -r requirements/ci.txt + # tox +coverage[toml]==7.2.3 # via # -r requirements/ci.txt # -r requirements/quality.txt - # codecov # pytest-cov -diff-cover==6.5.1 +diff-cover==7.5.0 # via -r requirements/dev.in -dill==0.3.5.1 +dill==0.3.6 # via # -r requirements/quality.txt # pylint -distlib==0.3.5 +distlib==0.3.6 # via # -r requirements/ci.txt # virtualenv -django==3.2.15 +django==3.2.18 # via # -r requirements/quality.txt + # djangorestframework # edx-i18n-tools +djangorestframework==3.14.0 + # via -r requirements/quality.txt docutils==0.19 # via # -r requirements/quality.txt # readme-renderer -edx-i18n-tools==0.9.1 +edx-i18n-tools==0.9.2 # via -r requirements/dev.in -edx-lint==5.2.4 +edx-lint==5.3.4 # via -r requirements/quality.txt -filelock==3.7.1 +exceptiongroup==1.1.1 + # via + # -r requirements/quality.txt + # pytest +filelock==3.11.0 # via # -r requirements/ci.txt # tox # virtualenv -grimp==1.2.3 +grimp==2.3 # via # -r requirements/ci.txt # import-linter -idna==3.3 +idna==3.4 # via - # -r requirements/ci.txt # -r requirements/quality.txt # requests -import-linter==1.2.7 +import-linter==1.8.0 # via -r requirements/ci.txt -importlib-metadata==4.12.0 +importlib-metadata==6.3.0 # via # -r requirements/quality.txt # keyring # twine -iniconfig==1.1.1 +importlib-resources==5.12.0 + # via + # -r requirements/quality.txt + # keyring +iniconfig==2.0.0 # via # -r requirements/quality.txt # pytest -isort==5.10.1 +isort==5.12.0 # via # -r requirements/quality.txt # pylint +jaraco-classes==3.2.3 + # via + # -r requirements/quality.txt + # keyring jinja2==3.1.2 # via # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.8.1 +keyring==23.13.1 # via # -r requirements/quality.txt # twine -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt # astroid -markupsafe==2.1.1 +markdown-it-py==2.2.0 + # via + # -r requirements/quality.txt + # rich +markupsafe==2.1.2 # via # -r requirements/quality.txt # jinja2 @@ -139,39 +155,41 @@ mccabe==0.7.0 # via # -r requirements/quality.txt # pylint -networkx==2.8.5 +mdurl==0.1.2 # via - # -r requirements/ci.txt - # grimp -packaging==21.3 + # -r requirements/quality.txt + # markdown-it-py +more-itertools==9.1.0 + # via + # -r requirements/quality.txt + # jaraco-classes +packaging==23.1 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # build + # pyproject-api # pytest # tox -path==16.4.0 +path==16.6.0 # via edx-i18n-tools -pbr==5.9.0 +pbr==5.11.1 # via # -r requirements/quality.txt # stevedore -pep517==0.13.0 - # via - # -r requirements/pip-tools.txt - # build -pip-tools==6.8.0 +pip-tools==6.13.0 # via -r requirements/pip-tools.txt -pkginfo==1.8.3 +pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==2.5.2 +platformdirs==3.2.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint + # tox # virtualenv pluggy==1.0.0 # via @@ -180,25 +198,19 @@ pluggy==1.0.0 # diff-cover # pytest # tox -polib==1.1.1 +polib==1.2.0 # via edx-i18n-tools -py==1.11.0 - # via - # -r requirements/ci.txt - # -r requirements/quality.txt - # pytest - # tox -pycodestyle==2.9.1 +pycodestyle==2.10.0 # via -r requirements/quality.txt -pydocstyle==6.1.1 +pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.12.0 +pygments==2.15.0 # via # -r requirements/quality.txt # diff-cover # readme-renderer # rich -pylint==2.14.5 +pylint==2.17.2 # via # -r requirements/quality.txt # edx-lint @@ -218,46 +230,47 @@ pylint-plugin-utils==0.7 # -r requirements/quality.txt # pylint-celery # pylint-django -pyparsing==3.0.9 +pyproject-api==1.5.1 # via # -r requirements/ci.txt + # tox +pyproject-hooks==1.0.0 + # via # -r requirements/pip-tools.txt - # -r requirements/quality.txt - # packaging -pytest==7.1.2 + # build +pytest==7.3.0 # via # -r requirements/quality.txt # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/quality.txt pytest-django==4.5.2 # via -r requirements/quality.txt -python-slugify==6.1.2 +python-slugify==8.0.1 # via # -r requirements/quality.txt # code-annotations -pytz==2022.1 +pytz==2023.3 # via # -r requirements/quality.txt # django + # djangorestframework pyyaml==6.0 # via # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==36.0 +readme-renderer==37.3 # via # -r requirements/quality.txt # twine -requests==2.28.1 +requests==2.28.2 # via - # -r requirements/ci.txt # -r requirements/quality.txt - # codecov # requests-toolbelt # twine -requests-toolbelt==0.9.1 +requests-toolbelt==0.10.1 # via # -r requirements/quality.txt # twine @@ -265,26 +278,24 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==12.5.1 +rich==13.3.4 # via # -r requirements/quality.txt # twine six==1.16.0 # via - # -r requirements/ci.txt # -r requirements/quality.txt # bleach # edx-lint - # tox snowballstemmer==2.2.0 # via # -r requirements/quality.txt # pydocstyle -sqlparse==0.4.2 +sqlparse==0.4.3 # via # -r requirements/quality.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via # -r requirements/quality.txt # code-annotations @@ -292,44 +303,46 @@ text-unidecode==1.3 # via # -r requirements/quality.txt # python-slugify -toml==0.10.2 - # via - # -r requirements/ci.txt - # tox tomli==2.0.1 # via + # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # build # coverage - # pep517 + # import-linter # pylint + # pyproject-api + # pyproject-hooks # pytest -tomlkit==0.11.1 + # tox +tomlkit==0.11.7 # via # -r requirements/quality.txt # pylint -tox==3.25.1 +tox==4.4.12 # via # -r requirements/ci.txt # tox-battery tox-battery==0.6.1 # via -r requirements/dev.in -twine==4.0.1 +twine==4.0.2 # via -r requirements/quality.txt -typing-extensions==4.3.0 +typing-extensions==4.5.0 # via + # -r requirements/ci.txt # -r requirements/quality.txt # astroid + # grimp + # import-linter # pylint # rich -urllib3==1.26.11 +urllib3==1.26.15 # via - # -r requirements/ci.txt # -r requirements/quality.txt # requests # twine -virtualenv==20.16.3 +virtualenv==20.21.0 # via # -r requirements/ci.txt # tox @@ -337,18 +350,19 @@ webencodings==0.5.1 # via # -r requirements/quality.txt # bleach -wheel==0.37.1 +wheel==0.40.0 # via # -r requirements/pip-tools.txt # pip-tools -wrapt==1.14.1 +wrapt==1.15.0 # via # -r requirements/quality.txt # astroid -zipp==3.8.1 +zipp==3.15.0 # via # -r requirements/quality.txt # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/doc.txt b/requirements/doc.txt index 342d77336..bd988df39 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,6 +1,6 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.12 # via sphinx -asgiref==3.5.2 +asgiref==3.6.0 # via # -r requirements/test.txt # django @@ -28,9 +28,9 @@ beautifulsoup4==4.12.2 # via pydata-sphinx-theme bleach==5.0.1 # via readme-renderer -certifi==2022.6.15 +certifi==2022.12.7 # via requests -charset-normalizer==2.1.0 +charset-normalizer==3.1.0 # via requests click==8.1.3 # via @@ -38,13 +38,18 @@ click==8.1.3 # code-annotations code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==6.4.3 +coverage[toml]==7.2.3 # via # -r requirements/test.txt # pytest-cov -django==3.2.15 +django==3.2.18 + # via + # -r requirements/test.txt + # djangorestframework + # sphinxcontrib-django +djangorestframework==3.14.0 # via -r requirements/test.txt -doc8==1.0.0 +doc8==1.1.1 # via -r requirements/doc.in docutils==0.19 # via @@ -57,9 +62,9 @@ idna==3.3 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==4.12.0 +importlib-metadata==6.3.0 # via sphinx -iniconfig==1.1.1 +iniconfig==2.0.0 # via # -r requirements/test.txt # pytest @@ -68,17 +73,17 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -markupsafe==2.1.1 +markupsafe==2.1.2 # via # -r requirements/test.txt # jinja2 -packaging==21.3 +packaging==23.1 # via # -r requirements/test.txt # pydata-sphinx-theme # pytest # sphinx -pbr==5.9.0 +pbr==5.11.1 # via # -r requirements/test.txt # stevedore @@ -99,35 +104,32 @@ pygments==2.13.0 # pydata-sphinx-theme # readme-renderer # sphinx -pyparsing==3.0.9 - # via - # -r requirements/test.txt - # packaging -pytest==7.1.2 +pytest==7.3.0 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt -python-slugify==6.1.2 +python-slugify==8.0.1 # via # -r requirements/test.txt # code-annotations -pytz==2022.1 +pytz==2023.3 # via # -r requirements/test.txt # babel # django + # djangorestframework pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==36.0 +readme-renderer==37.3 # via -r requirements/doc.in -requests==2.28.1 +requests==2.28.2 # via sphinx restructuredtext-lint==1.4.0 # via doc8 @@ -148,9 +150,9 @@ sphinxcontrib-applehelp==1.0.2 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-django==0.5.1 +sphinxcontrib-django==2.3 # via -r requirements/doc.in -sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-htmlhelp==2.0.1 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx @@ -158,11 +160,11 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlparse==0.4.2 +sqlparse==0.4.3 # via # -r requirements/test.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via # -r requirements/test.txt # code-annotations @@ -183,5 +185,5 @@ urllib3==1.26.12 # via requests webencodings==0.5.1 # via bleach -zipp==3.8.1 +zipp==3.15.0 # via importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index ebb8aa6a3..fd0cc1c78 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,26 +1,22 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -build==0.8.0 +build==0.10.0 # via pip-tools click==8.1.3 # via pip-tools -packaging==21.3 +packaging==23.1 # via build -pep517==0.13.0 - # via build -pip-tools==6.8.0 +pip-tools==6.13.0 # via -r requirements/pip-tools.in -pyparsing==3.0.9 - # via packaging +pyproject-hooks==1.0.0 + # via build tomli==2.0.1 - # via - # build - # pep517 -wheel==0.37.1 + # via build +wheel==0.40.0 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/quality.txt b/requirements/quality.txt index 5b541835a..134d429a1 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,26 +1,22 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 +asgiref==3.6.0 # via # -r requirements/test.txt # django -astroid==2.11.7 +astroid==2.15.2 # via # pylint # pylint-celery -attrs==22.1.0 - # via - # -r requirements/test.txt - # pytest -bleach==5.0.1 +bleach==6.0.0 # via readme-renderer -certifi==2022.6.15 +certifi==2022.12.7 # via requests -charset-normalizer==2.1.0 +charset-normalizer==3.1.0 # via requests click==8.1.3 # via @@ -34,77 +30,89 @@ code-annotations==1.3.0 # via # -r requirements/test.txt # edx-lint -commonmark==0.9.1 - # via rich -coverage[toml]==6.4.3 +coverage[toml]==7.2.3 # via # -r requirements/test.txt # pytest-cov -dill==0.3.5.1 +dill==0.3.6 # via pylint -django==3.2.15 +django==3.2.18 + # via + # -r requirements/test.txt + # djangorestframework +djangorestframework==3.14.0 # via -r requirements/test.txt docutils==0.19 # via readme-renderer -edx-lint==5.2.4 +edx-lint==5.3.4 # via -r requirements/quality.in -idna==3.3 +exceptiongroup==1.1.1 + # via + # -r requirements/test.txt + # pytest +idna==3.4 # via requests -importlib-metadata==4.12.0 +importlib-metadata==6.3.0 # via # keyring # twine -iniconfig==1.1.1 +importlib-resources==5.12.0 + # via keyring +iniconfig==2.0.0 # via # -r requirements/test.txt # pytest -isort==5.10.1 +isort==5.12.0 # via # -r requirements/quality.in # pylint +jaraco-classes==3.2.3 + # via keyring jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.8.1 +keyring==23.13.1 # via twine -lazy-object-proxy==1.7.1 +lazy-object-proxy==1.9.0 # via astroid -markupsafe==2.1.1 +markdown-it-py==2.2.0 + # via rich +markupsafe==2.1.2 # via # -r requirements/test.txt # jinja2 mccabe==0.7.0 # via pylint -packaging==21.3 +mdurl==0.1.2 + # via markdown-it-py +more-itertools==9.1.0 + # via jaraco-classes +packaging==23.1 # via # -r requirements/test.txt # pytest -pbr==5.9.0 +pbr==5.11.1 # via # -r requirements/test.txt # stevedore -pkginfo==1.8.3 +pkginfo==1.9.6 # via twine -platformdirs==2.5.2 +platformdirs==3.2.0 # via pylint pluggy==1.0.0 # via # -r requirements/test.txt # pytest -py==1.11.0 - # via - # -r requirements/test.txt - # pytest -pycodestyle==2.9.1 +pycodestyle==2.10.0 # via -r requirements/quality.in -pydocstyle==6.1.1 +pydocstyle==6.3.0 # via -r requirements/quality.in -pygments==2.12.0 +pygments==2.15.0 # via # readme-renderer # rich -pylint==2.14.5 +pylint==2.17.2 # via # edx-lint # pylint-celery @@ -118,42 +126,39 @@ pylint-plugin-utils==0.7 # via # pylint-celery # pylint-django -pyparsing==3.0.9 - # via - # -r requirements/test.txt - # packaging -pytest==7.1.2 +pytest==7.3.0 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt -python-slugify==6.1.2 +python-slugify==8.0.1 # via # -r requirements/test.txt # code-annotations -pytz==2022.1 +pytz==2023.3 # via # -r requirements/test.txt # django + # djangorestframework pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==36.0 +readme-renderer==37.3 # via twine -requests==2.28.1 +requests==2.28.2 # via # requests-toolbelt # twine -requests-toolbelt==0.9.1 +requests-toolbelt==0.10.1 # via twine rfc3986==2.0.0 # via twine -rich==12.5.1 +rich==13.3.4 # via twine six==1.16.0 # via @@ -161,11 +166,11 @@ six==1.16.0 # edx-lint snowballstemmer==2.2.0 # via pydocstyle -sqlparse==0.4.2 +sqlparse==0.4.3 # via # -r requirements/test.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via # -r requirements/test.txt # code-annotations @@ -179,25 +184,24 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.11.1 +tomlkit==0.11.7 # via pylint -twine==4.0.1 +twine==4.0.2 # via -r requirements/quality.in -typing-extensions==4.3.0 +typing-extensions==4.5.0 # via # astroid # pylint # rich -urllib3==1.26.11 +urllib3==1.26.15 # via # requests # twine webencodings==0.5.1 # via bleach -wrapt==1.14.1 +wrapt==1.15.0 # via astroid -zipp==3.8.1 - # via importlib-metadata - -# The following packages are considered to be unsafe in a requirements file: -# setuptools +zipp==3.15.0 + # via + # importlib-metadata + # importlib-resources diff --git a/requirements/test.txt b/requirements/test.txt index 3146dfa4b..9410f3893 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,61 +1,60 @@ # -# This file is autogenerated by pip-compile with python 3.8 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.8 +# by the following command: # # make upgrade # -asgiref==3.5.2 +asgiref==3.6.0 # via # -r requirements/base.txt # django -attrs==22.1.0 - # via - # -r requirements/base.txt - # pytest click==8.1.3 # via code-annotations code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==6.4.3 +coverage[toml]==7.2.3 # via pytest-cov + # via + # -r requirements/base.txt + # djangorestframework +djangorestframework==3.14.0 # via -r requirements/base.txt -iniconfig==1.1.1 +exceptiongroup==1.1.1 + # via pytest +iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -markupsafe==2.1.1 +markupsafe==2.1.2 # via jinja2 -packaging==21.3 +packaging==23.1 # via pytest -pbr==5.9.0 +pbr==5.11.1 # via stevedore pluggy==1.0.0 # via pytest -py==1.11.0 - # via pytest -pyparsing==3.0.9 - # via packaging -pytest==7.1.2 +pytest==7.3.0 # via # pytest-cov # pytest-django -pytest-cov==3.0.0 +pytest-cov==4.0.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in -python-slugify==6.1.2 +python-slugify==8.0.1 # via code-annotations -pytz==2022.1 +pytz==2023.3 # via # -r requirements/base.txt # django + # djangorestframework pyyaml==6.0 # via code-annotations -sqlparse==0.4.2 +sqlparse==0.4.3 # via # -r requirements/base.txt # django -stevedore==4.0.0 +stevedore==5.0.0 # via code-annotations text-unidecode==1.3 # via python-slugify From f6f7b323707dd2b15d31e561305845b0e690c953 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 19 Apr 2023 11:20:06 -0400 Subject: [PATCH 005/282] feat: centralize publishing models This is a major rethinking of the core data apps: * Parts of the components app were extracted into publishing and a new contents app. * The publishing app models are now entirely responsible for the draft and publish state management. * There is now a startup registration step for content models that want to build off of publishing's PublishableEntity and PublishableEntityVersion (register_content_models). See the ComponentsConfig class for an example of it in use. * Many model and field names were changed to make their intent clearer. (See comments in the models and ADRs in this commit for details.) * Apps now carry an "oel_" prefix in their labels. * An interim API layer was created, though this is still very much a work in progress, and should not be considered stable. --- .importlinter | 1 + README.rst | 4 +- .../management/commands/load_components.py | 199 +++----- openedx_learning/contrib/media_server/urls.py | 4 +- .../contrib/media_server/views.py | 4 +- openedx_learning/core/components/admin.py | 165 ++----- openedx_learning/core/components/api.py | 113 ++++- openedx_learning/core/components/apps.py | 10 + .../components/migrations/0001_initial.py | 292 ++---------- openedx_learning/core/components/models.py | 433 ++++-------------- openedx_learning/core/contents/__init__.py | 0 openedx_learning/core/contents/admin.py | 53 +++ openedx_learning/core/contents/api.py | 85 ++++ openedx_learning/core/contents/apps.py | 12 + .../core/contents/migrations/0001_initial.py | 103 +++++ .../core/contents/migrations/__init__.py | 0 openedx_learning/core/contents/models.py | 176 +++++++ openedx_learning/core/publishing/admin.py | 133 +++++- openedx_learning/core/publishing/api.py | 192 ++++++++ openedx_learning/core/publishing/apps.py | 1 + .../publishing/migrations/0001_initial.py | 270 ++++++++++- .../core/publishing/model_mixins.py | 276 +++++++++++ openedx_learning/core/publishing/models.py | 424 ++++++++++++++++- openedx_learning/lib/fields.py | 6 +- projects/dev.py | 9 + projects/urls.py | 6 +- test_settings.py | 3 +- tests/__init__.py | 0 tests/openedx_learning/__init__.py | 0 tests/openedx_learning/core/__init__.py | 0 .../core/components/__init__.py | 0 .../core/components/test_models.py | 43 ++ .../core/publishing/__init__.py | 0 .../core/publishing/test_api.py | 66 +++ tests/test_models.py | 13 - 35 files changed, 2130 insertions(+), 966 deletions(-) create mode 100644 openedx_learning/core/contents/__init__.py create mode 100644 openedx_learning/core/contents/admin.py create mode 100644 openedx_learning/core/contents/api.py create mode 100644 openedx_learning/core/contents/apps.py create mode 100644 openedx_learning/core/contents/migrations/0001_initial.py create mode 100644 openedx_learning/core/contents/migrations/__init__.py create mode 100644 openedx_learning/core/contents/models.py create mode 100644 openedx_learning/core/publishing/api.py create mode 100644 openedx_learning/core/publishing/model_mixins.py create mode 100644 tests/__init__.py create mode 100644 tests/openedx_learning/__init__.py create mode 100644 tests/openedx_learning/core/__init__.py create mode 100644 tests/openedx_learning/core/components/__init__.py create mode 100644 tests/openedx_learning/core/components/test_models.py create mode 100644 tests/openedx_learning/core/publishing/__init__.py create mode 100644 tests/openedx_learning/core/publishing/test_api.py delete mode 100644 tests/test_models.py diff --git a/.importlinter b/.importlinter index 82d881926..8dbba0cc7 100644 --- a/.importlinter +++ b/.importlinter @@ -41,4 +41,5 @@ name = Core App Dependency Layering type = layers layers= openedx_learning.core.components + openedx_learning.core.contents openedx_learning.core.publishing diff --git a/README.rst b/README.rst index 566537c0b..ff6dbbddc 100644 --- a/README.rst +++ b/README.rst @@ -45,8 +45,8 @@ We have a few different identifier types in the schema, and we try to avoid ``_i * ``id`` is the auto-generated, internal row ID and primary key. This never changes. Data models should make foreign keys to this field, as per Django convention. * ``uuid`` is a randomly generated UUID4. This is the stable way to refer to a row/resource from an external service. This never changes. This is separate from ``id`` mostly because there are performance penalties when using UUIDs as primary keys with MySQL. -* ``identifier`` is intended to be a case-sensitive, alphanumeric identifier, which holds some meaning to library clients. This is usually stable, but can be changed, depending on the business logic of the client. The apps in this repo should make no assumptions about it being stable. It can be used as a suffix. -* ``num`` is like ``identifier``, but for use when it's strictly numeric. It can also be used as a suffix. +* ``key`` is intended to be a case-sensitive, alphanumeric key, which holds some meaning to library clients. This is usually stable, but can be changed, depending on the business logic of the client. The apps in this repo should make no assumptions about it being stable. It can be used as a suffix. +* ``num`` is like ``key``, but for use when it's strictly numeric. It can also be used as a suffix. See Also diff --git a/olx_importer/management/commands/load_components.py b/olx_importer/management/commands/load_components.py index 08fc47cc6..d987751b8 100644 --- a/olx_importer/management/commands/load_components.py +++ b/olx_importer/management/commands/load_components.py @@ -18,33 +18,24 @@ changed?" in order to decide if we really make a new ComponentVersion or not. """ from datetime import datetime, timezone -import codecs import logging import mimetypes import pathlib import re import xml.etree.ElementTree as ET -from django.core.files.base import ContentFile -from django.core.management.base import BaseCommand +from django.core.management.base import BaseCommand, CommandError from django.db import transaction -from openedx_learning.core.publishing.models import LearningPackage, PublishLogEntry -from openedx_learning.core.components.models import ( - Component, - ComponentVersion, - ComponentVersionRawContent, - ComponentPublishLogEntry, - PublishedComponent, - RawContent, - TextContent, -) -from openedx_learning.lib.fields import create_hash_digest +# Model references to remove +from openedx_learning.core.components import api as components_api +from openedx_learning.core.contents import api as contents_api +from openedx_learning.core.publishing import api as publishing_api SUPPORTED_TYPES = ["problem", "video", "html"] logger = logging.getLogger(__name__) - + class Command(BaseCommand): help = "Load sample Component data from course export" @@ -70,12 +61,12 @@ def init_known_types(self): def add_arguments(self, parser): parser.add_argument("course_data_path", type=pathlib.Path) - parser.add_argument("learning_package_identifier", type=str) + parser.add_argument("learning_package_key", type=str) - def handle(self, course_data_path, learning_package_identifier, **options): + def handle(self, course_data_path, learning_package_key, **options): self.course_data_path = course_data_path - self.learning_package_identifier = learning_package_identifier - self.load_course_data(learning_package_identifier) + self.learning_package_key = learning_package_key + self.load_course_data(learning_package_key) def get_course_title(self): course_type_dir = self.course_data_path / "course" @@ -83,74 +74,62 @@ def get_course_title(self): course_root = ET.parse(course_xml_file).getroot() return course_root.attrib.get("display_name", "Unknown Course") - def load_course_data(self, learning_package_identifier): + def load_course_data(self, learning_package_key): print(f"Importing course from: {self.course_data_path}") now = datetime.now(timezone.utc) title = self.get_course_title() + if publishing_api.learning_package_exists(learning_package_key): + raise CommandError( + f"{learning_package_key} already exists. " + "This command currently only supports initial import." + ) + with transaction.atomic(): - learning_package, _created = LearningPackage.objects.get_or_create( - identifier=learning_package_identifier, - defaults={ - "title": title, - "created": now, - "updated": now, - }, + self.learning_package = publishing_api.create_learning_package( + learning_package_key, title, created=now, ) - self.learning_package = learning_package + for block_type in SUPPORTED_TYPES: + self.import_block_type(block_type, now) #, publish_log_entry) - publish_log_entry = PublishLogEntry.objects.create( - learning_package=learning_package, - message="Initial Import", - published_at=now, - published_by=None, + publishing_api.publish_all_drafts( + self.learning_package.id, + message="Initial Import from load_components script" ) - for block_type in SUPPORTED_TYPES: - self.import_block_type(block_type, now, publish_log_entry) def create_content(self, static_local_path, now, component_version): - identifier = pathlib.Path("static") / static_local_path - real_path = self.course_data_path / identifier - mime_type, _encoding = mimetypes.guess_type(identifier) + key = pathlib.Path("static") / static_local_path + real_path = self.course_data_path / key + mime_type, _encoding = mimetypes.guess_type(key) if mime_type is None: logger.error( - f" no mimetype found for {real_path}, defaulting to application/binary" + f' no mimetype found for "{real_path}", defaulting to application/binary' ) mime_type = "application/binary" try: data_bytes = real_path.read_bytes() except FileNotFoundError: - logger.warning(f" Static reference not found: {real_path}") + logger.warning(f' Static reference not found: "{real_path}"') return # Might as well bail if we can't find the file. - hash_digest = create_hash_digest(data_bytes) - - raw_content, created = RawContent.objects.get_or_create( - learning_package=self.learning_package, + raw_content, _created = contents_api.get_or_create_raw_content( + learning_package_id=self.learning_package.id, + data_bytes=data_bytes, mime_type=mime_type, - hash_digest=hash_digest, - defaults=dict( - size=len(data_bytes), - created=now, - ), + created=now, ) - if created: - raw_content.file.save( - f"{raw_content.learning_package.uuid}/{hash_digest}", - ContentFile(data_bytes), - ) - - ComponentVersionRawContent.objects.get_or_create( - component_version=component_version, - raw_content=raw_content, - identifier=identifier, + components_api.add_content_to_component_version( + component_version, + raw_content_id=raw_content.id, + key=key, learner_downloadable=True, ) - def import_block_type(self, block_type, now, publish_log_entry): + def import_block_type(self, block_type, now): # , publish_log_entry): components_found = 0 + components_skipped = 0 # Find everything that looks like a reference to a static file appearing # in attribute quotes, stripping off the querystring at the end. This is @@ -158,90 +137,50 @@ def import_block_type(self, block_type, now, publish_log_entry): # outside of tag declarations as well. static_files_regex = re.compile(r"""['"]\/static\/(.+?)["'\?]""") block_data_path = self.course_data_path / block_type + namespace="xblock.v1" for xml_file_path in block_data_path.glob("*.xml"): components_found += 1 - identifier = xml_file_path.stem - - # Find or create the Component itself - component, _created = Component.objects.get_or_create( - learning_package=self.learning_package, - namespace="xblock.v1", - type=block_type, - identifier=identifier, - defaults={ - "created": now, - }, - ) - - # Create the RawContent entry for the raw data... - data_bytes = xml_file_path.read_bytes() - hash_digest = create_hash_digest(data_bytes) - - raw_content, created = RawContent.objects.get_or_create( - learning_package=self.learning_package, - mime_type=f"application/vnd.openedx.xblock.v1.{block_type}+xml", - hash_digest=hash_digest, - defaults=dict( - # text=data_str, - size=len(data_bytes), - created=now, - ), - ) - if created: - raw_content.file.save( - f"{raw_content.learning_package.uuid}/{hash_digest}", - ContentFile(data_bytes), - ) - - # Decode as utf-8-sig in order to strip any BOM from the data. - data_str = codecs.decode(data_bytes, "utf-8-sig") - TextContent.objects.create( - raw_content=raw_content, - text=data_str, - length=len(data_str), - ) - - # TODO: Get associated file contents, both with the static regex, as - # well as with XBlock-specific code that examines attributes in - # video and HTML tag definitions. + local_key = xml_file_path.stem + # Do some basic parsing of the content to see if it's even well + # constructed enough to add (or whether we should skip/error on it). try: - block_root = ET.fromstring(data_str) + block_root = ET.parse(xml_file_path).getroot() except ET.ParseError as err: logger.error(f"Parse error for {xml_file_path}: {err}") + components_skipped += 1 continue - + display_name = block_root.attrib.get("display_name", "") - - # Create the ComponentVersion - component_version = ComponentVersion.objects.create( - component=component, - version_num=1, # This only works for initial import + _component, component_version = components_api.create_component_and_version( + learning_package_id=self.learning_package.id, + namespace=namespace, + type=block_type, + local_key=local_key, title=display_name, created=now, created_by=None, ) - ComponentVersionRawContent.objects.create( - component_version=component_version, - raw_content=raw_content, - identifier="source.xml", - learner_downloadable=False, - ) - static_files_found = static_files_regex.findall(data_str) - for static_local_path in static_files_found: - self.create_content(static_local_path, now, component_version) - # Mark that Component as Published - component_publish_log_entry = ComponentPublishLogEntry.objects.create( - component=component, - component_version=component_version, - publish_log_entry=publish_log_entry, + # Create the RawContent entry for the raw data... + data_bytes = xml_file_path.read_bytes() + text_content, _created = contents_api.get_or_create_text_content_from_bytes( + learning_package_id=self.learning_package.id, + data_bytes=data_bytes, + mime_type=f"application/vnd.openedx.xblock.v1.{block_type}+xml", + created=now, ) - PublishedComponent.objects.create( - component=component, - component_version=component_version, - component_publish_log_entry=component_publish_log_entry, + # Add the OLX source text to the ComponentVersion + components_api.add_content_to_component_version( + component_version, + raw_content_id=text_content.pk, + key="source.xml", + learner_downloadable=False ) - print(f"{block_type}: {components_found}") + # Cycle through static assets references and add those as well... + for static_local_path in static_files_regex.findall(text_content.text): + self.create_content(static_local_path, now, component_version) + + print(f"{block_type}: {components_found} (skipped: {components_skipped})") diff --git a/openedx_learning/contrib/media_server/urls.py b/openedx_learning/contrib/media_server/urls.py index 554966ace..a76155dec 100644 --- a/openedx_learning/contrib/media_server/urls.py +++ b/openedx_learning/contrib/media_server/urls.py @@ -6,8 +6,8 @@ path( ( "component_asset/" - "/" - "/" + "/" + "/" "/" "" ), diff --git a/openedx_learning/contrib/media_server/views.py b/openedx_learning/contrib/media_server/views.py index 86e92d131..d376283eb 100644 --- a/openedx_learning/contrib/media_server/views.py +++ b/openedx_learning/contrib/media_server/views.py @@ -8,7 +8,7 @@ def component_asset( - request, learning_package_identifier, component_identifier, version_num, asset_path + request, learning_package_key, component_key, version_num, asset_path ): """ Serve the ComponentVersion asset data. @@ -25,7 +25,7 @@ def component_asset( """ try: cvc = get_component_version_content( - learning_package_identifier, component_identifier, version_num, asset_path + learning_package_key, component_key, version_num, asset_path ) except ObjectDoesNotExist: raise Http404("File not found") diff --git a/openedx_learning/core/components/admin.py b/openedx_learning/core/components/admin.py index 10712d6a9..34f091d92 100644 --- a/openedx_learning/core/components/admin.py +++ b/openedx_learning/core/components/admin.py @@ -1,5 +1,4 @@ from django.contrib import admin -from django.db.models.aggregates import Count, Sum from django.template.defaultfilters import filesizeformat from django.urls import reverse from django.utils.html import format_html @@ -8,8 +7,6 @@ Component, ComponentVersion, ComponentVersionRawContent, - PublishedComponent, - RawContent, ) from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin @@ -17,13 +14,13 @@ class ComponentVersionInline(admin.TabularInline): model = ComponentVersion fields = ["version_num", "created", "title", "format_uuid"] - readonly_fields = ["version_num", "created", "title", "format_uuid"] + readonly_fields = ["version_num", "created", "title", "uuid", "format_uuid"] extra = 0 def format_uuid(self, cv_obj): return format_html( '{}', - reverse("admin:components_componentversion_change", args=(cv_obj.id,)), + reverse("admin:oel_components_componentversion_change", args=(cv_obj.pk,)), cv_obj.uuid, ) @@ -32,113 +29,44 @@ def format_uuid(self, cv_obj): @admin.register(Component) class ComponentAdmin(ReadOnlyModelAdmin): - list_display = ("identifier", "uuid", "namespace", "type", "created") + list_display = ("key", "uuid", "namespace", "type", "created") readonly_fields = [ "learning_package", "uuid", "namespace", "type", - "identifier", + "key", "created", ] list_filter = ("type", "learning_package") - search_fields = ["uuid", "identifier"] + search_fields = ["publishable_entity__uuid", "publishable_entity__key"] inlines = [ComponentVersionInline] -@admin.register(PublishedComponent) -class PublishedComponentAdmin(ReadOnlyModelAdmin): - model = PublishedComponent +class RawContentInline(admin.TabularInline): + model = ComponentVersion.raw_contents.through def get_queryset(self, request): queryset = super().get_queryset(request) - return ( - queryset.select_related( - "component", - "component__learning_package", - "component_version", - "component_publish_log_entry__publish_log_entry", - ) - .annotate(size=Sum("component_version__raw_contents__size")) - .annotate(content_count=Count("component_version__raw_contents")) + return queryset.select_related( + "raw_content", + "raw_content__learning_package", + "raw_content__text_content", + "component_version", + "component_version__publishable_entity_version", + "component_version__component", + "component_version__component__publishable_entity", ) - readonly_fields = ["component", "component_version", "component_publish_log_entry"] - list_display = [ - "identifier", - "version", - "title", - "published_at", - "type", - "content_count", - "size", - "learning_package", - ] - list_filter = ["component__type", "component__learning_package"] - search_fields = [ - "component__uuid", - "component__identifier", - "component_version__uuid", - "component_version__title", - ] - - def learning_package(self, pc): - return pc.component.learning_package.identifier - - def published_at(self, pc): - return pc.component_publish_log_entry.publish_log_entry.published_at - - def identifier(self, pc): - """ - Link to published ComponentVersion with Component identifier as text. - - This is a little weird in that we're showing the Component identifier, - but linking to the published ComponentVersion. But this is what you want - to link to most of the time, as the link to the Component has almost no - information in it (and can be accessed from the ComponentVersion details - page anyhow). - """ - return format_html( - '{}', - reverse( - "admin:components_componentversion_change", - args=(pc.component_version_id,), - ), - pc.component.identifier, - ) - - def content_count(self, pc): - return pc.content_count - - content_count.short_description = "#" - - def size(self, pc): - return filesizeformat(pc.size) - - def namespace(self, pc): - return pc.component.namespace - - def type(self, pc): - return pc.component.type - - def version(self, pc): - return pc.component_version.version_num - - def title(self, pc): - return pc.component_version.title - - -class RawContentInline(admin.TabularInline): - model = ComponentVersion.raw_contents.through fields = [ - "format_identifier", + "format_key", "format_size", "learner_downloadable", "rendered_data", ] readonly_fields = [ "raw_content", - "format_identifier", + "format_key", "format_size", "rendered_data", ] @@ -152,15 +80,15 @@ def format_size(self, cvc_obj): format_size.short_description = "Size" - def format_identifier(self, cvc_obj): + def format_key(self, cvc_obj): return format_html( '{}', link_for_cvc(cvc_obj), # reverse("admin:components_content_change", args=(cvc_obj.content_id,)), - cvc_obj.identifier, + cvc_obj.key, ) - format_identifier.short_description = "Identifier" + format_key.short_description = "Key" @admin.register(ComponentVersion) @@ -180,41 +108,16 @@ class ComponentVersionAdmin(ReadOnlyModelAdmin): "version_num", "created", ] + list_display = ["component", "version_num", "uuid", "created"] inlines = [RawContentInline] - -@admin.register(RawContent) -class RawContentAdmin(ReadOnlyModelAdmin): - list_display = [ - "hash_digest", - "learning_package", - "mime_type", - "size", - "created", - ] - fields = [ - "learning_package", - "hash_digest", - "mime_type", - "size", - "created", - "text_preview", - ] - readonly_fields = [ - "learning_package", - "hash_digest", - "mime_type", - "size", - "created", - "text_preview", - ] - list_filter = ("mime_type", "learning_package") - search_fields = ("hash_digest",) - - def text_preview(self, raw_content_obj): - if hasattr(raw_content_obj, "text_content"): - return format_text_for_admin_display(raw_content_obj.text_content.text) - return "" + def get_queryset(self, request): + queryset = super().get_queryset(request) + return queryset.select_related( + "component", + "component__publishable_entity", + "publishable_entity_version", + ) def is_displayable_text(mime_type): @@ -241,10 +144,10 @@ def is_displayable_text(mime_type): def link_for_cvc(cvc_obj: ComponentVersionRawContent): return "/media_server/component_asset/{}/{}/{}/{}".format( - cvc_obj.raw_content.learning_package.identifier, - cvc_obj.component_version.component.identifier, + cvc_obj.raw_content.learning_package.key, + cvc_obj.component_version.component.key, cvc_obj.component_version.version_num, - cvc_obj.identifier, + cvc_obj.key, ) @@ -263,10 +166,10 @@ def content_preview(cvc_obj: ComponentVersionRawContent): '', # TODO: configure with settings value: "/media_server/component_asset/{}/{}/{}/{}".format( - cvc_obj.raw_content.learning_package.identifier, - cvc_obj.component_version.component.identifier, + cvc_obj.raw_content.learning_package.key, + cvc_obj.component_version.component.key, cvc_obj.component_version.version_num, - cvc_obj.identifier, + cvc_obj.key, ), ) diff --git a/openedx_learning/core/components/api.py b/openedx_learning/core/components/api.py index 41d92f462..1a3972dec 100644 --- a/openedx_learning/core/components/api.py +++ b/openedx_learning/core/components/api.py @@ -1,29 +1,90 @@ """ -Public API for querying and manipulating Components. +Components API (warning: UNSTABLE, in progress API) -This API is still under construction and should not be considered "stable" until -this repo hits a 1.0 release. +These functions are often going to be simple-looking write operations, but there +is bookkeeping logic needed across multiple models to keep state consistent. You +can read from the models directly for various queries if necessary–we do this in +the Django Admin for instance. But you should NEVER mutate this app's models +directly, since there might be other related models that you may not know about. + +Please look at the models.py file for more information about the kinds of data +are stored in this app. """ -from django.db.models import Q from pathlib import Path -from .models import ComponentVersionRawContent +from django.db.models import Q +from django.db.transaction import atomic + +from ..publishing.api import ( + create_publishable_entity, + create_publishable_entity_version, +) +from .models import ComponentVersionRawContent, Component, ComponentVersion + + +def create_component( + learning_package_id, namespace, type, local_key, created, created_by +): + key = f"{namespace}:{type}@{local_key}" + with atomic(): + publishable_entity = create_publishable_entity( + learning_package_id, key, created, created_by + ) + component = Component.objects.create( + publishable_entity=publishable_entity, + learning_package_id=learning_package_id, + namespace=namespace, + type=type, + local_key=local_key, + ) + return component + + +def create_component_version(component_pk, version_num, title, created, created_by): + with atomic(): + publishable_entity_version = create_publishable_entity_version( + entity_id=component_pk, + version_num=version_num, + title=title, + created=created, + created_by=created_by, + ) + component_version = ComponentVersion.objects.create( + publishable_entity_version=publishable_entity_version, + component_id=component_pk, + ) + return component_version + + +def create_component_and_version( + learning_package_id, namespace, type, local_key, title, created, created_by +): + with atomic(): + component = create_component( + learning_package_id, namespace, type, local_key, created, created_by + ) + component_version = create_component_version( + component.pk, + version_num=1, + title=title, + created=created, + created_by=created_by, + ) + return (component, component_version) + + +def get_component_by_pk(component_pk): + return Component.objects.get(pk=component_pk) def get_component_version_content( - learning_package_identifier: str, - component_identifier: str, + learning_package_key: str, + component_key: str, version_num: int, - identifier: Path, + key: Path, ) -> ComponentVersionRawContent: """ - Look up ComponentVersionRawContent by human readable identifiers. - - Notes: - - 1. This function is returning a model, which we generally frown upon. - 2. I'd like to experiment with different lookup methods - (see https://github.com/openedx/openedx-learning/issues/34) + Look up ComponentVersionRawContent by human readable keys. Can raise a django.core.exceptions.ObjectDoesNotExist error if there is no matching ComponentVersionRawContent. @@ -34,10 +95,20 @@ def get_component_version_content( "component_version__component", "component_version__component__learning_package", ).get( - Q( - component_version__component__learning_package__identifier=learning_package_identifier - ) - & Q(component_version__component__identifier=component_identifier) - & Q(component_version__version_num=version_num) - & Q(identifier=identifier) + Q(component_version__component__learning_package__key=learning_package_key) + & Q(component_version__component__publishable_entity__key=component_key) + & Q(component_version__publishable_entity_version__version_num=version_num) + & Q(key=key) + ) + + +def add_content_to_component_version( + component_version, raw_content_id, key, learner_downloadable=False +): + cvrc, _created = ComponentVersionRawContent.objects.get_or_create( + component_version=component_version, + raw_content_id=raw_content_id, + key=key, + learner_downloadable=learner_downloadable, ) + return cvrc diff --git a/openedx_learning/core/components/apps.py b/openedx_learning/core/components/apps.py index 0354fcb71..9b25954ac 100644 --- a/openedx_learning/core/components/apps.py +++ b/openedx_learning/core/components/apps.py @@ -9,3 +9,13 @@ class ComponentsConfig(AppConfig): name = "openedx_learning.core.components" verbose_name = "Learning Core: Components" default_auto_field = "django.db.models.BigAutoField" + label = "oel_components" + + def ready(self): + """ + Register Component and ComponentVersion. + """ + from ..publishing.api import register_content_models + from .models import Component, ComponentVersion + + register_content_models(Component, ComponentVersion) diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py index f4a2c284b..ac379c67d 100644 --- a/openedx_learning/core/components/migrations/0001_initial.py +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -1,10 +1,7 @@ -# Generated by Django 4.1.6 on 2023-04-14 00:12 +# Generated by Django 3.2.18 on 2023-05-11 02:07 -from django.conf import settings -import django.core.validators from django.db import migrations, models import django.db.models.deletion -import openedx_learning.lib.validators import uuid @@ -12,8 +9,8 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ("publishing", "0001_initial"), - migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("oel_publishing", "0001_initial"), + ("oel_contents", "0001_initial"), ] operations = [ @@ -21,39 +18,22 @@ class Migration(migrations.Migration): name="Component", fields=[ ( - "id", - models.BigAutoField( - auto_created=True, + "publishable_entity", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", + to="oel_publishing.publishableentity", ), ), ("namespace", models.CharField(max_length=100)), ("type", models.CharField(blank=True, max_length=100)), - ("identifier", models.CharField(max_length=255)), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), + ("local_key", models.CharField(max_length=255)), ( "learning_package", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - to="publishing.learningpackage", + to="oel_publishing.learningpackage", ), ), ], @@ -66,51 +46,20 @@ class Migration(migrations.Migration): name="ComponentVersion", fields=[ ( - "id", - models.BigAutoField( - auto_created=True, + "publishable_entity_version", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("title", models.CharField(blank=True, default="", max_length=1000)), - ( - "version_num", - models.PositiveBigIntegerField( - validators=[django.core.validators.MinValueValidator(1)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] + to="oel_publishing.publishableentityversion", ), ), ( "component", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - to="components.component", - ), - ), - ( - "created_by", - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, + related_name="versions", + to="oel_components.component", ), ), ], @@ -119,83 +68,6 @@ class Migration(migrations.Migration): "verbose_name_plural": "Component Versions", }, ), - migrations.CreateModel( - name="RawContent", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("hash_digest", models.CharField(editable=False, max_length=40)), - ("mime_type", models.CharField(max_length=255)), - ( - "size", - models.PositiveBigIntegerField( - validators=[django.core.validators.MaxValueValidator(50000000)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ("file", models.FileField(null=True, upload_to="")), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="publishing.learningpackage", - ), - ), - ], - options={ - "verbose_name": "Raw Content", - "verbose_name_plural": "Raw Contents", - }, - ), - migrations.CreateModel( - name="PublishedComponent", - fields=[ - ( - "component", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="components.component", - ), - ), - ], - options={ - "verbose_name": "Published Component", - "verbose_name_plural": "Published Components", - }, - ), - migrations.CreateModel( - name="TextContent", - fields=[ - ( - "raw_content", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - related_name="text_content", - serialize=False, - to="components.rawcontent", - ), - ), - ("text", models.TextField(blank=True, max_length=100000)), - ("length", models.PositiveIntegerField()), - ], - ), migrations.CreateModel( name="ComponentVersionRawContent", fields=[ @@ -217,20 +89,20 @@ class Migration(migrations.Migration): verbose_name="UUID", ), ), - ("identifier", models.CharField(max_length=255)), + ("key", models.CharField(max_length=255)), ("learner_downloadable", models.BooleanField(default=False)), ( "component_version", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - to="components.componentversion", + to="oel_components.componentversion", ), ), ( "raw_content", models.ForeignKey( on_delete=django.db.models.deletion.RESTRICT, - to="components.rawcontent", + to="oel_contents.rawcontent", ), ), ], @@ -240,148 +112,42 @@ class Migration(migrations.Migration): name="raw_contents", field=models.ManyToManyField( related_name="component_versions", - through="components.ComponentVersionRawContent", - to="components.rawcontent", - ), - ), - migrations.CreateModel( - name="ComponentPublishLogEntry", - fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "component", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="components.component", - ), - ), - ( - "component_version", - models.ForeignKey( - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="components.componentversion", - ), - ), - ( - "publish_log_entry", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="publishing.publishlogentry", - ), - ), - ], - ), - migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "mime_type"], - name="content_idx_lp_mime_type", - ), - ), - migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-size"], name="content_idx_lp_rsize" - ), - ), - migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-created"], name="content_idx_lp_rcreated" - ), - ), - migrations.AddConstraint( - model_name="rawcontent", - constraint=models.UniqueConstraint( - fields=("learning_package", "mime_type", "hash_digest"), - name="content_uniq_lc_mime_type_hash_digest", - ), - ), - migrations.AddField( - model_name="publishedcomponent", - name="component_publish_log_entry", - field=models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="components.componentpublishlogentry", - ), - ), - migrations.AddField( - model_name="publishedcomponent", - name="component_version", - field=models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="components.componentversion", + through="oel_components.ComponentVersionRawContent", + to="oel_contents.RawContent", ), ), migrations.AddIndex( model_name="componentversionrawcontent", index=models.Index( - fields=["raw_content", "component_version"], name="cvrawcontent_c_cv" + fields=["raw_content", "component_version"], + name="oel_cvrawcontent_c_cv", ), ), migrations.AddIndex( model_name="componentversionrawcontent", index=models.Index( - fields=["component_version", "raw_content"], name="cvrawcontent_cv_d" + fields=["component_version", "raw_content"], + name="oel_cvrawcontent_cv_d", ), ), migrations.AddConstraint( model_name="componentversionrawcontent", constraint=models.UniqueConstraint( - fields=("component_version", "identifier"), - name="cvrawcontent_uniq_cv_id", + fields=("component_version", "key"), name="oel_cvrawcontent_uniq_cv_key" ), ), - migrations.AddIndex( - model_name="componentversion", - index=models.Index( - fields=["component", "-created"], name="cv_idx_component_rcreated" - ), - ), - migrations.AddIndex( - model_name="componentversion", - index=models.Index(fields=["title"], name="cv_idx_title"), - ), - migrations.AddConstraint( - model_name="componentversion", - constraint=models.UniqueConstraint( - fields=("component", "version_num"), - name="cv_uniq_component_version_num", - ), - ), - migrations.AddIndex( - model_name="component", - index=models.Index( - fields=["learning_package", "identifier"], - name="component_idx_lp_identifier", - ), - ), - migrations.AddIndex( - model_name="component", - index=models.Index(fields=["identifier"], name="component_idx_identifier"), - ), migrations.AddIndex( model_name="component", index=models.Index( - fields=["learning_package", "-created"], - name="component_idx_lp_rcreated", + fields=["learning_package", "namespace", "type", "local_key"], + name="oel_component_idx_lc_ns_t_lk", ), ), migrations.AddConstraint( model_name="component", constraint=models.UniqueConstraint( - fields=("learning_package", "namespace", "type", "identifier"), - name="component_uniq_lc_ns_type_identifier", + fields=("learning_package", "namespace", "type", "local_key"), + name="oel_component_uniq_lc_ns_t_lk", ), ), ] diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index b1d683394..b773c3f92 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -6,33 +6,29 @@ Problem), but little beyond that. Components have one or more ComponentVersions, which represent saved versions of -that Component. At any time, there is at most one published ComponentVersion for -a Component in a LearningPackage (there can be zero if it's unpublished). The -publish status is tracked in PublishedComponent, with historical publish data in -ComponentPublishLogEntry. - -RawContent is a simple model holding unversioned, raw data, along with some -simple metadata like size and MIME type. +that Component. Managing the publishing of these versions is handled through the +publishing app. Component maps 1:1 to PublishableEntity and ComponentVersion +maps 1:1 to PublishableEntityVersion. Multiple pieces of RawContent may be associated with a ComponentVersion, through the ComponentVersionRawContent model. ComponentVersionRawContent allows to -specify a Component-local identifier. We're using this like a file path by -convention, but it's possible we might want to have special identifiers later. +specify a ComponentVersion-local identifier. We're using this like a file path +by convention, but it's possible we might want to have special identifiers +later. """ from django.db import models -from django.conf import settings -from django.core.validators import MinValueValidator, MaxValueValidator - -from openedx_learning.lib.fields import ( - hash_field, - identifier_field, - immutable_uuid_field, - manual_date_time_field, +from django.db.models import F, Q + +from openedx_learning.lib.fields import key_field, immutable_uuid_field +from ..publishing.models import LearningPackage +from ..publishing.model_mixins import ( + PublishableEntityMixin, + PublishableEntityVersionMixin, ) -from ..publishing.models import LearningPackage, PublishLogEntry +from ..contents.models import RawContent -class Component(models.Model): +class Component(PublishableEntityMixin): """ This represents any Component that has ever existed in a LearningPackage. @@ -47,33 +43,36 @@ class Component(models.Model): associated with the ComponentVersion model and the RawContent that ComponentVersions are associated with. - A Component belongs to one and only one LearningPackage. + A Component belongs to exactly one LearningPackage. - How to use this model - --------------------- + Identifiers + ----------- - Make a foreign key to the Component model when you need a stable reference - that will exist for as long as the LearningPackage itself exists. It is - possible for an Component to have no published ComponentVersion, either - because it was never published or because it's been "deleted" (made - unavailable) at some point, but the Component will continue to exist. + Components have a ``publishable_entity`` OneToOneField to the ``publishing`` + app's PublishableEntity field, and it uses this as its primary key. Please + see PublishableEntity's docstring for how you should use its ``uuid`` and + ``key`` fields. - The UUID should be treated as immutable. + State Consistency + ----------------- - The identifier field *is* mutable, but changing it will affect all - ComponentVersions. + The ``key`` field on Component's ``publishable_entity`` is dervied from the + ``(namespace, type, local_key)`` fields in this model. We don't support + changing the keys yet, but if we do, those values need to be kept in sync. - If you are referencing this model from within the same process, use a - foreign key to the id. If you are referencing this Component from an - external system/service, use the UUID. The identifier is the part that is - most likely to be human-readable, and may be exported/copied, but try not to - rely on it, since this value may change. + How build on this model + ----------------------- - Note: When we actually implement the ability to change identifiers, we - should make a history table and a modified attribute on this model. + Make a foreign key to the Component model when you need a stable reference + that will exist for as long as the LearningPackage itself exists. """ - uuid = immutable_uuid_field() + # This foreign key is technically redundant because we're already locked to + # a single LearningPackage through our publishable_entity relation. However, having + # this foreign key directly allows us to make indexes that efficiently + # query by other Component fields within a given LearningPackage, which is + # going to be a common use case (and we can't make a compound index using + # columns from different tables). learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) # namespace and type work together to help figure out what Component needs @@ -87,53 +86,42 @@ class Component(models.Model): # the UsageKey. type = models.CharField(max_length=100, null=False, blank=True) - # identifier is local to a learning_package + namespace + type. For XBlocks, - # this is the block_id part of the UsageKey, which usually shows up in the - # OLX as the url_name attribute. - identifier = identifier_field() - - created = manual_date_time_field() + # local_key is an identifier that is local to the (namespace, type). The + # publishable.key should be calculated as a combination of (namespace, type, + # local_key). + local_key = key_field() class Meta: constraints = [ - # The combination of (namespace, type, identifier) is unique within + # The combination of (namespace, type, local_key) is unique within # a given LearningPackage. Note that this means it is possible to - # have two Components that have the exact same identifier. An XBlock + # have two Components that have the exact same local_key. An XBlock # would be modeled as namespace="xblock.v1" with the type as the - # block_type, so the identifier would only be the block_id (the + # block_type, so the local_key would only be the block_id (the # very last part of the UsageKey). models.UniqueConstraint( fields=[ "learning_package", "namespace", "type", - "identifier", + "local_key", ], - name="component_uniq_lc_ns_type_identifier", - ) + name="oel_component_uniq_lc_ns_t_lk", + ), ] indexes = [ - # LearningPackage Identifier Index: - # * Search by identifier (without having to specify namespace and - # type). This kind of freeform search will likely be common. + # Global Namespace/Type/Local-Key Index: + # * Search by the different Components fields across all Learning + # Packages on the site. This would be a support-oriented tool + # from Django Admin. models.Index( - fields=["learning_package", "identifier"], - name="component_idx_lp_identifier", - ), - # Global Identifier Index: - # * Search by identifier across all Components on the site. This - # would be a support-oriented tool from Django Admin. - models.Index( - fields=["identifier"], - name="component_idx_identifier", - ), - # LearningPackage (reverse) Created Index: - # * Search for most recently *created* Components for a given - # LearningPackage, since they're the most likely to be actively - # worked on. - models.Index( - fields=["learning_package", "-created"], - name="component_idx_lp_rcreated", + fields=[ + "learning_package", + "namespace", + "type", + "local_key", + ], + name="oel_component_idx_lc_ns_t_lk", ), ] @@ -142,296 +130,37 @@ class Meta: verbose_name_plural = "Components" def __str__(self): - return f"{self.identifier}" + return f"{self.namespace}:{self.type}:{self.local_key}" -class ComponentVersion(models.Model): +class ComponentVersion(PublishableEntityVersionMixin): """ A particular version of a Component. - This holds the title (because that's versioned information) and the contents - via a M:M relationship with RawContent via ComponentVersionRawContent. - - * Each ComponentVersion belongs to one and only one Component. - * ComponentVersions have a version_num that should increment by one with - each new version. + This holds the content using a M:M relationship with RawContent via + ComponentVersionRawContent. """ - uuid = immutable_uuid_field() - component = models.ForeignKey(Component, on_delete=models.CASCADE) - - # Blank titles are allowed because some Components are built to be used from - # a particular Unit, and the title would be redundant in that context (e.g. - # a "Welcome" video in a "Welcome" Unit). - title = models.CharField(max_length=1000, default="", null=False, blank=True) - - # The version_num starts at 1 and increments by 1 with each new version for - # a given Component. Doing it this way makes it more convenient for users to - # refer to than a hash or UUID value. - version_num = models.PositiveBigIntegerField( - null=False, - validators=[MinValueValidator(1)], - ) - - # All ComponentVersions created as part of the same publish should have the - # exact same created datetime (not off by a handful of microseconds). - created = manual_date_time_field() - - # User who created the ContentVersion. This can be null if the user is later - # removed. Open edX in general doesn't let you remove users, but we should - # try to model it so that this is possible eventually. - created_by = models.ForeignKey( - settings.AUTH_USER_MODEL, - on_delete=models.SET_NULL, - null=True, + # This is technically redundant, since we can get this through + # publishable_entity_version.publishable.component, but this is more + # convenient. + component = models.ForeignKey( + Component, on_delete=models.CASCADE, related_name="versions" ) - # The contents hold the actual interesting data associated with this + # The raw_contents hold the actual interesting data associated with this # ComponentVersion. raw_contents = models.ManyToManyField( - "RawContent", + RawContent, through="ComponentVersionRawContent", related_name="component_versions", ) - def __str__(self): - return f"v{self.version_num}: {self.title}" - class Meta: - constraints = [ - # Prevent the situation where we have multiple ComponentVersions - # claiming to be the same version_num for a given Component. This - # can happen if there's a race condition between concurrent editors - # in different browsers, working on the same Component. With this - # constraint, one of those processes will raise an IntegrityError. - models.UniqueConstraint( - fields=[ - "component", - "version_num", - ], - name="cv_uniq_component_version_num", - ) - ] - indexes = [ - # LearningPackage (reverse) Created Index: - # * Make it cheap to find the most recently created - # ComponentVersions for a given LearningPackage. This represents - # the most recently saved work for a LearningPackage and would - # be the most likely areas to get worked on next. - models.Index( - fields=["component", "-created"], - name="cv_idx_component_rcreated", - ), - # Title Index: - # * Search by title. - models.Index( - fields=[ - "title", - ], - name="cv_idx_title", - ), - ] - - # These are for the Django Admin UI. verbose_name = "Component Version" verbose_name_plural = "Component Versions" -class ComponentPublishLogEntry(models.Model): - """ - This is a historical record of Component publishing. - - When a ComponentVersion is initially created, it's considered a draft. The - act of publishing means we're marking a ContentVersion as the official, - ready-for-use version of this Component. - """ - - publish_log_entry = models.ForeignKey(PublishLogEntry, on_delete=models.CASCADE) - component = models.ForeignKey(Component, on_delete=models.RESTRICT) - component_version = models.ForeignKey( - ComponentVersion, on_delete=models.RESTRICT, null=True - ) - - -class PublishedComponent(models.Model): - """ - For any given Component, what is the currently published ComponentVersion. - - It may be possible for a Component to exist only as a Draft (and thus not - show up in this table). There is only ever one published ComponentVersion - per Component at any given time. - - TODO: Do we need to create a (redundant) title field in this model so that - we can more efficiently search across titles within a LearningPackage? - Probably not an immediate concern because the number of rows currently - shouldn't be > 10,000 in the more extreme cases. - """ - - component = models.OneToOneField( - Component, on_delete=models.RESTRICT, primary_key=True - ) - component_version = models.OneToOneField( - ComponentVersion, - on_delete=models.RESTRICT, - null=True, - ) - component_publish_log_entry = models.ForeignKey( - ComponentPublishLogEntry, - on_delete=models.RESTRICT, - ) - - class Meta: - verbose_name = "Published Component" - verbose_name_plural = "Published Components" - - -class RawContent(models.Model): - """ - This is the most basic piece of raw content data, with no version metadata. - - RawContent stores data using the "file" field. This data is not - auto-normalized in any way, meaning that pieces of content that are - semantically equivalent (e.g. differently spaced/sorted JSON) may result in - new entries. This model is intentionally ignorant of what these things mean, - because it expects supplemental data models to build on top of it. - - Two RawContent instances _can_ have the same hash_digest if they are of - different MIME types. For instance, an empty text file and an empty SRT file - will both hash the same way, but be considered different entities. - - The other fields on RawContent are for data that is intrinsic to the file - data itself (e.g. the size). Any smart parsing of the contents into more - structured metadata should happen in other models that hang off of - RawContent. - - RawContent models are not versioned in any way. The concept of versioning - only exists at a higher level. - - RawContent is optimized for cheap storage, not low latency. It stores - content in a FileField. If you need faster text access across multiple rows, - add a TextContent entry that corresponds to the relevant RawContent. - - If you need to transform this RawContent into more structured data for your - application, create a model with a OneToOneField(primary_key=True) - relationship to RawContent. Just remember that *you should always create the - RawContent entry* first, to ensure content is always exportable, even if - your app goes away in the future. - """ - - # 50 MB is our current limit, based on the current Open edX Studio file - # upload size limit. - MAX_FILE_SIZE = 50_000_000 - - learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) - - # This hash value may be calculated using create_hash_digest from the - # openedx.lib.fields module. - hash_digest = hash_field() - - # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type - # and sub-type may each be 127 chars, making a max of 255 (including the "/" - # in between). - # - # DO NOT STORE parameters here, e.g. "charset=". We can make a new field if - # that becomes necessary. - mime_type = models.CharField(max_length=255, blank=False, null=False) - - # This is the size of the raw data file in bytes. This can be different than - # the character length, since UTF-8 encoding can use anywhere between 1-4 - # bytes to represent any given character. - size = models.PositiveBigIntegerField( - validators=[MaxValueValidator(MAX_FILE_SIZE)], - ) - - # This should be manually set so that multiple RawContent rows being set in - # the same transaction are created with the same timestamp. The timestamp - # should be UTC. - created = manual_date_time_field() - - # All content for the LearningPackage should be stored in files. See model - # docstring for more details on how to store this data in supplementary data - # models that offer better latency guarantees. - file = models.FileField( - null=True, - storage=settings.OPENEDX_LEARNING.get( - "STORAGE", - settings.DEFAULT_FILE_STORAGE, - ), - ) - - class Meta: - constraints = [ - # Make sure we don't store duplicates of this raw data within the - # same LearningPackage, unless they're of different MIME types. - models.UniqueConstraint( - fields=[ - "learning_package", - "mime_type", - "hash_digest", - ], - name="content_uniq_lc_mime_type_hash_digest", - ), - ] - indexes = [ - # LearningPackage MIME type Index: - # * Break down Content counts by type/subtype with in a - # LearningPackage. - # * Find all the Content in a LearningPackage that matches a - # certain MIME type (e.g. "image/png", "application/pdf". - models.Index( - fields=["learning_package", "mime_type"], - name="content_idx_lp_mime_type", - ), - # LearningPackage (reverse) Size Index: - # * Find largest Content in a LearningPackage. - # * Find the sum of Content size for a given LearningPackage. - models.Index( - fields=["learning_package", "-size"], - name="content_idx_lp_rsize", - ), - # LearningPackage (reverse) Created Index: - # * Find most recently added Content. - models.Index( - fields=["learning_package", "-created"], - name="content_idx_lp_rcreated", - ), - ] - verbose_name = "Raw Content" - verbose_name_plural = "Raw Contents" - - -class TextContent(models.Model): - """ - TextContent supplements RawContent to give an in-table text copy. - - This model exists so that we can have lower-latency access to this data, - particularly if we're pulling back multiple rows at once. - - Apps are encouraged to create their own data models that further extend this - one with a more intelligent, parsed data model. For example, individual - XBlocks might parse the OLX in this model into separate data models for - VideoBlock, ProblemBlock, etc. - - The reason this is built directly into the Learning Core data model is - because we want to be able to easily access and browse this data even if the - app-extended models get deleted (e.g. if they are deprecated and removed). - """ - - # 100K is our limit for text data, like OLX. This means 100K *characters*, - # not bytes. Since UTF-8 encodes characters using as many as 4 bytes, this - # couled be as much as 400K of data if we had nothing but emojis. - MAX_TEXT_LENGTH = 100_000 - - raw_content = models.OneToOneField( - RawContent, - on_delete=models.CASCADE, - primary_key=True, - related_name="text_content", - ) - text = models.TextField(null=False, blank=True, max_length=MAX_TEXT_LENGTH) - length = models.PositiveIntegerField(null=False) - - class ComponentVersionRawContent(models.Model): """ Determines the RawContent for a given ComponentVersion. @@ -441,9 +170,9 @@ class ComponentVersionRawContent(models.Model): transcripts in different languages. When RawContent is associated with an ComponentVersion, it has some local - identifier that is unique within the the context of that ComponentVersion. - This allows the ComponentVersion to do things like store an image file and - reference it by a "path" identifier. + key that is unique within the the context of that ComponentVersion. This + allows the ComponentVersion to do things like store an image file and + reference it by a "path" key. RawContent is immutable and sharable across multiple ComponentVersions and even across LearningPackages. @@ -453,8 +182,10 @@ class ComponentVersionRawContent(models.Model): component_version = models.ForeignKey(ComponentVersion, on_delete=models.CASCADE) uuid = immutable_uuid_field() - identifier = identifier_field() + key = key_field() + # Long explanation for the ``learner_downloadable`` field: + # # Is this RawContent downloadable during the learning experience? This is # NOT about public vs. private permissions on course assets, as that will be # a policy that can be changed independently of new versions of the content. @@ -492,21 +223,21 @@ class ComponentVersionRawContent(models.Model): class Meta: constraints = [ - # Uniqueness is only by ComponentVersion and identifier. If for some - # reason a ComponentVersion wants to associate the same piece of - # content with two different identifiers, that is permitted. + # Uniqueness is only by ComponentVersion and key. If for some reason + # a ComponentVersion wants to associate the same piece of content + # with two different identifiers, that is permitted. models.UniqueConstraint( - fields=["component_version", "identifier"], - name="cvrawcontent_uniq_cv_id", + fields=["component_version", "key"], + name="oel_cvrawcontent_uniq_cv_key", ), ] indexes = [ models.Index( fields=["raw_content", "component_version"], - name="cvrawcontent_c_cv", + name="oel_cvrawcontent_c_cv", ), models.Index( fields=["component_version", "raw_content"], - name="cvrawcontent_cv_d", + name="oel_cvrawcontent_cv_d", ), ] diff --git a/openedx_learning/core/contents/__init__.py b/openedx_learning/core/contents/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_learning/core/contents/admin.py b/openedx_learning/core/contents/admin.py new file mode 100644 index 000000000..81230d3d2 --- /dev/null +++ b/openedx_learning/core/contents/admin.py @@ -0,0 +1,53 @@ +from django.contrib import admin +from django.utils.html import format_html + +from .models import RawContent + +from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin + + +@admin.register(RawContent) +class RawContentAdmin(ReadOnlyModelAdmin): + list_display = [ + "hash_digest", + "file_link", + "learning_package", + "mime_type", + "size", + "created", + ] + fields = [ + "learning_package", + "hash_digest", + "mime_type", + "size", + "created", + "file_link", + "text_preview", + ] + readonly_fields = [ + "learning_package", + "hash_digest", + "mime_type", + "size", + "created", + "file_link", + "text_preview", + ] + list_filter = ("mime_type", "learning_package") + search_fields = ("hash_digest",) + + def file_link(self, raw_content): + return format_html( + 'Download', + raw_content.file.url, + ) + + def text_preview(self, raw_content): + if not hasattr(raw_content, "text_content"): + return "(not available)" + + return format_html( + '
\n{}\n
', + raw_content.text_content.text, + ) diff --git a/openedx_learning/core/contents/api.py b/openedx_learning/core/contents/api.py new file mode 100644 index 000000000..282bb820f --- /dev/null +++ b/openedx_learning/core/contents/api.py @@ -0,0 +1,85 @@ +""" +Low Level Contents API (warning: UNSTABLE, in progress API) + +Please look at the models.py file for more information about the kinds of data +are stored in this app. +""" +import codecs + +from django.core.files.base import ContentFile +from django.db.transaction import atomic + +from openedx_learning.lib.fields import create_hash_digest +from .models import RawContent, TextContent + + +def create_raw_content( + learning_package_id, data_bytes, mime_type, created, hash_digest=None +): + hash_digest = hash_digest or create_hash_digest(data_bytes) + raw_content = RawContent.objects.create( + learning_package_id=learning_package_id, + mime_type=mime_type, + hash_digest=hash_digest, + size=len(data_bytes), + created=created, + ) + raw_content.file.save( + f"{raw_content.learning_package.uuid}/{hash_digest}", + ContentFile(data_bytes), + ) + return raw_content + + +def create_text_from_raw_content(raw_content, encoding="utf-8-sig"): + text = codecs.decode(raw_content.file.open().read(), "utf-8-sig") + return TextContent.objects.create( + raw_content=raw_content, + text=text, + length=len(text), + ) + + +def get_or_create_raw_content( + learning_package_id, data_bytes, mime_type, created, hash_digest=None +): + hash_digest = hash_digest or create_hash_digest(data_bytes) + try: + raw_content = RawContent.objects.get( + learning_package_id=learning_package_id, hash_digest=hash_digest + ) + created = False + except RawContent.DoesNotExist: + raw_content = create_raw_content( + learning_package_id, data_bytes, mime_type, created, hash_digest + ) + created = True + + return raw_content, created + + +def get_or_create_text_content_from_bytes( + learning_package_id, + data_bytes, + mime_type, + created, + hash_digest=None, + encoding="utf-8-sig", +): + with atomic(): + raw_content, rc_created = get_or_create_raw_content( + learning_package_id, data_bytes, mime_type, created, hash_digest=None + ) + if rc_created or not hasattr(raw_content, "text_content"): + text = codecs.decode(data_bytes, "utf-8-sig") + text_content = TextContent.objects.create( + raw_content=raw_content, + text=text, + length=len(text), + ) + tc_created = True + else: + text_content = raw_content.text_content + tc_created = False + + return (text_content, tc_created) diff --git a/openedx_learning/core/contents/apps.py b/openedx_learning/core/contents/apps.py new file mode 100644 index 000000000..4c736b812 --- /dev/null +++ b/openedx_learning/core/contents/apps.py @@ -0,0 +1,12 @@ +from django.apps import AppConfig + + +class ContentsConfig(AppConfig): + """ + Configuration for the Contents Django application. + """ + + name = "openedx_learning.core.contents" + verbose_name = "Learning Core: Contents" + default_auto_field = "django.db.models.BigAutoField" + label = "oel_contents" diff --git a/openedx_learning/core/contents/migrations/0001_initial.py b/openedx_learning/core/contents/migrations/0001_initial.py new file mode 100644 index 000000000..f106553e7 --- /dev/null +++ b/openedx_learning/core/contents/migrations/0001_initial.py @@ -0,0 +1,103 @@ +# Generated by Django 3.2.18 on 2023-05-11 02:07 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import openedx_learning.lib.validators + + +class Migration(migrations.Migration): + initial = True + + dependencies = [ + ("oel_publishing", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="RawContent", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("hash_digest", models.CharField(editable=False, max_length=40)), + ("mime_type", models.CharField(max_length=255)), + ( + "size", + models.PositiveBigIntegerField( + validators=[django.core.validators.MaxValueValidator(50000000)] + ), + ), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), + ("file", models.FileField(null=True, upload_to="")), + ( + "learning_package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="oel_publishing.learningpackage", + ), + ), + ], + options={ + "verbose_name": "Raw Content", + "verbose_name_plural": "Raw Contents", + }, + ), + migrations.CreateModel( + name="TextContent", + fields=[ + ( + "raw_content", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + related_name="text_content", + serialize=False, + to="oel_contents.rawcontent", + ), + ), + ("text", models.TextField(blank=True, max_length=100000)), + ("length", models.PositiveIntegerField()), + ], + ), + migrations.AddIndex( + model_name="rawcontent", + index=models.Index( + fields=["learning_package", "mime_type"], + name="oel_content_idx_lp_mime_type", + ), + ), + migrations.AddIndex( + model_name="rawcontent", + index=models.Index( + fields=["learning_package", "-size"], name="oel_content_idx_lp_rsize" + ), + ), + migrations.AddIndex( + model_name="rawcontent", + index=models.Index( + fields=["learning_package", "-created"], + name="oel_content_idx_lp_rcreated", + ), + ), + migrations.AddConstraint( + model_name="rawcontent", + constraint=models.UniqueConstraint( + fields=("learning_package", "mime_type", "hash_digest"), + name="oel_content_uniq_lc_mime_type_hash_digest", + ), + ), + ] diff --git a/openedx_learning/core/contents/migrations/__init__.py b/openedx_learning/core/contents/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_learning/core/contents/models.py b/openedx_learning/core/contents/models.py new file mode 100644 index 000000000..e19f151cb --- /dev/null +++ b/openedx_learning/core/contents/models.py @@ -0,0 +1,176 @@ +""" +These models are the most basic pieces of content we support. Think of them as +the simplest building blocks to store data with. They need to be composed into +more intelligent data models to be useful. +""" +from django.db import models +from django.conf import settings +from django.core.validators import MaxValueValidator + +from openedx_learning.lib.fields import hash_field, manual_date_time_field + +from ..publishing.models import LearningPackage + + +class RawContent(models.Model): + """ + This is the most basic piece of raw content data, with no version metadata. + + RawContent stores data using the "file" field. This data is not + auto-normalized in any way, meaning that pieces of content that are + semantically equivalent (e.g. differently spaced/sorted JSON) may result in + new entries. This model is intentionally ignorant of what these things mean, + because it expects supplemental data models to build on top of it. + + Two RawContent instances _can_ have the same hash_digest if they are of + different MIME types. For instance, an empty text file and an empty SRT file + will both hash the same way, but be considered different entities. + + The other fields on RawContent are for data that is intrinsic to the file + data itself (e.g. the size). Any smart parsing of the contents into more + structured metadata should happen in other models that hang off of + RawContent. + + RawContent models are not versioned in any way. The concept of versioning + only exists at a higher level. + + RawContent is optimized for cheap storage, not low latency. It stores + content in a FileField. If you need faster text access across multiple rows, + add a TextContent entry that corresponds to the relevant RawContent. + + If you need to transform this RawContent into more structured data for your + application, create a model with a OneToOneField(primary_key=True) + relationship to RawContent. Just remember that *you should always create the + RawContent entry* first, to ensure content is always exportable, even if + your app goes away in the future. + + Operational Notes + ----------------- + + RawContent stores data using a FileField, which you'd typically want to back + with something like S3 when running in a production environment. That file + storage backend will not support rollback, meaning that if you start the + import process and things break halfway through, the RawContent model rows + will be rolled back, but the uploaded files will still remain on your file + storage system. The files are based on a hash of the contents though, so it + should still work later on when the import succeeds (it'll just have to + upload fewer files). + + TODO: Write about cleaning up accidental uploads of really large/unnecessary + files. Pruning of unreferenced (never published, or currently unused) + component versions and assets, and how that ties in? + """ + + # 50 MB is our current limit, based on the current Open edX Studio file + # upload size limit. + MAX_FILE_SIZE = 50_000_000 + + learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + + # This hash value may be calculated using create_hash_digest from the + # openedx.lib.fields module. + hash_digest = hash_field() + + # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type + # and sub-type may each be 127 chars, making a max of 255 (including the "/" + # in between). + # + # DO NOT STORE parameters here, e.g. "charset=". We can make a new field if + # that becomes necessary. + mime_type = models.CharField(max_length=255, blank=False, null=False) + + # This is the size of the raw data file in bytes. This can be different than + # the character length, since UTF-8 encoding can use anywhere between 1-4 + # bytes to represent any given character. + size = models.PositiveBigIntegerField( + validators=[MaxValueValidator(MAX_FILE_SIZE)], + ) + + # This should be manually set so that multiple RawContent rows being set in + # the same transaction are created with the same timestamp. The timestamp + # should be UTC. + created = manual_date_time_field() + + # All content for the LearningPackage should be stored in files. See model + # docstring for more details on how to store this data in supplementary data + # models that offer better latency guarantees. + file = models.FileField( + null=True, + storage=settings.OPENEDX_LEARNING.get( + "STORAGE", + settings.DEFAULT_FILE_STORAGE, + ), + ) + + class Meta: + constraints = [ + # Make sure we don't store duplicates of this raw data within the + # same LearningPackage, unless they're of different MIME types. + models.UniqueConstraint( + fields=[ + "learning_package", + "mime_type", + "hash_digest", + ], + name="oel_content_uniq_lc_mime_type_hash_digest", + ), + ] + indexes = [ + # LearningPackage MIME type Index: + # * Break down Content counts by type/subtype with in a + # LearningPackage. + # * Find all the Content in a LearningPackage that matches a + # certain MIME type (e.g. "image/png", "application/pdf". + models.Index( + fields=["learning_package", "mime_type"], + name="oel_content_idx_lp_mime_type", + ), + # LearningPackage (reverse) Size Index: + # * Find largest Content in a LearningPackage. + # * Find the sum of Content size for a given LearningPackage. + models.Index( + fields=["learning_package", "-size"], + name="oel_content_idx_lp_rsize", + ), + # LearningPackage (reverse) Created Index: + # * Find most recently added Content. + models.Index( + fields=["learning_package", "-created"], + name="oel_content_idx_lp_rcreated", + ), + ] + verbose_name = "Raw Content" + verbose_name_plural = "Raw Contents" + + +class TextContent(models.Model): + """ + TextContent supplements RawContent to give an in-table text copy. + + This model exists so that we can have lower-latency access to this data, + particularly if we're pulling back multiple rows at once. + + Apps are encouraged to create their own data models that further extend this + one with a more intelligent, parsed data model. For example, individual + XBlocks might parse the OLX in this model into separate data models for + VideoBlock, ProblemBlock, etc. You can do this by making your supplementary + model linked to this model via OneToOneField with primary_key=True. + + The reason this is built directly into the Learning Core data model is + because we want to be able to easily access and browse this data even if the + app-extended models get deleted (e.g. if they are deprecated and removed). + """ + + # 100K is our limit for text data, like OLX. This means 100K *characters*, + # not bytes. Since UTF-8 encodes characters using as many as 4 bytes, this + # could be as much as 400K of data if we had nothing but emojis. + MAX_TEXT_LENGTH = 100_000 + + raw_content = models.OneToOneField( + RawContent, + on_delete=models.CASCADE, + primary_key=True, + related_name="text_content", + ) + text = models.TextField(null=False, blank=True, max_length=MAX_TEXT_LENGTH) + length = models.PositiveIntegerField(null=False) diff --git a/openedx_learning/core/publishing/admin.py b/openedx_learning/core/publishing/admin.py index 7040f9008..258b6ffaf 100644 --- a/openedx_learning/core/publishing/admin.py +++ b/openedx_learning/core/publishing/admin.py @@ -1,29 +1,122 @@ from django.contrib import admin -from .models import LearningPackage, PublishLogEntry +from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin +from .models import ( + Draft, + LearningPackage, + PublishableEntity, + Published, + PublishLog, + PublishLogRecord, +) @admin.register(LearningPackage) -class LearningPackageAdmin(admin.ModelAdmin): - fields = ("identifier", "title", "uuid", "created", "updated") - readonly_fields = ("identifier", "title", "uuid", "created", "updated") - list_display = ("identifier", "title", "uuid", "created", "updated") +class LearningPackageAdmin(ReadOnlyModelAdmin): + fields = ["key", "title", "uuid", "created", "updated"] + readonly_fields = ["key", "title", "uuid", "created", "updated"] + list_display = ["key", "title", "uuid", "created", "updated"] + search_fields = ["key", "title", "uuid"] -@admin.register(PublishLogEntry) -class PublishLogEntryAdmin(admin.ModelAdmin): - fields = ("uuid", "learning_package", "published_at", "published_by", "message") - readonly_fields = ( - "uuid", - "learning_package", - "published_at", - "published_by", - "message", - ) - list_display = ( +class PublishLogRecordTabularInline(admin.TabularInline): + model = PublishLogRecord + fields = [ + "entity", + "title", + "old_version_num", + "new_version_num", + ] + readonly_fields = fields + + def get_queryset(self, request): + queryset = super().get_queryset(request) + return queryset.select_related("entity", "old_version", "new_version") + + def old_version_num(self, pl_record: PublishLogRecord): + if pl_record.old_version is None: + return "-" + return pl_record.old_version.version_num + + def new_version_num(self, pl_record: PublishLogRecord): + if pl_record.new_version is None: + return "-" + return pl_record.new_version.version_num + + def title(self, pl_record: PublishLogRecord): + if pl_record.new_version: + return pl_record.new_version.title + if pl_record.old_version: + return pl_record.old_version.title + return "" + + +@admin.register(PublishLog) +class PublishLogAdmin(ReadOnlyModelAdmin): + inlines = [PublishLogRecordTabularInline] + + fields = ["uuid", "learning_package", "published_at", "published_by", "message"] + readonly_fields = fields + list_display = fields + list_filter = ["learning_package"] + + +@admin.register(PublishableEntity) +class PublishableEntityAdmin(ReadOnlyModelAdmin): + fields = [ + "key", + "draft_version", + "published_version", "uuid", "learning_package", - "published_at", - "published_by", - "message", - ) + "created", + "created_by", + ] + readonly_fields = fields + list_display = fields + list_filter = ["learning_package"] + search_fields = ["key", "uuid"] + + def get_queryset(self, request): + queryset = super().get_queryset(request) + return queryset.select_related( + "learning_package", "published__version", "draft__version" + ) + + def draft_version(self, entity): + return entity.draft.version.version_num + + def published_version(self, entity): + return entity.published.version.version_num + + +@admin.register(Published) +class PublishedAdmin(ReadOnlyModelAdmin): + fields = ["entity", "version_num", "previous", "published_at", "message"] + list_display = fields + + def get_queryset(self, request): + queryset = super().get_queryset(request) + return queryset.select_related( + "entity", + "version", + "publish_log_record", + "publish_log_record__old_version", + "publish_log_record__publish_log", + ) + + def version_num(self, published_obj): + return published_obj.version.version_num + + def previous(self, published_obj): + old_version = published_obj.publish_log_record.old_version + # if there was no previous old version, old version is None + if not old_version: + return old_version + return old_version.version_num + + def published_at(self, published_obj): + return published_obj.publish_log_record.publish_log.published_at + + def message(self, published_obj): + return published_obj.publish_log_record.publish_log.message diff --git a/openedx_learning/core/publishing/api.py b/openedx_learning/core/publishing/api.py new file mode 100644 index 000000000..52ac7a90e --- /dev/null +++ b/openedx_learning/core/publishing/api.py @@ -0,0 +1,192 @@ +""" +Publishing API (warning: UNSTABLE, in progress API) + +Please look at the models.py file for more information about the kinds of data +are stored in this app. +""" +from datetime import datetime, timezone +from typing import Optional + +from django.core.exceptions import ObjectDoesNotExist +from django.db.models import F, QuerySet +from django.db.transaction import atomic + +from .models import ( + Draft, + LearningPackage, + Published, + PublishLog, + PublishLogRecord, + PublishableEntity, + PublishableEntityVersion, +) +from .model_mixins import PublishableContentModelRegistry + + +def create_learning_package( + key: str, title: str, created: Optional[datetime] = None +) -> LearningPackage: + """ + Create a new LearningPackage. + + The ``key`` must be unique. + + Errors that can be raised: + + * django.core.exceptions.ValidationError + """ + if not created: + created = datetime.now(tz=timezone.utc) + + package = LearningPackage( + key=key, + title=title, + created=created, + updated=created, + ) + package.full_clean() + package.save() + + return package + + +def create_publishable_entity(learning_package_id, key, created, created_by): + """ + Create a PublishableEntity. + + You'd typically want to call this right before creating your own content + model that points to it. + """ + return PublishableEntity.objects.create( + learning_package_id=learning_package_id, + key=key, + created=created, + created_by=created_by, + ) + + +def create_publishable_entity_version( + entity_id, version_num, title, created, created_by +): + """ + Create a PublishableEntityVersion. + + You'd typically want to call this right before creating your own content + version model that points to it. + """ + with atomic(): + version = PublishableEntityVersion.objects.create( + entity_id=entity_id, + version_num=version_num, + title=title, + created=created, + created_by=created_by, + ) + Draft.objects.create( + entity_id=entity_id, + version=version, + ) + return version + + +def learning_package_exists(key: str) -> bool: + """ + Check whether a LearningPackage with a particular key exists. + """ + LearningPackage.objects.filter(key=key).exists() + + +def publish_all_drafts( + learning_package_id, message="", published_at=None, published_by=None +): + """ + Publish everything that is a Draft and is not already published. + """ + draft_qset = ( + Draft.objects.select_related("entity__published") + .filter(entity__learning_package_id=learning_package_id) + .exclude(entity__published__version_id=F("version_id")) + ) + return publish_from_drafts( + learning_package_id, draft_qset, message, published_at, published_by + ) + + +def publish_from_drafts( + learning_package_id: int, # LearningPackage.id + draft_qset: QuerySet, + message: str = "", + published_at: Optional[datetime] = None, + published_by: Optional[int] = None, # User.id +) -> PublishLog: + """ + Publish the rows in the ``draft_model_qsets`` args passed in. + """ + if published_at is None: + published_at = datetime.now(tz=timezone.utc) + + with atomic(): + # One PublishLog for this entire publish operation. + publish_log = PublishLog( + learning_package_id=learning_package_id, + message=message, + published_at=published_at, + published_by=published_by, + ) + publish_log.full_clean() + publish_log.save(force_insert=True) + + for draft in draft_qset.select_related("entity__published__version"): + try: + old_version = draft.entity.published.version + except ObjectDoesNotExist: + # This means there is no published version yet. + old_version = None + + # Create a record describing publishing this particular Publishable + # (useful for auditing and reverting). + publish_log_record = PublishLogRecord( + publish_log=publish_log, + entity=draft.entity, + old_version=old_version, + new_version=draft.version, + ) + publish_log_record.full_clean() + publish_log_record.save(force_insert=True) + + # Update the lookup we use to fetch the published versions + Published.objects.update_or_create( + entity=draft.entity, + defaults={ + "version": draft.version, + "publish_log_record": publish_log_record, + }, + ) + + return publish_log + + +def register_content_models(content_model_cls, content_version_model_cls): + """ + Register what content model maps to what content version model. + + This is so that we can provide convenience links between content models and + content version models *through* the publishing apps, so that you can do + things like finding the draft version of a content model more easily. See + the model_mixins.py module for more details. + + This should only be imported and run from the your app's AppConfig.ready() + method. For example, in the components app, this looks like: + + def ready(self): + from ..publishing.api import register_content_models + from .models import Component, ComponentVersion + + register_content_models(Component, ComponentVersion) + + There may be a more clever way to introspect this information from the model + metadata, but this is simple and explicit. + """ + return PublishableContentModelRegistry.register( + content_model_cls, content_version_model_cls + ) diff --git a/openedx_learning/core/publishing/apps.py b/openedx_learning/core/publishing/apps.py index 4a98507cc..fbd42a0e4 100644 --- a/openedx_learning/core/publishing/apps.py +++ b/openedx_learning/core/publishing/apps.py @@ -13,3 +13,4 @@ class PublishingConfig(AppConfig): name = "openedx_learning.core.publishing" verbose_name = "Learning Core: Publishing" default_auto_field = "django.db.models.BigAutoField" + label = "oel_publishing" diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index 350a1849a..76e02b316 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,6 +1,7 @@ -# Generated by Django 4.1.6 on 2023-04-14 00:12 +# Generated by Django 3.2.18 on 2023-05-11 02:07 from django.conf import settings +import django.core.validators from django.db import migrations, models import django.db.models.deletion import openedx_learning.lib.validators @@ -36,7 +37,7 @@ class Migration(migrations.Migration): verbose_name="UUID", ), ), - ("identifier", models.CharField(max_length=255)), + ("key", models.CharField(max_length=255)), ("title", models.CharField(max_length=1000)), ( "created", @@ -61,7 +62,118 @@ class Migration(migrations.Migration): }, ), migrations.CreateModel( - name="PublishLogEntry", + name="PublishableEntity", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), + ("key", models.CharField(max_length=255)), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "learning_package", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="oel_publishing.learningpackage", + ), + ), + ], + options={ + "verbose_name": "Publishable Entity", + "verbose_name_plural": "Publishable Entities", + }, + ), + migrations.CreateModel( + name="PublishableEntityVersion", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "uuid", + models.UUIDField( + default=uuid.uuid4, + editable=False, + unique=True, + verbose_name="UUID", + ), + ), + ("title", models.CharField(blank=True, default="", max_length=1000)), + ( + "version_num", + models.PositiveBigIntegerField( + validators=[django.core.validators.MinValueValidator(1)] + ), + ), + ( + "created", + models.DateTimeField( + validators=[ + openedx_learning.lib.validators.validate_utc_datetime + ] + ), + ), + ( + "created_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "entity", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="versions", + to="oel_publishing.publishableentity", + ), + ), + ], + options={ + "verbose_name": "Publishable Entity Version", + "verbose_name_plural": "Publishable Entity Versions", + }, + ), + migrations.CreateModel( + name="PublishLog", fields=[ ( "id", @@ -94,12 +206,13 @@ class Migration(migrations.Migration): "learning_package", models.ForeignKey( on_delete=django.db.models.deletion.CASCADE, - to="publishing.learningpackage", + to="oel_publishing.learningpackage", ), ), ( "published_by", models.ForeignKey( + blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL, @@ -107,14 +220,157 @@ class Migration(migrations.Migration): ), ], options={ - "verbose_name": "Publish Log Entry", - "verbose_name_plural": "Publish Log Entries", + "verbose_name": "Publish Log", + "verbose_name_plural": "Publish Logs", + }, + ), + migrations.CreateModel( + name="Draft", + fields=[ + ( + "entity", + models.OneToOneField( + on_delete=django.db.models.deletion.RESTRICT, + primary_key=True, + serialize=False, + to="oel_publishing.publishableentity", + ), + ), + ], + ), + migrations.CreateModel( + name="Published", + fields=[ + ( + "entity", + models.OneToOneField( + on_delete=django.db.models.deletion.RESTRICT, + primary_key=True, + serialize=False, + to="oel_publishing.publishableentity", + ), + ), + ], + options={ + "verbose_name": "Published Entity", + "verbose_name_plural": "Published Entities", + }, + ), + migrations.CreateModel( + name="PublishLogRecord", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "entity", + models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + to="oel_publishing.publishableentity", + ), + ), + ( + "new_version", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to="oel_publishing.publishableentityversion", + ), + ), + ( + "old_version", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.RESTRICT, + related_name="+", + to="oel_publishing.publishableentityversion", + ), + ), + ( + "publish_log", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="oel_publishing.publishlog", + ), + ), + ], + options={ + "verbose_name": "Publish Log Record", + "verbose_name_plural": "Publish Log Records", }, ), migrations.AddConstraint( model_name="learningpackage", constraint=models.UniqueConstraint( - fields=("identifier",), name="lp_uniq_identifier" + fields=("key",), name="oel_publishing_lp_uniq_key" + ), + ), + migrations.AddField( + model_name="published", + name="publish_log_record", + field=models.ForeignKey( + on_delete=django.db.models.deletion.RESTRICT, + to="oel_publishing.publishlogrecord", + ), + ), + migrations.AddField( + model_name="published", + name="version", + field=models.OneToOneField( + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to="oel_publishing.publishableentityversion", + ), + ), + migrations.AddIndex( + model_name="publishableentityversion", + index=models.Index( + fields=["entity", "-created"], name="oel_pv_idx_entity_rcreated" + ), + ), + migrations.AddIndex( + model_name="publishableentityversion", + index=models.Index(fields=["title"], name="oel_pv_idx_title"), + ), + migrations.AddConstraint( + model_name="publishableentityversion", + constraint=models.UniqueConstraint( + fields=("entity", "version_num"), name="oel_pv_uniq_entity_version_num" + ), + ), + migrations.AddIndex( + model_name="publishableentity", + index=models.Index(fields=["key"], name="oel_pub_ent_idx_key"), + ), + migrations.AddIndex( + model_name="publishableentity", + index=models.Index( + fields=["learning_package", "-created"], + name="oel_pub_ent_idx_lp_rcreated", + ), + ), + migrations.AddConstraint( + model_name="publishableentity", + constraint=models.UniqueConstraint( + fields=("learning_package", "key"), name="oel_pub_ent_uniq_lp_key" + ), + ), + migrations.AddField( + model_name="draft", + name="version", + field=models.OneToOneField( + blank=True, + null=True, + on_delete=django.db.models.deletion.RESTRICT, + to="oel_publishing.publishableentityversion", ), ), ] diff --git a/openedx_learning/core/publishing/model_mixins.py b/openedx_learning/core/publishing/model_mixins.py new file mode 100644 index 000000000..9d39c645c --- /dev/null +++ b/openedx_learning/core/publishing/model_mixins.py @@ -0,0 +1,276 @@ +""" +Helper mixin classes for content apps that want to use the publishing app. +""" +from functools import cached_property + +from django.core.exceptions import ImproperlyConfigured +from django.db import models +from django.db.models.query import QuerySet + +from .models import Draft, Published, PublishableEntity, PublishableEntityVersion + + +class PublishableEntityMixin(models.Model): + """ + Convenience mixin to link your models against PublishableEntity. + + Please see docstring for PublishableEntity for more details. + + If you use this class, you *MUST* also use PublishableEntityVersionMixin and + the publishing app's api.register_content_models (see its docstring for + details). + """ + + class PublishableEntityMixinManager(models.Manager): + def get_queryset(self) -> QuerySet: + return super().get_queryset().select_related("publishable_entity") + + objects = PublishableEntityMixinManager() + + publishable_entity = models.OneToOneField( + PublishableEntity, on_delete=models.CASCADE, primary_key=True + ) + + @cached_property + def versioning(self): + return self.VersioningHelper(self) + + @property + def uuid(self): + return self.publishable_entity.uuid + + @property + def key(self): + return self.publishable_entity.key + + @property + def created(self): + return self.publishable_entity.created + + class Meta: + abstract = True + + class VersioningHelper: + """ + Helper class to link content models to their versions. + + The publishing app has PublishableEntity and PublishableEntityVersion. + This is a helper class so that if you mix PublishableEntityMixin into + a content model like Component, then you can do something like:: + + component.versioning.draft # current draft ComponentVersion + component.versioning.published # current published ComponentVersion + + It links the relationships between content models and their versioned + counterparts *through* the connection between PublishableEntity and + PublishableEntityVersion. So ``component.versioning.draft`` ends up + querying: Component -> PublishableEntity -> Draft -> + PublishableEntityVersion -> ComponentVersion. But the people writing + Component don't need to understand how the publishing models work to do + these common queries. + + Caching Warning + --------------- + Note that because we're just using the underlying model's relations, + calling this a second time will returned the cached relation, and + not cause a fetch of new data from the database. So for instance, if + you do:: + + # Create a new Component + ComponentVersion + component, component_version = create_component_and_version( + learning_package_id=learning_package.id, + namespace="xblock.v1", + type="problem", + local_key="monty_hall", + title="Monty Hall Problem", + created=now, + created_by=None, + ) + + # This will work, because it's never been published + assert component.versioning.published is None + + # Publishing happens + publish_all_drafts(learning_package.id, published_at=now) + + # This will FAIL because it's going to use the relation value + # cached on component instead of going to the database again. + # You need to re-fetch the component for this to work. + assert component.versioning.published == component_version + + # You need to manually refetch it from the database to see the new + # publish status: + component = get_component_by_pk(component.pk) + + # Now this will work: + assert component.versioning.published == component_version + + TODO: This probably means we should use a custom Manager to select + related fields. + """ + + def __init__(self, content_obj): + self.content_obj = content_obj + + self.content_version_model_cls = ( + PublishableContentModelRegistry.get_versioned_model_cls( + type(content_obj) + ) + ) + # Get the field that points from the *versioned* content model + # (e.g. ComponentVersion) to the PublishableEntityVersion. + field_to_pev = self.content_version_model_cls._meta.get_field( + "publishable_entity_version" + ) + + # Now that we know the field that leads to PublishableEntityVersion, + # get the reverse related field name so that we can use that later. + self.related_name = field_to_pev.related_query_name() + + @property + def draft(self): + """ + Return the content version object that is the current draft. + """ + try: + draft = Draft.objects.get(entity_id=self.content_obj.publishable_entity_id) + except Draft.DoesNotExist: + # If no Draft object exists at all, then no + # PublishableEntityVersion was ever created (just the + # PublishableEntity). + return None + + draft_pub_ent_version = draft.version + + # There is an entry for this Draft, but the entry is None. This means + # there was a Draft at some point, but it's been "deleted" (though the + # actual history is still preserved). + if draft_pub_ent_version is None: + return None + + # If we've gotten this far, then draft_pub_ent_version points to an + # actual PublishableEntityVersion, but we want to follow that and go to + # the matching content version model. + return getattr(draft_pub_ent_version, self.related_name) + + @property + def published(self): + """ + Return the content version object that is currently published. + """ + try: + published = Published.objects.get(entity_id=self.content_obj.publishable_entity_id) + except Published.DoesNotExist: + # This means it's never been published. + return None + + published_pub_ent_version = published.version + + # There is a Published entry, but the entry is None. This means that + # it was published at some point, but it's been "deleted" or + # unpublished (though the actual history is still preserved). + if published_pub_ent_version is None: + return None + + # If we've gotten this far, then published_pub_ent_version points to an + # actual PublishableEntityVersion, but we want to follow that and go to + # the matching content version model. + return getattr(published_pub_ent_version, self.related_name) + + @property + def versions(self): + """ + Return a QuerySet of content version models for this content model. + + Example: If you mix PublishableEntityMixin into a Component model, + This would return you a QuerySet of ComponentVersion models. + """ + pub_ent = self.content_obj.publishable_entity + return self.content_version_model_cls.objects.filter( + publishable_entity_version__entity_id=pub_ent.id + ) + + +class PublishableEntityVersionMixin(models.Model): + """ + Convenience mixin to link your models against PublishableEntityVersion. + + Please see docstring for PublishableEntityVersion for more details. + + If you use this class, you *MUST* also use PublishableEntityMixin and the + publishing app's api.register_content_models (see its docstring for + details). + """ + + class PublishableEntityVersionMixinManager(models.Manager): + def get_queryset(self) -> QuerySet: + return ( + super() + .get_queryset() + .select_related( + "publishable_entity_version", + ) + ) + + objects = PublishableEntityVersionMixinManager() + + publishable_entity_version = models.OneToOneField( + PublishableEntityVersion, on_delete=models.CASCADE, primary_key=True + ) + + @property + def uuid(self): + return self.publishable_entity_version.uuid + + @property + def title(self): + return self.publishable_entity_version.title + + @property + def created(self): + return self.publishable_entity_version.created + + @property + def version_num(self): + return self.publishable_entity_version.version_num + + class Meta: + abstract = True + + +class PublishableContentModelRegistry: + """ + This class tracks content models built on PublishableEntity(Version). + """ + + _unversioned_to_versioned = {} + _versioned_to_unversioned = {} + + @classmethod + def register(cls, content_model_cls, content_version_model_cls): + """ + Register what content model maps to what content version model. + + If you want to call this from another app, please use the + ``register_content_models`` function in this app's ``api`` module + instead. + """ + if not issubclass(content_model_cls, PublishableEntityMixin): + raise ImproperlyConfigured( + f"{content_model_cls} must inherit from PublishableEntityMixin" + ) + if not issubclass(content_version_model_cls, PublishableEntityVersionMixin): + raise ImproperlyConfigured( + f"{content_version_model_cls} must inherit from PublishableEntityMixin" + ) + + cls._unversioned_to_versioned[content_model_cls] = content_version_model_cls + cls._versioned_to_unversioned[content_version_model_cls] = content_model_cls + + @classmethod + def get_versioned_model_cls(cls, content_model_cls): + return cls._unversioned_to_versioned[content_model_cls] + + @classmethod + def get_unversioned_model_cls(cls, content_version_model_cls): + return cls._versioned_to_unversioned[content_version_model_cls] diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 293555ffa..0ce3cc2b7 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -1,48 +1,340 @@ """ -Idea: This app has _only_ things related to Publishing any kind of content -associated with a LearningPackage. +The data models here are intended to be used by other apps to publish different +types of content, such as Components, Units, Sections, etc. These models should +support the logic for the management of the publishing process: + +* The relationship between publishable entities and their many versions. +* The management of drafts. +* Publishing specific versions of publishable entities. +* Finding the currently published versions. +* The act of publishing, and doing so atomically. +* Managing reverts. +* Storing and querying publish history. """ from django.db import models + from django.conf import settings +from django.core.validators import MinValueValidator from openedx_learning.lib.fields import ( - identifier_field, + key_field, immutable_uuid_field, manual_date_time_field, ) class LearningPackage(models.Model): + """ + Top level container for a grouping of authored content. + + Each PublishableEntity belongs to exactly one LearningPackage. + """ + uuid = immutable_uuid_field() - identifier = identifier_field() + key = key_field() title = models.CharField(max_length=1000, null=False, blank=False) - created = manual_date_time_field() updated = manual_date_time_field() def __str__(self): - return f"{self.identifier}: {self.title}" + return f"{self.key}" class Meta: constraints = [ - # LearningPackage identifiers must be globally unique. This is - # something that might be relaxed in the future if this system were - # to be extensible to something like multi-tenancy, in which case - # we'd tie it to something like a Site or Org. - models.UniqueConstraint(fields=["identifier"], name="lp_uniq_identifier") + # LearningPackage keys must be globally unique. This is something + # that might be relaxed in the future if this system were to be + # extensible to something like multi-tenancy, in which case we'd tie + # it to something like a Site or Org. + models.UniqueConstraint( + fields=["key"], + name="oel_publishing_lp_uniq_key", + ) ] verbose_name = "Learning Package" verbose_name_plural = "Learning Packages" -class PublishLogEntry(models.Model): +class PublishableEntity(models.Model): """ - This model tracks Publishing activity. + This represents any publishable thing that has ever existed in a + LearningPackage. It serves as a stable model that will not go away even if + these things are later unpublished or deleted. + + A PublishableEntity belongs to exactly one LearningPackage. + + Examples of Publishable Entities + -------------------------------- + + Components (e.g. VideoBlock, ProblemBlock), Units, and Sections/Subsections + would all be considered Publishable Entites. But anything that can be + imported, exported, published, and reverted in a course or library could be + modeled as a PublishableEntity, including things like Grading Policy or + possibly Taxonomies (?). + + How to use this model + --------------------- + + The publishing app understands that publishable entities exist, along with + their drafts and published versions. It has some basic metadata, such as + identifiers, who created it, and when it was created. It's meant to + encapsulate the draft and publishing related aspects of your content, but + the ``publishing`` app doesn't know anything about the actual content being + referenced. + + You have to provide actual meaning to PublishableEntity by creating your own + models that will represent your particular content and associating them to + PublishableEntity via a OneToOneField with primary_key=True. The easiest way + to do this is to have your model inherit from PublishableEntityMixin. + + Identifiers + ----------- + The UUID is globally unique and should be treated as immutable. + + The key field *is* mutable, but changing it will affect all + PublishedEntityVersions. They are locally unique within the LearningPackage. - It is expected that other apps make foreign keys to this table to mark when - their content gets published. This is to allow us to tie together many - different entities (e.g. Components, Units, etc.) that are all published at - the same time. + If you are referencing this model from within the same process, use a + foreign key to the id. If you are referencing this PublishedEntity from an + external system/service, use the UUID. The key is the part that is most + likely to be human-readable, and may be exported/copied, but try not to rely + on it, since this value may change. + + Note: When we actually implement the ability to change identifiers, we + should make a history table and a modified attribute on this model. + + Why are Identifiers in this Model? + ---------------------------------- + + A PublishableEntity never stands alone–it's always intended to be used with + a 1:1 model like Component or Unit. So why have all the identifiers in this + model instead of storing them in those other models? Two reasons: + + * Published things need to have the right identifiers so they can be used + throughout the system, and the UUID is serving the role of ISBN in physical + book publishing. + * We want to be able to enforce the idea that "key" is locally unique across + all PublishableEntities within a given LearningPackage. Component and Unit + can't do that without a shared model. + + That being said, models that build on PublishableEntity are free to add + their own identifiers if it's useful to do so. + + Why not Inherit from this Model? + -------------------------------- + + Django supports multi-table inheritance: + + https://docs.djangoproject.com/en/4.2/topics/db/models/#multi-table-inheritance + + We don't use that, primarily because we want to more clearly decouple + publishing concerns from the rest of the logic around Components, Units, + etc. If you made a Component and ComponentVersion models that subclassed + PublishableEntity and PublishableEntityVersion, and then accessed + ``component.versions``, you might expect ComponentVersions to come back and + be surprised when you get EntityVersions instead. + + In general, we want freedom to add new Publishing models, fields, and + methods without having to worry about the downstream name collisions with + other apps (many of which might live in other repositories). The helper + mixins will provide a little syntactic sugar to make common access patterns + more convenient, like file access. + """ + + uuid = immutable_uuid_field() + learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) + key = key_field() + created = manual_date_time_field() + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + ) + + class Meta: + constraints = [ + # Keys are unique within a given LearningPackage. + models.UniqueConstraint( + fields=[ + "learning_package", + "key", + ], + name="oel_pub_ent_uniq_lp_key", + ) + ] + indexes = [ + # Global Key Index: + # * Search by key across all PublishableEntities on the site. This + # would be a support-oriented tool from Django Admin. + models.Index( + fields=["key"], + name="oel_pub_ent_idx_key", + ), + # LearningPackage (reverse) Created Index: + # * Search for most recently *created* PublishableEntities for a + # given LearningPackage, since they're the most likely to be + # actively worked on. + models.Index( + fields=["learning_package", "-created"], + name="oel_pub_ent_idx_lp_rcreated", + ), + ] + # These are for the Django Admin UI. + verbose_name = "Publishable Entity" + verbose_name_plural = "Publishable Entities" + + def __str__(self): + return f"{self.key}" + + +class PublishableEntityVersion(models.Model): + """ + A particular version of a PublishableEntity. + + This model has its own ``uuid`` so that it can be referenced directly. The + ``uuid`` should be treated as immutable. + + PublishableEntityVersions are created once and never updated. So for + instance, the ``title`` should never be modified. + + Like PublishableEntity, the data in this model is only enough to cover the + parts that are most important for the actual process of managing drafts and + publishes. You will want to create your own models to represent the actual + content data that's associated with this PublishableEntityVersion, and + connect them using a OneToOneField with primary_key=True. The easiest way to + do this is to inherit from PublishableEntityVersionMixin. Be sure to treat + these versioned models in your app as immutable as well. + """ + + uuid = immutable_uuid_field() + entity = models.ForeignKey( + PublishableEntity, on_delete=models.CASCADE, related_name="versions" + ) + + # Most publishable things will have some sort of title, but blanks are + # allowed for those that don't require one. + title = models.CharField(max_length=1000, default="", null=False, blank=True) + + # The version_num starts at 1 and increments by 1 with each new version for + # a given PublishableEntity. Doing it this way makes it more convenient for + # users to refer to than a hash or UUID value. It also helps us catch race + # conditions on save, by setting a unique constraint on the entity and + # version_num. + version_num = models.PositiveBigIntegerField( + null=False, + validators=[MinValueValidator(1)], + ) + + # All PublishableEntityVersions created as part of the same publish should + # have the exact same created datetime (not off by a handful of + # microseconds). + created = manual_date_time_field() + + # User who created the PublishableEntityVersion. This can be null if the + # user is later removed. Open edX in general doesn't let you remove users, + # but we should try to model it so that this is possible eventually. + created_by = models.ForeignKey( + settings.AUTH_USER_MODEL, + on_delete=models.SET_NULL, + null=True, + blank=True, + ) + + class Meta: + constraints = [ + # Prevent the situation where we have multiple + # PublishableEntityVersions claiming to be the same version_num for + # a given PublishableEntity. This can happen if there's a race + # condition between concurrent editors in different browsers, + # working on the same Publishable. With this constraint, one of + # those processes will raise an IntegrityError. + models.UniqueConstraint( + fields=[ + "entity", + "version_num", + ], + name="oel_pv_uniq_entity_version_num", + ) + ] + indexes = [ + # LearningPackage (reverse) Created Index: + # * Make it cheap to find the most recently created + # PublishableEntityVersions for a given LearningPackage. This + # represents the most recently saved work for a LearningPackage + # and would be the most likely areas to get worked on next. + models.Index( + fields=["entity", "-created"], + name="oel_pv_idx_entity_rcreated", + ), + # Title Index: + # * Search by title. + models.Index( + fields=[ + "title", + ], + name="oel_pv_idx_title", + ), + ] + + # These are for the Django Admin UI. + verbose_name = "Publishable Entity Version" + verbose_name_plural = "Publishable Entity Versions" + + +class Draft(models.Model): + """ + Find the active draft version of an entity (usually most recently created). + + This model mostly only exists to allow us to join against a bunch of + PublishableEntity objects at once and get all their latest drafts. You might + use this together with Published in order to see which Drafts haven't been + published yet. + + A Draft entry should be created whenever a new PublishableEntityVersion is + created. This means there are three possible states: + + 1. No Draft entry for a PublishableEntity: This means a PublishableEntity + was created, but no PublishableEntityVersion was ever made for it, so + there was never a Draft version. + 2. A Draft entry exists and points to a PublishableEntityVersion: This is + the most common state. + 3. A Draft entry exists and points to a null version: This means a version + used to be the draft, but it's been functionally "deleted". The versions + still exist in our history, but we're done using it. + + It would have saved a little space to add this data to the Published model + (and possibly call the combined model something else). Split Modulestore did + this with its active_versions table. I keep it separate here to get a better + separation of lifecycle events: i.e. this table *only* changes when drafts + are updated, not when publishing happens. The Published model only changes + when something is published. + """ + + entity = models.OneToOneField( + PublishableEntity, + on_delete=models.RESTRICT, + primary_key=True, + ) + version = models.OneToOneField( + PublishableEntityVersion, + on_delete=models.RESTRICT, + null=True, + blank=True, + ) + + +class PublishLog(models.Model): + """ + There is one row in this table for every time content is published. + + Each PublishLog has 0 or more PublishLogRecords describing exactly which + PublishableEntites were published and what the version changes are. A + PublishLog is like a git commit in that sense, with individual + PublishLogRecords representing the files changed. + + Open question: Empty publishes are allowed at this time, and might be useful + for "fake" publishes that are necessary to invoke other post-publish + actions. It's not clear at this point how useful this will actually be. """ uuid = immutable_uuid_field() @@ -53,8 +345,102 @@ class PublishLogEntry(models.Model): settings.AUTH_USER_MODEL, on_delete=models.SET_NULL, null=True, + blank=True, + ) + + class Meta: + verbose_name = "Publish Log" + verbose_name_plural = "Publish Logs" + + +class PublishLogRecord(models.Model): + """ + A record for each publishable entity version changed, for each publish. + + To revert a publish, we would make a new publish that swaps ``old_version`` + and ``new_version`` field values. + """ + + publish_log = models.ForeignKey(PublishLog, on_delete=models.CASCADE) + entity = models.ForeignKey(PublishableEntity, on_delete=models.RESTRICT) + old_version = models.ForeignKey( + PublishableEntityVersion, + on_delete=models.RESTRICT, + null=True, + blank=True, + related_name="+", + ) + new_version = models.ForeignKey( + PublishableEntityVersion, on_delete=models.RESTRICT, null=True, blank=True + ) + + class Meta: + constraints = [ + # A Publishable can have only one PublishLogRecord per PublishLog. + # You can't simultaneously publish two different versions of the + # same publishable. + models.UniqueConstraint( + fields=[ + "publish_log", + "entity", + ], + name="oel_plr_uniq_pl_publishable", + ) + ] + indexes = [ + # Publishable (reverse) Publish Log Index: + # * Find the history of publishes for a given Publishable, + # starting with the most recent (since IDs are ascending ints). + models.Index( + fields=["entity", "-publish_log"], + name="oel_plr_idx_entity_rplr", + ), + ] + + class Meta: + verbose_name = "Publish Log Record" + verbose_name_plural = "Publish Log Records" + + +class Published(models.Model): + """ + Find the currently published version of an entity. + + Notes: + + * There is only ever one published PublishableEntityVersion per + PublishableEntity at any given time. + * It may be possible for a PublishableEntity to exist only as a Draft (and thus + not show up in this table). + * If a row exists for a PublishableEntity, but the ``version`` field is + None, it means that the entity was published at some point, but is no + longer published now–i.e. it's functionally "deleted", even though all + the version history is preserved behind the scenes. + + TODO: Do we need to create a (redundant) title field in this model so that + we can more efficiently search across titles within a LearningPackage? + Probably not an immediate concern because the number of rows currently + shouldn't be > 10,000 in the more extreme cases. + + TODO: Do we need to make a "most_recently" published version when an entry + is unpublished/deleted? + """ + + entity = models.OneToOneField( + PublishableEntity, + on_delete=models.RESTRICT, + primary_key=True, + ) + version = models.OneToOneField( + PublishableEntityVersion, + on_delete=models.RESTRICT, + null=True, + ) + publish_log_record = models.ForeignKey( + PublishLogRecord, + on_delete=models.RESTRICT, ) class Meta: - verbose_name = "Publish Log Entry" - verbose_name_plural = "Publish Log Entries" + verbose_name = "Published Entity" + verbose_name_plural = "Published Entities" diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index 4955ad11c..71273afa0 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -8,8 +8,8 @@ https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0038-Data-Modeling.html * The UUID fields are intended to be globally unique identifiers that other services can store and rely on staying the same. -* The "identifier" fields can be more human-friendly strings, but these may only - be unique within a given context. These values should be treated as mutable, +* The "key" fields can be more human-friendly strings, but these may only + be unique within a given scope. These values should be treated as mutable, even if they rarely change in practice. TODO: @@ -30,7 +30,7 @@ from .validators import validate_utc_datetime -def identifier_field(): +def key_field(): """ Externally created Identifier fields. diff --git a/projects/dev.py b/projects/dev.py index 13c351d02..8bc2f2524 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -31,6 +31,7 @@ "django.contrib.admindocs", # Learning Core Apps "openedx_learning.core.components.apps.ComponentsConfig", + "openedx_learning.core.contents.apps.ContentsConfig", "openedx_learning.core.publishing.apps.PublishingConfig", # Learning Contrib Apps "openedx_learning.contrib.media_server.apps.MediaServerConfig", @@ -40,9 +41,14 @@ # REST API "rest_framework", "openedx_learning.rest_api.apps.RESTAPIConfig", + + # Debugging + "debug_toolbar", ) MIDDLEWARE = [ + "debug_toolbar.middleware.DebugToolbarMiddleware", + "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", @@ -95,3 +101,6 @@ # STORAGES setting in Django >= 4.2 "STORAGE": None, } +INTERNAL_IPS = [ + "127.0.0.1", +] diff --git a/projects/urls.py b/projects/urls.py index 67364cfcb..e27da7532 100644 --- a/projects/urls.py +++ b/projects/urls.py @@ -1,9 +1,13 @@ +from django.conf import settings from django.contrib import admin from django.urls import include, path +from django.conf.urls.static import static + urlpatterns = [ path("admin/doc/", include("django.contrib.admindocs.urls")), path("admin/", admin.site.urls), path("media_server/", include("openedx_learning.contrib.media_server.urls")), path("rest_api/", include("openedx_learning.rest_api.urls")), -] + path('__debug__/', include('debug_toolbar.urls')), +] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/test_settings.py b/test_settings.py index d2d0820fc..896092ca5 100644 --- a/test_settings.py +++ b/test_settings.py @@ -36,8 +36,9 @@ def root(*args): # 'django.contrib.admin', # 'django.contrib.admindocs', # Our own apps - "openedx_learning.core.publishing.apps.PublishingConfig", "openedx_learning.core.components.apps.ComponentsConfig", + "openedx_learning.core.contents.apps.ContentsConfig", + "openedx_learning.core.publishing.apps.PublishingConfig", ] LOCALE_PATHS = [ diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_learning/__init__.py b/tests/openedx_learning/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_learning/core/__init__.py b/tests/openedx_learning/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_learning/core/components/__init__.py b/tests/openedx_learning/core/components/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_learning/core/components/test_models.py b/tests/openedx_learning/core/components/test_models.py new file mode 100644 index 000000000..f41df14d0 --- /dev/null +++ b/tests/openedx_learning/core/components/test_models.py @@ -0,0 +1,43 @@ +from datetime import datetime, timezone + +from django.test.testcases import TestCase + +from openedx_learning.core.publishing.api import ( + create_learning_package, + publish_all_drafts, +) +from openedx_learning.core.components.api import create_component_and_version + + +class TestModelVersioningQueries(TestCase): + """ + Test that Component/ComponentVersion are registered with the publishing app. + """ + + @classmethod + def setUpTestData(cls): + cls.learning_package = create_learning_package( + "components.TestVersioning", + "Learning Package for Testing Component Versioning", + ) + cls.now = datetime(2023, 5, 8, tzinfo=timezone.utc) + + def test_latest_version(self): + component, component_version = create_component_and_version( + learning_package_id=self.learning_package.id, + namespace="xblock.v1", + type="problem", + local_key="monty_hall", + title="Monty Hall Problem", + created=self.now, + created_by=None, + ) + assert component.versioning.draft == component_version + assert component.versioning.published is None + publish_all_drafts(self.learning_package.pk, published_at=self.now) + + # Force the re-fetch from the database + assert component.versioning.published == component_version + + # Grabbing the list of versions for this component + assert list(component.versioning.versions) == [component_version] diff --git a/tests/openedx_learning/core/publishing/__init__.py b/tests/openedx_learning/core/publishing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_learning/core/publishing/test_api.py b/tests/openedx_learning/core/publishing/test_api.py new file mode 100644 index 000000000..ebfdf05e1 --- /dev/null +++ b/tests/openedx_learning/core/publishing/test_api.py @@ -0,0 +1,66 @@ +from datetime import datetime, timezone +from uuid import UUID + +from django.core.exceptions import ValidationError +from django.test import TestCase +import pytest + +from openedx_learning.core.publishing.api import create_learning_package + + +class CreateLearningPackageTestCase(TestCase): + def test_normal(self): + """Normal flow with no errors.""" + key = "my_key" + title = "My Excellent Title with Emoji 🔥" + created = datetime(2023, 4, 2, 15, 9, 0, tzinfo=timezone.utc) + package = create_learning_package(key, title, created) + + assert package.key == "my_key" + assert package.title == "My Excellent Title with Emoji 🔥" + assert package.created == created + assert package.updated == created + + # Should be auto-generated + assert isinstance(package.uuid, UUID) + + # Having an actual value here means we were persisted to the database. + assert isinstance(package.id, int) + + def test_auto_datetime(self): + """Auto-generated created datetime works as expected.""" + key = "my_key" + title = "My Excellent Title with Emoji 🔥" + package = create_learning_package(key, title) + + assert package.key == "my_key" + assert package.title == "My Excellent Title with Emoji 🔥" + + # Auto-generated datetime checking... + assert isinstance(package.created, datetime) + assert package.created == package.updated + assert package.created.tzinfo == timezone.utc + + # Should be auto-generated + assert isinstance(package.uuid, UUID) + + # Having an actual value here means we were persisted to the database. + assert isinstance(package.id, int) + + def test_non_utc_time(self): + """Require UTC timezone for created.""" + with pytest.raises(ValidationError) as excinfo: + create_learning_package("my_key", "A Title", datetime(2023, 4, 2)) + message_dict = excinfo.value.message_dict + + # Both datetime fields should be marked as invalid + assert "created" in message_dict + assert "updated" in message_dict + + def test_already_exists(self): + """Raises ValidationError for duplicate keys.""" + create_learning_package("my_key", "Original") + with pytest.raises(ValidationError) as excinfo: + create_learning_package("my_key", "Duplicate") + message_dict = excinfo.value.message_dict + assert "key" in message_dict diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index 0e2faf6e6..000000000 --- a/tests/test_models.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -""" -Tests for the `openedx-learning` models module. -""" - - -class TestPublishedLearningPackage: - """ - Tests of the PublishedLearningPackage model. - """ - - def test_something(self): - """TODO: Write real test cases.""" From 1b907bfc3183507a624f6d48882f75aa12165b99 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 10 May 2023 22:36:49 -0400 Subject: [PATCH 006/282] feat: add ADRs around identifier conventions, app labels --- .../decisions/0005-identifier-conventions.rst | 39 +++++++++++++++++++ docs/decisions/0006-app-label-prefix.rst | 13 +++++++ 2 files changed, 52 insertions(+) create mode 100644 docs/decisions/0005-identifier-conventions.rst create mode 100644 docs/decisions/0006-app-label-prefix.rst diff --git a/docs/decisions/0005-identifier-conventions.rst b/docs/decisions/0005-identifier-conventions.rst new file mode 100644 index 000000000..8a53bca4b --- /dev/null +++ b/docs/decisions/0005-identifier-conventions.rst @@ -0,0 +1,39 @@ +5. Identifier Conventions +========================= + +Context +------- + +We want a common set of conventions for how to reference models in various situations, such as: + +* From a model in a different app, but in the same Python process. +* From a different server. + +Decision +-------- + +The content-related data models in our system will use the following convention for identifier fields: + +Primary Key + The primary key will be a BigAutoField. This will usually be ``id``, but it can be different if the model builds off another one via a ``OneToOneField`` primary key. Other apps that are running in the same process should directly make foreign key field references to this. This value will not change. + +UUID + The ``uuid`` field is a randomly generated UUID4 that is globally unique and should be treated as immutable. If you are making a reference to this record in another system (e.g. an external service), this is the identifier to store. + +Key + The ``key`` field is chosen by apps or users, and is the most likely piece to be human-readable (though it doesn't have to be). These values are only locally unique within a given scope, such as a particular LearningPackage or ComponentVersion. + + The correlates most closely to OpaqueKeys, though they are not precisely the same. In particular, we don't want to directly store BlockUsageKeys that have the LearningContextKey baked into them, because that makes it much harder to change the LearningContextKey later, e.g. if we ever want to change a CourseKey for a course. Different models can choose whether the ``key`` value is mutable or not, but outside users should assume that it can change, even if it rarely does in practice. + +Implementation Notes +-------------------- + +Helpers to generate these field types are in the ``openedx_learning.lib.fields`` module. + +Rejected Alternatives +--------------------- + +A number of names were considered for ``key``, and were rejected for different reasons: + +* ``identifier`` is used in some standards like QTI, but it's too easily confused with ``id`` or the general concept of the three types of identity-related fields we have. +* ``slug`` was considered, but ultimately rejected because that implied these fields would be human-readable, and that's not guaranteed. Most XBlock content that comes from MongoDB will be using GUIDs, for instance. diff --git a/docs/decisions/0006-app-label-prefix.rst b/docs/decisions/0006-app-label-prefix.rst new file mode 100644 index 000000000..376d079e5 --- /dev/null +++ b/docs/decisions/0006-app-label-prefix.rst @@ -0,0 +1,13 @@ +6. App Label Prefix +=================== + +Context +------- + +We want this repo to be useful in different Django projects outside of just edx-platform, and we want to avoid downstream collisions with our app names. + + +Decision +-------- + +All apps in this repo will create a default AppConfig that sets the ``label`` to have a prefix of ``oel_`` before the app name. So if the app name is ``publishing``, the ``label`` will be ``oel_publishing``. This means that all generated database tables will similarly be prefixed with ``oel_``. From c0d36d9d82ba75264a9e47b11a0a95712a16bfa5 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Wed, 10 May 2023 22:37:27 -0400 Subject: [PATCH 007/282] feat: ignore dev.db-related files Enabling WAL on dev.db will generate extra dev.db-shm and dev.db-wal files that we want git to ignore.. --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 9cabbe5db..888019d47 100644 --- a/.gitignore +++ b/.gitignore @@ -61,7 +61,7 @@ requirements/private.in requirements/private.txt # database file -dev.db +dev.db* .vscode From 369270c32e91b8110e08fde33f666cd2c340bbce Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Fri, 12 May 2023 11:43:25 -0400 Subject: [PATCH 008/282] fix: make docs generate again Doc generation was failing to a number of errors because I wasn't running it regularly. This fixes things so the docs at least generate successfully. They're not that useful just yet, but they generate. --- README.rst | 4 ++-- docs/decisions/0002-content-flexibility.rst | 2 +- docs/decisions/0004-content-tagging.rst | 4 ++-- docs/decisions/0005-identifier-conventions.rst | 2 +- requirements/constraints.txt | 4 ++++ 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.rst b/README.rst index ff6dbbddc..4bf7e06da 100644 --- a/README.rst +++ b/README.rst @@ -129,8 +129,8 @@ Reporting Security Issues Please do not report security issues in public. Please email security@edx.org. -Getting Help ------------- +Help +---- If you're having trouble, we have discussion forums at https://discuss.openedx.org where you can connect with others in the community. diff --git a/docs/decisions/0002-content-flexibility.rst b/docs/decisions/0002-content-flexibility.rst index 12c9a123a..cea33e9c5 100644 --- a/docs/decisions/0002-content-flexibility.rst +++ b/docs/decisions/0002-content-flexibility.rst @@ -25,7 +25,7 @@ Unit A Unit is an ordered list of one or more Components that is typically displayed together. A common use case might be to display some introductory Text, a Video, and some followup Problem (all separate Components). An individual Component in a Unit may or may not make sense when taken outside of that Unit–e.g. a Video may be reusable elsewhere, but the Problem referencing the video might not be. Sequence - A Sequence is a collection of Units that are presented one after the other, either to assess student understanding or to achieve some learning objective. + A Sequence is a collection of Units that are presented one after the other, either to assess student understanding or to achieve some learning objective. A Sequence is analogous to a Subsection in a traditional Open edX course. diff --git a/docs/decisions/0004-content-tagging.rst b/docs/decisions/0004-content-tagging.rst index dff13aec1..0f43fd4eb 100644 --- a/docs/decisions/0004-content-tagging.rst +++ b/docs/decisions/0004-content-tagging.rst @@ -13,9 +13,9 @@ Decision Implement the new tagging service as a pluggable django app, and installed alongside learning-core as a dependency in the edx-platform. -Tagging data models will follow the guidelines for this repository, and focus on extensibility and flexibility. +Tagging data models will follow the guidelines for this repository, and focus on extensibility and flexibility. -Since some use cases for content tagging are not considered "kernel" (like providing data for a marketing site), a generic mechanism to differentiate those uses cases will be built, and proper Python and REST APIs will be provided, to different taxonomies/tags for the same content. +Since some use cases for content tagging are not considered "kernel" (like providing data for a marketing site), a generic mechanism to differentiate those uses cases will be built, and proper Python and REST APIs will be provided, to different taxonomies/tags for the same content. Rejected Alternatives diff --git a/docs/decisions/0005-identifier-conventions.rst b/docs/decisions/0005-identifier-conventions.rst index 8a53bca4b..6aae93c4f 100644 --- a/docs/decisions/0005-identifier-conventions.rst +++ b/docs/decisions/0005-identifier-conventions.rst @@ -22,7 +22,7 @@ UUID Key The ``key`` field is chosen by apps or users, and is the most likely piece to be human-readable (though it doesn't have to be). These values are only locally unique within a given scope, such as a particular LearningPackage or ComponentVersion. - + The correlates most closely to OpaqueKeys, though they are not precisely the same. In particular, we don't want to directly store BlockUsageKeys that have the LearningContextKey baked into them, because that makes it much harder to change the LearningContextKey later, e.g. if we ever want to change a CourseKey for a course. Different models can choose whether the ``key`` value is mutable or not, but outside users should assume that it can change, even if it rarely does in practice. Implementation Notes diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 117189356..4f2f47ff4 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -10,3 +10,7 @@ # Common constraints for edx repos # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + +# tox>4.0.0 isn't yet compatible with many tox plugins, causing CI failures in almost all repos. +# Details can be found in this discussion: https://github.com/tox-dev/tox/discussions/1810 +tox<4.0.0 \ No newline at end of file From 7cc330d8698b232442edbf86ac7ff5d33cca540c Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Fri, 12 May 2023 20:46:47 -0400 Subject: [PATCH 009/282] chore: update dependencies --- projects/dev.py | 2 +- requirements/base.in | 2 +- requirements/base.txt | 5 ++- requirements/ci.txt | 35 +++++++----------- requirements/constraints.txt | 5 ++- requirements/dev.in | 1 + requirements/dev.txt | 72 +++++++++++++++++------------------- requirements/doc.txt | 50 ++++++++++++------------- requirements/quality.txt | 33 +++++++++-------- requirements/test.txt | 7 ++-- tox.ini | 3 +- 11 files changed, 103 insertions(+), 112 deletions(-) diff --git a/projects/dev.py b/projects/dev.py index 8bc2f2524..7be663f44 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -48,7 +48,6 @@ MIDDLEWARE = [ "debug_toolbar.middleware.DebugToolbarMiddleware", - "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", @@ -56,6 +55,7 @@ "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", + # Admin-specific "django.contrib.admindocs.middleware.XViewMiddleware", ] diff --git a/requirements/base.in b/requirements/base.in index f5ebd9403..aa00dddd7 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,6 @@ # Core requirements for using this application -c constraints.txt -Django<4.0 # Web application framework +Django<5.0 # Web application framework djangorestframework<4.0 # REST API diff --git a/requirements/base.txt b/requirements/base.txt index 7c80551d3..d48934464 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,8 +6,9 @@ # asgiref==3.6.0 # via django -django==3.2.18 +django==3.2.19 # via + # -c requirements/constraints.txt # -r requirements/base.in # djangorestframework djangorestframework==3.14.0 @@ -16,5 +17,5 @@ pytz==2023.3 # via # django # djangorestframework -sqlparse==0.4.3 +sqlparse==0.4.4 # via django diff --git a/requirements/ci.txt b/requirements/ci.txt index e6ca7b96a..ee3eb376d 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,48 +4,41 @@ # # make upgrade # -cachetools==5.3.0 - # via tox -chardet==5.1.0 - # via tox click==8.1.3 # via import-linter -colorama==0.4.6 - # via tox -coverage==7.2.3 +coverage==7.2.5 # via -r requirements/ci.in distlib==0.3.6 # via virtualenv -filelock==3.11.0 +filelock==3.12.0 # via # tox # virtualenv -grimp==2.3 +grimp==2.4 # via import-linter import-linter==1.8.0 # via -r requirements/ci.in packaging==23.1 - # via - # pyproject-api - # tox -platformdirs==3.2.0 - # via - # tox - # virtualenv + # via tox +platformdirs==3.5.1 + # via virtualenv pluggy==1.0.0 # via tox -pyproject-api==1.5.1 +py==1.11.0 + # via tox +six==1.16.0 # via tox tomli==2.0.1 # via # import-linter - # pyproject-api # tox -tox==4.4.12 - # via -r requirements/ci.in +tox==3.28.0 + # via + # -c requirements/constraints.txt + # -r requirements/ci.in typing-extensions==4.5.0 # via # grimp # import-linter -virtualenv==20.21.0 +virtualenv==20.23.0 # via tox diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 4f2f47ff4..6014c0923 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -13,4 +13,7 @@ # tox>4.0.0 isn't yet compatible with many tox plugins, causing CI failures in almost all repos. # Details can be found in this discussion: https://github.com/tox-dev/tox/discussions/1810 -tox<4.0.0 \ No newline at end of file +tox<4.0.0 + +# Develop primarily on Django 3.2, but we'll want to start testing against 4.2 +Django<4.0 diff --git a/requirements/dev.in b/requirements/dev.in index 4a7645b38..2c9492480 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -8,3 +8,4 @@ diff-cover # Changeset diff test coverage edx-i18n-tools # For i18n_tool dummy tox-battery # Makes tox aware of requirements file changes +django-debug-toolbar # Debugging DB queries primarily diff --git a/requirements/dev.txt b/requirements/dev.txt index 2fbb634cf..0ca1afd83 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ asgiref==3.6.0 # via # -r requirements/quality.txt # django -astroid==2.15.2 +astroid==2.15.4 # via # -r requirements/quality.txt # pylint @@ -21,19 +21,12 @@ build==0.10.0 # via # -r requirements/pip-tools.txt # pip-tools -cachetools==5.3.0 - # via - # -r requirements/ci.txt - # tox -certifi==2022.12.7 +certifi==2023.5.7 # via # -r requirements/quality.txt # requests chardet==5.1.0 - # via - # -r requirements/ci.txt - # diff-cover - # tox + # via diff-cover charset-normalizer==3.1.0 # via # -r requirements/quality.txt @@ -56,11 +49,7 @@ code-annotations==1.3.0 # via # -r requirements/quality.txt # edx-lint -colorama==0.4.6 - # via - # -r requirements/ci.txt - # tox -coverage[toml]==7.2.3 +coverage[toml]==7.2.5 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -75,14 +64,18 @@ distlib==0.3.6 # via # -r requirements/ci.txt # virtualenv -django==3.2.18 +django==3.2.19 # via + # -c requirements/constraints.txt # -r requirements/quality.txt + # django-debug-toolbar # djangorestframework # edx-i18n-tools +django-debug-toolbar==4.0.0 + # via -r requirements/dev.in djangorestframework==3.14.0 # via -r requirements/quality.txt -docutils==0.19 +docutils==0.20 # via # -r requirements/quality.txt # readme-renderer @@ -94,12 +87,12 @@ exceptiongroup==1.1.1 # via # -r requirements/quality.txt # pytest -filelock==3.11.0 +filelock==3.12.0 # via # -r requirements/ci.txt # tox # virtualenv -grimp==2.3 +grimp==2.4 # via # -r requirements/ci.txt # import-linter @@ -109,7 +102,7 @@ idna==3.4 # requests import-linter==1.8.0 # via -r requirements/ci.txt -importlib-metadata==6.3.0 +importlib-metadata==6.6.0 # via # -r requirements/quality.txt # keyring @@ -169,7 +162,6 @@ packaging==23.1 # -r requirements/pip-tools.txt # -r requirements/quality.txt # build - # pyproject-api # pytest # tox path==16.6.0 @@ -184,12 +176,11 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.2.0 +platformdirs==3.5.1 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint - # tox # virtualenv pluggy==1.0.0 # via @@ -200,17 +191,21 @@ pluggy==1.0.0 # tox polib==1.2.0 # via edx-i18n-tools +py==1.11.0 + # via + # -r requirements/ci.txt + # tox pycodestyle==2.10.0 # via -r requirements/quality.txt pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.15.0 +pygments==2.15.1 # via # -r requirements/quality.txt # diff-cover # readme-renderer # rich -pylint==2.17.2 +pylint==2.17.4 # via # -r requirements/quality.txt # edx-lint @@ -230,15 +225,11 @@ pylint-plugin-utils==0.7 # -r requirements/quality.txt # pylint-celery # pylint-django -pyproject-api==1.5.1 - # via - # -r requirements/ci.txt - # tox pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.3.0 +pytest==7.3.1 # via # -r requirements/quality.txt # pytest-cov @@ -265,12 +256,12 @@ readme-renderer==37.3 # via # -r requirements/quality.txt # twine -requests==2.28.2 +requests==2.30.0 # via # -r requirements/quality.txt # requests-toolbelt # twine -requests-toolbelt==0.10.1 +requests-toolbelt==1.0.0 # via # -r requirements/quality.txt # twine @@ -278,23 +269,26 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.3.4 +rich==13.3.5 # via # -r requirements/quality.txt # twine six==1.16.0 # via + # -r requirements/ci.txt # -r requirements/quality.txt # bleach # edx-lint + # tox snowballstemmer==2.2.0 # via # -r requirements/quality.txt # pydocstyle -sqlparse==0.4.3 +sqlparse==0.4.4 # via # -r requirements/quality.txt # django + # django-debug-toolbar stevedore==5.0.0 # via # -r requirements/quality.txt @@ -312,16 +306,16 @@ tomli==2.0.1 # coverage # import-linter # pylint - # pyproject-api # pyproject-hooks # pytest # tox -tomlkit==0.11.7 +tomlkit==0.11.8 # via # -r requirements/quality.txt # pylint -tox==4.4.12 +tox==3.28.0 # via + # -c requirements/constraints.txt # -r requirements/ci.txt # tox-battery tox-battery==0.6.1 @@ -337,12 +331,12 @@ typing-extensions==4.5.0 # import-linter # pylint # rich -urllib3==1.26.15 +urllib3==2.0.2 # via # -r requirements/quality.txt # requests # twine -virtualenv==20.21.0 +virtualenv==20.23.0 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index bd988df39..3ae643a37 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -6,29 +6,21 @@ # accessible-pygments==0.0.4 # via pydata-sphinx-theme -alabaster==0.7.12 +alabaster==0.7.13 # via sphinx asgiref==3.6.0 # via # -r requirements/test.txt # django -attrs==22.1.0 - # via - # -r requirements/test.txt - # pytest -babel==2.10.3 +babel==2.12.1 # via # pydata-sphinx-theme # sphinx -backports-zoneinfo==0.2.1 - # via - # -r requirements/test.txt - # django beautifulsoup4==4.12.2 # via pydata-sphinx-theme -bleach==5.0.1 +bleach==6.0.0 # via readme-renderer -certifi==2022.12.7 +certifi==2023.5.7 # via requests charset-normalizer==3.1.0 # via requests @@ -38,12 +30,13 @@ click==8.1.3 # code-annotations code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==7.2.3 +coverage[toml]==7.2.5 # via # -r requirements/test.txt # pytest-cov -django==3.2.18 +django==3.2.19 # via + # -c requirements/constraints.txt # -r requirements/test.txt # djangorestframework # sphinxcontrib-django @@ -58,11 +51,15 @@ docutils==0.19 # readme-renderer # restructuredtext-lint # sphinx -idna==3.3 +exceptiongroup==1.1.1 + # via + # -r requirements/test.txt + # pytest +idna==3.4 # via requests imagesize==1.4.1 # via sphinx -importlib-metadata==6.3.0 +importlib-metadata==6.6.0 # via sphinx iniconfig==2.0.0 # via @@ -91,20 +88,18 @@ pluggy==1.0.0 # via # -r requirements/test.txt # pytest -py==1.11.0 - # via - # -r requirements/test.txt - # pytest +pprintpp==0.4.0 + # via sphinxcontrib-django pydata-sphinx-theme==0.13.3 # via sphinx-book-theme -pygments==2.13.0 +pygments==2.15.1 # via # accessible-pygments # doc8 # pydata-sphinx-theme # readme-renderer # sphinx -pytest==7.3.0 +pytest==7.3.1 # via # -r requirements/test.txt # pytest-cov @@ -129,7 +124,7 @@ pyyaml==6.0 # code-annotations readme-renderer==37.3 # via -r requirements/doc.in -requests==2.28.2 +requests==2.30.0 # via sphinx restructuredtext-lint==1.4.0 # via doc8 @@ -139,14 +134,15 @@ snowballstemmer==2.2.0 # via sphinx soupsieve==2.4.1 # via beautifulsoup4 -sphinx==5.1.1 +sphinx==6.2.1 # via # -r requirements/doc.in # pydata-sphinx-theme # sphinx-book-theme + # sphinxcontrib-django sphinx-book-theme==1.0.1 # via -r requirements/doc.in -sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx @@ -160,7 +156,7 @@ sphinxcontrib-qthelp==1.0.3 # via sphinx sphinxcontrib-serializinghtml==1.1.5 # via sphinx -sqlparse==0.4.3 +sqlparse==0.4.4 # via # -r requirements/test.txt # django @@ -181,7 +177,7 @@ tomli==2.0.1 # pytest typing-extensions==4.5.0 # via pydata-sphinx-theme -urllib3==1.26.12 +urllib3==2.0.2 # via requests webencodings==0.5.1 # via bleach diff --git a/requirements/quality.txt b/requirements/quality.txt index 134d429a1..012149d1f 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,13 +8,13 @@ asgiref==3.6.0 # via # -r requirements/test.txt # django -astroid==2.15.2 +astroid==2.15.4 # via # pylint # pylint-celery bleach==6.0.0 # via readme-renderer -certifi==2022.12.7 +certifi==2023.5.7 # via requests charset-normalizer==3.1.0 # via requests @@ -30,19 +30,20 @@ code-annotations==1.3.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.2.3 +coverage[toml]==7.2.5 # via # -r requirements/test.txt # pytest-cov dill==0.3.6 # via pylint -django==3.2.18 +django==3.2.19 # via + # -c requirements/constraints.txt # -r requirements/test.txt # djangorestframework djangorestframework==3.14.0 # via -r requirements/test.txt -docutils==0.19 +docutils==0.20 # via readme-renderer edx-lint==5.3.4 # via -r requirements/quality.in @@ -52,7 +53,7 @@ exceptiongroup==1.1.1 # pytest idna==3.4 # via requests -importlib-metadata==6.3.0 +importlib-metadata==6.6.0 # via # keyring # twine @@ -98,7 +99,7 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.2.0 +platformdirs==3.5.1 # via pylint pluggy==1.0.0 # via @@ -108,11 +109,11 @@ pycodestyle==2.10.0 # via -r requirements/quality.in pydocstyle==6.3.0 # via -r requirements/quality.in -pygments==2.15.0 +pygments==2.15.1 # via # readme-renderer # rich -pylint==2.17.2 +pylint==2.17.4 # via # edx-lint # pylint-celery @@ -126,7 +127,7 @@ pylint-plugin-utils==0.7 # via # pylint-celery # pylint-django -pytest==7.3.0 +pytest==7.3.1 # via # -r requirements/test.txt # pytest-cov @@ -150,15 +151,15 @@ pyyaml==6.0 # code-annotations readme-renderer==37.3 # via twine -requests==2.28.2 +requests==2.30.0 # via # requests-toolbelt # twine -requests-toolbelt==0.10.1 +requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.3.4 +rich==13.3.5 # via twine six==1.16.0 # via @@ -166,7 +167,7 @@ six==1.16.0 # edx-lint snowballstemmer==2.2.0 # via pydocstyle -sqlparse==0.4.3 +sqlparse==0.4.4 # via # -r requirements/test.txt # django @@ -184,7 +185,7 @@ tomli==2.0.1 # coverage # pylint # pytest -tomlkit==0.11.7 +tomlkit==0.11.8 # via pylint twine==4.0.2 # via -r requirements/quality.in @@ -193,7 +194,7 @@ typing-extensions==4.5.0 # astroid # pylint # rich -urllib3==1.26.15 +urllib3==2.0.2 # via # requests # twine diff --git a/requirements/test.txt b/requirements/test.txt index 9410f3893..b5cc57ce5 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -12,9 +12,10 @@ click==8.1.3 # via code-annotations code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==7.2.3 +coverage[toml]==7.2.5 # via pytest-cov # via + # -c requirements/constraints.txt # -r requirements/base.txt # djangorestframework djangorestframework==3.14.0 @@ -33,7 +34,7 @@ pbr==5.11.1 # via stevedore pluggy==1.0.0 # via pytest -pytest==7.3.0 +pytest==7.3.1 # via # pytest-cov # pytest-django @@ -50,7 +51,7 @@ pytz==2023.3 # djangorestframework pyyaml==6.0 # via code-annotations -sqlparse==0.4.3 +sqlparse==0.4.4 # via # -r requirements/base.txt # django diff --git a/tox.ini b/tox.ini index 6e84a0ace..44168b281 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38-django{32}, quality, docs, pii_check +envlist = py38-django{32,42}, quality, docs, pii_check [doc8] ; D001 = Line too long @@ -37,6 +37,7 @@ norecursedirs = .* docs requirements site-packages [testenv] deps = django32: Django>=3.2,<3.3 + django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt commands = pytest {posargs} From 892fb4b032d0f825b0870e41e696f9449f2aa837 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 14 May 2023 10:00:47 -0400 Subject: [PATCH 010/282] fix: add unique constraint and index to PublishLogRecord These were meant to be in there from the beginning, but were missing because I redefined the Meta class for that model. I saw this when I went to set up the linting for this repo (which makes a very compelling argument for linting). --- .../0002_add_publish_log_constraints.py | 24 +++++++++++++++++++ openedx_learning/core/publishing/models.py | 2 -- 2 files changed, 24 insertions(+), 2 deletions(-) create mode 100644 openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py diff --git a/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py b/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py new file mode 100644 index 000000000..986683c47 --- /dev/null +++ b/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.19 on 2023-05-14 13:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_publishing", "0001_initial"), + ] + + operations = [ + migrations.AddIndex( + model_name="publishlogrecord", + index=models.Index( + fields=["entity", "-publish_log"], name="oel_plr_idx_entity_rplr" + ), + ), + migrations.AddConstraint( + model_name="publishlogrecord", + constraint=models.UniqueConstraint( + fields=("publish_log", "entity"), name="oel_plr_uniq_pl_publishable" + ), + ), + ] diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 0ce3cc2b7..0f9a50fdd 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -396,8 +396,6 @@ class Meta: name="oel_plr_idx_entity_rplr", ), ] - - class Meta: verbose_name = "Publish Log Record" verbose_name_plural = "Publish Log Records" From e6c039db752068f8746e74f96c3ac6de80d33f7d Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Sun, 14 May 2023 10:03:55 -0400 Subject: [PATCH 011/282] chore: remove unused imports and reformat for linter compliance --- openedx_learning/core/components/models.py | 1 - openedx_learning/core/publishing/admin.py | 1 - openedx_learning/core/publishing/model_mixins.py | 8 ++++++-- openedx_learning/rest_api/v1/components.py | 1 - 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index b773c3f92..b3a848f78 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -17,7 +17,6 @@ later. """ from django.db import models -from django.db.models import F, Q from openedx_learning.lib.fields import key_field, immutable_uuid_field from ..publishing.models import LearningPackage diff --git a/openedx_learning/core/publishing/admin.py b/openedx_learning/core/publishing/admin.py index 258b6ffaf..9ce6a9ef6 100644 --- a/openedx_learning/core/publishing/admin.py +++ b/openedx_learning/core/publishing/admin.py @@ -2,7 +2,6 @@ from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin from .models import ( - Draft, LearningPackage, PublishableEntity, Published, diff --git a/openedx_learning/core/publishing/model_mixins.py b/openedx_learning/core/publishing/model_mixins.py index 9d39c645c..1623bb256 100644 --- a/openedx_learning/core/publishing/model_mixins.py +++ b/openedx_learning/core/publishing/model_mixins.py @@ -133,7 +133,9 @@ def draft(self): Return the content version object that is the current draft. """ try: - draft = Draft.objects.get(entity_id=self.content_obj.publishable_entity_id) + draft = Draft.objects.get( + entity_id=self.content_obj.publishable_entity_id + ) except Draft.DoesNotExist: # If no Draft object exists at all, then no # PublishableEntityVersion was ever created (just the @@ -159,7 +161,9 @@ def published(self): Return the content version object that is currently published. """ try: - published = Published.objects.get(entity_id=self.content_obj.publishable_entity_id) + published = Published.objects.get( + entity_id=self.content_obj.publishable_entity_id + ) except Published.DoesNotExist: # This means it's never been published. return None diff --git a/openedx_learning/rest_api/v1/components.py b/openedx_learning/rest_api/v1/components.py index 4fdc2261d..571753b3f 100644 --- a/openedx_learning/rest_api/v1/components.py +++ b/openedx_learning/rest_api/v1/components.py @@ -2,7 +2,6 @@ This is just an example REST API endpoint. """ from rest_framework import viewsets -from rest_framework.response import Response from openedx_learning.core.components.models import Component From c11a531e510752de4c432510b7d76041a9c7c367 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Thu, 18 May 2023 16:58:02 -0400 Subject: [PATCH 012/282] docs: fix outdated comments in .importlinter There were references to itemstore, which hasn't existed in a while. I also reorganized the comments to be inline with the apps in the Core App Dependency Layering section, since this section is going to grow over time, and it's easier to understand if the commments are right next to the app paths. --- .importlinter | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.importlinter b/.importlinter index 8dbba0cc7..75eef9ee0 100644 --- a/.importlinter +++ b/.importlinter @@ -26,20 +26,22 @@ layers= openedx_learning.core openedx_learning.lib -# This is layering within our Core apps. -# -# The lowest layer is "publishing", which holds the basic primitives needed to -# create LearningPackages and versioning. -# -# One layer above that is "itemstore" which stores single Items (e.g. Problem, -# Video). -# -# Above "itemstore" are apps that can compose those Items into more interesting -# structures (like Units). +# This is layering within our Core apps. Every new Core app should be added to +# this list when it it created. [importlinter:contract:core_apps_layering] name = Core App Dependency Layering type = layers layers= + # The "components" app is responsible for storing versioned Components, + # which is Open edX Studio terminology maps to things like individual + # Problems, Videos, and blocks of HTML text. This is also the type we would + # associate with a single "leaf" XBlock–one that is not a container type and + # has no child elements. openedx_learning.core.components + # The "contents" app stores the simplest pieces of binary and text data, + # without versioning information. These belong to a single Learning Package. openedx_learning.core.contents + # The lowest layer is "publishing", which holds the basic primitives needed + # to create Learning Packages and manage the draft and publish states for + # various types of content. openedx_learning.core.publishing From 1d4019636cf12ebc5de831e09b76a89e0ad85f3e Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Fri, 12 May 2023 23:25:50 -0400 Subject: [PATCH 013/282] test: enable unit tests in GitHub CI This runs the unit tests anywhere in the /tests directory using pytest and tox configured for both Django 3.2 and Django 4.2, using Python 3.8. It also tests using both Ubuntu and MacOS, which might be overkill, but I've been developing this repo using MacOS so far. --- .github/workflows/ci.yml | 37 +++++++++++++++++++++++++++++++++++++ Makefile | 2 +- requirements/ci.in | 2 -- requirements/ci.txt | 16 +--------------- requirements/dev.txt | 19 ++++++++----------- requirements/doc.txt | 16 ++++++++++++++-- requirements/quality.txt | 19 +++++++++++++++---- requirements/test.in | 4 ++++ requirements/test.txt | 20 +++++++++++++++++--- 9 files changed, 97 insertions(+), 38 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..1eff1b8e4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: Python CI + +on: + push: + branches: [ main ] + pull_request: + branches: + - '**' + + +jobs: + run_tests: + name: tests + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ['3.8'] + toxenv: ["py38-django32", "py38-django42"] + steps: + - uses: actions/checkout@v3 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + + - name: Install pip + run: pip install -r requirements/pip.txt + + - name: Install Dependencies + run: pip install -r requirements/ci.txt + + - name: Run Tests + env: + TOXENV: ${{ matrix.toxenv }} + run: tox -e ${{ matrix.toxenv }} + diff --git a/Makefile b/Makefile index 2ad767893..2440de3e1 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ requirements: ## install development environment requirements pip-sync requirements/dev.txt requirements/private.* test: clean ## run tests in the current virtualenv - pytest + pytest tests diff_cover: test ## find diff lines that need test coverage diff-cover coverage.xml diff --git a/requirements/ci.in b/requirements/ci.in index 8776af102..6959b416c 100644 --- a/requirements/ci.in +++ b/requirements/ci.in @@ -1,6 +1,4 @@ # Requirements for running tests on CI -c constraints.txt -coverage # Code coverage reporting tox # Virtualenv management for tests -import-linter # Track our internal dependencies diff --git a/requirements/ci.txt b/requirements/ci.txt index ee3eb376d..ca1b37b97 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,20 +4,12 @@ # # make upgrade # -click==8.1.3 - # via import-linter -coverage==7.2.5 - # via -r requirements/ci.in distlib==0.3.6 # via virtualenv filelock==3.12.0 # via # tox # virtualenv -grimp==2.4 - # via import-linter -import-linter==1.8.0 - # via -r requirements/ci.in packaging==23.1 # via tox platformdirs==3.5.1 @@ -29,16 +21,10 @@ py==1.11.0 six==1.16.0 # via tox tomli==2.0.1 - # via - # import-linter - # tox + # via tox tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.in -typing-extensions==4.5.0 - # via - # grimp - # import-linter virtualenv==20.23.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 0ca1afd83..0cf172808 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -8,7 +8,7 @@ asgiref==3.6.0 # via # -r requirements/quality.txt # django -astroid==2.15.4 +astroid==2.15.5 # via # -r requirements/quality.txt # pylint @@ -33,7 +33,6 @@ charset-normalizer==3.1.0 # requests click==8.1.3 # via - # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # click-log @@ -51,7 +50,6 @@ code-annotations==1.3.0 # edx-lint coverage[toml]==7.2.5 # via - # -r requirements/ci.txt # -r requirements/quality.txt # pytest-cov diff-cover==7.5.0 @@ -71,11 +69,11 @@ django==3.2.19 # django-debug-toolbar # djangorestframework # edx-i18n-tools -django-debug-toolbar==4.0.0 +django-debug-toolbar==4.1.0 # via -r requirements/dev.in djangorestframework==3.14.0 # via -r requirements/quality.txt -docutils==0.20 +docutils==0.20.1 # via # -r requirements/quality.txt # readme-renderer @@ -94,14 +92,14 @@ filelock==3.12.0 # virtualenv grimp==2.4 # via - # -r requirements/ci.txt + # -r requirements/quality.txt # import-linter idna==3.4 # via # -r requirements/quality.txt # requests -import-linter==1.8.0 - # via -r requirements/ci.txt +import-linter==1.9.0 + # via -r requirements/quality.txt importlib-metadata==6.6.0 # via # -r requirements/quality.txt @@ -220,7 +218,7 @@ pylint-django==2.5.3 # via # -r requirements/quality.txt # edx-lint -pylint-plugin-utils==0.7 +pylint-plugin-utils==0.8.1 # via # -r requirements/quality.txt # pylint-celery @@ -289,7 +287,7 @@ sqlparse==0.4.4 # -r requirements/quality.txt # django # django-debug-toolbar -stevedore==5.0.0 +stevedore==5.1.0 # via # -r requirements/quality.txt # code-annotations @@ -324,7 +322,6 @@ twine==4.0.2 # via -r requirements/quality.txt typing-extensions==4.5.0 # via - # -r requirements/ci.txt # -r requirements/quality.txt # astroid # grimp diff --git a/requirements/doc.txt b/requirements/doc.txt index 3ae643a37..aa92d8e59 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -28,6 +28,7 @@ click==8.1.3 # via # -r requirements/test.txt # code-annotations + # import-linter code-annotations==1.3.0 # via -r requirements/test.txt coverage[toml]==7.2.5 @@ -55,10 +56,16 @@ exceptiongroup==1.1.1 # via # -r requirements/test.txt # pytest +grimp==2.4 + # via + # -r requirements/test.txt + # import-linter idna==3.4 # via requests imagesize==1.4.1 # via sphinx +import-linter==1.9.0 + # via -r requirements/test.txt importlib-metadata==6.6.0 # via sphinx iniconfig==2.0.0 @@ -160,7 +167,7 @@ sqlparse==0.4.4 # via # -r requirements/test.txt # django -stevedore==5.0.0 +stevedore==5.1.0 # via # -r requirements/test.txt # code-annotations @@ -174,9 +181,14 @@ tomli==2.0.1 # -r requirements/test.txt # coverage # doc8 + # import-linter # pytest typing-extensions==4.5.0 - # via pydata-sphinx-theme + # via + # -r requirements/test.txt + # grimp + # import-linter + # pydata-sphinx-theme urllib3==2.0.2 # via requests webencodings==0.5.1 diff --git a/requirements/quality.txt b/requirements/quality.txt index 012149d1f..b36f07abd 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -8,7 +8,7 @@ asgiref==3.6.0 # via # -r requirements/test.txt # django -astroid==2.15.4 +astroid==2.15.5 # via # pylint # pylint-celery @@ -24,6 +24,7 @@ click==8.1.3 # click-log # code-annotations # edx-lint + # import-linter click-log==0.4.0 # via edx-lint code-annotations==1.3.0 @@ -43,7 +44,7 @@ django==3.2.19 # djangorestframework djangorestframework==3.14.0 # via -r requirements/test.txt -docutils==0.20 +docutils==0.20.1 # via readme-renderer edx-lint==5.3.4 # via -r requirements/quality.in @@ -51,8 +52,14 @@ exceptiongroup==1.1.1 # via # -r requirements/test.txt # pytest +grimp==2.4 + # via + # -r requirements/test.txt + # import-linter idna==3.4 # via requests +import-linter==1.9.0 + # via -r requirements/test.txt importlib-metadata==6.6.0 # via # keyring @@ -123,7 +130,7 @@ pylint-celery==0.3 # via edx-lint pylint-django==2.5.3 # via edx-lint -pylint-plugin-utils==0.7 +pylint-plugin-utils==0.8.1 # via # pylint-celery # pylint-django @@ -171,7 +178,7 @@ sqlparse==0.4.4 # via # -r requirements/test.txt # django -stevedore==5.0.0 +stevedore==5.1.0 # via # -r requirements/test.txt # code-annotations @@ -183,6 +190,7 @@ tomli==2.0.1 # via # -r requirements/test.txt # coverage + # import-linter # pylint # pytest tomlkit==0.11.8 @@ -191,7 +199,10 @@ twine==4.0.2 # via -r requirements/quality.in typing-extensions==4.5.0 # via + # -r requirements/test.txt # astroid + # grimp + # import-linter # pylint # rich urllib3==2.0.2 diff --git a/requirements/test.in b/requirements/test.in index 6797160bf..65ac63e81 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -3,6 +3,10 @@ -r base.txt # Core dependencies for this package +coverage # Code coverage reporting +import-linter # Track our internal dependencies + +pytest pytest-cov # pytest extension for code coverage statistics pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. diff --git a/requirements/test.txt b/requirements/test.txt index b5cc57ce5..ae4cfa040 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -9,11 +9,15 @@ asgiref==3.6.0 # -r requirements/base.txt # django click==8.1.3 - # via code-annotations + # via + # code-annotations + # import-linter code-annotations==1.3.0 # via -r requirements/test.in coverage[toml]==7.2.5 - # via pytest-cov + # via + # -r requirements/test.in + # pytest-cov # via # -c requirements/constraints.txt # -r requirements/base.txt @@ -22,6 +26,10 @@ djangorestframework==3.14.0 # via -r requirements/base.txt exceptiongroup==1.1.1 # via pytest +grimp==2.4 + # via import-linter +import-linter==1.9.0 + # via -r requirements/test.in iniconfig==2.0.0 # via pytest jinja2==3.1.2 @@ -36,6 +44,7 @@ pluggy==1.0.0 # via pytest pytest==7.3.1 # via + # -r requirements/test.in # pytest-cov # pytest-django pytest-cov==4.0.0 @@ -55,11 +64,16 @@ sqlparse==0.4.4 # via # -r requirements/base.txt # django -stevedore==5.0.0 +stevedore==5.1.0 # via code-annotations text-unidecode==1.3 # via python-slugify tomli==2.0.1 # via # coverage + # import-linter # pytest +typing-extensions==4.5.0 + # via + # grimp + # import-linter From df7f234296e31135fbd270f9fda3124a951bb0af Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Thu, 18 May 2023 15:30:24 -0400 Subject: [PATCH 014/282] test: add lint-imports to GitHub CI --- .github/workflows/lint-imports.yml | 29 +++++++++++++++++++++++++++++ requirements/ci.in | 1 + requirements/ci.txt | 14 +++++++++++++- requirements/dev.txt | 7 ++++++- tox.ini | 2 +- 5 files changed, 50 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lint-imports.yml diff --git a/.github/workflows/lint-imports.yml b/.github/workflows/lint-imports.yml new file mode 100644 index 000000000..d40ce01f6 --- /dev/null +++ b/.github/workflows/lint-imports.yml @@ -0,0 +1,29 @@ +name: Lint Imports + +on: + push: + branches: [ main ] + pull_request: + branches: + - '**' + + +jobs: + lint-imports: + name: Lint Python Imports + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: setup python + uses: actions/setup-python@v2 + with: + python-version: '3.8' + + - name: Install pip + run: pip install -r requirements/pip.txt + + - name: Install Dependencies + run: pip install -r requirements/ci.txt + + - name: Analyze imports + run: lint-imports diff --git a/requirements/ci.in b/requirements/ci.in index 6959b416c..d5f09d0cb 100644 --- a/requirements/ci.in +++ b/requirements/ci.in @@ -2,3 +2,4 @@ -c constraints.txt tox # Virtualenv management for tests +import-linter # Track our internal dependencies diff --git a/requirements/ci.txt b/requirements/ci.txt index ca1b37b97..d851c07e2 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,12 +4,18 @@ # # make upgrade # +click==8.1.3 + # via import-linter distlib==0.3.6 # via virtualenv filelock==3.12.0 # via # tox # virtualenv +grimp==2.4 + # via import-linter +import-linter==1.9.0 + # via -r requirements/ci.in packaging==23.1 # via tox platformdirs==3.5.1 @@ -21,10 +27,16 @@ py==1.11.0 six==1.16.0 # via tox tomli==2.0.1 - # via tox + # via + # import-linter + # tox tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.in +typing-extensions==4.5.0 + # via + # grimp + # import-linter virtualenv==20.23.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 0cf172808..9f6bd5332 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -33,6 +33,7 @@ charset-normalizer==3.1.0 # requests click==8.1.3 # via + # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # click-log @@ -92,6 +93,7 @@ filelock==3.12.0 # virtualenv grimp==2.4 # via + # -r requirements/ci.txt # -r requirements/quality.txt # import-linter idna==3.4 @@ -99,7 +101,9 @@ idna==3.4 # -r requirements/quality.txt # requests import-linter==1.9.0 - # via -r requirements/quality.txt + # via + # -r requirements/ci.txt + # -r requirements/quality.txt importlib-metadata==6.6.0 # via # -r requirements/quality.txt @@ -322,6 +326,7 @@ twine==4.0.2 # via -r requirements/quality.txt typing-extensions==4.5.0 # via + # -r requirements/ci.txt # -r requirements/quality.txt # astroid # grimp diff --git a/tox.ini b/tox.ini index 44168b281..6e592fa87 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py38-django{32,42}, quality, docs, pii_check +envlist = py38-django{32,42}, quality, docs, pii_check, lint-imports [doc8] ; D001 = Line too long From 74892fbeb0cf369abf8f031ebab22f0a70ea174e Mon Sep 17 00:00:00 2001 From: Sarina Canelake Date: Wed, 24 May 2023 13:53:28 -0400 Subject: [PATCH 015/282] docs: Update tcril-engineering to axim-engineering for backstage --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index a875b1a8b..d57c3aa29 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -5,4 +5,4 @@ metadata: spec: type: other lifecycle: unknown - owner: tcril-engineering + owner: axim-engineering From f1b8579b80dde8aa6bba6b3c799860ec60dafba4 Mon Sep 17 00:00:00 2001 From: Jillian Date: Wed, 14 Jun 2023 04:25:50 +0930 Subject: [PATCH 016/282] docs: document tagging architecture decisions (#55) --- .gitignore | 1 + docs/decisions/0007-tagging-app.rst | 43 +++++++++ .../decisions/0008-tagging-tree-data-arch.rst | 53 +++++++++++ .../decisions/0009-tagging-administrators.rst | 39 ++++++++ .../0010-taxonomy-enable-context.rst | 90 +++++++++++++++++++ docs/decisions/0011-tag-changes.rst | 35 ++++++++ openedx_tagging/__init__.py | 1 + openedx_tagging/core/tagging/__init__.py | 0 openedx_tagging/core/tagging/admin.py | 6 ++ openedx_tagging/core/tagging/apps.py | 16 ++++ .../core/tagging/migrations/0001_initial.py | 23 +++++ .../core/tagging/migrations/__init__.py | 0 openedx_tagging/core/tagging/models.py | 39 ++++++++ openedx_tagging/core/tagging/readme.rst | 26 ++++++ projects/dev.py | 2 + setup.py | 5 +- test_settings.py | 1 + tests/openedx_tagging/__init__.py | 0 tests/openedx_tagging/core/__init__.py | 0 .../openedx_tagging/core/tagging/__init__.py | 0 .../core/tagging/test_models.py | 17 ++++ tox.ini | 8 +- 22 files changed, 399 insertions(+), 6 deletions(-) create mode 100644 docs/decisions/0007-tagging-app.rst create mode 100644 docs/decisions/0008-tagging-tree-data-arch.rst create mode 100644 docs/decisions/0009-tagging-administrators.rst create mode 100644 docs/decisions/0010-taxonomy-enable-context.rst create mode 100644 docs/decisions/0011-tag-changes.rst create mode 100644 openedx_tagging/__init__.py create mode 100644 openedx_tagging/core/tagging/__init__.py create mode 100644 openedx_tagging/core/tagging/admin.py create mode 100644 openedx_tagging/core/tagging/apps.py create mode 100644 openedx_tagging/core/tagging/migrations/0001_initial.py create mode 100644 openedx_tagging/core/tagging/migrations/__init__.py create mode 100644 openedx_tagging/core/tagging/models.py create mode 100644 openedx_tagging/core/tagging/readme.rst create mode 100644 tests/openedx_tagging/__init__.py create mode 100644 tests/openedx_tagging/core/__init__.py create mode 100644 tests/openedx_tagging/core/tagging/__init__.py create mode 100644 tests/openedx_tagging/core/tagging/test_models.py diff --git a/.gitignore b/.gitignore index 888019d47..0fc03c7df 100644 --- a/.gitignore +++ b/.gitignore @@ -64,6 +64,7 @@ requirements/private.txt dev.db* .vscode +venv/ # Media files (for uploads) media/ diff --git a/docs/decisions/0007-tagging-app.rst b/docs/decisions/0007-tagging-app.rst new file mode 100644 index 000000000..7dbc45b0b --- /dev/null +++ b/docs/decisions/0007-tagging-app.rst @@ -0,0 +1,43 @@ +7. Tagging App structure +======================== + +Context +------- + +We want the openedx_tagging app to be useful in different Django projects outside of just openedx-learning and edx-platform. + + +Decisions +--------- + +The openedx_tagging data structures and code will stand alone with no dependencies on other Open edX projects. + +Classes which require dependencies on other Open edX projects should be defined within a ``tagging`` module inside those projects. + +Taxonomy +~~~~~~~~ + +The ``openedx_tagging`` module defines ``openedx_tagging.core.models.Taxonomy``, whose data and functionality are self-contained to the ``openedx_tagging`` app. However in Studio, we need to be able to limit access to some Taxonomy by organization, using the same "course creator" access which limits course creation for an organization to a defined set of users. + +So in edx-platform, we will create the ``openedx.features.tagging`` app, to contain ``models.OrgTaxonomy``. OrgTaxonomy subclasses ``openedx_tagging.core.models.Taxonomy``, employing Django's `multi-table inheritance`_ feature, which allows the base Tag class to keep foreign keys to the Taxonomy, while allowing OrgTaxonomy to store foreign keys into Studio's Organization table. + +ObjectTag +~~~~~~~~~ + +Similarly, the ``openedx_tagging`` module defined ``openedx_tagging.core.models.ObjectTag``, also self-contained to the +``openedx_tagging`` app. + +But to tag content in the LMS/Studio, we create ``openedx.features.tagging.models.ContentTag``, which subclasses ``ObjectTag``, and can then reference functionality available in the platform code. + +Rejected Alternatives +--------------------- + +Embed in edx-platform +~~~~~~~~~~~~~~~~~~~~~ + +Embedding the logic in edx-platform would provide the content tagging logic specifically required for the MVP. + +However, we plan to extend tagging to other object types (e.g. People) and contexts (e.g. Marketing), and so a generic, standalone library is preferable in the log run. + + +.. _multi-table inheritance: https://docs.djangoproject.com/en/3.2/topics/db/models/#multi-table-inheritance diff --git a/docs/decisions/0008-tagging-tree-data-arch.rst b/docs/decisions/0008-tagging-tree-data-arch.rst new file mode 100644 index 000000000..f4a879fef --- /dev/null +++ b/docs/decisions/0008-tagging-tree-data-arch.rst @@ -0,0 +1,53 @@ +8. Tag tree data structure +========================== + +Context +------- + +A taxonomy groups a set of related tags under a namespace and a set of usage rules. Some taxonomies require tags to be selected from a list or nested tree of valid options. + +Do we need a formal tree data structure to represent hierarchical tags? + +Decision +-------- + +No, a simplistic tree structure is sufficient for the MVP and forseeable feature requests. + +Existing tree data structures are designed to support very dynamic and deeply nested trees (e.g. forum threads) which are traversed frequently, and this feature set is overkill for taxonomy trees. + +Taxonomy trees have a maximum depth of 3 levels, which limits the depth of node traversal, and simplifies the UI/UX required to tag or search filter content with nested tags. + +Taxonomy trees only require simple operations, and infrequent traversals. Frequent operations (like viewing content tags) will only display the leaf tag value, not its full lineage, to minimize tree traversal. Full trees can be fetched quickly enough during content tag editing. Taxonomy tree changes themselves will also be infrequent. + +Rejected Alternatives +--------------------- + +All taxonomies are trees +~~~~~~~~~~~~~~~~~~~~~~~~ + +We could use a tree structure for all taxonomies: flat taxonomies would have only 1 level of tags under the root, while nested taxonomies can be deeper. + +To implement this, we'd link each taxonomy to a root tag, with the user-visible tags underneath. + +It was simpler instead to link the tag to the taxonomy, which removes the need for the unseen root tag. + +Closure tables +~~~~~~~~~~~~~~ + +https://coderwall.com/p/lixing/closure-tables-for-browsing-trees-in-sql + +Implementing the taxonomy tree using closure tables allows for tree traversals in O(N) time or less, where N is the total number of tags in the taxonomy. So the tree depth isn't as much of a performance concern as the total number of tags. + +Options include: + +* `django-tree-queries `_ + + Simple, performant, and well-maintained tree data structure. However it uses RECURSIVE CTE queries, which aren't supported until MySQL 8.0. + +* `django-mptt `_ + + Already an edx-platform dependency, but no longer maintained. It can be added retroactively to an existing tree-like model. + +* `django-closuretree `_ + + Another a good reference implementation for closure tables which can be added retroactively to an existing tree-like model. It is not actively maintained. diff --git a/docs/decisions/0009-tagging-administrators.rst b/docs/decisions/0009-tagging-administrators.rst new file mode 100644 index 000000000..652dcd243 --- /dev/null +++ b/docs/decisions/0009-tagging-administrators.rst @@ -0,0 +1,39 @@ +9. Taxonomy administrators +========================== + +Context +------- + +Taxonomy Administrators have the right to create, edit, populate, and delete taxonomies available globably for a given instance, or for a specific organization. + +How should these users be identified and their access granted? + +Decision +-------- + +In the Studio context, a modified version of "course creator" access will be used to identify Taxonomy Administrators (ref `get_organizations`_): + +#. Global staff and superusers can create/edit/populate/delete Taxonomies for the instance or for any org key. + +#. Users who can create courses for "any organization" access can create/edit/populate/delete Taxonomies for the instance or for any org key. + +#. Users who can create courses only for specific organizations can create/edit/populate/delete Taxonomies with only these org keys. + + +Permission #1 requires no external access, so can be enforced by the ``openedx_tagging`` app. + +But because permissions #2 + #3 require access to the edx-platform CMS model `CourseCreator`_, this access can only be enforced in Studio, and so will live under `cms.djangoapps.tagging` along with the ``ContentTag`` class. Tagging MVP must work for libraries v1, v2 and courses created in Studio, and so tying these permissions to Studio is reasonable for the MVP. + +Per `OEP-9`_, ``openedx_tagging`` will allow applications to use the standard Django API to query permissions, for example: ``user.has_perm('openedx_tagging.edit_taxonomy', taxonomy)``, and the appropriate permissions will be applied in that application's context. + +Rejected Alternatives +--------------------- + +Django users & groups +~~~~~~~~~~~~~~~~~~~~~ + +This is a standard way to grant access in Django apps, but it is not used in Open edX. + +.. _get_organizations: https://github.com/openedx/edx-platform/blob/4dc35c73ffa6d6a1dcb6e9ea1baa5bed40721125/cms/djangoapps/contentstore/views/course.py#L1958 +.. _CourseCreator: https://github.com/openedx/edx-platform/blob/4dc35c73ffa6d6a1dcb6e9ea1baa5bed40721125/cms/djangoapps/course_creators/models.py#L27 +.. _OEP-9: https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0009-bp-permissions.html diff --git a/docs/decisions/0010-taxonomy-enable-context.rst b/docs/decisions/0010-taxonomy-enable-context.rst new file mode 100644 index 000000000..8e9a854ae --- /dev/null +++ b/docs/decisions/0010-taxonomy-enable-context.rst @@ -0,0 +1,90 @@ +10. Taxonomy enabled for context +================================ + +Context +------- + +The MVP specification says that taxonomies need to be able to be enabled/disabled for the following contexts: instance, organization, and course. + +Taxonomy Administrators must be able to enable a taxonomy globally for all organizations in an instance, or to set a list of organizations who can use the taxonomy. + +Content Authors must be able to turn taxonomies (instance and org-levels) on/off at the course level. + +Decision +-------- + +When is a taxonomy field shown to course authors in a given course? + ++-------------+-----------------------------+--------------------------+-------------------------------+ +| tax.enabled | tax.enabled_for(course.org) | course enables all tax's | Is taxonomy shown for course? | ++=============+=============================+==========================+===============================+ +| True | True | True | True | ++-------------+-----------------------------+--------------------------+-------------------------------+ +| False | True | True | False | ++-------------+-----------------------------+--------------------------+-------------------------------+ +| True | False | True | False | ++-------------+-----------------------------+--------------------------+-------------------------------+ +| True | True | False | False | ++-------------+-----------------------------+--------------------------+-------------------------------+ +| False | True | False | False | ++-------------+-----------------------------+--------------------------+-------------------------------+ +| False | False | True | False | ++-------------+-----------------------------+--------------------------+-------------------------------+ + +.. _Course +Course +~~~~~~ + +We will add a Course `Advanced Settings`_ that allows course authors to enable/disable *all available taxonomies* for a given course. + +In order for a given taxonomy to be "available to a course", it must be enabled in the :ref:`Instance` context and the course's :ref:`Organization` context. + +Disabling taxonomies for a course will remove/hide the taxonomy fields from the course edit page and unit edit page(s), and tags will not be shown in Studio for that course. LMS use of tags is outside of this MVP. + +Future versions may add more granularity to these settings, to be determined by user needs. + +.. _Instance +Instance +~~~~~~~~ + +Taxonomy contains a boolean ``enabled`` field. + +A Taxonomy can be disabled for all contexts by setting ``enabled = False``. +If ``enabled = True``, then the :ref:`Organization` and :ref:`Course` contexts determine whether a taxonomy will be shown to course authors. + +.. _Organization +Organization +~~~~~~~~~~~~ + +OrgTaxonomy has a many-to-many relationship with the Organization model, accessed via the ``org_owners`` field. OrgTaxonomy lives under `cms.djangoapps.tagging` and so has access to the Organization model and logic in Studio. + +An OrgTaxonomy is enabled for all organizations if ``org_owners == []``. +If there are any ``org_owners`` set, then the OrgTaxonomy is only enabled for those orgas, i.e. only courses in these orgs will see the taxonomy field in Studio. + +Allowing multiple orgs to access a taxonomy reduces redundancy in data and maintenance. + +Rejected Alternatives +--------------------- + +Single org per taxonomy +~~~~~~~~~~~~~~~~~~~~~~~ + +Having a single org on a taxonomy is simpler from an implementation perspective, but the UI/UX frames demonstrated that it is simpler for the user to maintain a single taxonomy for multiple orgs. + +Course Waffle Flags +~~~~~~~~~~~~~~~~~~~ + +Use `Course Waffle Flags`_ to enable/disable all taxonomies for a given course. + +Waffle flags can only be changed by instance superusers, but the MVP specifically requires that content authors have control over this switch. + + +Link courses to taxonomies +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Link individual courses as enabled/disabled to specific taxonomies. +This was deemed too granular for the MVP, and the data structures and UI can be simplified by using a broader on/off flag. + + +.. _Advanced Settings: https://github.com/openedx/edx-platform/blob/4dc35c73ffa6d6a1dcb6e9ea1baa5bed40721125/cms/djangoapps/models/settings/course_metadata.py#L28 +.. _Course Waffle Flags: https://github.com/openedx/edx-platform/blob/4dc35c73ffa6d6a1dcb6e9ea1baa5bed40721125/openedx/core/djangoapps/waffle_utils/models.py#L14 diff --git a/docs/decisions/0011-tag-changes.rst b/docs/decisions/0011-tag-changes.rst new file mode 100644 index 000000000..6e1ea7d6d --- /dev/null +++ b/docs/decisions/0011-tag-changes.rst @@ -0,0 +1,35 @@ +11. Taxonomy and tag changes +============================ + +Context +------- + +Tagging content may be a labor-intensive, and the data produced is precious, both for human and automated users. Content tags should be structured and namespaced according to the needs of the instance's taxonomy administrators. But taxonomies themselves need to allow for changes: their tags can be overridden with a single import, they can be deleted, reworked, and their rules changed on the fly. + +What happens to the existing content tags if a Taxonomy or Tag is renamed, moved, or deleted? + +Decision +-------- + +Preserve content tag name:value pairs even if the associated taxonomy or tag is removed. +Reflect name:value changes from the linked taxonomy:tag immediately to the user. + +Content tags (through their base ObjectTag class) store a foreign key to their Taxonomy and Tag (if relevant), but they also store a copy of the Taxonomy.name and Tag.value, which can be used if there is no Taxonomy or Tag available. + +We consult the authoritative Taxonomy.name and Tag.value whenever displaying a content tag, so any changes are immediately reflected to the user. + +If a Taxonomy or Tag is deleted, the linked content tags will remain, and cached copies of the name:value pair will be displayed. + +This cached "value" field enables content tags (through their base ObjectTag class) to store free-text tag values, so that the free-text Taxonomy itself need not be modified when new free-text tags are added. + +This extra storage also allows tagged content to be imported independently of a taxonomy. The taxonomy (and appropriate tags) can be constructed later, and content tags validated and re-synced by editing the content tag or by running a maintenance command. + +Rejected Alternatives +--------------------- + +Require foreign keys +~~~~~~~~~~~~~~~~~~~~ + +Require a foreign key from a content tag to Taxonomy for the name, and Tag for the value. + +Only using foreign keys puts the labor-intensive content tag data at risk during taxonomy changes, and requires free-text tags to be made part of a taxonomy. diff --git a/openedx_tagging/__init__.py b/openedx_tagging/__init__.py new file mode 100644 index 000000000..6b99f0677 --- /dev/null +++ b/openedx_tagging/__init__.py @@ -0,0 +1 @@ +"""Open edX Tagging app.""" diff --git a/openedx_tagging/core/tagging/__init__.py b/openedx_tagging/core/tagging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_tagging/core/tagging/admin.py b/openedx_tagging/core/tagging/admin.py new file mode 100644 index 000000000..e993198a7 --- /dev/null +++ b/openedx_tagging/core/tagging/admin.py @@ -0,0 +1,6 @@ +""" Tagging app admin """ +from django.contrib import admin + +from .models import TagContent + +admin.site.register(TagContent) diff --git a/openedx_tagging/core/tagging/apps.py b/openedx_tagging/core/tagging/apps.py new file mode 100644 index 000000000..2cb90974f --- /dev/null +++ b/openedx_tagging/core/tagging/apps.py @@ -0,0 +1,16 @@ +""" +tagging Django application initialization. +""" + +from django.apps import AppConfig + + +class TaggingConfig(AppConfig): + """ + Configuration for the tagging Django application. + """ + + name = "openedx_tagging.core.tagging" + verbose_name = "Tagging" + default_auto_field = 'django.db.models.BigAutoField' + label = "oel_tagging" diff --git a/openedx_tagging/core/tagging/migrations/0001_initial.py b/openedx_tagging/core/tagging/migrations/0001_initial.py new file mode 100644 index 000000000..f3b336ef3 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0001_initial.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.19 on 2023-05-25 21:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='TagContent', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('content_id', models.CharField(max_length=255)), + ('name', models.CharField(max_length=255)), + ('value', models.CharField(max_length=765)), + ], + ), + ] diff --git a/openedx_tagging/core/tagging/migrations/__init__.py b/openedx_tagging/core/tagging/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_tagging/core/tagging/models.py b/openedx_tagging/core/tagging/models.py new file mode 100644 index 000000000..8521b65e7 --- /dev/null +++ b/openedx_tagging/core/tagging/models.py @@ -0,0 +1,39 @@ +""" Tagging app data models """ +from django.db import models + + +class TagContent(models.Model): + """ + TODO: This is a basic representation of TagContent, it may change + depending on the discovery in: + https://docs.google.com/document/d/13zfsGDfomSTCp_G-_ncevQHAb4Y4UW0d_6N8R2PdHlA/edit#heading=h.o6fm1hktwp7b + + This is the representation of a tag that is linked to a content + + Name-Value pair + ----------------- + + We use a Field-Value Pair representation, where `name` functions + as the constant that defines the field data set, and `value` functions + as the variable tag lineage that belong to the set. + + Value lineage + --------------- + + `value` are stored as null character-delimited strings to preserve the tag's lineage. + We use the null character to avoid choosing a character that may exist in a tag's name. + We don't use the Taxonomy's user-facing delimiter so that this delimiter can be changed + without disrupting stored tags. + """ + + # Content id to which this tag is associated + content_id = models.CharField(max_length=255) + + # It's not usually large + name = models.CharField(max_length=255) + + # Tag lineage value. + # + # The lineage can be large. + # TODO: The length is under discussion + value = models.CharField(max_length=765) diff --git a/openedx_tagging/core/tagging/readme.rst b/openedx_tagging/core/tagging/readme.rst new file mode 100644 index 000000000..5ce22340d --- /dev/null +++ b/openedx_tagging/core/tagging/readme.rst @@ -0,0 +1,26 @@ +Tagging App +============== + +The ``tagging`` app will enable content authors to tag pieces of content and quickly +filter for ease of re-use. + +Motivation +---------- + +Tagging content is central to enable content re-use, facilitate the implementation +of flexible content structures different from the current implementation and +allow adaptive learning in the Open edX platform. + +This service has been implemented as pluggable django app. Since it is necessary for +it to work independently of the content to which it links to. + +Setup +--------- + +**TODO:** We need to wait the discussion of the `Taxonomy discovery `_. +to build a proper setup for linking different pieces of content. + +The current approach is to save the id of the content through a generic string, +so that the tag can be linked to any type of content, no matter what type of ID have. +For example, with this approach we can link standalone blocks and library blocks, +both on Denver (v3) and Blockstore Content Libraries (v2). diff --git a/projects/dev.py b/projects/dev.py index 7be663f44..99903c3f7 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -41,6 +41,8 @@ # REST API "rest_framework", "openedx_learning.rest_api.apps.RESTAPIConfig", + # Tagging Core Apps + "openedx_tagging.core.tagging.apps.TaggingConfig", # Debugging "debug_toolbar", diff --git a/setup.py b/setup.py index aba8d0163..4fef021c4 100755 --- a/setup.py +++ b/setup.py @@ -71,9 +71,10 @@ def is_requirement(line): long_description=README + '\n\n' + CHANGELOG, author='David Ormsbee', author_email='dave@tcril.org', - url='https://github.com/ormsbee/openedx-learning', + url='https://github.com/openedx/openedx-learning', packages=[ - 'openedx_learning' + 'openedx_learning', + 'openedx_tagging', ], include_package_data=True, install_requires=load_requirements('requirements/base.in'), diff --git a/test_settings.py b/test_settings.py index 896092ca5..46e52fa59 100644 --- a/test_settings.py +++ b/test_settings.py @@ -39,6 +39,7 @@ def root(*args): "openedx_learning.core.components.apps.ComponentsConfig", "openedx_learning.core.contents.apps.ContentsConfig", "openedx_learning.core.publishing.apps.PublishingConfig", + "openedx_tagging.core.tagging.apps.TaggingConfig", ] LOCALE_PATHS = [ diff --git a/tests/openedx_tagging/__init__.py b/tests/openedx_tagging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_tagging/core/__init__.py b/tests/openedx_tagging/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_tagging/core/tagging/__init__.py b/tests/openedx_tagging/core/tagging/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py new file mode 100644 index 000000000..d1a4d76a4 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -0,0 +1,17 @@ +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.models import TagContent + + +class TestModelTagContent(TestCase): + """ + Test that TagContent objects can be created and edited. + """ + + def test_tag_content(self): + content_tag = TagContent.objects.create( + content_id="lb:Axim:video:abc", + name="Subject areas", + value="Chemistry", + ) + assert content_tag.id diff --git a/tox.ini b/tox.ini index 6e592fa87..362718418 100644 --- a/tox.ini +++ b/tox.ini @@ -70,11 +70,11 @@ deps = -r{toxinidir}/requirements/quality.txt commands = touch tests/__init__.py - pylint openedx_learning tests test_utils manage.py setup.py + pylint openedx_learning openedx_tagging tests test_utils manage.py setup.py rm tests/__init__.py - pycodestyle openedx_learning tests manage.py setup.py - pydocstyle openedx_learning tests manage.py setup.py - isort --check-only --diff --recursive tests test_utils openedx_learning manage.py setup.py test_settings.py + pycodestyle openedx_learning openedx_tagging tests manage.py setup.py + pydocstyle openedx_learning openedx_tagging tests manage.py setup.py + isort --check-only --diff --recursive tests test_utils openedx_learning openedx_tagging manage.py setup.py test_settings.py make selfcheck [testenv:pii_check] From 084ec21a0292584f42f79930b6d6fe3bcf0ef08b Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 15 May 2023 23:43:19 -0400 Subject: [PATCH 017/282] feat: adjust models to properly support MySQL This codebase was originally developed and tested using only SQLite. In this commit, we're adding proper support for MySQL. In particular: * The size of ``key`` fields and titles were reduced from 1000 to 500. This is to accommodate MySQL's index size limit (3072 bytes which translates to 768 unicode code points in 4-byte encoding). Saving a little headroom for future compound indexes. * fields.py now has helper classes to support multiple collations so that we can specify utf8mb4 for the charset in MySQL. * fields.py now has helper functions that allow us to normalize case sensitivity across databases. By default, fields would otherwise be case sensitive in SQLite and case insensitive in MySQL. This is important for correctness, since ``key`` fields are meant to be be case sensitive for the purposes of uniqueness constraints. --- .../components/migrations/0001_initial.py | 149 ++----- openedx_learning/core/components/models.py | 10 +- .../core/contents/migrations/0001_initial.py | 112 ++--- openedx_learning/core/contents/models.py | 34 +- .../publishing/migrations/0001_initial.py | 383 ++++-------------- .../0002_add_publish_log_constraints.py | 24 -- openedx_learning/core/publishing/models.py | 9 +- openedx_learning/lib/collations.py | 116 ++++++ openedx_learning/lib/fields.py | 137 +++++-- setup.py | 10 +- 10 files changed, 425 insertions(+), 559 deletions(-) delete mode 100644 openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py create mode 100644 openedx_learning/lib/collations.py diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py index ac379c67d..1d3707476 100644 --- a/openedx_learning/core/components/migrations/0001_initial.py +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -1,153 +1,80 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import uuid class Migration(migrations.Migration): + initial = True dependencies = [ - ("oel_publishing", "0001_initial"), - ("oel_contents", "0001_initial"), + ('oel_publishing', '0001_initial'), + ('oel_contents', '0001_initial'), ] operations = [ migrations.CreateModel( - name="Component", + name='Component', fields=[ - ( - "publishable_entity", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), - ("namespace", models.CharField(max_length=100)), - ("type", models.CharField(blank=True, max_length=100)), - ("local_key", models.CharField(max_length=255)), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('publishable_entity', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), + ('namespace', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100)), + ('type', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100)), + ('local_key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Component", - "verbose_name_plural": "Components", + 'verbose_name': 'Component', + 'verbose_name_plural': 'Components', }, ), migrations.CreateModel( - name="ComponentVersion", + name='ComponentVersion', fields=[ - ( - "publishable_entity_version", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentityversion", - ), - ), - ( - "component", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="versions", - to="oel_components.component", - ), - ), + ('publishable_entity_version', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentityversion')), + ('component', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='oel_components.component')), ], options={ - "verbose_name": "Component Version", - "verbose_name_plural": "Component Versions", + 'verbose_name': 'Component Version', + 'verbose_name_plural': 'Component Versions', }, ), migrations.CreateModel( - name="ComponentVersionRawContent", + name='ComponentVersionRawContent', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ("learner_downloadable", models.BooleanField(default=False)), - ( - "component_version", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_components.componentversion", - ), - ), - ( - "raw_content", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_contents.rawcontent", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('learner_downloadable', models.BooleanField(default=False)), + ('component_version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_components.componentversion')), + ('raw_content', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_contents.rawcontent')), ], ), migrations.AddField( - model_name="componentversion", - name="raw_contents", - field=models.ManyToManyField( - related_name="component_versions", - through="oel_components.ComponentVersionRawContent", - to="oel_contents.RawContent", - ), + model_name='componentversion', + name='raw_contents', + field=models.ManyToManyField(related_name='component_versions', through='oel_components.ComponentVersionRawContent', to='oel_contents.RawContent'), ), migrations.AddIndex( - model_name="componentversionrawcontent", - index=models.Index( - fields=["raw_content", "component_version"], - name="oel_cvrawcontent_c_cv", - ), + model_name='componentversionrawcontent', + index=models.Index(fields=['raw_content', 'component_version'], name='oel_cvrawcontent_c_cv'), ), migrations.AddIndex( - model_name="componentversionrawcontent", - index=models.Index( - fields=["component_version", "raw_content"], - name="oel_cvrawcontent_cv_d", - ), + model_name='componentversionrawcontent', + index=models.Index(fields=['component_version', 'raw_content'], name='oel_cvrawcontent_cv_d'), ), migrations.AddConstraint( - model_name="componentversionrawcontent", - constraint=models.UniqueConstraint( - fields=("component_version", "key"), name="oel_cvrawcontent_uniq_cv_key" - ), + model_name='componentversionrawcontent', + constraint=models.UniqueConstraint(fields=('component_version', 'key'), name='oel_cvrawcontent_uniq_cv_key'), ), migrations.AddIndex( - model_name="component", - index=models.Index( - fields=["learning_package", "namespace", "type", "local_key"], - name="oel_component_idx_lc_ns_t_lk", - ), + model_name='component', + index=models.Index(fields=['learning_package', 'namespace', 'type', 'local_key'], name='oel_component_idx_lc_ns_t_lk'), ), migrations.AddConstraint( - model_name="component", - constraint=models.UniqueConstraint( - fields=("learning_package", "namespace", "type", "local_key"), - name="oel_component_uniq_lc_ns_t_lk", - ), + model_name='component', + constraint=models.UniqueConstraint(fields=('learning_package', 'namespace', 'type', 'local_key'), name='oel_component_uniq_lc_ns_t_lk'), ), ] diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index b3a848f78..2d2f21391 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -18,7 +18,11 @@ """ from django.db import models -from openedx_learning.lib.fields import key_field, immutable_uuid_field +from openedx_learning.lib.fields import ( + case_sensitive_char_field, + immutable_uuid_field, + key_field, +) from ..publishing.models import LearningPackage from ..publishing.model_mixins import ( PublishableEntityMixin, @@ -77,13 +81,13 @@ class Component(PublishableEntityMixin): # namespace and type work together to help figure out what Component needs # to handle this data. A namespace is *required*. The namespace for XBlocks # is "xblock.v1" (to match the setup.py entrypoint naming scheme). - namespace = models.CharField(max_length=100, null=False, blank=False) + namespace = case_sensitive_char_field(max_length=100, blank=False) # type is a way to help sub-divide namespace if that's convenient. This # field cannot be null, but it can be blank if it's not necessary. For an # XBlock, type corresponds to tag, e.g. "video". It's also the block_type in # the UsageKey. - type = models.CharField(max_length=100, null=False, blank=True) + type = case_sensitive_char_field(max_length=100, blank=True) # local_key is an identifier that is local to the (namespace, type). The # publishable.key should be calculated as a combination of (namespace, type, diff --git a/openedx_learning/core/contents/migrations/0001_initial.py b/openedx_learning/core/contents/migrations/0001_initial.py index f106553e7..768730389 100644 --- a/openedx_learning/core/contents/migrations/0001_initial.py +++ b/openedx_learning/core/contents/migrations/0001_initial.py @@ -1,103 +1,75 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import openedx_learning.lib.validators +def use_compressed_table_format(apps, schema_editor): + """ + Use the COMPRESSED row format for TextContent if we're using MySQL. + + This table will hold a lot of OLX, which compresses very well using MySQL's + built-in zlib compression. This is especially important because we're + keeping so much version history. + """ + if schema_editor.connection.vendor == 'mysql': + table_name = apps.get_model("oel_contents", "TextContent")._meta.db_table + sql = f"ALTER TABLE {table_name} ROW_FORMAT=COMPRESSED;" + schema_editor.execute(sql) + + class Migration(migrations.Migration): + initial = True dependencies = [ - ("oel_publishing", "0001_initial"), + ('oel_publishing', '0001_initial'), ] operations = [ migrations.CreateModel( - name="RawContent", + name='RawContent', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ("hash_digest", models.CharField(editable=False, max_length=40)), - ("mime_type", models.CharField(max_length=255)), - ( - "size", - models.PositiveBigIntegerField( - validators=[django.core.validators.MaxValueValidator(50000000)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ("file", models.FileField(null=True, upload_to="")), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('hash_digest', models.CharField(editable=False, max_length=40)), + ('mime_type', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, max_length=255)), + ('size', models.PositiveBigIntegerField(validators=[django.core.validators.MaxValueValidator(50000000)])), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('file', models.FileField(null=True, upload_to='')), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Raw Content", - "verbose_name_plural": "Raw Contents", + 'verbose_name': 'Raw Content', + 'verbose_name_plural': 'Raw Contents', }, ), migrations.CreateModel( - name="TextContent", + name='TextContent', fields=[ - ( - "raw_content", - models.OneToOneField( - on_delete=django.db.models.deletion.CASCADE, - primary_key=True, - related_name="text_content", - serialize=False, - to="oel_contents.rawcontent", - ), - ), - ("text", models.TextField(blank=True, max_length=100000)), - ("length", models.PositiveIntegerField()), + ('raw_content', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, related_name='text_content', serialize=False, to='oel_contents.rawcontent')), + ('text', openedx_learning.lib.fields.MultiCollationTextField(blank=True, db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=100000)), + ('length', models.PositiveIntegerField()), ], ), + # Call out to custom code here to change row format for TextContent + migrations.RunPython(use_compressed_table_format, reverse_code=migrations.RunPython.noop, atomic=False), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "mime_type"], - name="oel_content_idx_lp_mime_type", - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', 'mime_type'], name='oel_content_idx_lp_mime_type'), ), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-size"], name="oel_content_idx_lp_rsize" - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', '-size'], name='oel_content_idx_lp_rsize'), ), migrations.AddIndex( - model_name="rawcontent", - index=models.Index( - fields=["learning_package", "-created"], - name="oel_content_idx_lp_rcreated", - ), + model_name='rawcontent', + index=models.Index(fields=['learning_package', '-created'], name='oel_content_idx_lp_rcreated'), ), migrations.AddConstraint( - model_name="rawcontent", - constraint=models.UniqueConstraint( - fields=("learning_package", "mime_type", "hash_digest"), - name="oel_content_uniq_lc_mime_type_hash_digest", - ), + model_name='rawcontent', + constraint=models.UniqueConstraint(fields=('learning_package', 'mime_type', 'hash_digest'), name='oel_content_uniq_lc_mime_type_hash_digest'), ), ] diff --git a/openedx_learning/core/contents/models.py b/openedx_learning/core/contents/models.py index e19f151cb..6794f1b23 100644 --- a/openedx_learning/core/contents/models.py +++ b/openedx_learning/core/contents/models.py @@ -5,9 +5,15 @@ """ from django.db import models from django.conf import settings +from django.core.files.storage import default_storage from django.core.validators import MaxValueValidator -from openedx_learning.lib.fields import hash_field, manual_date_time_field +from openedx_learning.lib.fields import ( + case_insensitive_char_field, + hash_field, + manual_date_time_field, + MultiCollationTextField, +) from ..publishing.models import LearningPackage @@ -73,11 +79,13 @@ class RawContent(models.Model): # MIME type, such as "text/html", "image/png", etc. Per RFC 4288, MIME type # and sub-type may each be 127 chars, making a max of 255 (including the "/" - # in between). + # in between). Also, while MIME types are almost always written in lowercase + # as a matter of convention, by spec they are NOT case sensitive. # # DO NOT STORE parameters here, e.g. "charset=". We can make a new field if - # that becomes necessary. - mime_type = models.CharField(max_length=255, blank=False, null=False) + # that becomes necessary. If we do decide to store parameters and values + # later, note that those *may be* case sensitive. + mime_type = case_insensitive_char_field(max_length=255, blank=False) # This is the size of the raw data file in bytes. This can be different than # the character length, since UTF-8 encoding can use anywhere between 1-4 @@ -96,10 +104,7 @@ class RawContent(models.Model): # models that offer better latency guarantees. file = models.FileField( null=True, - storage=settings.OPENEDX_LEARNING.get( - "STORAGE", - settings.DEFAULT_FILE_STORAGE, - ), + storage=settings.OPENEDX_LEARNING.get("STORAGE", default_storage), ) class Meta: @@ -172,5 +177,16 @@ class TextContent(models.Model): primary_key=True, related_name="text_content", ) - text = models.TextField(null=False, blank=True, max_length=MAX_TEXT_LENGTH) + text = MultiCollationTextField( + blank=True, + max_length=MAX_TEXT_LENGTH, + + # We don't really expect to ever sort by the text column. This is here + # primarily to force the column to be created as utf8mb4 on MySQL. I'm + # using the binary collation because it's a little cheaper/faster. + db_collations={ + "sqlite": "BINARY", + "mysql": "utf8mb4_bin", + } + ) length = models.PositiveIntegerField(null=False) diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index 76e02b316..3f41cd45a 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,14 +1,16 @@ -# Generated by Django 3.2.18 on 2023-05-11 02:07 +# Generated by Django 3.2.19 on 2023-06-15 14:43 from django.conf import settings import django.core.validators from django.db import migrations, models import django.db.models.deletion +import openedx_learning.lib.fields import openedx_learning.lib.validators import uuid class Migration(migrations.Migration): + initial = True dependencies = [ @@ -17,360 +19,145 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name="LearningPackage", + name='LearningPackage', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ("title", models.CharField(max_length=1000)), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "updated", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('title', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, max_length=500)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), ], options={ - "verbose_name": "Learning Package", - "verbose_name_plural": "Learning Packages", + 'verbose_name': 'Learning Package', + 'verbose_name_plural': 'Learning Packages', }, ), migrations.CreateModel( - name="PublishableEntity", + name='PublishableEntity', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("key", models.CharField(max_length=255)), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "created_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, max_length=500)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), ], options={ - "verbose_name": "Publishable Entity", - "verbose_name_plural": "Publishable Entities", + 'verbose_name': 'Publishable Entity', + 'verbose_name_plural': 'Publishable Entities', }, ), migrations.CreateModel( - name="PublishableEntityVersion", + name='PublishableEntityVersion', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("title", models.CharField(blank=True, default="", max_length=1000)), - ( - "version_num", - models.PositiveBigIntegerField( - validators=[django.core.validators.MinValueValidator(1)] - ), - ), - ( - "created", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "created_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), - ( - "entity", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - related_name="versions", - to="oel_publishing.publishableentity", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('title', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, default='', max_length=500)), + ('version_num', models.PositiveBigIntegerField(validators=[django.core.validators.MinValueValidator(1)])), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('created_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), + ('entity', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='versions', to='oel_publishing.publishableentity')), ], options={ - "verbose_name": "Publishable Entity Version", - "verbose_name_plural": "Publishable Entity Versions", + 'verbose_name': 'Publishable Entity Version', + 'verbose_name_plural': 'Publishable Entity Versions', }, ), migrations.CreateModel( - name="PublishLog", + name='PublishLog', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "uuid", - models.UUIDField( - default=uuid.uuid4, - editable=False, - unique=True, - verbose_name="UUID", - ), - ), - ("message", models.CharField(blank=True, default="", max_length=1000)), - ( - "published_at", - models.DateTimeField( - validators=[ - openedx_learning.lib.validators.validate_utc_datetime - ] - ), - ), - ( - "learning_package", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.learningpackage", - ), - ), - ( - "published_by", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - to=settings.AUTH_USER_MODEL, - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('message', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, default='', max_length=500)), + ('published_at', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('learning_package', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.learningpackage')), + ('published_by', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)), ], options={ - "verbose_name": "Publish Log", - "verbose_name_plural": "Publish Logs", + 'verbose_name': 'Publish Log', + 'verbose_name_plural': 'Publish Logs', }, ), migrations.CreateModel( - name="Draft", + name='Draft', fields=[ - ( - "entity", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), + ('entity', models.OneToOneField(on_delete=django.db.models.deletion.RESTRICT, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), ], ), migrations.CreateModel( - name="Published", + name='Published', fields=[ - ( - "entity", - models.OneToOneField( - on_delete=django.db.models.deletion.RESTRICT, - primary_key=True, - serialize=False, - to="oel_publishing.publishableentity", - ), - ), + ('entity', models.OneToOneField(on_delete=django.db.models.deletion.RESTRICT, primary_key=True, serialize=False, to='oel_publishing.publishableentity')), ], options={ - "verbose_name": "Published Entity", - "verbose_name_plural": "Published Entities", + 'verbose_name': 'Published Entity', + 'verbose_name_plural': 'Published Entities', }, ), migrations.CreateModel( - name="PublishLogRecord", + name='PublishLogRecord', fields=[ - ( - "id", - models.BigAutoField( - auto_created=True, - primary_key=True, - serialize=False, - verbose_name="ID", - ), - ), - ( - "entity", - models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentity", - ), - ), - ( - "new_version", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), - ), - ( - "old_version", - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - related_name="+", - to="oel_publishing.publishableentityversion", - ), - ), - ( - "publish_log", - models.ForeignKey( - on_delete=django.db.models.deletion.CASCADE, - to="oel_publishing.publishlog", - ), - ), + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('entity', models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentity')), + ('new_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion')), + ('old_version', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, related_name='+', to='oel_publishing.publishableentityversion')), + ('publish_log', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='oel_publishing.publishlog')), ], options={ - "verbose_name": "Publish Log Record", - "verbose_name_plural": "Publish Log Records", + 'verbose_name': 'Publish Log Record', + 'verbose_name_plural': 'Publish Log Records', }, ), migrations.AddConstraint( - model_name="learningpackage", - constraint=models.UniqueConstraint( - fields=("key",), name="oel_publishing_lp_uniq_key" - ), + model_name='learningpackage', + constraint=models.UniqueConstraint(fields=('key',), name='oel_publishing_lp_uniq_key'), + ), + migrations.AddIndex( + model_name='publishlogrecord', + index=models.Index(fields=['entity', '-publish_log'], name='oel_plr_idx_entity_rplr'), + ), + migrations.AddConstraint( + model_name='publishlogrecord', + constraint=models.UniqueConstraint(fields=('publish_log', 'entity'), name='oel_plr_uniq_pl_publishable'), ), migrations.AddField( - model_name="published", - name="publish_log_record", - field=models.ForeignKey( - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishlogrecord", - ), + model_name='published', + name='publish_log_record', + field=models.ForeignKey(on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishlogrecord'), ), migrations.AddField( - model_name="published", - name="version", - field=models.OneToOneField( - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), + model_name='published', + name='version', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion'), ), migrations.AddIndex( - model_name="publishableentityversion", - index=models.Index( - fields=["entity", "-created"], name="oel_pv_idx_entity_rcreated" - ), + model_name='publishableentityversion', + index=models.Index(fields=['entity', '-created'], name='oel_pv_idx_entity_rcreated'), ), migrations.AddIndex( - model_name="publishableentityversion", - index=models.Index(fields=["title"], name="oel_pv_idx_title"), + model_name='publishableentityversion', + index=models.Index(fields=['title'], name='oel_pv_idx_title'), ), migrations.AddConstraint( - model_name="publishableentityversion", - constraint=models.UniqueConstraint( - fields=("entity", "version_num"), name="oel_pv_uniq_entity_version_num" - ), + model_name='publishableentityversion', + constraint=models.UniqueConstraint(fields=('entity', 'version_num'), name='oel_pv_uniq_entity_version_num'), ), migrations.AddIndex( - model_name="publishableentity", - index=models.Index(fields=["key"], name="oel_pub_ent_idx_key"), + model_name='publishableentity', + index=models.Index(fields=['key'], name='oel_pub_ent_idx_key'), ), migrations.AddIndex( - model_name="publishableentity", - index=models.Index( - fields=["learning_package", "-created"], - name="oel_pub_ent_idx_lp_rcreated", - ), + model_name='publishableentity', + index=models.Index(fields=['learning_package', '-created'], name='oel_pub_ent_idx_lp_rcreated'), ), migrations.AddConstraint( - model_name="publishableentity", - constraint=models.UniqueConstraint( - fields=("learning_package", "key"), name="oel_pub_ent_uniq_lp_key" - ), + model_name='publishableentity', + constraint=models.UniqueConstraint(fields=('learning_package', 'key'), name='oel_pub_ent_uniq_lp_key'), ), migrations.AddField( - model_name="draft", - name="version", - field=models.OneToOneField( - blank=True, - null=True, - on_delete=django.db.models.deletion.RESTRICT, - to="oel_publishing.publishableentityversion", - ), + model_name='draft', + name='version', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.RESTRICT, to='oel_publishing.publishableentityversion'), ), ] diff --git a/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py b/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py deleted file mode 100644 index 986683c47..000000000 --- a/openedx_learning/core/publishing/migrations/0002_add_publish_log_constraints.py +++ /dev/null @@ -1,24 +0,0 @@ -# Generated by Django 3.2.19 on 2023-05-14 13:58 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ("oel_publishing", "0001_initial"), - ] - - operations = [ - migrations.AddIndex( - model_name="publishlogrecord", - index=models.Index( - fields=["entity", "-publish_log"], name="oel_plr_idx_entity_rplr" - ), - ), - migrations.AddConstraint( - model_name="publishlogrecord", - constraint=models.UniqueConstraint( - fields=("publish_log", "entity"), name="oel_plr_uniq_pl_publishable" - ), - ), - ] diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 0f9a50fdd..78f19b4ba 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -17,8 +17,9 @@ from django.core.validators import MinValueValidator from openedx_learning.lib.fields import ( - key_field, + case_insensitive_char_field, immutable_uuid_field, + key_field, manual_date_time_field, ) @@ -32,7 +33,7 @@ class LearningPackage(models.Model): uuid = immutable_uuid_field() key = key_field() - title = models.CharField(max_length=1000, null=False, blank=False) + title = case_insensitive_char_field(max_length=500, blank=False) created = manual_date_time_field() updated = manual_date_time_field() @@ -213,7 +214,7 @@ class PublishableEntityVersion(models.Model): # Most publishable things will have some sort of title, but blanks are # allowed for those that don't require one. - title = models.CharField(max_length=1000, default="", null=False, blank=True) + title = case_insensitive_char_field(max_length=500, blank=True, default="") # The version_num starts at 1 and increments by 1 with each new version for # a given PublishableEntity. Doing it this way makes it more convenient for @@ -339,7 +340,7 @@ class PublishLog(models.Model): uuid = immutable_uuid_field() learning_package = models.ForeignKey(LearningPackage, on_delete=models.CASCADE) - message = models.CharField(max_length=1000, null=False, blank=True, default="") + message = case_insensitive_char_field(max_length=500, blank=True, default="") published_at = manual_date_time_field() published_by = models.ForeignKey( settings.AUTH_USER_MODEL, diff --git a/openedx_learning/lib/collations.py b/openedx_learning/lib/collations.py new file mode 100644 index 000000000..2afdcc377 --- /dev/null +++ b/openedx_learning/lib/collations.py @@ -0,0 +1,116 @@ +""" +This module has collation-related code to allow us to attach collation settings +to specific fields on a per-database-vendor basis. This used by the ``fields`` +module in order to specify field types have have normalized behavior between +SQLite and MySQL (see fields.py for more details). +""" +from django.db import models + + +class MultiCollationMixin: + """ + Mixin to enable multiple, database-vendor-specific collations. + + This should be mixed into new subclasses of CharField and TextField, since + they're the only Field types that store text data. + """ + + def __init__(self, *args, db_collations=None, db_collation=None, **kwargs): + """ + Init like any field but add db_collations and disallow db_collation + + The ``db_collations`` param should be a dict of vendor names to + collations, like:: + + { + 'msyql': 'utf8mb4_bin', + 'sqlite': 'BINARY' + } + + It is an error to pass in a CharField-style ``db_collation``. I + originally wanted to use this attribute name, but I needed to preserve + it for Django 3.2 compatibility (see the ``db_collation`` method + docstring for details). + """ + if db_collation is not None: + raise ValueError( + f"Cannot use db_collation with {self.__class__.__name__}. " + + "Please use a db_collations dict instead." + ) + + super().__init__(*args, **kwargs) + self.db_collations = db_collations or {} + + # This is part of a hack to get this to work for Django < 4.1. Please + # see comments in the db_collation method for details. + self._vendor = None + + @property + def db_collation(self): + """ + Return the db_collation, understanding that it varies by vendor. + + This method is a hack for Django 3.2 compatibility and should be removed + after we move to 4.2. + + Description of why this is hacky: + + In Django 4.2, the schema builder pulls the collation settings from the + field using the value returned from the ``db_parameters`` method, and + this does what we want it to do. In Django 3.2, field.db_parameters is + called, but any collation value sent back is ignored and the code grabs + the value of db_collation directly from the field: + + https://github.com/django/django/blob/stable/3.2.x/django/db/backends/base/schema.py#L214-L224 + + But this call to get the ``field.db_collation`` attribute happens almost + immediately after the ``field.db_parameters`` method call. So our + fragile hack is to set ``self._vendor`` in the ``db_parameters`` method, + using the value we get from the connection that is passed in there. We + can then use ``self._vendor`` to return the right value when Django + calls ``field.db_collation`` (which is this property method). + + This method, the corresponding setter, and all references to + ``self._vendor`` should be removed after we've cut over to Django 4.2. + """ + return self.db_collations.get(self._vendor) + + @db_collation.setter + def db_collation(self, value): + """ + Don't allow db_collation to be set manually (just ignore). + + This can be removed when we move to Django 4.2. + """ + pass + + def db_parameters(self, connection): + """ + Return database parameters for this field. This adds collation info. + + We examine this field's ``db_collations`` attribute and return the + collation that maps to ``connection.vendor``. This will typically be + 'mysql' or 'sqlite'. + """ + db_params = models.Field.db_parameters(self, connection) + + # Remove once we no longer need to support Django < 4.1 + self._vendor = connection.vendor + + # Now determine collation based on DB vendor (e.g. 'sqlite', 'mysql') + if connection.vendor in self.db_collations: + db_params["collation"] = self.db_collations[connection.vendor] + + return db_params + + def deconstruct(self): + """ + How to serialize our Field for the migration file. + + For our mixin fields, this is just doing what the field's superclass + would do and then tacking on our custom ``db_collations`` dict data. + """ + name, path, args, kwargs = super().deconstruct() + if self.db_collations: + kwargs["db_collations"] = self.db_collations + return name, path, args, kwargs diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index 71273afa0..0e231e3a6 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -1,51 +1,82 @@ """ Convenience functions to make consistent field conventions easier. -Field conventions: - -* Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a - primary key + separate UUID column. +Per OEP-38, we're using the MySQL-friendly convention of BigInt ID as a +primary key + separate UUID column. https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0038-Data-Modeling.html -* The UUID fields are intended to be globally unique identifiers that other - services can store and rely on staying the same. -* The "key" fields can be more human-friendly strings, but these may only - be unique within a given scope. These values should be treated as mutable, - even if they rarely change in practice. - -TODO: -* Try making a CaseSensitiveCharField and CaseInsensitiveCharField -* Investigate more efficient UUID binary encoding + search in MySQL - -Other data thoughts: -* It would be good to make a data-dumping sort of script that exported part of - the data to SQLite3. -* identifiers support stable import/export with serialized formats -* UUIDs will import/export in the SQLite3 dumps, but are not there in other contexts + +We have helpers to make case sensitivity consistent across backends. MySQL is +case-insensitive by default, SQLite and Postgres are case-sensitive. + + """ import hashlib import uuid from django.db import models +from .collations import MultiCollationMixin from .validators import validate_utc_datetime -def key_field(): +def create_hash_digest(data_bytes): + return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() + + +def case_insensitive_char_field(**kwargs): """ - Externally created Identifier fields. + Return a case-insensitive ``MultiCollationCharField``. - These will often be local to a particular scope, like within a - LearningPackage. It's up to the application as to whether they're - semantically meaningful or look more machine-generated. + This means that entries will sort in a case-insensitive manner, and that + unique indexes will be case insensitive, e.g. you would not be able to + insert "abc" and "ABC" into the same table field if you put a unique index + on this field. - Other apps should *not* make references to these values directly, since - these values may in theory change. + You may override any argument that you would normally pass into + ``MultiCollationCharField`` (which is itself a subclass of ``CharField``). """ - return models.CharField( - max_length=255, - blank=False, - null=False, - ) + # Set our default arguments + final_kwargs = { + "null": False, + "db_collations": { + "sqlite": "NOCASE", + # We're using utf8mb4_unicode_ci to keep MariaDB compatibility, + # since their collation support diverges after this. MySQL is now on + # utf8mb4_0900_ai_ci based on Unicode 9, while MariaDB has + # uca1400_ai_ci based on Unicode 14. + "mysql": "utf8mb4_unicode_ci", + }, + } + # Override our defaults with whatever is passed in. + final_kwargs.update(kwargs) + + return MultiCollationCharField(**final_kwargs) + + +def case_sensitive_char_field(**kwargs): + """ + Return a case-sensitive ``MultiCollationCharField``. + + This means that entries will sort in a case-sensitive manner, and that + unique indexes will be case sensitive, e.g. "abc" and "ABC" would be + distinct and you would not get a unique constraint violation by adding them + both to the same table field. + + You may override any argument that you would normally pass into + ``MultiCollationCharField`` (which is itself a subclass of ``CharField``). + """ + # Set our default arguments + final_kwargs = { + "null": False, + "db_collations": { + "sqlite": "BINARY", + "mysql": "utf8mb4_bin", + }, + } + # Override our defaults with whatever is passed in. + final_kwargs.update(kwargs) + + return MultiCollationCharField(**final_kwargs) def immutable_uuid_field(): @@ -66,6 +97,20 @@ def immutable_uuid_field(): ) +def key_field(): + """ + Externally created Identifier fields. + + These will often be local to a particular scope, like within a + LearningPackage. It's up to the application as to whether they're + semantically meaningful or look more machine-generated. + + Other apps should *not* make references to these values directly, since + these values may in theory change (even if this is rare in practice). + """ + return case_sensitive_char_field(max_length=500, blank=False) + + def hash_field(): """ Holds a hash digest meant to identify a piece of content. @@ -85,10 +130,6 @@ def hash_field(): ) -def create_hash_digest(data_bytes): - return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() - - def manual_date_time_field(): """ DateTimeField that does not auto-generate values. @@ -117,3 +158,29 @@ def manual_date_time_field(): validate_utc_datetime, ], ) + + +class MultiCollationCharField(MultiCollationMixin, models.CharField): + """ + CharField subclass with per-database-vendor collation settings. + + Django's CharField already supports specifying the database collation, but + that only works with a single value. So there would be no way to say, "Use + utf8mb4_bin for MySQL, and BINARY if we're running SQLite." This is a + problem because we run tests in SQLite (and may potentially run more later). + It's also a problem if we ever want to support other database backends, like + PostgreSQL. Even MariaDB is starting to diverge from MySQL in terms of what + collations are supported. + """ + pass + + +class MultiCollationTextField(MultiCollationMixin, models.TextField): + """ + TextField subclass with per-database-vendor collation settings. + + We don't ever really want to _sort_ by a TextField, but setting a collation + forces the compatible charset to be set in MySQL, and that's the part that + matters for our purposes. So for example, if you set + """ + pass diff --git a/setup.py b/setup.py index 4fef021c4..ab642620a 100755 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ import re import sys -from setuptools import setup +from setuptools import find_packages, setup def get_version(*file_paths): @@ -72,10 +72,10 @@ def is_requirement(line): author='David Ormsbee', author_email='dave@tcril.org', url='https://github.com/openedx/openedx-learning', - packages=[ - 'openedx_learning', - 'openedx_tagging', - ], + packages=find_packages( + include=['openedx_learning*', 'openedx_tagging*'], + exclude=['*.test', '*.tests'] + ), include_package_data=True, install_requires=load_requirements('requirements/base.in'), python_requires=">=3.8", From 8ba5c3beef9682ae7680d05d34c6cacad76d2182 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Tue, 20 Jun 2023 12:19:32 -0400 Subject: [PATCH 018/282] test: use MySQL when running unit tests in CI This will run all CI tests against MySQL 8.0. It changes the tox py38-django{32|42} envs to use a new MySQL settings file. Running pytest manually will continue to use the in-memory SQLite database. --- .github/workflows/ci.yml | 23 ++++++++++++++++++++++- mysql_test_settings.py | 26 ++++++++++++++++++++++++++ requirements/base.txt | 4 +++- requirements/ci.txt | 8 ++++---- requirements/dev.txt | 39 +++++++++++++++++++++------------------ requirements/doc.txt | 23 +++++++++++++---------- requirements/quality.txt | 33 ++++++++++++++++++--------------- requirements/test.in | 2 ++ requirements/test.txt | 16 ++++++++++------ tox.ini | 4 ++++ 10 files changed, 123 insertions(+), 55 deletions(-) create mode 100644 mysql_test_settings.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1eff1b8e4..432111450 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,9 +14,30 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] # Add macos-latest later? python-version: ['3.8'] toxenv: ["py38-django32", "py38-django42"] + # We're only testing against MySQL 8 right now because 5.7 is + # incompatible with Djagno 4.2. We'd have to make the tox.ini file more + # complicated than it's worth given the short expected shelf-life of + # MySQL 5.7 in our stack. + mysql-version: ["8"] + services: + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "test_oel_db" + MYSQL_USER: "test_oel_user" + MYSQL_PASSWORD: "test_oel_pass" + MYSQL_RANDOM_ROOT_PASSWORD: true + # these options are blatantly copied from edx-platform's values + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: - uses: actions/checkout@v3 - name: setup python diff --git a/mysql_test_settings.py b/mysql_test_settings.py new file mode 100644 index 000000000..ae8e37d3c --- /dev/null +++ b/mysql_test_settings.py @@ -0,0 +1,26 @@ +""" +This is an extension of the default test_settings.py file that uses MySQL for +the backend. While the openedx-learning apps should run fine using SQLite, they +also do some MySQL-specific things around charset/collation settings and row +compression. + +The tox targets for py38-django32 and py38-django42 will use this settings file. +For the most part, you can use test_settings.py instead (that's the default if +you just run "pytest" with no arguments). +""" + +from test_settings import * + +DATABASES = { + "default": { + "ENGINE": "django.db.backends.mysql", + "NAME": "oel_db", + "USER": "test_oel_user", + "PASSWORD": "test_oel_pass", + "HOST": "127.0.0.1", + "PORT": "3306", + "OPTIONS": { + "charset": "utf8mb4" + } + } +} diff --git a/requirements/base.txt b/requirements/base.txt index d48934464..97ec03540 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via django django==3.2.19 # via @@ -19,3 +19,5 @@ pytz==2023.3 # djangorestframework sqlparse==0.4.4 # via django +typing-extensions==4.6.3 + # via asgiref diff --git a/requirements/ci.txt b/requirements/ci.txt index d851c07e2..6e1dd7f92 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,7 +8,7 @@ click==8.1.3 # via import-linter distlib==0.3.6 # via virtualenv -filelock==3.12.0 +filelock==3.12.2 # via # tox # virtualenv @@ -18,7 +18,7 @@ import-linter==1.9.0 # via -r requirements/ci.in packaging==23.1 # via tox -platformdirs==3.5.1 +platformdirs==3.6.0 # via virtualenv pluggy==1.0.0 # via tox @@ -34,9 +34,9 @@ tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # grimp # import-linter -virtualenv==20.23.0 +virtualenv==20.23.1 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 9f6bd5332..b2dcdaf2e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/quality.txt # django @@ -49,11 +49,11 @@ code-annotations==1.3.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/quality.txt # pytest-cov -diff-cover==7.5.0 +diff-cover==7.6.0 # via -r requirements/dev.in dill==0.3.6 # via @@ -86,7 +86,7 @@ exceptiongroup==1.1.1 # via # -r requirements/quality.txt # pytest -filelock==3.12.0 +filelock==3.12.2 # via # -r requirements/ci.txt # tox @@ -104,7 +104,7 @@ import-linter==1.9.0 # via # -r requirements/ci.txt # -r requirements/quality.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # -r requirements/quality.txt # keyring @@ -130,7 +130,7 @@ jinja2==3.1.2 # -r requirements/quality.txt # code-annotations # diff-cover -keyring==23.13.1 +keyring==24.0.0 # via # -r requirements/quality.txt # twine @@ -138,11 +138,11 @@ lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt # astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via # -r requirements/quality.txt # rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/quality.txt # jinja2 @@ -158,6 +158,8 @@ more-itertools==9.1.0 # via # -r requirements/quality.txt # jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/quality.txt packaging==23.1 # via # -r requirements/ci.txt @@ -178,7 +180,7 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -222,7 +224,7 @@ pylint-django==2.5.3 # via # -r requirements/quality.txt # edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # -r requirements/quality.txt # pylint-celery @@ -231,12 +233,12 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/quality.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/quality.txt pytest-django==4.5.2 # via -r requirements/quality.txt @@ -254,11 +256,11 @@ pyyaml==6.0 # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==37.3 +readme-renderer==40.0 # via # -r requirements/quality.txt # twine -requests==2.30.0 +requests==2.31.0 # via # -r requirements/quality.txt # requests-toolbelt @@ -271,7 +273,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.3.5 +rich==13.4.2 # via # -r requirements/quality.txt # twine @@ -324,21 +326,22 @@ tox-battery==0.6.1 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/quality.txt -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/ci.txt # -r requirements/quality.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # -r requirements/quality.txt # requests # twine -virtualenv==20.23.0 +virtualenv==20.23.1 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index aa92d8e59..d475be069 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.txt -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -66,7 +66,7 @@ imagesize==1.4.1 # via sphinx import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via sphinx iniconfig==2.0.0 # via @@ -77,10 +77,12 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,12 +108,12 @@ pygments==2.15.1 # pydata-sphinx-theme # readme-renderer # sphinx -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -129,9 +131,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via -r requirements/doc.in -requests==2.30.0 +requests==2.31.0 # via sphinx restructuredtext-lint==1.4.0 # via doc8 @@ -183,13 +185,14 @@ tomli==2.0.1 # doc8 # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # grimp # import-linter # pydata-sphinx-theme -urllib3==2.0.2 +urllib3==2.0.3 # via requests webencodings==0.5.1 # via bleach diff --git a/requirements/quality.txt b/requirements/quality.txt index b36f07abd..ab8f3977d 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/test.txt # django @@ -31,7 +31,7 @@ code-annotations==1.3.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov @@ -60,7 +60,7 @@ idna==3.4 # via requests import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.6.0 +importlib-metadata==6.7.0 # via # keyring # twine @@ -80,13 +80,13 @@ jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==23.13.1 +keyring==24.0.0 # via twine lazy-object-proxy==1.9.0 # via astroid -markdown-it-py==2.2.0 +markdown-it-py==3.0.0 # via rich -markupsafe==2.1.2 +markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 @@ -96,6 +96,8 @@ mdurl==0.1.2 # via markdown-it-py more-itertools==9.1.0 # via jaraco-classes +mysqlclient==2.1.1 + # via -r requirements/test.txt packaging==23.1 # via # -r requirements/test.txt @@ -106,7 +108,7 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.5.1 +platformdirs==3.6.0 # via pylint pluggy==1.0.0 # via @@ -130,16 +132,16 @@ pylint-celery==0.3 # via edx-lint pylint-django==2.5.3 # via edx-lint -pylint-plugin-utils==0.8.1 +pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.txt # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt @@ -156,9 +158,9 @@ pyyaml==6.0 # via # -r requirements/test.txt # code-annotations -readme-renderer==37.3 +readme-renderer==40.0 # via twine -requests==2.30.0 +requests==2.31.0 # via # requests-toolbelt # twine @@ -166,7 +168,7 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.3.5 +rich==13.4.2 # via twine six==1.16.0 # via @@ -197,15 +199,16 @@ tomlkit==0.11.8 # via pylint twine==4.0.2 # via -r requirements/quality.in -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via # -r requirements/test.txt + # asgiref # astroid # grimp # import-linter # pylint # rich -urllib3==2.0.2 +urllib3==2.0.3 # via # requests # twine diff --git a/requirements/test.in b/requirements/test.in index 65ac63e81..48446899b 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -3,6 +3,8 @@ -r base.txt # Core dependencies for this package +mysqlclient<3.0 # MySQL support + coverage # Code coverage reporting import-linter # Track our internal dependencies diff --git a/requirements/test.txt b/requirements/test.txt index ae4cfa040..8865d16b8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ # # make upgrade # -asgiref==3.6.0 +asgiref==3.7.2 # via # -r requirements/base.txt # django @@ -14,7 +14,7 @@ click==8.1.3 # import-linter code-annotations==1.3.0 # via -r requirements/test.in -coverage[toml]==7.2.5 +coverage[toml]==7.2.7 # via # -r requirements/test.in # pytest-cov @@ -34,20 +34,22 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -markupsafe==2.1.2 +markupsafe==2.1.3 # via jinja2 +mysqlclient==2.1.1 + # via -r requirements/test.in packaging==23.1 # via pytest pbr==5.11.1 # via stevedore pluggy==1.0.0 # via pytest -pytest==7.3.1 +pytest==7.3.2 # via # -r requirements/test.in # pytest-cov # pytest-django -pytest-cov==4.0.0 +pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in @@ -73,7 +75,9 @@ tomli==2.0.1 # coverage # import-linter # pytest -typing-extensions==4.5.0 +typing-extensions==4.6.3 # via + # -r requirements/base.txt + # asgiref # grimp # import-linter diff --git a/tox.ini b/tox.ini index 362718418..23e2650ad 100644 --- a/tox.ini +++ b/tox.ini @@ -39,6 +39,10 @@ deps = django32: Django>=3.2,<3.3 django42: Django>=4.2,<4.3 -r{toxinidir}/requirements/test.txt +setenv = + # Note that the django32/django42 targets use MySQL, but running pytest by + # itself (without tox) will run tests in SQLite for developer convenience. + DJANGO_SETTINGS_MODULE = mysql_test_settings commands = pytest {posargs} From 1495822b9f90b573e2d48ce9a7d0d268fad1a4b9 Mon Sep 17 00:00:00 2001 From: XnpioChV Date: Mon, 12 Jun 2023 19:25:02 -0500 Subject: [PATCH 019/282] docs: ADRs for System-defined taxonomies --- .../0012-system-taxonomy-creation.rst | 44 +++++++++++++++++++ .../0013-system-taxonomy-auto-tagging.rst | 30 +++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 docs/decisions/0012-system-taxonomy-creation.rst create mode 100644 docs/decisions/0013-system-taxonomy-auto-tagging.rst diff --git a/docs/decisions/0012-system-taxonomy-creation.rst b/docs/decisions/0012-system-taxonomy-creation.rst new file mode 100644 index 000000000..24744a84e --- /dev/null +++ b/docs/decisions/0012-system-taxonomy-creation.rst @@ -0,0 +1,44 @@ +12. System-defined Taxonomy & Tags creation +============================================ + +Context +-------- + +The System-defined are closed taxonomies created by the system. Some of this are totally static (e.g Language) +and some depends on a core data model (e.g. Organizations). It is necessary to define how to create and validate +the System-defined taxonomies and their tags. + + +Decision +--------- + +Create a ``Content-side`` class that lives on ``openedx.features.tagging``. +You must create a content-side model for each content and taxonomy to be used. +You can use ``ContentSystemTaxonomyMixin`` and the mixin of the Taxonomy (e.g. ``LanguageSystemTaxonomy``) to create the new class. +Then, all system-defined taxonomies must be created initially in a fixture. + +Each Taxonomy mixin has ``get_tags``; to configure the valid tags, and ``validate_tags``; to check if a list of tags are valid. + +We have two ways to handle tags in this type of taxonomies: + +Hardcoded by fixtures/migrations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +#. If the tags don't change over the time, you can create all on a fixture (e.g Languages). +#. If the tags change over the time, you can create all on a migration. If you edit, delete, or add new tags, you should also do it in a migration. + +Free-form tags +~~~~~~~~~~~~~~ + +This taxonomy depends on a core data model, but simplifies the creation of Tags by allowing free-form tags, +but we can validate the tags using the ``validate_tags`` method. For example we can put the Author taxonomy associated with +the ``User`` model and use the ``ID`` field as tags. Also we can validate if an User still exists or has been deleted over time. + + +Rejected Options +----------------- + +Auto-generated from the codebase +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Taxonomies that depend on a core data model and it is necessary to create a Tag for each object created. diff --git a/docs/decisions/0013-system-taxonomy-auto-tagging.rst b/docs/decisions/0013-system-taxonomy-auto-tagging.rst new file mode 100644 index 000000000..4a0e1ea7a --- /dev/null +++ b/docs/decisions/0013-system-taxonomy-auto-tagging.rst @@ -0,0 +1,30 @@ +13. System-defined automatic tagging +===================================== + +Context +-------- + +One main characteristic of the System-defined is the automatic content tagging. +It is necessary to implement this functionality when the associated content is created or edited. + +Decision +--------- + +Use `openedx-filters`_ to call the auto tagging function after content creation/edition. +After create the ``Content-side`` class, `register a pipeline`_ with the respective filter. +Some filters must to be created, like ``CourseCreation``, ``LibraryCreation``, etc. + +All this logic will be live on ``openedx.features.tagging``. + +Rejected Options +----------------- + +Django Signals +~~~~~~~~~~~~~~ + +Implement a function to add the tag from the content metadata and register that function +as a Django signal. Use openedx-filters is better in the edx context, but if there is +other no-edX project that need to use ``openedx-tagging``, can use the Django Signals approach. + +.. _openedx-filters: https://github.com/openedx/openedx-filters/tree/a4a192e1cac0b70bed31e0db8e4c4b058848c5c4 +.. _register a pipeline: https://github.com/openedx/edx-platform/blob/40613ae3f47eb470aff87359a952ed7e79ad8555/docs/guides/hooks/filters.rst#implement-pipeline-steps From 70401a6059ae4a08a0f3086765ce34ad623531e0 Mon Sep 17 00:00:00 2001 From: XnpioChV Date: Tue, 13 Jun 2023 13:15:28 -0500 Subject: [PATCH 020/282] docs: Updating System taxonomy creation --- .../0012-system-taxonomy-creation.rst | 36 +++++++++++-------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/decisions/0012-system-taxonomy-creation.rst b/docs/decisions/0012-system-taxonomy-creation.rst index 24744a84e..db9500eee 100644 --- a/docs/decisions/0012-system-taxonomy-creation.rst +++ b/docs/decisions/0012-system-taxonomy-creation.rst @@ -12,33 +12,41 @@ the System-defined taxonomies and their tags. Decision --------- -Create a ``Content-side`` class that lives on ``openedx.features.tagging``. -You must create a content-side model for each content and taxonomy to be used. -You can use ``ContentSystemTaxonomyMixin`` and the mixin of the Taxonomy (e.g. ``LanguageSystemTaxonomy``) to create the new class. -Then, all system-defined taxonomies must be created initially in a fixture. +System-defined Taxonomy creation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Each Taxonomy mixin has ``get_tags``; to configure the valid tags, and ``validate_tags``; to check if a list of tags are valid. +Each System-defined Taxonomy has its own class, which is used for tag validation (e.g. ``LanguageSystemTaxonomy``, ``OrganizationSystemTaxonomy``). +Each has ``get_tags``; to configure the valid tags, and ``validate_tags``; to check if a list of tags are valid. +We need to create an instance of each System-defined Taxonomy in a fixture. This instances will be used on different APIs. -We have two ways to handle tags in this type of taxonomies: +Later, we need to create a ``Content-side`` class that lives on ``openedx.features.tagging``for each content and taxonomy to be used +(eg. ``CourseLanguageSystemTaxonomy``, ``CourseOrganizationSystemTaxonomy``). +This new class is used to configure the automatic content tagging. You can read the `document number 0013`_ to see this configuration. -Hardcoded by fixtures/migrations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Tags creation +~~~~~~~~~~~~~~ + +We have two ways to handle Tags in this type of taxonomies: + +**Hardcoded by fixtures/migrations** #. If the tags don't change over the time, you can create all on a fixture (e.g Languages). #. If the tags change over the time, you can create all on a migration. If you edit, delete, or add new tags, you should also do it in a migration. -Free-form tags -~~~~~~~~~~~~~~ +**Free-form tags** This taxonomy depends on a core data model, but simplifies the creation of Tags by allowing free-form tags, -but we can validate the tags using the ``validate_tags`` method. For example we can put the Author taxonomy associated with -the ``User`` model and use the ``ID`` field as tags. Also we can validate if an User still exists or has been deleted over time. +but we can validate the tags using the ``validate_tags`` method. For example we can put the ``AuthorSystemTaxonomy`` associated with +the ``User`` model and use the ``ID`` field as tags. Also we can validate if an ``User`` still exists or has been deleted over time. Rejected Options ----------------- -Auto-generated from the codebase -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Tags created by Auto-generated from the codebase +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Taxonomies that depend on a core data model and it is necessary to create a Tag for each object created. + + +.. _document number 0013: https://github.com/openedx/openedx-learning/blob/main/docs/decisions/0013-system-taxonomy-auto-tagging.rst From 8b14e52e6d6e247be3663a5ddbaa9d1203fb5f4d Mon Sep 17 00:00:00 2001 From: XnpioChV Date: Tue, 13 Jun 2023 14:13:32 -0500 Subject: [PATCH 021/282] docs: Auto tagging doc updated --- .../0012-system-taxonomy-creation.rst | 16 +++++++++---- .../0013-system-taxonomy-auto-tagging.rst | 23 +++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/docs/decisions/0012-system-taxonomy-creation.rst b/docs/decisions/0012-system-taxonomy-creation.rst index db9500eee..93034b940 100644 --- a/docs/decisions/0012-system-taxonomy-creation.rst +++ b/docs/decisions/0012-system-taxonomy-creation.rst @@ -16,17 +16,19 @@ System-defined Taxonomy creation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Each System-defined Taxonomy has its own class, which is used for tag validation (e.g. ``LanguageSystemTaxonomy``, ``OrganizationSystemTaxonomy``). -Each has ``get_tags``; to configure the valid tags, and ``validate_tags``; to check if a list of tags are valid. +Each can overwrite ``get_tags``; to configure the valid tags, and ``validate_object_tag``; to check if a list of tags are valid. +Both functions are implemented on the ``Taxonomy`` base class, but can be overwritten to handle special cases. + We need to create an instance of each System-defined Taxonomy in a fixture. This instances will be used on different APIs. -Later, we need to create a ``Content-side`` class that lives on ``openedx.features.tagging``for each content and taxonomy to be used +Later, we need to create a ``Content-side`` class that lives on ``openedx.features.tagging`` for each content and taxonomy to be used (eg. ``CourseLanguageSystemTaxonomy``, ``CourseOrganizationSystemTaxonomy``). This new class is used to configure the automatic content tagging. You can read the `document number 0013`_ to see this configuration. Tags creation ~~~~~~~~~~~~~~ -We have two ways to handle Tags in this type of taxonomies: +We have two ways to handle Tags creation and validation for System-defined Taxonomies: **Hardcoded by fixtures/migrations** @@ -36,7 +38,7 @@ We have two ways to handle Tags in this type of taxonomies: **Free-form tags** This taxonomy depends on a core data model, but simplifies the creation of Tags by allowing free-form tags, -but we can validate the tags using the ``validate_tags`` method. For example we can put the ``AuthorSystemTaxonomy`` associated with +but we can validate the tags using the ``validate_object_tag`` method. For example we can put the ``AuthorSystemTaxonomy`` associated with the ``User`` model and use the ``ID`` field as tags. Also we can validate if an ``User`` still exists or has been deleted over time. @@ -46,7 +48,11 @@ Rejected Options Tags created by Auto-generated from the codebase ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Taxonomies that depend on a core data model and it is necessary to create a Tag for each object created. +Taxonomies that depend on a core data model could create a Tag for each eligible object. +Maintaining this dynamic list of available Tags is cumbersome: we'd need triggers for creation, editing, and deletion. +And if it's a large list of objects (e.g. Users), then copying that list into the Tag table is overkill. +It is better to dynamically generate the list of available Tags, and/or dynamically validate a submitted object tag than +to store the options in the database. .. _document number 0013: https://github.com/openedx/openedx-learning/blob/main/docs/decisions/0013-system-taxonomy-auto-tagging.rst diff --git a/docs/decisions/0013-system-taxonomy-auto-tagging.rst b/docs/decisions/0013-system-taxonomy-auto-tagging.rst index 4a0e1ea7a..0c3aa1082 100644 --- a/docs/decisions/0013-system-taxonomy-auto-tagging.rst +++ b/docs/decisions/0013-system-taxonomy-auto-tagging.rst @@ -11,10 +11,21 @@ Decision --------- Use `openedx-filters`_ to call the auto tagging function after content creation/edition. -After create the ``Content-side`` class, `register a pipeline`_ with the respective filter. -Some filters must to be created, like ``CourseCreation``, ``LibraryCreation``, etc. -All this logic will be live on ``openedx.features.tagging``. +Filters +~~~~~~~~ + +It is necessary to create `filters`_ for each content for creation/edition: ``CourseCreation``, ``LibraryCreation``, etc. +This filters will live on ``openedx-filters``. + +Pipelines +~~~~~~~~~~ + +After create the ``Content-side`` class for each content and taxonomy, +create a function inside each class with the logic of the auto tagging. +Then you need to register each class as `a pipeline`_ with the respective filter. + +This pipelines will live on ``openedx.features.tagging`` Rejected Options ----------------- @@ -23,8 +34,10 @@ Django Signals ~~~~~~~~~~~~~~ Implement a function to add the tag from the content metadata and register that function -as a Django signal. Use openedx-filters is better in the edx context, but if there is +as a Django signal. This works for Django database models, but some of the content lives in Mongo, +outside of the Django models. Also, using openedx-filters is better in the edx context, but if there is other no-edX project that need to use ``openedx-tagging``, can use the Django Signals approach. .. _openedx-filters: https://github.com/openedx/openedx-filters/tree/a4a192e1cac0b70bed31e0db8e4c4b058848c5c4 -.. _register a pipeline: https://github.com/openedx/edx-platform/blob/40613ae3f47eb470aff87359a952ed7e79ad8555/docs/guides/hooks/filters.rst#implement-pipeline-steps +.. _filters: https://github.com/openedx/openedx-filters/blob/a4a192e1cac0b70bed31e0db8e4c4b058848c5c4/openedx_filters/learning/filters.py +.. _a pipeline: https://github.com/openedx/edx-platform/blob/40613ae3f47eb470aff87359a952ed7e79ad8555/docs/guides/hooks/filters.rst#implement-pipeline-steps From eb593383891bad4271661801d6743439f64c51af Mon Sep 17 00:00:00 2001 From: XnpioChV Date: Wed, 14 Jun 2023 11:23:34 -0500 Subject: [PATCH 022/282] docs: Nits --- docs/decisions/0012-system-taxonomy-creation.rst | 2 +- docs/decisions/0013-system-taxonomy-auto-tagging.rst | 11 ++++------- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/docs/decisions/0012-system-taxonomy-creation.rst b/docs/decisions/0012-system-taxonomy-creation.rst index 93034b940..fe67b6e34 100644 --- a/docs/decisions/0012-system-taxonomy-creation.rst +++ b/docs/decisions/0012-system-taxonomy-creation.rst @@ -4,7 +4,7 @@ Context -------- -The System-defined are closed taxonomies created by the system. Some of this are totally static (e.g Language) +System-defined taxonomies are closed taxonomies created by the system. Some of these are totally static (e.g Language) and some depends on a core data model (e.g. Organizations). It is necessary to define how to create and validate the System-defined taxonomies and their tags. diff --git a/docs/decisions/0013-system-taxonomy-auto-tagging.rst b/docs/decisions/0013-system-taxonomy-auto-tagging.rst index 0c3aa1082..74b36bc1f 100644 --- a/docs/decisions/0013-system-taxonomy-auto-tagging.rst +++ b/docs/decisions/0013-system-taxonomy-auto-tagging.rst @@ -4,7 +4,7 @@ Context -------- -One main characteristic of the System-defined is the automatic content tagging. +One main characteristic of system-defined taxonomies is automatic content tagging. It is necessary to implement this functionality when the associated content is created or edited. Decision @@ -21,11 +21,8 @@ This filters will live on ``openedx-filters``. Pipelines ~~~~~~~~~~ -After create the ``Content-side`` class for each content and taxonomy, -create a function inside each class with the logic of the auto tagging. -Then you need to register each class as `a pipeline`_ with the respective filter. - -This pipelines will live on ``openedx.features.tagging`` +Auto-tagging pipelines will live under ``openedx.features.tagging``, +registered as `a pipeline`_ with the respective filter. Rejected Options ----------------- @@ -38,6 +35,6 @@ as a Django signal. This works for Django database models, but some of the conte outside of the Django models. Also, using openedx-filters is better in the edx context, but if there is other no-edX project that need to use ``openedx-tagging``, can use the Django Signals approach. -.. _openedx-filters: https://github.com/openedx/openedx-filters/tree/a4a192e1cac0b70bed31e0db8e4c4b058848c5c4 +.. _openedx-filters: https://github.com/openedx/openedx-filters .. _filters: https://github.com/openedx/openedx-filters/blob/a4a192e1cac0b70bed31e0db8e4c4b058848c5c4/openedx_filters/learning/filters.py .. _a pipeline: https://github.com/openedx/edx-platform/blob/40613ae3f47eb470aff87359a952ed7e79ad8555/docs/guides/hooks/filters.rst#implement-pipeline-steps From e2b4f021376476ee644eb36248205671a03c2391 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Tue, 13 Jun 2023 13:48:06 +0930 Subject: [PATCH 023/282] build: added openedx_tagging to build and test config files --- .coveragerc | 4 +++- MANIFEST.in | 1 + Makefile | 5 +++++ README.rst | 1 + tox.ini | 2 +- 5 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.coveragerc b/.coveragerc index 368d35820..32fc62c81 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,9 @@ [run] branch = True data_file = .coverage -source=openedx_learning +source = + openedx_learning + openedx_tagging omit = test_settings *migrations* diff --git a/MANIFEST.in b/MANIFEST.in index 6ce1f3b20..8cbdc3a65 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include LICENSE.txt include README.rst include requirements/base.in recursive-include openedx_learning *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py +recursive-include openedx_tagging *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py diff --git a/Makefile b/Makefile index 2440de3e1..3b2e8f1eb 100644 --- a/Makefile +++ b/Makefile @@ -77,12 +77,16 @@ extract_translations: ## extract strings to be translated, outputting .mo files rm -rf docs/_build cd openedx_learning && ../manage.py makemessages -l en -v1 -d django cd openedx_learning && ../manage.py makemessages -l en -v1 -d djangojs + cd openedx_tagging && ../manage.py makemessages -l en -v1 -d django + cd openedx_tagging && ../manage.py makemessages -l en -v1 -d djangojs compile_translations: ## compile translation files, outputting .po files for each supported language cd openedx_learning && ../manage.py compilemessages + cd openedx_tagging && ../manage.py compilemessages detect_changed_source_translations: cd openedx_learning && i18n_tool changed + cd openedx_tagging && i18n_tool changed pull_translations: ## pull translations from Transifex tx pull -a -f -t --mode reviewed @@ -92,6 +96,7 @@ push_translations: ## push source translation files (.po) from Transifex dummy_translations: ## generate dummy translation (.po) files cd openedx_learning && i18n_tool dummy + cd openedx_tagging && i18n_tool dummy build_dummy_translations: extract_translations dummy_translations compile_translations ## generate and compile dummy translation files diff --git a/README.rst b/README.rst index 4bf7e06da..5ad362c6b 100644 --- a/README.rst +++ b/README.rst @@ -27,6 +27,7 @@ Parts * ``openedx_learning.lib`` is for shared utilities, and may include things like custom field types, plugin registration code, etc. * ``openedx_learning.core`` contains our Core Django apps, where foundational data structures and APIs will live. +* ``openedx_tagging.core`` contains the core Tagging app, which provides data structures and apis for tagging Open edX objects. App Dependencies ~~~~~~~~~~~~~~~~ diff --git a/tox.ini b/tox.ini index 23e2650ad..ad8e2cb5e 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ match-dir = (?!migrations) [pytest] DJANGO_SETTINGS_MODULE = test_settings -addopts = --cov openedx_learning --cov-report term-missing --cov-report xml +addopts = --cov openedx_learning --cov openedx_tagging --cov-report term-missing --cov-report xml norecursedirs = .* docs requirements site-packages [testenv] From 6c49368639cc5dd5e2a850d52bd7da10824721cf Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Tue, 13 Jun 2023 13:49:30 +0930 Subject: [PATCH 024/282] feat: adds Taxonomy, Tag, ObjectTag models, APIs and tests Also: * Adds compile-requirements make target and updates (without upgrading) the requirements, which pulled in some other requirements, since it's been a while since they were compiled. * Adds ddt as a test dependency. * Replaces the placeholder TagContent class, and squashes migrations down to a single file * use the openedx_learning.lib multi-collation char and text fields --- Makefile | 9 +- openedx_tagging/core/tagging/admin.py | 6 +- openedx_tagging/core/tagging/api.py | 109 ++++ openedx_tagging/core/tagging/apps.py | 2 +- .../core/tagging/migrations/0001_initial.py | 207 +++++++- openedx_tagging/core/tagging/models.py | 469 +++++++++++++++++- requirements/dev.txt | 23 + requirements/doc.txt | 2 + requirements/quality.txt | 14 + requirements/test.in | 1 + requirements/test.txt | 2 + .../core/fixtures/tagging.yaml | 156 ++++++ .../openedx_tagging/core/tagging/test_api.py | 190 +++++++ .../core/tagging/test_models.py | 326 +++++++++++- 14 files changed, 1467 insertions(+), 49 deletions(-) create mode 100644 openedx_tagging/core/tagging/api.py create mode 100644 tests/openedx_tagging/core/fixtures/tagging.yaml create mode 100644 tests/openedx_tagging/core/tagging/test_api.py diff --git a/Makefile b/Makefile index 3b2e8f1eb..34eadfc08 100644 --- a/Makefile +++ b/Makefile @@ -30,10 +30,10 @@ docs: ## generate Sphinx HTML documentation, including API docs $(BROWSER)docs/_build/html/index.html # Define PIP_COMPILE_OPTS=-v to get more information during make upgrade. -PIP_COMPILE = pip-compile --rebuild --upgrade $(PIP_COMPILE_OPTS) +PIP_COMPILE = pip-compile --rebuild $(PIP_COMPILE_OPTS) -upgrade: export CUSTOM_COMPILE_COMMAND=make upgrade -upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in +compile-requirements: export CUSTOM_COMPILE_COMMAND=make upgrade +compile-requirements: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in pip install -qr requirements/pip-tools.txt # Make sure to compile files after any other files they include! $(PIP_COMPILE) -o requirements/pip-tools.txt requirements/pip-tools.in @@ -47,6 +47,9 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy sed '/^[dD]jango==/d' requirements/test.txt > requirements/test.tmp mv requirements/test.tmp requirements/test.txt +upgrade: ## update the pip requirements files to use the latest releases satisfying our constraints + make compile-requirements PIP_COMPILE_OPTS="--upgrade" + quality: ## check coding style with pycodestyle and pylint tox -e quality diff --git a/openedx_tagging/core/tagging/admin.py b/openedx_tagging/core/tagging/admin.py index e993198a7..91b1d753d 100644 --- a/openedx_tagging/core/tagging/admin.py +++ b/openedx_tagging/core/tagging/admin.py @@ -1,6 +1,8 @@ """ Tagging app admin """ from django.contrib import admin -from .models import TagContent +from .models import ObjectTag, Tag, Taxonomy -admin.site.register(TagContent) +admin.site.register(Taxonomy) +admin.site.register(Tag) +admin.site.register(ObjectTag) diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py new file mode 100644 index 000000000..6b9bf2a04 --- /dev/null +++ b/openedx_tagging/core/tagging/api.py @@ -0,0 +1,109 @@ +"""" +Tagging API + +Anyone using the openedx_tagging app should use these APIs instead of creating +or modifying the models directly, since there might be other related model +changes that you may not know about. + +No permissions/rules are enforced by these methods -- these must be enforced in the views. + +Please look at the models.py file for more information about the kinds of data +are stored in this app. +""" +from typing import List, Type + +from django.db.models import QuerySet +from django.utils.translation import gettext_lazy as _ + +from .models import ObjectTag, Tag, Taxonomy + + +def create_taxonomy( + name, + description=None, + enabled=True, + required=False, + allow_multiple=False, + allow_free_text=False, +) -> Taxonomy: + """ + Creates, saves, and returns a new Taxonomy with the given attributes. + """ + return Taxonomy.objects.create( + name=name, + description=description, + enabled=enabled, + required=required, + allow_multiple=allow_multiple, + allow_free_text=allow_free_text, + ) + + +def get_taxonomies(enabled=True) -> QuerySet: + """ + Returns a queryset containing the enabled taxonomies, sorted by name. + If you want the disabled taxonomies, pass enabled=False. + If you want all taxonomies (both enabled and disabled), pass enabled=None. + """ + queryset = Taxonomy.objects.order_by("name", "id") + if enabled is None: + return queryset.all() + return queryset.filter(enabled=enabled) + + +def get_tags(taxonomy: Taxonomy) -> List[Tag]: + """ + Returns a list of predefined tags for the given taxonomy. + + Note that if the taxonomy allows free-text tags, then the returned list will be empty. + """ + return taxonomy.get_tags() + + +def resync_object_tags(object_tags: QuerySet = None) -> int: + """ + Reconciles ObjectTag entries with any changes made to their associated taxonomies and tags. + + By default, we iterate over all ObjectTags. Pass a filtered ObjectTags queryset to limit which tags are resynced. + """ + if not object_tags: + object_tags = ObjectTag.objects.all() + + num_changed = 0 + for object_tag in object_tags: + changed = object_tag.resync() + if changed: + object_tag.save() + num_changed += 1 + return num_changed + + +def get_object_tags( + taxonomy: Taxonomy, object_id: str, object_type: str, valid_only=True +) -> List[ObjectTag]: + """ + Returns a list of tags for a given taxonomy + content. + + Pass valid_only=False when displaying tags to content authors, so they can see invalid tags too. + Invalid tags will likely be hidden from learners. + """ + tags = ObjectTag.objects.filter( + taxonomy=taxonomy, object_id=object_id, object_type=object_type + ).order_by("id") + return [tag for tag in tags if not valid_only or taxonomy.validate_object_tag(tag)] + + +def tag_object( + taxonomy: Taxonomy, tags: List, object_id: str, object_type: str +) -> List[ObjectTag]: + """ + Replaces the existing ObjectTag entries for the given taxonomy + object_id with the given list of tags. + + If taxonomy.allows_free_text, then the list should be a list of tag values. + Otherwise, it should be a list of existing Tag IDs. + + Raised ValueError if the proposed tags are invalid for this taxonomy. + Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. + """ + + return taxonomy.tag_object(tags, object_id, object_type) diff --git a/openedx_tagging/core/tagging/apps.py b/openedx_tagging/core/tagging/apps.py index 2cb90974f..d5877dfee 100644 --- a/openedx_tagging/core/tagging/apps.py +++ b/openedx_tagging/core/tagging/apps.py @@ -12,5 +12,5 @@ class TaggingConfig(AppConfig): name = "openedx_tagging.core.tagging" verbose_name = "Tagging" - default_auto_field = 'django.db.models.BigAutoField' + default_auto_field = "django.db.models.BigAutoField" label = "oel_tagging" diff --git a/openedx_tagging/core/tagging/migrations/0001_initial.py b/openedx_tagging/core/tagging/migrations/0001_initial.py index f3b336ef3..1599f8769 100644 --- a/openedx_tagging/core/tagging/migrations/0001_initial.py +++ b/openedx_tagging/core/tagging/migrations/0001_initial.py @@ -1,23 +1,212 @@ -# Generated by Django 3.2.19 on 2023-05-25 21:08 +# Generated by Django 3.2.19 on 2023-06-22 07:37 +import django.db.models.deletion from django.db import migrations, models +import openedx_learning.lib.fields -class Migration(migrations.Migration): +class Migration(migrations.Migration): initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='TagContent', + name="Taxonomy", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ( + "name", + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={ + "mysql": "utf8mb4_unicode_ci", + "sqlite": "NOCASE", + }, + db_index=True, + help_text="User-facing label used when applying tags from this taxonomy to Open edX objects.", + max_length=255, + ), + ), + ( + "description", + openedx_learning.lib.fields.MultiCollationTextField( + blank=True, + help_text="Provides extra information for the user when applying tags from this taxonomy to an object.", + null=True, + ), + ), + ( + "enabled", + models.BooleanField( + default=True, + help_text="Only enabled taxonomies will be shown to authors.", + ), + ), + ( + "required", + models.BooleanField( + default=False, + help_text="Indicates that one or more tags from this taxonomy must be added to an object.", + ), + ), + ( + "allow_multiple", + models.BooleanField( + default=False, + help_text="Indicates that multiple tags from this taxonomy may be added to an object.", + ), + ), + ( + "allow_free_text", + models.BooleanField( + default=False, + help_text="Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values.", + ), + ), + ], + options={ + "verbose_name_plural": "Taxonomies", + }, + ), + migrations.CreateModel( + name="Tag", fields=[ - ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('content_id', models.CharField(max_length=255)), - ('name', models.CharField(max_length=255)), - ('value', models.CharField(max_length=765)), + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ( + "value", + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={ + "mysql": "utf8mb4_unicode_ci", + "sqlite": "NOCASE", + }, + help_text="Content of a given tag, occupying the 'value' part of the key:value pair.", + max_length=500, + ), + ), + ( + "external_id", + openedx_learning.lib.fields.MultiCollationCharField( + blank=True, + db_collations={ + "mysql": "utf8mb4_unicode_ci", + "sqlite": "NOCASE", + }, + help_text="Used to link an Open edX Tag with a tag in an externally-defined taxonomy.", + max_length=255, + null=True, + ), + ), + ( + "parent", + models.ForeignKey( + default=None, + help_text="Tag that lives one level up from the current tag, forming a hierarchy.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="children", + to="oel_tagging.tag", + ), + ), + ( + "taxonomy", + models.ForeignKey( + default=None, + help_text="Namespace and rules for using a given set of tags.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="oel_tagging.taxonomy", + ), + ), ], ), + migrations.CreateModel( + name="ObjectTag", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ( + "object_id", + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={ + "mysql": "utf8mb4_unicode_ci", + "sqlite": "NOCASE", + }, + help_text="Identifier for the object being tagged", + max_length=255, + ), + ), + ( + "object_type", + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={ + "mysql": "utf8mb4_unicode_ci", + "sqlite": "NOCASE", + }, + help_text="Type of object being tagged", + max_length=255, + ), + ), + ( + "_name", + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={ + "mysql": "utf8mb4_unicode_ci", + "sqlite": "NOCASE", + }, + help_text="User-facing label used for this tag, stored in case taxonomy is (or becomes) null. If the taxonomy field is set, then taxonomy.name takes precedence over this field.", + max_length=255, + ), + ), + ( + "_value", + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={ + "mysql": "utf8mb4_unicode_ci", + "sqlite": "NOCASE", + }, + help_text="User-facing value used for this tag, stored in case tag is null, e.g if taxonomy is free text, or if it becomes null (e.g. if the Tag is deleted). If the tag field is set, then tag.value takes precedence over this field.", + max_length=500, + ), + ), + ( + "tag", + models.ForeignKey( + default=None, + help_text="Tag associated with this object tag. Provides the tag's 'value' if set.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="oel_tagging.tag", + ), + ), + ( + "taxonomy", + models.ForeignKey( + default=None, + help_text="Taxonomy that this object tag belongs to. Used for validating the tag and provides the tag's 'name' if set.", + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="oel_tagging.taxonomy", + ), + ), + ], + ), + migrations.AddIndex( + model_name="tag", + index=models.Index( + fields=["taxonomy", "value"], name="oel_tagging_taxonom_89e779_idx" + ), + ), + migrations.AddIndex( + model_name="tag", + index=models.Index( + fields=["taxonomy", "external_id"], + name="oel_tagging_taxonom_44e355_idx", + ), + ), + migrations.AddIndex( + model_name="objecttag", + index=models.Index( + fields=["taxonomy", "_value"], name="oel_tagging_taxonom_3668ec_idx" + ), + ), ] diff --git a/openedx_tagging/core/tagging/models.py b/openedx_tagging/core/tagging/models.py index 8521b65e7..c90f8a34a 100644 --- a/openedx_tagging/core/tagging/models.py +++ b/openedx_tagging/core/tagging/models.py @@ -1,39 +1,458 @@ """ Tagging app data models """ +from typing import List, Type + from django.db import models +from django.utils.translation import gettext_lazy as _ + +from openedx_learning.lib.fields import MultiCollationTextField, case_insensitive_char_field + + +# Maximum depth allowed for a hierarchical taxonomy's tree of tags. +TAXONOMY_MAX_DEPTH = 3 + +# Ancestry of a given tag; the Tag.value fields of a given tag and its parents, starting from the root. +# Will contain 0...TAXONOMY_MAX_DEPTH elements. +Lineage = List[str] + + +class Tag(models.Model): + """ + Represents a single value in a list or tree of values which can be applied to a particular Open edX object. + + Open edX tags are "name:value" pairs which can be applied to objects like content libraries, units, or people. + Tag.taxonomy.name provides the "name" and the Tag.value provides the "value". + (And an ObjectTag links a Tag with an object.) + """ + + id = models.BigAutoField(primary_key=True) + taxonomy = models.ForeignKey( + "Taxonomy", + null=True, + default=None, + on_delete=models.SET_NULL, + help_text=_("Namespace and rules for using a given set of tags."), + ) + parent = models.ForeignKey( + "self", + null=True, + default=None, + on_delete=models.SET_NULL, + related_name="children", + help_text=_( + "Tag that lives one level up from the current tag, forming a hierarchy." + ), + ) + value = case_insensitive_char_field( + max_length=500, + help_text=_( + "Content of a given tag, occupying the 'value' part of the key:value pair." + ), + ) + external_id = case_insensitive_char_field( + max_length=255, + null=True, + blank=True, + help_text=_( + "Used to link an Open edX Tag with a tag in an externally-defined taxonomy." + ), + ) + + class Meta: + indexes = [ + models.Index(fields=["taxonomy", "value"]), + models.Index(fields=["taxonomy", "external_id"]), + ] + + def __repr__(self): + """ + Developer-facing representation of a Tag. + """ + return str(self) + + def __str__(self): + """ + User-facing string representation of a Tag. + """ + return f"Tag ({self.id}) {self.value}" + + def get_lineage(self) -> Lineage: + """ + Queries and returns the lineage of the current tag as a list of Tag.value strings. + + The root Tag.value is first, followed by its child.value, and on down to self.value. + + Performance note: may perform as many as TAXONOMY_MAX_DEPTH select queries. + """ + lineage: Lineage = [] + tag = self + depth = TAXONOMY_MAX_DEPTH + while tag and depth > 0: + lineage.insert(0, tag.value) + tag = tag.parent + depth -= 1 + return lineage -class TagContent(models.Model): +class Taxonomy(models.Model): """ - TODO: This is a basic representation of TagContent, it may change - depending on the discovery in: - https://docs.google.com/document/d/13zfsGDfomSTCp_G-_ncevQHAb4Y4UW0d_6N8R2PdHlA/edit#heading=h.o6fm1hktwp7b + Represents a namespace and rules for a group of tags which can be applied to a particular Open edX object. + """ + + id = models.BigAutoField(primary_key=True) + name = case_insensitive_char_field( + null=False, + max_length=255, + db_index=True, + help_text=_( + "User-facing label used when applying tags from this taxonomy to Open edX objects." + ), + ) + description = MultiCollationTextField( + null=True, + blank=True, + help_text=_( + "Provides extra information for the user when applying tags from this taxonomy to an object." + ), + ) + enabled = models.BooleanField( + default=True, + help_text=_("Only enabled taxonomies will be shown to authors."), + ) + required = models.BooleanField( + default=False, + help_text=_( + "Indicates that one or more tags from this taxonomy must be added to an object." + ), + ) + allow_multiple = models.BooleanField( + default=False, + help_text=_( + "Indicates that multiple tags from this taxonomy may be added to an object." + ), + ) + allow_free_text = models.BooleanField( + default=False, + help_text=_( + "Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values." + ), + ) + + class Meta: + verbose_name_plural = "Taxonomies" + + @property + def system_defined(self) -> bool: + """ + Base taxonomies are user-defined, not system-defined. + + System-defined taxonomies cannot be edited by ordinary users. + + Subclasses should override this property as required. + """ + return False + + def get_tags(self) -> List[Tag]: + """ + Returns a list of all Tags in the current taxonomy, from the root(s) down to TAXONOMY_MAX_DEPTH tags, in tree order. + + Annotates each returned Tag with its ``depth`` in the tree (starting at 0). + + Performance note: may perform as many as TAXONOMY_MAX_DEPTH select queries. + """ + tags = [] + if self.allow_free_text: + return tags + + parents = None + for depth in range(TAXONOMY_MAX_DEPTH): + filtered_tags = self.tag_set.prefetch_related("parent") + if parents is None: + filtered_tags = filtered_tags.filter(parent=None) + else: + filtered_tags = filtered_tags.filter(parent__in=parents) + next_parents = list( + filtered_tags.annotate( + annotated_field=models.Value( + depth, output_field=models.IntegerField() + ) + ) + .order_by("parent__value", "value", "id") + .all() + ) + tags.extend(next_parents) + parents = next_parents + if not parents: + break + return tags + + def validate_object_tag( + self, + object_tag: "ObjectTag", + check_taxonomy=True, + check_tag=True, + check_object=True, + ) -> bool: + """ + Returns True if the given object tag is valid for the current Taxonomy. + + Subclasses can override this method to perform their own validation checks, e.g. against dynamically generated + tag lists. + + If `check_taxonomy` is False, then we skip validating the object tag's taxonomy reference. + If `check_tag` is False, then we skip validating the object tag's tag reference. + If `check_object` is False, then we skip validating the object ID/type. + """ + # Must be linked to this taxonomy + if check_taxonomy and ( + not object_tag.taxonomy_id or object_tag.taxonomy_id != self.id + ): + return False + + # Must be linked to a Tag unless its a free-text taxonomy + if check_tag and (not self.allow_free_text and not object_tag.tag_id): + return False + + # Must have a valid object id/type: + if check_object and (not object_tag.object_id or not object_tag.object_type): + return False + + return True - This is the representation of a tag that is linked to a content + def tag_object( + self, tags: List, object_id: str, object_type: str + ) -> List["ObjectTag"]: + """ + Replaces the existing ObjectTag entries for the current taxonomy + object_id with the given list of tags. - Name-Value pair - ----------------- + If self.allows_free_text, then the list should be a list of tag values. + Otherwise, it should be a list of existing Tag IDs. - We use a Field-Value Pair representation, where `name` functions - as the constant that defines the field data set, and `value` functions - as the variable tag lineage that belong to the set. + Raised ValueError if the proposed tags are invalid for this taxonomy. + Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. + """ - Value lineage - --------------- + if not self.allow_multiple and len(tags) > 1: + raise ValueError(_(f"Taxonomy ({self.id}) only allows one tag per object.")) - `value` are stored as null character-delimited strings to preserve the tag's lineage. - We use the null character to avoid choosing a character that may exist in a tag's name. - We don't use the Taxonomy's user-facing delimiter so that this delimiter can be changed - without disrupting stored tags. + if self.required and len(tags) == 0: + raise ValueError( + _(f"Taxonomy ({self.id}) requires at least one tag per object.") + ) + + current_tags = { + tag.tag_ref: tag + for tag in ObjectTag.objects.filter( + taxonomy=self, object_id=object_id, object_type=object_type + ) + } + updated_tags = [] + for tag_ref in tags: + if tag_ref in current_tags: + object_tag = current_tags.pop(tag_ref) + else: + object_tag = ObjectTag( + taxonomy=self, + object_id=object_id, + object_type=object_type, + ) + + try: + object_tag.tag = self.tag_set.get( + id=tag_ref, + ) + except (ValueError, Tag.DoesNotExist): + # This might be ok, e.g. if self.allow_free_text. + # We'll validate below before saving. + object_tag.value = tag_ref + + object_tag.resync() + if not self.validate_object_tag(object_tag): + raise ValueError( + _(f"Invalid object tag for taxonomy ({self.id}): {tag_ref}") + ) + updated_tags.append(object_tag) + + # Save all updated tags at once to avoid partial updates + for object_tag in updated_tags: + object_tag.save() + + # ...and delete any omitted existing tags + for old_tag in current_tags.values(): + old_tag.delete() + + return updated_tags + + +class ObjectTag(models.Model): """ + Represents the association between a tag and an Open edX object. + + Tagging content in Open edX involves linking the object to a particular name:value "tag", where the "name" is the + tag's label, and the value is the content of the tag itself. + + Tagging objects can be time-consuming for users, and so we guard against the deletion of Taxonomies and Tags by + providing fields to cache the name:value stored for an object. + + However, sometimes Taxonomy names or Tag values change (e.g if there's a typo, or a policy change about how a label + is used), and so we still store a link to the original Taxonomy and Tag, so that these changes will take precedence + over the original name:value used. + + Also, if an ObjectTag is associated with free-text Taxonomy, then the tag's value won't be stored as a standalone + Tag in the database -- it'll be stored here. + """ + + id = models.BigAutoField(primary_key=True) + object_id = case_insensitive_char_field( + max_length=255, + help_text=_("Identifier for the object being tagged"), + ) + object_type = case_insensitive_char_field( + max_length=255, + help_text=_("Type of object being tagged"), + ) + taxonomy = models.ForeignKey( + Taxonomy, + null=True, + default=None, + on_delete=models.SET_NULL, + help_text=_( + "Taxonomy that this object tag belongs to. Used for validating the tag and provides the tag's 'name' if set." + ), + ) + tag = models.ForeignKey( + Tag, + null=True, + default=None, + on_delete=models.SET_NULL, + help_text=_( + "Tag associated with this object tag. Provides the tag's 'value' if set." + ), + ) + _name = case_insensitive_char_field( + null=False, + max_length=255, + help_text=_( + "User-facing label used for this tag, stored in case taxonomy is (or becomes) null." + " If the taxonomy field is set, then taxonomy.name takes precedence over this field." + ), + ) + _value = case_insensitive_char_field( + null=False, + max_length=500, + help_text=_( + "User-facing value used for this tag, stored in case tag is null, e.g if taxonomy is free text, or if it" + " becomes null (e.g. if the Tag is deleted)." + " If the tag field is set, then tag.value takes precedence over this field." + ), + ) + + class Meta: + indexes = [ + models.Index(fields=["taxonomy", "_value"]), + ] + + @property + def name(self) -> str: + """ + Returns this tag's name/label. + + If taxonomy is set, then returns its name. + Otherwise, returns the cached _name field. + """ + return self.taxonomy.name if self.taxonomy_id else self._name + + @name.setter + def name(self, name: str): + """ + Stores to the _name field. + """ + self._name = name + + @property + def value(self) -> str: + """ + Returns this tag's value. + + If tag is set, then returns its value. + Otherwise, returns the cached _value field. + """ + return self.tag.value if self.tag_id else self._value + + @value.setter + def value(self, value: str): + """ + Stores to the _value field. + """ + self._value = value + + @property + def tag_ref(self) -> str: + """ + Returns this tag's reference string. + + If tag is set, then returns its id. + Otherwise, returns the cached _value field. + """ + return self.tag.id if self.tag_id else self._value + + @property + def is_valid(self) -> bool: + """ + Returns True if this ObjectTag represents a valid taxonomy tag. + + A valid ObjectTag must be linked to a Taxonomy, and be a valid tag in that taxonomy. + """ + return self.taxonomy_id and self.taxonomy.validate_object_tag(self) + + def get_lineage(self) -> Lineage: + """ + Returns the lineage of the current tag as a list of value strings. + + If linked to a Tag, returns its lineage. + Otherwise, returns an array containing its value string. + """ + return self.tag.get_lineage() if self.tag_id else [self._value] + + def resync(self) -> bool: + """ + Reconciles the stored ObjectTag properties with any changes made to its associated taxonomy or tag. + + This method is useful to propagate changes to a Taxonomy name or Tag value. + + It's also useful for a set of ObjectTags are imported from an external source prior to when a Taxonomy exists to + validate or store its available Tags. + + Returns True if anything was changed, False otherwise. + """ + changed = False + + # Locate a taxonomy matching _name + if not self.taxonomy_id: + for taxonomy in Taxonomy.objects.filter(name=self.name, enabled=True): + # Make sure this taxonomy will accept object tags like this. + self.taxonomy = taxonomy + if taxonomy.validate_object_tag(self, check_tag=False): + changed = True + break + # If not, try the next one + else: + self.taxonomy = None + + # Sync the stored _name with the taxonomy.name + elif self._name != self.taxonomy.name: + self.name = self.taxonomy.name + changed = True - # Content id to which this tag is associated - content_id = models.CharField(max_length=255) + # Locate a tag matching _value + if self.taxonomy and not self.tag_id and not self.taxonomy.allow_free_text: + tag = self.taxonomy.tag_set.filter(value=self.value).first() + if tag: + self.tag = tag + changed = True - # It's not usually large - name = models.CharField(max_length=255) + # Sync the stored _value with the tag.name + elif self.tag and self._value != self.tag.value: + self.value = self.tag.value + changed = True - # Tag lineage value. - # - # The lineage can be large. - # TODO: The length is under discussion - value = models.CharField(max_length=765) + return changed diff --git a/requirements/dev.txt b/requirements/dev.txt index b2dcdaf2e..e816277f9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -25,6 +25,10 @@ certifi==2023.5.7 # via # -r requirements/quality.txt # requests +cffi==1.15.1 + # via + # -r requirements/quality.txt + # cryptography chardet==5.1.0 # via diff-cover charset-normalizer==3.1.0 @@ -53,6 +57,12 @@ coverage[toml]==7.2.7 # via # -r requirements/quality.txt # pytest-cov +cryptography==41.0.1 + # via + # -r requirements/quality.txt + # secretstorage +ddt==1.6.0 + # via -r requirements/quality.txt diff-cover==7.6.0 # via -r requirements/dev.in dill==0.3.6 @@ -125,6 +135,11 @@ jaraco-classes==3.2.3 # via # -r requirements/quality.txt # keyring +jeepney==0.8.0 + # via + # -r requirements/quality.txt + # keyring + # secretstorage jinja2==3.1.2 # via # -r requirements/quality.txt @@ -201,6 +216,10 @@ py==1.11.0 # tox pycodestyle==2.10.0 # via -r requirements/quality.txt +pycparser==2.21 + # via + # -r requirements/quality.txt + # cffi pydocstyle==6.3.0 # via -r requirements/quality.txt pygments==2.15.1 @@ -277,6 +296,10 @@ rich==13.4.2 # via # -r requirements/quality.txt # twine +secretstorage==3.3.3 + # via + # -r requirements/quality.txt + # keyring six==1.16.0 # via # -r requirements/ci.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index d475be069..b4e6b5d5b 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -35,6 +35,8 @@ coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov +ddt==1.6.0 + # via -r requirements/test.txt django==3.2.19 # via # -c requirements/constraints.txt diff --git a/requirements/quality.txt b/requirements/quality.txt index ab8f3977d..c70f66a72 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -16,6 +16,8 @@ bleach==6.0.0 # via readme-renderer certifi==2023.5.7 # via requests +cffi==1.15.1 + # via cryptography charset-normalizer==3.1.0 # via requests click==8.1.3 @@ -35,6 +37,10 @@ coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov +cryptography==41.0.1 + # via secretstorage +ddt==1.6.0 + # via -r requirements/test.txt dill==0.3.6 # via pylint django==3.2.19 @@ -76,6 +82,10 @@ isort==5.12.0 # pylint jaraco-classes==3.2.3 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # -r requirements/test.txt @@ -116,6 +126,8 @@ pluggy==1.0.0 # pytest pycodestyle==2.10.0 # via -r requirements/quality.in +pycparser==2.21 + # via cffi pydocstyle==6.3.0 # via -r requirements/quality.in pygments==2.15.1 @@ -170,6 +182,8 @@ rfc3986==2.0.0 # via twine rich==13.4.2 # via twine +secretstorage==3.3.3 + # via keyring six==1.16.0 # via # bleach diff --git a/requirements/test.in b/requirements/test.in index 48446899b..6b4e096d4 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -12,3 +12,4 @@ pytest pytest-cov # pytest extension for code coverage statistics pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. +ddt # supports data driven tests diff --git a/requirements/test.txt b/requirements/test.txt index 8865d16b8..dafcddb06 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -18,6 +18,8 @@ coverage[toml]==7.2.7 # via # -r requirements/test.in # pytest-cov +ddt==1.6.0 + # via -r requirements/test.in # via # -c requirements/constraints.txt # -r requirements/base.txt diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml new file mode 100644 index 000000000..50b9459bb --- /dev/null +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -0,0 +1,156 @@ +- model: oel_tagging.tag + pk: 1 + fields: + taxonomy: 1 + parent: null + value: Bacteria + external_id: null +- model: oel_tagging.tag + pk: 2 + fields: + taxonomy: 1 + parent: null + value: Archaea + external_id: null +- model: oel_tagging.tag + pk: 3 + fields: + taxonomy: 1 + parent: null + value: Eukaryota + external_id: null +- model: oel_tagging.tag + pk: 4 + fields: + taxonomy: 1 + parent: 1 + value: Eubacteria + external_id: null +- model: oel_tagging.tag + pk: 5 + fields: + taxonomy: 1 + parent: 1 + value: Archaebacteria + external_id: null +- model: oel_tagging.tag + pk: 6 + fields: + taxonomy: 1 + parent: 2 + value: DPANN + external_id: null +- model: oel_tagging.tag + pk: 7 + fields: + taxonomy: 1 + parent: 2 + value: Euryarchaeida + external_id: null +- model: oel_tagging.tag + pk: 8 + fields: + taxonomy: 1 + parent: 2 + value: Proteoarchaeota + external_id: null +- model: oel_tagging.tag + pk: 9 + fields: + taxonomy: 1 + parent: 3 + value: Animalia + external_id: null +- model: oel_tagging.tag + pk: 10 + fields: + taxonomy: 1 + parent: 3 + value: Plantae + external_id: null +- model: oel_tagging.tag + pk: 11 + fields: + taxonomy: 1 + parent: 3 + value: Fungi + external_id: null +- model: oel_tagging.tag + pk: 12 + fields: + taxonomy: 1 + parent: 3 + value: Protista + external_id: null +- model: oel_tagging.tag + pk: 13 + fields: + taxonomy: 1 + parent: 3 + value: Monera + external_id: null +- model: oel_tagging.tag + pk: 14 + fields: + taxonomy: 1 + parent: 9 + value: Arthropoda + external_id: null +- model: oel_tagging.tag + pk: 15 + fields: + taxonomy: 1 + parent: 9 + value: Chordata + external_id: null +- model: oel_tagging.tag + pk: 16 + fields: + taxonomy: 1 + parent: 9 + value: Gastrotrich + external_id: null +- model: oel_tagging.tag + pk: 17 + fields: + taxonomy: 1 + parent: 9 + value: Cnidaria + external_id: null +- model: oel_tagging.tag + pk: 18 + fields: + taxonomy: 1 + parent: 9 + value: Ctenophora + external_id: null +- model: oel_tagging.tag + pk: 19 + fields: + taxonomy: 1 + parent: 9 + value: Placozoa + external_id: null +- model: oel_tagging.tag + pk: 20 + fields: + taxonomy: 1 + parent: 9 + value: Porifera + external_id: null +- model: oel_tagging.tag + pk: 21 + fields: + taxonomy: 1 + parent: 15 + value: Mammalia + external_id: null +- model: oel_tagging.taxonomy + pk: 1 + fields: + name: Life on Earth + description: null + enabled: true + required: false + allow_multiple: false + allow_free_text: false diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py new file mode 100644 index 000000000..7a418e687 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -0,0 +1,190 @@ +""" Test the tagging APIs """ + +from unittest.mock import patch + +from django.test.testcases import TestCase + +import openedx_tagging.core.tagging.api as tagging_api +from openedx_tagging.core.tagging.models import ObjectTag, Tag + +from .test_models import TestTagTaxonomyMixin + + +class TestApiTagging(TestTagTaxonomyMixin, TestCase): + """ + Test the Tagging API methods. + """ + + def test_create_taxonomy(self): + params = { + "name": "Difficulty", + "description": "This taxonomy contains tags describing the difficulty of an activity", + "enabled": False, + "required": True, + "allow_multiple": True, + "allow_free_text": True, + } + taxonomy = tagging_api.create_taxonomy(**params) + for param, value in params.items(): + assert getattr(taxonomy, param) == value + + def test_get_taxonomies(self): + tax1 = tagging_api.create_taxonomy("Enabled") + tax2 = tagging_api.create_taxonomy("Disabled", enabled=False) + enabled = tagging_api.get_taxonomies() + assert list(enabled) == [tax1, self.taxonomy] + + disabled = tagging_api.get_taxonomies(enabled=False) + assert list(disabled) == [tax2] + + both = tagging_api.get_taxonomies(enabled=None) + assert list(both) == [tax2, tax1, self.taxonomy] + + def test_get_tags(self): + self.setup_tag_depths() + assert tagging_api.get_tags(self.taxonomy) == [ + *self.domain_tags, + *self.kingdom_tags, + *self.phylum_tags, + ] + + def check_object_tag(self, object_tag, taxonomy, tag, name, value): + """ + Verifies that the properties of the given object_tag (once refreshed from the database) match those given. + """ + object_tag.refresh_from_db() + assert object_tag.taxonomy == taxonomy + assert object_tag.tag == tag + assert object_tag.name == name + assert object_tag.value == value + + def test_resync_object_tags(self): + missing_links = ObjectTag(object_id="abc", object_type="alpha") + missing_links.name = self.taxonomy.name + missing_links.value = self.mammalia.value + missing_links.save() + changed_links = ObjectTag( + object_id="def", + object_type="alpha", + taxonomy=self.taxonomy, + tag=self.mammalia, + ) + changed_links.name = "Life" + changed_links.value = "Animals" + changed_links.save() + + no_changes = ObjectTag( + object_id="ghi", + object_type="beta", + taxonomy=self.taxonomy, + tag=self.mammalia, + ) + no_changes.name = self.taxonomy.name + no_changes.value = self.mammalia.value + no_changes.save() + + changed = tagging_api.resync_object_tags() + assert changed == 2 + for object_tag in (missing_links, changed_links, no_changes): + self.check_object_tag( + object_tag, self.taxonomy, self.mammalia, "Life on Earth", "Mammalia" + ) + + # Once all tags are resynced, they stay that way + changed = tagging_api.resync_object_tags() + assert changed == 0 + + # ObjectTag value preserved even if linked tag is deleted + self.mammalia.delete() + for object_tag in (missing_links, changed_links, no_changes): + self.check_object_tag( + object_tag, self.taxonomy, None, "Life on Earth", "Mammalia" + ) + + # ObjectTag name preserved even if linked taxonomy is deleted + self.taxonomy.delete() + for object_tag in (missing_links, changed_links, no_changes): + self.check_object_tag(object_tag, None, None, "Life on Earth", "Mammalia") + + # Resyncing the tags for code coverage + changed = tagging_api.resync_object_tags() + assert changed == 0 + + # Recreate the taxonomy and resync some tags + first_taxonomy = tagging_api.create_taxonomy("Life on Earth") + second_taxonomy = tagging_api.create_taxonomy("Life on Earth") + new_tag = Tag.objects.create( + value="Mammalia", + taxonomy=second_taxonomy, + ) + + with patch( + "openedx_tagging.core.tagging.models.Taxonomy.validate_object_tag", + side_effect=[False, True, False, True], + ): + changed = tagging_api.resync_object_tags( + ObjectTag.objects.filter(object_type="alpha") + ) + assert changed == 2 + for object_tag in (missing_links, changed_links): + self.check_object_tag( + object_tag, second_taxonomy, new_tag, "Life on Earth", "Mammalia" + ) + + # Ensure the omitted tag was not updated + self.check_object_tag(no_changes, None, None, "Life on Earth", "Mammalia") + + # Update that one too (without the patching) + changed = tagging_api.resync_object_tags( + ObjectTag.objects.filter(object_type="beta") + ) + assert changed == 1 + self.check_object_tag( + no_changes, first_taxonomy, None, "Life on Earth", "Mammalia" + ) + + def test_tag_object(self): + self.taxonomy.allow_multiple = True + test_tags = [ + [ + self.archaea.id, + self.eubacteria.id, + self.chordata.id, + ], + [ + self.chordata.id, + self.archaebacteria.id, + ], + [ + self.archaebacteria.id, + self.archaea.id, + ], + ] + + # Tag and re-tag the object, checking that the expected tags are returned and deleted + for tag_list in test_tags: + object_tags = tagging_api.tag_object( + self.taxonomy, + tag_list, + "biology101", + "course", + ) + + # Ensure the expected number of tags exist in the database + assert ( + tagging_api.get_object_tags( + taxonomy=self.taxonomy, + object_id="biology101", + object_type="course", + ) + == object_tags + ) + # And the expected number of tags were returned + assert len(object_tags) == len(tag_list) + for index, object_tag in enumerate(object_tags): + assert object_tag.tag_id == tag_list[index] + assert object_tag.is_valid + assert object_tag.taxonomy == self.taxonomy + assert object_tag.name == self.taxonomy.name + assert object_tag.object_id == "biology101" + assert object_tag.object_type == "course" diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index d1a4d76a4..d74c041db 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -1,17 +1,325 @@ +""" Test the tagging models """ + +import ddt from django.test.testcases import TestCase -from openedx_tagging.core.tagging.models import TagContent +from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy + + +def get_tag(value): + """ + Fetches and returns the tag with the given value. + """ + return Tag.objects.get(value=value) + + +class TestTagTaxonomyMixin: + """ + Base class that uses the taxonomy fixture to load a base taxonomy and tags for testing. + """ + + fixtures = ["tests/openedx_tagging/core/fixtures/tagging.yaml"] + + def setUp(self): + super().setUp() + self.taxonomy = Taxonomy.objects.get(name="Life on Earth") + self.archaea = get_tag("Archaea") + self.archaebacteria = get_tag("Archaebacteria") + self.bacteria = get_tag("Bacteria") + self.eubacteria = get_tag("Eubacteria") + self.chordata = get_tag("Chordata") + self.mammalia = get_tag("Mammalia") + # Domain tags (depth=0) + # https://en.wikipedia.org/wiki/Domain_(biology) + self.domain_tags = [ + get_tag("Archaea"), + get_tag("Bacteria"), + get_tag("Eukaryota"), + ] + # Kingdom tags (depth=1) + self.kingdom_tags = [ + # Kingdoms of https://en.wikipedia.org/wiki/Archaea + get_tag("DPANN"), + get_tag("Euryarchaeida"), + get_tag("Proteoarchaeota"), + # Kingdoms of https://en.wikipedia.org/wiki/Bacterial_taxonomy + get_tag("Archaebacteria"), + get_tag("Eubacteria"), + # Kingdoms of https://en.wikipedia.org/wiki/Eukaryote + get_tag("Animalia"), + get_tag("Fungi"), + get_tag("Monera"), + get_tag("Plantae"), + get_tag("Protista"), + ] + # Phylum tags (depth=2) + self.phylum_tags = [ + # Some phyla of https://en.wikipedia.org/wiki/Animalia + get_tag("Arthropoda"), + get_tag("Chordata"), + get_tag("Cnidaria"), + get_tag("Ctenophora"), + get_tag("Gastrotrich"), + get_tag("Placozoa"), + get_tag("Porifera"), + ] -class TestModelTagContent(TestCase): + def setup_tag_depths(self): + """ + Annotate our tags with depth so we can compare them. + """ + for tag in self.domain_tags: + tag.depth = 0 + for tag in self.kingdom_tags: + tag.depth = 1 + for tag in self.phylum_tags: + tag.depth = 2 + + +@ddt.ddt +class TestModelTagTaxonomy(TestTagTaxonomyMixin, TestCase): """ - Test that TagContent objects can be created and edited. + Test the Tag and Taxonomy models' properties and methods. """ - def test_tag_content(self): - content_tag = TagContent.objects.create( - content_id="lb:Axim:video:abc", - name="Subject areas", - value="Chemistry", + def test_system_defined(self): + assert not self.taxonomy.system_defined + + def test_representations(self): + assert str(self.bacteria) == "Tag (1) Bacteria" + assert repr(self.bacteria) == "Tag (1) Bacteria" + + @ddt.data( + # Root tags just return their own value + ("bacteria", ["Bacteria"]), + # Second level tags return two levels + ("eubacteria", ["Bacteria", "Eubacteria"]), + # Third level tags return three levels + ("chordata", ["Eukaryota", "Animalia", "Chordata"]), + # Lineage beyond TAXONOMY_MAX_DEPTH won't trace back to the root + ("mammalia", ["Animalia", "Chordata", "Mammalia"]), + ) + @ddt.unpack + def test_get_lineage(self, tag_attr, lineage): + assert getattr(self, tag_attr).get_lineage() == lineage + + def test_get_tags(self): + self.setup_tag_depths() + assert self.taxonomy.get_tags() == [ + *self.domain_tags, + *self.kingdom_tags, + *self.phylum_tags, + ] + + def test_get_tags_free_text(self): + self.taxonomy.allow_free_text = True + with self.assertNumQueries(0): + assert self.taxonomy.get_tags() == [] + + def test_get_tags_shallow_taxonomy(self): + taxonomy = Taxonomy.objects.create(name="Difficulty") + tags = [ + Tag.objects.create(taxonomy=taxonomy, value="1. Easy"), + Tag.objects.create(taxonomy=taxonomy, value="2. Moderate"), + Tag.objects.create(taxonomy=taxonomy, value="3. Hard"), + ] + with self.assertNumQueries(2): + assert taxonomy.get_tags() == tags + + +class TestModelObjectTag(TestTagTaxonomyMixin, TestCase): + """ + Test the ObjectTag model and the related Taxonomy methods and fields. + """ + + def setUp(self): + super().setUp() + self.tag = self.bacteria + + def test_object_tag_name(self): + # ObjectTag's name defaults to its taxonomy's name + object_tag = ObjectTag.objects.create( + object_id="object:id", + object_type="any_old_object", + taxonomy=self.taxonomy, + ) + assert object_tag.name == self.taxonomy.name + + # Even if we overwrite the name, it still uses the taxonomy's name + object_tag.name = "Another tag" + assert object_tag.name == self.taxonomy.name + object_tag.save() + assert object_tag.name == self.taxonomy.name + + # But if the taxonomy is deleted, then the object_tag's name reverts to our cached name + self.taxonomy.delete() + object_tag.refresh_from_db() + assert object_tag.name == "Another tag" + + def test_object_tag_value(self): + # ObjectTag's value defaults to its tag's value + object_tag = ObjectTag.objects.create( + object_id="object:id", + object_type="any_old_object", + taxonomy=self.taxonomy, + tag=self.tag, + ) + assert object_tag.value == self.tag.value + + # Even if we overwrite the value, it still uses the tag's value + object_tag.value = "Another tag" + assert object_tag.value == self.tag.value + object_tag.save() + assert object_tag.value == self.tag.value + + # But if the tag is deleted, then the object_tag's value reverts to our cached value + self.tag.delete() + object_tag.refresh_from_db() + assert object_tag.value == "Another tag" + + def test_object_tag_lineage(self): + # ObjectTag's value defaults to its tag's lineage + object_tag = ObjectTag.objects.create( + object_id="object:id", + object_type="any_old_object", + taxonomy=self.taxonomy, + tag=self.tag, + ) + assert object_tag.get_lineage() == self.tag.get_lineage() + + # Even if we overwrite the value, it still uses the tag's lineage + object_tag.value = "Another tag" + assert object_tag.get_lineage() == self.tag.get_lineage() + object_tag.save() + assert object_tag.get_lineage() == self.tag.get_lineage() + + # But if the tag is deleted, then the object_tag's lineage reverts to our cached value + self.tag.delete() + object_tag.refresh_from_db() + assert object_tag.get_lineage() == ["Another tag"] + + def test_object_tag_is_valid(self): + object_tag = ObjectTag( + object_id="object:id", + object_type="any_old_object", + ) + assert not object_tag.is_valid + + object_tag.taxonomy = self.taxonomy + assert not object_tag.is_valid + + object_tag.tag = self.tag + assert object_tag.is_valid + + # or, we can have no tag, and a free-text taxonomy + object_tag.tag = None + self.taxonomy.allow_free_text = True + assert object_tag.is_valid + + def test_validate_object_tag_invalid(self): + taxonomy = Taxonomy.objects.create( + name="Another taxonomy", ) - assert content_tag.id + object_tag = ObjectTag( + taxonomy=self.taxonomy, + ) + assert not taxonomy.validate_object_tag(object_tag) + + object_tag.taxonomy = taxonomy + assert not taxonomy.validate_object_tag(object_tag) + + taxonomy.allow_free_text = True + assert not taxonomy.validate_object_tag(object_tag) + + object_tag.object_id = "object:id" + object_tag.object_type = "object:type" + assert taxonomy.validate_object_tag(object_tag) + + def test_tag_object(self): + self.taxonomy.allow_multiple = True + + test_tags = [ + [ + self.archaea.id, + self.eubacteria.id, + self.chordata.id, + ], + [ + self.archaebacteria.id, + self.chordata.id, + ], + [ + self.archaea.id, + self.archaebacteria.id, + ], + ] + + # Tag and re-tag the object, checking that the expected tags are returned and deleted + for tag_list in test_tags: + object_tags = self.taxonomy.tag_object( + tag_list, + "biology101", + "course", + ) + + # Ensure the expected number of tags exist in the database + assert ObjectTag.objects.filter( + taxonomy=self.taxonomy, + object_id="biology101", + object_type="course", + ).count() == len(tag_list) + # And the expected number of tags were returned + assert len(object_tags) == len(tag_list) + for index, object_tag in enumerate(object_tags): + assert object_tag.tag_id == tag_list[index] + assert object_tag.is_valid + assert object_tag.taxonomy == self.taxonomy + assert object_tag.name == self.taxonomy.name + assert object_tag.object_id == "biology101" + assert object_tag.object_type == "course" + + def test_tag_object_free_text(self): + self.taxonomy.allow_free_text = True + object_tags = self.taxonomy.tag_object( + ["Eukaryota Xenomorph"], + "biology101", + "course", + ) + assert len(object_tags) == 1 + object_tag = object_tags[0] + assert object_tag.is_valid + assert object_tag.taxonomy == self.taxonomy + assert object_tag.name == self.taxonomy.name + assert object_tag.tag_ref == "Eukaryota Xenomorph" + assert object_tag.get_lineage() == ["Eukaryota Xenomorph"] + assert object_tag.object_id == "biology101" + assert object_tag.object_type == "course" + + def test_tag_object_no_multiple(self): + with self.assertRaises(ValueError) as exc: + self.taxonomy.tag_object( + ["A", "B"], + "biology101", + "course", + ) + assert "only allows one tag per object" in str(exc.exception) + + def test_tag_object_required(self): + self.taxonomy.required = True + with self.assertRaises(ValueError) as exc: + self.taxonomy.tag_object( + [], + "biology101", + "course", + ) + assert "requires at least one tag per object" in str(exc.exception) + + def test_tag_object_invalid_tag(self): + with self.assertRaises(ValueError) as exc: + self.taxonomy.tag_object( + ["Eukaryota Xenomorph"], + "biology101", + "course", + ) + assert "Invalid object tag for taxonomy" in str(exc.exception) From 92f739cca4f2a64468bd7683efbd15615d7c41df Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Thu, 15 Jun 2023 11:12:37 +0930 Subject: [PATCH 025/282] feat: adds django-rules based permissions for tagging app Also: * Adds rules requirement and app settings to enable it * Adds mock to test requirements, so we can test system taxonomy rules * ADR: Clarifies that rules will be enforced in the views, not the model or APIs --- .../decisions/0009-tagging-administrators.rst | 3 + openedx_tagging/core/tagging/rules.py | 74 ++++++ projects/dev.py | 6 + requirements/base.in | 2 + requirements/base.txt | 2 + requirements/dev.txt | 4 + requirements/doc.txt | 4 + requirements/quality.txt | 4 + requirements/test.in | 1 + requirements/test.txt | 4 + test_settings.py | 6 + .../core/tagging/test_rules.py | 230 ++++++++++++++++++ 12 files changed, 340 insertions(+) create mode 100644 openedx_tagging/core/tagging/rules.py create mode 100644 tests/openedx_tagging/core/tagging/test_rules.py diff --git a/docs/decisions/0009-tagging-administrators.rst b/docs/decisions/0009-tagging-administrators.rst index 652dcd243..33b0ab523 100644 --- a/docs/decisions/0009-tagging-administrators.rst +++ b/docs/decisions/0009-tagging-administrators.rst @@ -26,6 +26,8 @@ But because permissions #2 + #3 require access to the edx-platform CMS model `Co Per `OEP-9`_, ``openedx_tagging`` will allow applications to use the standard Django API to query permissions, for example: ``user.has_perm('openedx_tagging.edit_taxonomy', taxonomy)``, and the appropriate permissions will be applied in that application's context. +These rules will be enforced in the tagging `views`_, not the API or models, so that external code using this library need not have a logged-in user in order to call the API. So please use with care. + Rejected Alternatives --------------------- @@ -37,3 +39,4 @@ This is a standard way to grant access in Django apps, but it is not used in Ope .. _get_organizations: https://github.com/openedx/edx-platform/blob/4dc35c73ffa6d6a1dcb6e9ea1baa5bed40721125/cms/djangoapps/contentstore/views/course.py#L1958 .. _CourseCreator: https://github.com/openedx/edx-platform/blob/4dc35c73ffa6d6a1dcb6e9ea1baa5bed40721125/cms/djangoapps/course_creators/models.py#L27 .. _OEP-9: https://open-edx-proposals.readthedocs.io/en/latest/best-practices/oep-0009-bp-permissions.html +.. _views: https://github.com/dfunckt/django-rules#permissions-in-views diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py new file mode 100644 index 000000000..7ef984b86 --- /dev/null +++ b/openedx_tagging/core/tagging/rules.py @@ -0,0 +1,74 @@ +"""Django rules-based permissions for tagging""" + +import rules + +# Global staff are taxonomy admins. +# (Superusers can already do anything) +is_taxonomy_admin = rules.is_staff + + +@rules.predicate +def can_view_taxonomy(user, taxonomy=None): + """ + Anyone can view an enabled taxonomy, + but only taxonomy admins can view a disabled taxonomy. + """ + return (taxonomy and taxonomy.enabled) or is_taxonomy_admin(user) + + +@rules.predicate +def can_change_taxonomy(user, taxonomy=None): + """ + Even taxonomy admins cannot change system taxonomies. + """ + return is_taxonomy_admin(user) and ( + not taxonomy or not taxonomy or (taxonomy and not taxonomy.system_defined) + ) + + +@rules.predicate +def can_change_taxonomy_tag(user, tag=None): + """ + Even taxonomy admins cannot add tags to system taxonomies (their tags are system-defined), or free-text taxonomies + (these don't have predefined tags). + """ + return is_taxonomy_admin(user) and ( + not tag + or not tag.taxonomy + or ( + tag.taxonomy + and not tag.taxonomy.allow_free_text + and not tag.taxonomy.system_defined + ) + ) + + +@rules.predicate +def can_change_object_tag(user, object_tag=None): + """ + Taxonomy admins can create or modify object tags on enabled taxonomies. + """ + return is_taxonomy_admin(user) and ( + not object_tag + or not object_tag.taxonomy + or (object_tag.taxonomy and object_tag.taxonomy.enabled) + ) + + +# Taxonomy +rules.add_perm("oel_tagging.add_taxonomy", can_change_taxonomy) +rules.add_perm("oel_tagging.change_taxonomy", can_change_taxonomy) +rules.add_perm("oel_tagging.delete_taxonomy", can_change_taxonomy) +rules.add_perm("oel_tagging.view_taxonomy", can_view_taxonomy) + +# Tag +rules.add_perm("oel_tagging.add_tag", can_change_taxonomy_tag) +rules.add_perm("oel_tagging.change_tag", can_change_taxonomy_tag) +rules.add_perm("oel_tagging.delete_tag", is_taxonomy_admin) +rules.add_perm("oel_tagging.view_tag", rules.always_allow) + +# ObjectTag +rules.add_perm("oel_tagging.add_object_tag", can_change_object_tag) +rules.add_perm("oel_tagging.change_object_tag", can_change_object_tag) +rules.add_perm("oel_tagging.delete_object_tag", is_taxonomy_admin) +rules.add_perm("oel_tagging.view_object_tag", rules.always_allow) diff --git a/projects/dev.py b/projects/dev.py index 99903c3f7..a96473d4b 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -41,6 +41,8 @@ # REST API "rest_framework", "openedx_learning.rest_api.apps.RESTAPIConfig", + # django-rules based authorization + 'rules.apps.AutodiscoverRulesConfig', # Tagging Core Apps "openedx_tagging.core.tagging.apps.TaggingConfig", @@ -48,6 +50,10 @@ "debug_toolbar", ) +AUTHENTICATION_BACKENDS = [ + 'rules.permissions.ObjectPermissionBackend', +] + MIDDLEWARE = [ "debug_toolbar.middleware.DebugToolbarMiddleware", "django.middleware.security.SecurityMiddleware", diff --git a/requirements/base.in b/requirements/base.in index aa00dddd7..d17e06397 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -4,3 +4,5 @@ Django<5.0 # Web application framework djangorestframework<4.0 # REST API + +rules<4.0 # Django extension for rules-based authorization checks diff --git a/requirements/base.txt b/requirements/base.txt index 97ec03540..8c0f00b63 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -17,6 +17,8 @@ pytz==2023.3 # via # django # djangorestframework +rules==3.3 + # via -r requirements/base.in sqlparse==0.4.4 # via django typing-extensions==4.6.3 diff --git a/requirements/dev.txt b/requirements/dev.txt index e816277f9..add9cf1de 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -169,6 +169,8 @@ mdurl==0.1.2 # via # -r requirements/quality.txt # markdown-it-py +mock==5.0.2 + # via -r requirements/quality.txt more-itertools==9.1.0 # via # -r requirements/quality.txt @@ -296,6 +298,8 @@ rich==13.4.2 # via # -r requirements/quality.txt # twine +rules==3.3 + # via -r requirements/quality.txt secretstorage==3.3.3 # via # -r requirements/quality.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index b4e6b5d5b..d9a4c3c2a 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -83,6 +83,8 @@ markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 +mock==5.0.2 + # via -r requirements/test.txt mysqlclient==2.1.1 # via -r requirements/test.txt packaging==23.1 @@ -139,6 +141,8 @@ requests==2.31.0 # via sphinx restructuredtext-lint==1.4.0 # via doc8 +rules==3.3 + # via -r requirements/test.txt six==1.16.0 # via bleach snowballstemmer==2.2.0 diff --git a/requirements/quality.txt b/requirements/quality.txt index c70f66a72..c580bd7e1 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -104,6 +104,8 @@ mccabe==0.7.0 # via pylint mdurl==0.1.2 # via markdown-it-py +mock==5.0.2 + # via -r requirements/test.txt more-itertools==9.1.0 # via jaraco-classes mysqlclient==2.1.1 @@ -182,6 +184,8 @@ rfc3986==2.0.0 # via twine rich==13.4.2 # via twine +rules==3.3 + # via -r requirements/test.txt secretstorage==3.3.3 # via keyring six==1.16.0 diff --git a/requirements/test.in b/requirements/test.in index 6b4e096d4..1d521351d 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -13,3 +13,4 @@ pytest-cov # pytest extension for code coverage statistics pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. ddt # supports data driven tests +mock # supports overriding classes and methods in tests diff --git a/requirements/test.txt b/requirements/test.txt index dafcddb06..e64d8190b 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -38,6 +38,8 @@ jinja2==3.1.2 # via code-annotations markupsafe==2.1.3 # via jinja2 +mock==5.0.2 + # via -r requirements/test.in mysqlclient==2.1.1 # via -r requirements/test.in packaging==23.1 @@ -64,6 +66,8 @@ pytz==2023.3 # djangorestframework pyyaml==6.0 # via code-annotations +rules==3.3 + # via -r requirements/base.txt sqlparse==0.4.4 # via # -r requirements/base.txt diff --git a/test_settings.py b/test_settings.py index 46e52fa59..2e41694e4 100644 --- a/test_settings.py +++ b/test_settings.py @@ -35,6 +35,8 @@ def root(*args): # Admin # 'django.contrib.admin', # 'django.contrib.admindocs', + # django-rules based authorization + 'rules.apps.AutodiscoverRulesConfig', # Our own apps "openedx_learning.core.components.apps.ComponentsConfig", "openedx_learning.core.contents.apps.ContentsConfig", @@ -42,6 +44,10 @@ def root(*args): "openedx_tagging.core.tagging.apps.TaggingConfig", ] +AUTHENTICATION_BACKENDS = [ + 'rules.permissions.ObjectPermissionBackend', +] + LOCALE_PATHS = [ root("openedx_learning", "conf", "locale"), ] diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py new file mode 100644 index 000000000..2c2b4e748 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -0,0 +1,230 @@ +"""Tests tagging rules-based permissions""" + +import ddt +from django.contrib.auth import get_user_model +from django.test.testcases import TestCase +from mock import Mock + +from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy + +from .test_models import TestTagTaxonomyMixin + +User = get_user_model() + + +@ddt.ddt +class TestRulesTagging(TestTagTaxonomyMixin, TestCase): + """ + Tests that the expected rules have been applied to the tagging models. + """ + + def setUp(self): + super().setUp() + self.superuser = User.objects.create( + username="superuser", + email="superuser@example.com", + is_superuser=True, + ) + self.staff = User.objects.create( + username="staff", + email="staff@example.com", + is_staff=True, + ) + self.learner = User.objects.create( + username="learner", + email="learner@example.com", + ) + + self.object_tag = ObjectTag( + taxonomy=self.taxonomy, + tag=self.bacteria, + ) + self.object_tag.resync() + self.object_tag.save() + + # Taxonomy + + @ddt.data( + "oel_tagging.add_taxonomy", + "oel_tagging.change_taxonomy", + ) + def test_add_change_taxonomy(self, perm): + """Taxonomy administrators can create or modify any Taxonomy""" + assert self.superuser.has_perm(perm) + assert self.superuser.has_perm(perm, self.taxonomy) + assert self.staff.has_perm(perm) + assert self.staff.has_perm(perm, self.taxonomy) + assert not self.learner.has_perm(perm) + assert not self.learner.has_perm(perm, self.taxonomy) + + @ddt.data( + "oel_tagging.add_taxonomy", + "oel_tagging.change_taxonomy", + "oel_tagging.delete_taxonomy", + ) + def test_system_taxonomy(self, perm): + """Taxonomy administrators cannot edit system taxonomies""" + # TODO: use SystemTaxonomy when available + system_taxonomy = Mock(spec=Taxonomy) + system_taxonomy.system_defined.return_value = True + assert self.superuser.has_perm(perm, system_taxonomy) + assert not self.staff.has_perm(perm, system_taxonomy) + assert not self.learner.has_perm(perm, system_taxonomy) + + @ddt.data( + True, + False, + ) + def test_delete_taxonomy(self, enabled): + """Taxonomy administrators can delete any Taxonomy""" + self.taxonomy.enabled = enabled + assert self.superuser.has_perm("oel_tagging.delete_taxonomy") + assert self.superuser.has_perm("oel_tagging.delete_taxonomy", self.taxonomy) + assert self.staff.has_perm("oel_tagging.delete_taxonomy") + assert self.staff.has_perm("oel_tagging.delete_taxonomy", self.taxonomy) + assert not self.learner.has_perm("oel_tagging.delete_taxonomy") + assert not self.learner.has_perm("oel_tagging.delete_taxonomy", self.taxonomy) + + @ddt.data( + True, + False, + ) + def test_view_taxonomy_enabled(self, enabled): + """Anyone can see enabled taxonomies, but learners cannot see disabled taxonomies""" + self.taxonomy.enabled = enabled + assert self.superuser.has_perm("oel_tagging.view_taxonomy") + assert self.superuser.has_perm("oel_tagging.view_taxonomy", self.taxonomy) + assert self.staff.has_perm("oel_tagging.view_taxonomy") + assert self.staff.has_perm("oel_tagging.view_taxonomy", self.taxonomy) + assert not self.learner.has_perm("oel_tagging.view_taxonomy") + assert ( + self.learner.has_perm("oel_tagging.view_taxonomy", self.taxonomy) == enabled + ) + + # Tag + + @ddt.data( + "oel_tagging.add_tag", + "oel_tagging.change_tag", + ) + def test_add_change_tag(self, perm): + """Taxonomy administrators can modify tags on non-free-text taxonomies""" + assert self.superuser.has_perm(perm) + assert self.superuser.has_perm(perm, self.bacteria) + assert self.staff.has_perm(perm) + assert self.staff.has_perm(perm, self.bacteria) + assert not self.learner.has_perm(perm) + assert not self.learner.has_perm(perm, self.bacteria) + + @ddt.data( + "oel_tagging.add_tag", + "oel_tagging.change_tag", + ) + def test_tag_free_text_taxonomy(self, perm): + """Taxonomy administrators cannot modify tags on a free-text Taxonomy""" + self.taxonomy.allow_free_text = True + self.taxonomy.save() + assert self.superuser.has_perm(perm, self.bacteria) + assert not self.staff.has_perm(perm, self.bacteria) + assert not self.learner.has_perm(perm, self.bacteria) + + @ddt.data( + "oel_tagging.add_tag", + "oel_tagging.change_tag", + "oel_tagging.delete_tag", + ) + def test_tag_no_taxonomy(self, perm): + """Taxonomy administrators can modify any Tag, even those with no Taxonnmy.""" + tag = Tag() + assert self.superuser.has_perm(perm, tag) + assert self.staff.has_perm(perm, tag) + assert not self.learner.has_perm(perm, tag) + + @ddt.data( + True, + False, + ) + def test_delete_tag(self, allow_free_text): + """Taxonomy administrators can delete any Tag, even those associated with a free-text Taxonomy.""" + self.taxonomy.allow_free_text = allow_free_text + self.taxonomy.save() + assert self.superuser.has_perm("oel_tagging.delete_tag") + assert self.superuser.has_perm("oel_tagging.delete_tag", self.bacteria) + assert self.staff.has_perm("oel_tagging.delete_tag") + assert self.staff.has_perm("oel_tagging.delete_tag", self.bacteria) + assert not self.learner.has_perm("oel_tagging.delete_tag") + assert not self.learner.has_perm("oel_tagging.delete_tag", self.bacteria) + + def test_view_tag(self): + """Anyone can view any Tag""" + assert self.superuser.has_perm("oel_tagging.view_tag") + assert self.superuser.has_perm("oel_tagging.view_tag", self.bacteria) + assert self.staff.has_perm("oel_tagging.view_tag") + assert self.staff.has_perm("oel_tagging.view_tag", self.bacteria) + assert self.learner.has_perm("oel_tagging.view_tag") + assert self.learner.has_perm("oel_tagging.view_tag", self.bacteria) + + # ObjectTag + + @ddt.data( + "oel_tagging.add_object_tag", + "oel_tagging.change_object_tag", + ) + def test_add_change_object_tag(self, perm): + """Taxonomy administrators can create/edit an ObjectTag with an enabled Taxonomy""" + assert self.superuser.has_perm(perm) + assert self.superuser.has_perm(perm, self.object_tag) + assert self.staff.has_perm(perm) + assert self.staff.has_perm(perm, self.object_tag) + assert not self.learner.has_perm(perm) + assert not self.learner.has_perm(perm, self.object_tag) + + @ddt.data( + "oel_tagging.add_object_tag", + "oel_tagging.change_object_tag", + ) + def test_object_tag_disabled_taxonomy(self, perm): + """Taxonomy administrators cannot create/edit an ObjectTag with a disabled Taxonomy""" + self.taxonomy.enabled = False + self.taxonomy.save() + assert self.superuser.has_perm(perm, self.object_tag) + assert not self.staff.has_perm(perm, self.object_tag) + assert not self.learner.has_perm(perm, self.object_tag) + + @ddt.data( + True, + False, + ) + def test_delete_object_tag(self, enabled): + """Taxonomy administrators can delete any ObjectTag, even those associated with a disabled Taxonomy.""" + self.taxonomy.enabled = enabled + self.taxonomy.save() + assert self.superuser.has_perm("oel_tagging.delete_object_tag") + assert self.superuser.has_perm("oel_tagging.delete_object_tag", self.object_tag) + assert self.staff.has_perm("oel_tagging.delete_object_tag") + assert self.staff.has_perm("oel_tagging.delete_object_tag", self.object_tag) + assert not self.learner.has_perm("oel_tagging.delete_object_tag") + assert not self.learner.has_perm( + "oel_tagging.delete_object_tag", self.object_tag + ) + + @ddt.data( + "oel_tagging.add_object_tag", + "oel_tagging.change_object_tag", + "oel_tagging.delete_object_tag", + ) + def test_object_tag_no_taxonomy(self, perm): + """Taxonomy administrators can modify an ObjectTag with no Taxonomy""" + object_tag = ObjectTag() + assert self.superuser.has_perm(perm, object_tag) + assert self.staff.has_perm(perm, object_tag) + assert not self.learner.has_perm(perm, object_tag) + + def test_view_object_tag(self): + """Anyone can view any ObjectTag""" + assert self.superuser.has_perm("oel_tagging.view_object_tag") + assert self.superuser.has_perm("oel_tagging.view_object_tag", self.object_tag) + assert self.staff.has_perm("oel_tagging.view_object_tag") + assert self.staff.has_perm("oel_tagging.view_object_tag", self.object_tag) + assert self.learner.has_perm("oel_tagging.view_object_tag") + assert self.learner.has_perm("oel_tagging.view_object_tag", self.object_tag) From 821a53c38da47addec3135d32ac1ce70544c5263 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Tue, 27 Jun 2023 07:19:32 +0930 Subject: [PATCH 026/282] fix: remove db_collation check Tests are failing in openedx on sqlite. --- openedx_learning/lib/collations.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/openedx_learning/lib/collations.py b/openedx_learning/lib/collations.py index 2afdcc377..561b1f316 100644 --- a/openedx_learning/lib/collations.py +++ b/openedx_learning/lib/collations.py @@ -32,11 +32,6 @@ def __init__(self, *args, db_collations=None, db_collation=None, **kwargs): it for Django 3.2 compatibility (see the ``db_collation`` method docstring for details). """ - if db_collation is not None: - raise ValueError( - f"Cannot use db_collation with {self.__class__.__name__}. " - + "Please use a db_collations dict instead." - ) super().__init__(*args, **kwargs) self.db_collations = db_collations or {} @@ -106,7 +101,7 @@ def db_parameters(self, connection): def deconstruct(self): """ How to serialize our Field for the migration file. - + For our mixin fields, this is just doing what the field's superclass would do and then tacking on our custom ``db_collations`` dict data. """ From a5b109f58dfe27c63033240c6b9458887014f517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 3 Jul 2023 22:30:02 -0300 Subject: [PATCH 027/282] feat: update automatic tagging decision (#61) Using openedx-events instead of openedx-filters. --- .../0013-system-taxonomy-auto-tagging.rst | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/docs/decisions/0013-system-taxonomy-auto-tagging.rst b/docs/decisions/0013-system-taxonomy-auto-tagging.rst index 74b36bc1f..ef506aaf2 100644 --- a/docs/decisions/0013-system-taxonomy-auto-tagging.rst +++ b/docs/decisions/0013-system-taxonomy-auto-tagging.rst @@ -4,25 +4,23 @@ Context -------- -One main characteristic of system-defined taxonomies is automatic content tagging. +One main characteristic of system-defined taxonomies is automatic content tagging. It is necessary to implement this functionality when the associated content is created or edited. Decision --------- -Use `openedx-filters`_ to call the auto tagging function after content creation/edition. - -Filters +Events ~~~~~~~~ -It is necessary to create `filters`_ for each content for creation/edition: ``CourseCreation``, ``LibraryCreation``, etc. -This filters will live on ``openedx-filters``. +It is necessary to create `events`_ for each content for creation/edition: ``CourseCreation``, ``LibraryCreation``, etc. +This events will live in `openedx-events`_. -Pipelines +Receivers ~~~~~~~~~~ -Auto-tagging pipelines will live under ``openedx.features.tagging``, -registered as `a pipeline`_ with the respective filter. +Auto-tagging receivers will live under ``openedx.features.tagging``, +registered as `a receiver`_ with the respective code. Rejected Options ----------------- @@ -31,10 +29,23 @@ Django Signals ~~~~~~~~~~~~~~ Implement a function to add the tag from the content metadata and register that function -as a Django signal. This works for Django database models, but some of the content lives in Mongo, +as a Django signal. This works for Django database models, but some of the content lives in Mongo, outside of the Django models. Also, using openedx-filters is better in the edx context, but if there is other no-edX project that need to use ``openedx-tagging``, can use the Django Signals approach. + +openedx-filters +~~~~~~~~~~~~~~~ +Use `openedx-filters`_ to create a `filter`_ which calls `the auto tagging pipeline`_ after content +creation/editing. Although this approach works, there are more suitable options. Filters are +used to act on the input data and provide means to block the flow. This is not necessary in the +auto tagging context. The `hooks documentation`_ suggests the use of `events`_ hooks to expand functionality. + + +.. _openedx-events: https://github.com/openedx/openedx-events .. _openedx-filters: https://github.com/openedx/openedx-filters -.. _filters: https://github.com/openedx/openedx-filters/blob/a4a192e1cac0b70bed31e0db8e4c4b058848c5c4/openedx_filters/learning/filters.py -.. _a pipeline: https://github.com/openedx/edx-platform/blob/40613ae3f47eb470aff87359a952ed7e79ad8555/docs/guides/hooks/filters.rst#implement-pipeline-steps +.. _filter: https://github.com/openedx/openedx-filters/blob/a4a192e1cac0b70bed31e0db8e4c4b058848c5c4/openedx_filters/learning/filters.py +.. _the auto tagging pipeline: https://github.com/openedx/edx-platform/blob/40613ae3f47eb470aff87359a952ed7e79ad8555/docs/guides/hooks/filters.rst#implement-pipeline-steps +.. _hooks documentation: https://github.com/openedx/edx-platform/blob/master/docs/guides/hooks/index.rst +.. _events: https://github.com/openedx/edx-platform/blob/master/docs/guides/hooks/events.rst +.. _a receiver: https://github.com/openedx/edx-platform/blob/master/docs/guides/hooks/events.rst#receiving-events From f5164bb030fa30c400b5999d0097f8b342ab7836 Mon Sep 17 00:00:00 2001 From: Jillian Date: Thu, 20 Jul 2023 09:45:30 +0930 Subject: [PATCH 028/282] Allow custom Taxonomy, ObjectTag subclasses to customize tagging behavior (#62) Adds support for custom Taxonomy subclasses to be stored against a Taxonomy, to be used by the python API when instantiating and returning Taxonomies. Also adds minimal support for ObjectTag subclasses. However, these are not stored against the ObjectTag instances; they can be instantiated by the Taxonomy subclasses if and when needed. Related: * docs: updates decisions to reflect this change * feat: adds api.get_taxonomy, which returns a Taxonomy cast to its subclass, when set * refactor: adds _check_taxonomy, _check_tag, and _check_object methods to the Taxonomy class, which can be overridden by subclasses when validating ObjectTags Added to support system-defined Taxonomies: * feat: adds un-editable Taxonomy.system_defined field so that system taxonomies can store this field and ensure no one edits them. * feat: adds Taxonomy.visible_to_authors, which is needed for fully automated tagging. Cleanup changes: * fix: updates Tag model to cascade delete if the Taxonomy or parent Tag is deleted. * style: adds missing type annotations to rules and python API --- docs/decisions/0007-tagging-app.rst | 7 +- .../decisions/0009-tagging-administrators.rst | 2 +- .../0012-system-taxonomy-creation.rst | 17 +- openedx_tagging/core/tagging/api.py | 63 +++-- .../migrations/0002_auto_20230718_2026.py | 79 ++++++ openedx_tagging/core/tagging/models.py | 257 +++++++++++++++--- openedx_tagging/core/tagging/rules.py | 14 +- .../core/fixtures/tagging.yaml | 11 + .../openedx_tagging/core/tagging/test_api.py | 225 ++++++++++++--- .../core/tagging/test_models.py | 185 +++++++++---- .../core/tagging/test_rules.py | 16 +- 11 files changed, 695 insertions(+), 181 deletions(-) create mode 100644 openedx_tagging/core/tagging/migrations/0002_auto_20230718_2026.py diff --git a/docs/decisions/0007-tagging-app.rst b/docs/decisions/0007-tagging-app.rst index 7dbc45b0b..9993a49bb 100644 --- a/docs/decisions/0007-tagging-app.rst +++ b/docs/decisions/0007-tagging-app.rst @@ -19,7 +19,7 @@ Taxonomy The ``openedx_tagging`` module defines ``openedx_tagging.core.models.Taxonomy``, whose data and functionality are self-contained to the ``openedx_tagging`` app. However in Studio, we need to be able to limit access to some Taxonomy by organization, using the same "course creator" access which limits course creation for an organization to a defined set of users. -So in edx-platform, we will create the ``openedx.features.tagging`` app, to contain ``models.OrgTaxonomy``. OrgTaxonomy subclasses ``openedx_tagging.core.models.Taxonomy``, employing Django's `multi-table inheritance`_ feature, which allows the base Tag class to keep foreign keys to the Taxonomy, while allowing OrgTaxonomy to store foreign keys into Studio's Organization table. +So in edx-platform, we will create the ``openedx.features.content_tagging`` app, to contain the models and logic for linking Organization owners to Taxonomies. Here, we can subclass ``Taxonomy`` as needed, preferably using proxy models. The APIs are responsible for ensuring that any ``Taxonomy`` instances are cast to the appropriate subclass. ObjectTag ~~~~~~~~~ @@ -27,7 +27,7 @@ ObjectTag Similarly, the ``openedx_tagging`` module defined ``openedx_tagging.core.models.ObjectTag``, also self-contained to the ``openedx_tagging`` app. -But to tag content in the LMS/Studio, we create ``openedx.features.tagging.models.ContentTag``, which subclasses ``ObjectTag``, and can then reference functionality available in the platform code. +But to tag content in the LMS/Studio, we need to enforce ``object_id`` as a CourseKey or UsageKey type. So to do this, we subclass ``ObjectTag``, and use this class when creating content object tags. Once the ``object_id`` is set, it is not editable, and so this key validation need not happen again. Rejected Alternatives --------------------- @@ -38,6 +38,3 @@ Embed in edx-platform Embedding the logic in edx-platform would provide the content tagging logic specifically required for the MVP. However, we plan to extend tagging to other object types (e.g. People) and contexts (e.g. Marketing), and so a generic, standalone library is preferable in the log run. - - -.. _multi-table inheritance: https://docs.djangoproject.com/en/3.2/topics/db/models/#multi-table-inheritance diff --git a/docs/decisions/0009-tagging-administrators.rst b/docs/decisions/0009-tagging-administrators.rst index 33b0ab523..cd5284232 100644 --- a/docs/decisions/0009-tagging-administrators.rst +++ b/docs/decisions/0009-tagging-administrators.rst @@ -22,7 +22,7 @@ In the Studio context, a modified version of "course creator" access will be use Permission #1 requires no external access, so can be enforced by the ``openedx_tagging`` app. -But because permissions #2 + #3 require access to the edx-platform CMS model `CourseCreator`_, this access can only be enforced in Studio, and so will live under `cms.djangoapps.tagging` along with the ``ContentTag`` class. Tagging MVP must work for libraries v1, v2 and courses created in Studio, and so tying these permissions to Studio is reasonable for the MVP. +But because permissions #2 + #3 require access to the edx-platform CMS model `CourseCreator`_, this access can only be enforced in Studio, and so will live under ``cms.djangoapps.content_tagging`` along with the ``ContentTag`` class. Tagging MVP must work for libraries v1, v2 and courses created in Studio, and so tying these permissions to Studio is reasonable for the MVP. Per `OEP-9`_, ``openedx_tagging`` will allow applications to use the standard Django API to query permissions, for example: ``user.has_perm('openedx_tagging.edit_taxonomy', taxonomy)``, and the appropriate permissions will be applied in that application's context. diff --git a/docs/decisions/0012-system-taxonomy-creation.rst b/docs/decisions/0012-system-taxonomy-creation.rst index fe67b6e34..2ed17886f 100644 --- a/docs/decisions/0012-system-taxonomy-creation.rst +++ b/docs/decisions/0012-system-taxonomy-creation.rst @@ -4,7 +4,7 @@ Context -------- -System-defined taxonomies are closed taxonomies created by the system. Some of these are totally static (e.g Language) +System-defined taxonomies are taxonomies created by the system. Some of these are totally static (e.g Language) and some depends on a core data model (e.g. Organizations). It is necessary to define how to create and validate the System-defined taxonomies and their tags. @@ -12,17 +12,15 @@ the System-defined taxonomies and their tags. Decision --------- -System-defined Taxonomy creation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +System Tag lists and validation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Each System-defined Taxonomy has its own class, which is used for tag validation (e.g. ``LanguageSystemTaxonomy``, ``OrganizationSystemTaxonomy``). -Each can overwrite ``get_tags``; to configure the valid tags, and ``validate_object_tag``; to check if a list of tags are valid. -Both functions are implemented on the ``Taxonomy`` base class, but can be overwritten to handle special cases. +Each System-defined Taxonomy will have its own ``ObjectTag`` subclass which is used for tag validation (e.g. ``LanguageObjectTag``, ``OrganizationObjectTag``). +Each subclass can overwrite ``get_tags``; to configure the valid tags, and ``is_valid``; to check if a list of tags are valid. Both functions are implemented on the ``ObjectTag`` base class, but can be overwritten to handle special cases. -We need to create an instance of each System-defined Taxonomy in a fixture. This instances will be used on different APIs. +We need to create an instance of each System-defined Taxonomy's ObjectTag in a fixture. This instances will be used on different APIs. -Later, we need to create a ``Content-side`` class that lives on ``openedx.features.tagging`` for each content and taxonomy to be used -(eg. ``CourseLanguageSystemTaxonomy``, ``CourseOrganizationSystemTaxonomy``). +Later, we need to create content-side ObjectTags that live on ``openedx.features.content_tagging`` for each content and taxonomy to be used (eg. ``CourseLanguageObjectTag``, ``CourseOrganizationObjectTag``). This new class is used to configure the automatic content tagging. You can read the `document number 0013`_ to see this configuration. Tags creation @@ -54,5 +52,4 @@ And if it's a large list of objects (e.g. Users), then copying that list into th It is better to dynamically generate the list of available Tags, and/or dynamically validate a submitted object tag than to store the options in the database. - .. _document number 0013: https://github.com/openedx/openedx-learning/blob/main/docs/decisions/0013-system-taxonomy-auto-tagging.rst diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 6b9bf2a04..5ca23c485 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -10,7 +10,7 @@ Please look at the models.py file for more information about the kinds of data are stored in this app. """ -from typing import List, Type +from typing import Iterator, List, Type, Union from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ @@ -19,17 +19,18 @@ def create_taxonomy( - name, - description=None, + name: str, + description: str = None, enabled=True, required=False, allow_multiple=False, allow_free_text=False, + taxonomy_class: Type = None, ) -> Taxonomy: """ Creates, saves, and returns a new Taxonomy with the given attributes. """ - return Taxonomy.objects.create( + taxonomy = Taxonomy( name=name, description=description, enabled=enabled, @@ -37,11 +38,27 @@ def create_taxonomy( allow_multiple=allow_multiple, allow_free_text=allow_free_text, ) + if taxonomy_class: + taxonomy.taxonomy_class = taxonomy_class + taxonomy.save() + return taxonomy.cast() + + +def get_taxonomy(id: int) -> Union[Taxonomy, None]: + """ + Returns a Taxonomy cast to the appropriate subclass which has the given ID. + """ + taxonomy = Taxonomy.objects.filter(id=id).first() + return taxonomy.cast() if taxonomy else None def get_taxonomies(enabled=True) -> QuerySet: """ Returns a queryset containing the enabled taxonomies, sorted by name. + + We return a QuerySet here for ease of use with Django Rest Framework and other query-based use cases. + So be sure to use `Taxonomy.cast()` to cast these instances to the appropriate subclass before use. + If you want the disabled taxonomies, pass enabled=False. If you want all taxonomies (both enabled and disabled), pass enabled=None. """ @@ -57,7 +74,7 @@ def get_tags(taxonomy: Taxonomy) -> List[Tag]: Note that if the taxonomy allows free-text tags, then the returned list will be empty. """ - return taxonomy.get_tags() + return taxonomy.cast().get_tags() def resync_object_tags(object_tags: QuerySet = None) -> int: @@ -67,7 +84,7 @@ def resync_object_tags(object_tags: QuerySet = None) -> int: By default, we iterate over all ObjectTags. Pass a filtered ObjectTags queryset to limit which tags are resynced. """ if not object_tags: - object_tags = ObjectTag.objects.all() + object_tags = ObjectTag.objects.select_related("tag", "taxonomy") num_changed = 0 for object_tag in object_tags: @@ -79,22 +96,35 @@ def resync_object_tags(object_tags: QuerySet = None) -> int: def get_object_tags( - taxonomy: Taxonomy, object_id: str, object_type: str, valid_only=True -) -> List[ObjectTag]: + object_id: str, taxonomy: Taxonomy = None, valid_only=True +) -> Iterator[ObjectTag]: """ - Returns a list of tags for a given taxonomy + content. + Generates a list of object tags for a given object. + + Pass taxonomy to limit the returned object_tags to a specific taxonomy. Pass valid_only=False when displaying tags to content authors, so they can see invalid tags too. - Invalid tags will likely be hidden from learners. + Invalid tags will (probably) be hidden from learners. """ - tags = ObjectTag.objects.filter( - taxonomy=taxonomy, object_id=object_id, object_type=object_type - ).order_by("id") - return [tag for tag in tags if not valid_only or taxonomy.validate_object_tag(tag)] + tags = ( + ObjectTag.objects.filter( + object_id=object_id, + ) + .select_related("tag", "taxonomy") + .order_by("id") + ) + if taxonomy: + tags = tags.filter(taxonomy=taxonomy) + + for object_tag in tags: + if not valid_only or object_tag.is_valid(): + yield object_tag def tag_object( - taxonomy: Taxonomy, tags: List, object_id: str, object_type: str + taxonomy: Taxonomy, + tags: List, + object_id: str, ) -> List[ObjectTag]: """ Replaces the existing ObjectTag entries for the given taxonomy + object_id with the given list of tags. @@ -105,5 +135,4 @@ def tag_object( Raised ValueError if the proposed tags are invalid for this taxonomy. Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. """ - - return taxonomy.tag_object(tags, object_id, object_type) + return taxonomy.cast().tag_object(tags, object_id) diff --git a/openedx_tagging/core/tagging/migrations/0002_auto_20230718_2026.py b/openedx_tagging/core/tagging/migrations/0002_auto_20230718_2026.py new file mode 100644 index 000000000..d0d14c938 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0002_auto_20230718_2026.py @@ -0,0 +1,79 @@ +# Generated by Django 3.2.19 on 2023-07-18 05:54 + +import django.db.models.deletion +from django.db import migrations, models + +import openedx_learning.lib.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0001_initial"), + ] + + operations = [ + migrations.AddField( + model_name="taxonomy", + name="system_defined", + field=models.BooleanField( + default=False, + editable=False, + help_text="Indicates that tags and metadata for this taxonomy are maintained by the system; taxonomy admins will not be permitted to modify them.", + ), + ), + migrations.AlterField( + model_name="tag", + name="parent", + field=models.ForeignKey( + default=None, + help_text="Tag that lives one level up from the current tag, forming a hierarchy.", + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="children", + to="oel_tagging.tag", + ), + ), + migrations.AlterField( + model_name="tag", + name="taxonomy", + field=models.ForeignKey( + default=None, + help_text="Namespace and rules for using a given set of tags.", + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="oel_tagging.taxonomy", + ), + ), + migrations.AddField( + model_name="taxonomy", + name="visible_to_authors", + field=models.BooleanField( + default=True, + editable=False, + help_text="Indicates whether this taxonomy should be visible to object authors.", + ), + ), + migrations.RemoveField( + model_name="objecttag", + name="object_type", + ), + migrations.AddField( + model_name="taxonomy", + name="_taxonomy_class", + field=models.CharField( + help_text="Taxonomy subclass used to instantiate this instance; must be a fully-qualified module and class name. If the module/class cannot be imported, an error is logged and the base Taxonomy class is used instead.", + max_length=255, + null=True, + ), + ), + migrations.AlterField( + model_name="objecttag", + name="object_id", + field=openedx_learning.lib.fields.MultiCollationCharField( + db_collations={"mysql": "utf8mb4_unicode_ci", "sqlite": "NOCASE"}, + editable=False, + help_text="Identifier for the object being tagged", + max_length=255, + ), + ), + ] diff --git a/openedx_tagging/core/tagging/models.py b/openedx_tagging/core/tagging/models.py index c90f8a34a..3c8222bcd 100644 --- a/openedx_tagging/core/tagging/models.py +++ b/openedx_tagging/core/tagging/models.py @@ -1,11 +1,15 @@ """ Tagging app data models """ -from typing import List, Type +import logging +from typing import List, Type, Union from django.db import models +from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from openedx_learning.lib.fields import MultiCollationTextField, case_insensitive_char_field +log = logging.getLogger(__name__) + # Maximum depth allowed for a hierarchical taxonomy's tree of tags. TAXONOMY_MAX_DEPTH = 3 @@ -29,14 +33,14 @@ class Tag(models.Model): "Taxonomy", null=True, default=None, - on_delete=models.SET_NULL, + on_delete=models.CASCADE, help_text=_("Namespace and rules for using a given set of tags."), ) parent = models.ForeignKey( "self", null=True, default=None, - on_delete=models.SET_NULL, + on_delete=models.CASCADE, related_name="children", help_text=_( "Tag that lives one level up from the current tag, forming a hierarchy." @@ -73,7 +77,7 @@ def __str__(self): """ User-facing string representation of a Tag. """ - return f"Tag ({self.id}) {self.value}" + return f"<{self.__class__.__name__}> ({self.id}) {self.value}" def get_lineage(self) -> Lineage: """ @@ -95,7 +99,7 @@ def get_lineage(self) -> Lineage: class Taxonomy(models.Model): """ - Represents a namespace and rules for a group of tags which can be applied to a particular Open edX object. + Represents a namespace and rules for a group of tags. """ id = models.BigAutoField(primary_key=True) @@ -136,20 +140,109 @@ class Taxonomy(models.Model): "Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values." ), ) + system_defined = models.BooleanField( + default=False, + editable=False, + help_text=_( + "Indicates that tags and metadata for this taxonomy are maintained by the system;" + " taxonomy admins will not be permitted to modify them.", + ), + ) + visible_to_authors = models.BooleanField( + default=True, + editable=False, + help_text=_( + "Indicates whether this taxonomy should be visible to object authors." + ), + ) + _taxonomy_class = models.CharField( + null=True, + max_length=255, + help_text=_( + "Taxonomy subclass used to instantiate this instance; must be a fully-qualified module and class name." + " If the module/class cannot be imported, an error is logged and the base Taxonomy class is used instead." + ), + ) class Meta: verbose_name_plural = "Taxonomies" + def __repr__(self): + """ + Developer-facing representation of a Taxonomy. + """ + return str(self) + + def __str__(self): + """ + User-facing string representation of a Taxonomy. + """ + return f"<{self.__class__.__name__}> ({self.id}) {self.name}" + @property - def system_defined(self) -> bool: + def taxonomy_class(self) -> Type: + """ + Returns the Taxonomy subclass associated with this instance, or None if none supplied. + + May raise ImportError if a custom taxonomy_class cannot be imported. + """ + if self._taxonomy_class: + return import_string(self._taxonomy_class) + return None + + @taxonomy_class.setter + def taxonomy_class(self, taxonomy_class: Union[Type, None]): + """ + Assigns the given taxonomy_class's module path.class to the field. + + Must be a subclass of Taxonomy, or raises a ValueError. + """ + if taxonomy_class: + if not issubclass(taxonomy_class, Taxonomy): + raise ValueError( + f"Unable to assign taxonomy_class for {self}: {taxonomy_class} must be a subclass of Taxonomy" + ) + + # ref: https://stackoverflow.com/a/2020083 + self._taxonomy_class = ".".join( + [taxonomy_class.__module__, taxonomy_class.__qualname__] + ) + else: + self._taxonomy_class = None + + def cast(self): """ - Base taxonomies are user-defined, not system-defined. + Returns the current Taxonomy instance cast into its taxonomy_class. + + If no taxonomy_class is set, or if we're unable to import it, then just returns self. + """ + try: + TaxonomyClass = self.taxonomy_class + if TaxonomyClass and not isinstance(self, TaxonomyClass): + return TaxonomyClass().copy(self) + except ImportError: + # Log error and continue + log.exception( + f"Unable to import taxonomy_class for {self}: {self._taxonomy_class}" + ) - System-defined taxonomies cannot be edited by ordinary users. + return self - Subclasses should override this property as required. + def copy(self, taxonomy: "Taxonomy") -> "Taxonomy": """ - return False + Copy the fields from the given Taxonomy into the current instance. + """ + self.id = taxonomy.id + self.name = taxonomy.name + self.description = taxonomy.description + self.enabled = taxonomy.enabled + self.required = taxonomy.required + self.allow_multiple = taxonomy.allow_multiple + self.allow_free_text = taxonomy.allow_free_text + self.system_defined = taxonomy.system_defined + self.visible_to_authors = taxonomy.visible_to_authors + self._taxonomy_class = taxonomy._taxonomy_class + return self def get_tags(self) -> List[Tag]: """ @@ -195,38 +288,72 @@ def validate_object_tag( """ Returns True if the given object tag is valid for the current Taxonomy. - Subclasses can override this method to perform their own validation checks, e.g. against dynamically generated - tag lists. + Subclasses should override the internal _validate* methods to perform their own validation checks, e.g. against + dynamically generated tag lists. If `check_taxonomy` is False, then we skip validating the object tag's taxonomy reference. If `check_tag` is False, then we skip validating the object tag's tag reference. If `check_object` is False, then we skip validating the object ID/type. """ - # Must be linked to this taxonomy - if check_taxonomy and ( - not object_tag.taxonomy_id or object_tag.taxonomy_id != self.id - ): + if check_taxonomy and not self._check_taxonomy(object_tag): return False - # Must be linked to a Tag unless its a free-text taxonomy - if check_tag and (not self.allow_free_text and not object_tag.tag_id): + if check_tag and not self._check_tag(object_tag): return False - # Must have a valid object id/type: - if check_object and (not object_tag.object_id or not object_tag.object_type): + if check_object and not self._check_object(object_tag): return False return True + def _check_taxonomy( + self, + object_tag: "ObjectTag", + ) -> bool: + """ + Returns True if the given object tag is valid for the current Taxonomy. + + Subclasses can override this method to perform their own taxonomy validation checks. + """ + # Must be linked to this taxonomy + return object_tag.taxonomy_id and object_tag.taxonomy_id == self.id + + def _check_tag( + self, + object_tag: "ObjectTag", + ) -> bool: + """ + Returns True if the given object tag's value is valid for the current Taxonomy. + + Subclasses can override this method to perform their own taxonomy validation checks. + """ + # Open taxonomies only need a value. + if self.allow_free_text: + return bool(object_tag.value) + + # Closed taxonomies need an associated tag in this taxonomy + return object_tag.tag_id and object_tag.tag.taxonomy_id == self.id + + def _check_object( + self, + object_tag: "ObjectTag", + ) -> bool: + """ + Returns True if the given object tag's object is valid for the current Taxonomy. + + Subclasses can override this method to perform their own taxonomy validation checks. + """ + return bool(object_tag.object_id) + def tag_object( - self, tags: List, object_id: str, object_type: str + self, + tags: List, + object_id: str, ) -> List["ObjectTag"]: """ Replaces the existing ObjectTag entries for the current taxonomy + object_id with the given list of tags. - If self.allows_free_text, then the list should be a list of tag values. Otherwise, it should be a list of existing Tag IDs. - Raised ValueError if the proposed tags are invalid for this taxonomy. Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. """ @@ -242,7 +369,8 @@ def tag_object( current_tags = { tag.tag_ref: tag for tag in ObjectTag.objects.filter( - taxonomy=self, object_id=object_id, object_type=object_type + taxonomy=self, + object_id=object_id, ) } updated_tags = [] @@ -253,7 +381,6 @@ def tag_object( object_tag = ObjectTag( taxonomy=self, object_id=object_id, - object_type=object_type, ) try: @@ -304,12 +431,9 @@ class ObjectTag(models.Model): id = models.BigAutoField(primary_key=True) object_id = case_insensitive_char_field( max_length=255, + editable=False, help_text=_("Identifier for the object being tagged"), ) - object_type = case_insensitive_char_field( - max_length=255, - help_text=_("Type of object being tagged"), - ) taxonomy = models.ForeignKey( Taxonomy, null=True, @@ -351,6 +475,18 @@ class Meta: models.Index(fields=["taxonomy", "_value"]), ] + def __repr__(self): + """ + Developer-facing representation of an ObjectTag. + """ + return str(self) + + def __str__(self): + """ + User-facing string representation of an ObjectTag. + """ + return f"<{self.__class__.__name__}> {self.object_id}: {self.name}={self.value}" + @property def name(self) -> str: """ @@ -395,7 +531,6 @@ def tag_ref(self) -> str: """ return self.tag.id if self.tag_id else self._value - @property def is_valid(self) -> bool: """ Returns True if this ObjectTag represents a valid taxonomy tag. @@ -426,25 +561,42 @@ def resync(self) -> bool: """ changed = False - # Locate a taxonomy matching _name + # Locate an enabled taxonomy matching _name, and maybe a tag matching _value if not self.taxonomy_id: - for taxonomy in Taxonomy.objects.filter(name=self.name, enabled=True): - # Make sure this taxonomy will accept object tags like this. - self.taxonomy = taxonomy - if taxonomy.validate_object_tag(self, check_tag=False): - changed = True - break - # If not, try the next one - else: - self.taxonomy = None + # Use the linked tag's taxonomy if there is one. + if self.tag_id: + self.taxonomy_id = self.tag.taxonomy_id + changed = True + else: + for taxonomy in Taxonomy.objects.filter( + name=self.name, enabled=True + ).order_by("allow_free_text", "id"): + # Cast to the subclass to preserve custom validation + taxonomy = taxonomy.cast() + + # Closed taxonomies require a tag matching _value, + # and we'd rather match a closed taxonomy than an open one. + # So see if there's a matching tag available in this taxonomy. + tag = taxonomy.tag_set.filter(value=self.value).first() + + # Make sure this taxonomy will accept object tags like this. + self.taxonomy = taxonomy + self.tag = tag + if taxonomy.validate_object_tag(self): + changed = True + break + # If not, undo those changes and try the next one + else: + self.taxonomy = None + self.tag = None # Sync the stored _name with the taxonomy.name - elif self._name != self.taxonomy.name: + if self.taxonomy_id and self._name != self.taxonomy.name: self.name = self.taxonomy.name changed = True - # Locate a tag matching _value - if self.taxonomy and not self.tag_id and not self.taxonomy.allow_free_text: + # Closed taxonomies require a tag matching _value + if self.taxonomy and not self.taxonomy.allow_free_text and not self.tag_id: tag = self.taxonomy.tag_set.filter(value=self.value).first() if tag: self.tag = tag @@ -456,3 +608,22 @@ def resync(self) -> bool: changed = True return changed + + @classmethod + def cast(cls, object_tag: "ObjectTag") -> "ObjectTag": + """ + Returns a cls instance with the same properties as the given ObjectTag. + """ + return cls().copy(object_tag) + + def copy(self, object_tag: "ObjectTag") -> "ObjectTag": + """ + Copy the fields from the given ObjectTag into the current instance. + """ + self.id = object_tag.id + self.tag = object_tag.tag + self.taxonomy = object_tag.taxonomy + self.object_id = object_tag.object_id + self._value = object_tag._value + self._name = object_tag._name + return self diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 7ef984b86..72b9fa6b4 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -1,6 +1,12 @@ """Django rules-based permissions for tagging""" import rules +from django.contrib.auth import get_user_model + +from .models import ObjectTag, Tag, Taxonomy + +User = get_user_model() + # Global staff are taxonomy admins. # (Superusers can already do anything) @@ -8,7 +14,7 @@ @rules.predicate -def can_view_taxonomy(user, taxonomy=None): +def can_view_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: """ Anyone can view an enabled taxonomy, but only taxonomy admins can view a disabled taxonomy. @@ -17,7 +23,7 @@ def can_view_taxonomy(user, taxonomy=None): @rules.predicate -def can_change_taxonomy(user, taxonomy=None): +def can_change_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: """ Even taxonomy admins cannot change system taxonomies. """ @@ -27,7 +33,7 @@ def can_change_taxonomy(user, taxonomy=None): @rules.predicate -def can_change_taxonomy_tag(user, tag=None): +def can_change_taxonomy_tag(user: User, tag: Tag = None) -> bool: """ Even taxonomy admins cannot add tags to system taxonomies (their tags are system-defined), or free-text taxonomies (these don't have predefined tags). @@ -44,7 +50,7 @@ def can_change_taxonomy_tag(user, tag=None): @rules.predicate -def can_change_object_tag(user, object_tag=None): +def can_change_object_tag(user: User, object_tag: ObjectTag = None) -> bool: """ Taxonomy admins can create or modify object tags on enabled taxonomies. """ diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index 50b9459bb..a56ece81a 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -154,3 +154,14 @@ required: false allow_multiple: false allow_free_text: false + system_defined: false +- model: oel_tagging.taxonomy + pk: 2 + fields: + name: System Languages + description: Allows tags for any language configured for use on the instance. + enabled: true + required: false + allow_multiple: false + allow_free_text: false + system_defined: true diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 7a418e687..798b4826b 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -1,7 +1,5 @@ """ Test the tagging APIs """ -from unittest.mock import patch - from django.test.testcases import TestCase import openedx_tagging.core.tagging.api as tagging_api @@ -27,18 +25,41 @@ def test_create_taxonomy(self): taxonomy = tagging_api.create_taxonomy(**params) for param, value in params.items(): assert getattr(taxonomy, param) == value + assert not taxonomy.system_defined + assert taxonomy.visible_to_authors + + def test_bad_taxonomy_class(self): + with self.assertRaises(ValueError) as exc: + tagging_api.create_taxonomy( + name="Bad class", + taxonomy_class=str, + ) + assert " must be a subclass of Taxonomy" in str(exc.exception) + + def test_get_taxonomy(self): + tax1 = tagging_api.get_taxonomy(1) + assert tax1 == self.taxonomy + no_tax = tagging_api.get_taxonomy(10) + assert no_tax is None def test_get_taxonomies(self): tax1 = tagging_api.create_taxonomy("Enabled") tax2 = tagging_api.create_taxonomy("Disabled", enabled=False) - enabled = tagging_api.get_taxonomies() - assert list(enabled) == [tax1, self.taxonomy] + with self.assertNumQueries(1): + enabled = list(tagging_api.get_taxonomies()) + assert enabled == [tax1, self.taxonomy, self.system_taxonomy] + assert str(enabled[0]) == f" ({tax1.id}) Enabled" + assert str(enabled[1]) == " (1) Life on Earth" + assert str(enabled[2]) == " (2) System Languages" - disabled = tagging_api.get_taxonomies(enabled=False) - assert list(disabled) == [tax2] + with self.assertNumQueries(1): + disabled = list(tagging_api.get_taxonomies(enabled=False)) + assert disabled == [tax2] + assert str(disabled[0]) == f" ({tax2.id}) Disabled" - both = tagging_api.get_taxonomies(enabled=None) - assert list(both) == [tax2, tax1, self.taxonomy] + with self.assertNumQueries(1): + both = list(tagging_api.get_taxonomies(enabled=None)) + assert both == [tax2, tax1, self.taxonomy, self.system_taxonomy] def test_get_tags(self): self.setup_tag_depths() @@ -59,23 +80,21 @@ def check_object_tag(self, object_tag, taxonomy, tag, name, value): assert object_tag.value == value def test_resync_object_tags(self): - missing_links = ObjectTag(object_id="abc", object_type="alpha") - missing_links.name = self.taxonomy.name - missing_links.value = self.mammalia.value - missing_links.save() - changed_links = ObjectTag( + missing_links = ObjectTag.objects.create( + object_id="abc", + _name=self.taxonomy.name, + _value=self.mammalia.value, + ) + changed_links = ObjectTag.objects.create( object_id="def", - object_type="alpha", taxonomy=self.taxonomy, tag=self.mammalia, ) changed_links.name = "Life" changed_links.value = "Animals" changed_links.save() - - no_changes = ObjectTag( + no_changes = ObjectTag.objects.create( object_id="ghi", - object_type="beta", taxonomy=self.taxonomy, tag=self.mammalia, ) @@ -94,14 +113,44 @@ def test_resync_object_tags(self): changed = tagging_api.resync_object_tags() assert changed == 0 + # Resync will use the tag's taxonomy if possible + changed_links.taxonomy = None + changed_links.save() + changed = tagging_api.resync_object_tags() + assert changed == 1 + for object_tag in (missing_links, changed_links, no_changes): + self.check_object_tag( + object_tag, self.taxonomy, self.mammalia, "Life on Earth", "Mammalia" + ) + + # Resync will use the taxonomy's tags if possible + changed_links.tag = None + changed_links.value = "Xenomorph" + changed_links.save() + changed = tagging_api.resync_object_tags() + assert changed == 0 + changed_links.value = "Mammalia" + changed_links.save() + # ObjectTag value preserved even if linked tag is deleted self.mammalia.delete() for object_tag in (missing_links, changed_links, no_changes): self.check_object_tag( object_tag, self.taxonomy, None, "Life on Earth", "Mammalia" ) + # Recreating the tag to test resyncing works + new_mammalia = Tag.objects.create( + value="Mammalia", + taxonomy=self.taxonomy, + ) + changed = tagging_api.resync_object_tags() + assert changed == 3 + for object_tag in (missing_links, changed_links, no_changes): + self.check_object_tag( + object_tag, self.taxonomy, new_mammalia, "Life on Earth", "Mammalia" + ) - # ObjectTag name preserved even if linked taxonomy is deleted + # ObjectTag name preserved even if linked taxonomy and its tags are deleted self.taxonomy.delete() for object_tag in (missing_links, changed_links, no_changes): self.check_object_tag(object_tag, None, None, "Life on Earth", "Mammalia") @@ -111,21 +160,21 @@ def test_resync_object_tags(self): assert changed == 0 # Recreate the taxonomy and resync some tags - first_taxonomy = tagging_api.create_taxonomy("Life on Earth") + first_taxonomy = tagging_api.create_taxonomy( + "Life on Earth", allow_free_text=True + ) second_taxonomy = tagging_api.create_taxonomy("Life on Earth") new_tag = Tag.objects.create( value="Mammalia", taxonomy=second_taxonomy, ) - with patch( - "openedx_tagging.core.tagging.models.Taxonomy.validate_object_tag", - side_effect=[False, True, False, True], - ): - changed = tagging_api.resync_object_tags( - ObjectTag.objects.filter(object_type="alpha") - ) - assert changed == 2 + # Ensure the resync prefers the closed taxonomy with the matching tag + changed = tagging_api.resync_object_tags( + ObjectTag.objects.filter(object_id__in=["abc", "def"]) + ) + assert changed == 2 + for object_tag in (missing_links, changed_links): self.check_object_tag( object_tag, second_taxonomy, new_tag, "Life on Earth", "Mammalia" @@ -134,17 +183,20 @@ def test_resync_object_tags(self): # Ensure the omitted tag was not updated self.check_object_tag(no_changes, None, None, "Life on Earth", "Mammalia") - # Update that one too (without the patching) + # Update that one too, to demonstrate the free-text tags are ok + no_changes.value = "Anamelia" + no_changes.save() changed = tagging_api.resync_object_tags( - ObjectTag.objects.filter(object_type="beta") + ObjectTag.objects.filter(id=no_changes.id) ) assert changed == 1 self.check_object_tag( - no_changes, first_taxonomy, None, "Life on Earth", "Mammalia" + no_changes, first_taxonomy, None, "Life on Earth", "Anamelia" ) def test_tag_object(self): self.taxonomy.allow_multiple = True + self.taxonomy.save() test_tags = [ [ self.archaea.id, @@ -167,15 +219,15 @@ def test_tag_object(self): self.taxonomy, tag_list, "biology101", - "course", ) # Ensure the expected number of tags exist in the database assert ( - tagging_api.get_object_tags( - taxonomy=self.taxonomy, - object_id="biology101", - object_type="course", + list( + tagging_api.get_object_tags( + taxonomy=self.taxonomy, + object_id="biology101", + ) ) == object_tags ) @@ -183,8 +235,107 @@ def test_tag_object(self): assert len(object_tags) == len(tag_list) for index, object_tag in enumerate(object_tags): assert object_tag.tag_id == tag_list[index] - assert object_tag.is_valid + assert object_tag.is_valid() assert object_tag.taxonomy == self.taxonomy assert object_tag.name == self.taxonomy.name assert object_tag.object_id == "biology101" - assert object_tag.object_type == "course" + + def test_tag_object_free_text(self): + self.taxonomy.allow_free_text = True + self.taxonomy.save() + object_tags = tagging_api.tag_object( + self.taxonomy, + ["Eukaryota Xenomorph"], + "biology101", + ) + assert len(object_tags) == 1 + object_tag = object_tags[0] + assert object_tag.is_valid() + assert object_tag.taxonomy == self.taxonomy + assert object_tag.name == self.taxonomy.name + assert object_tag.tag_ref == "Eukaryota Xenomorph" + assert object_tag.get_lineage() == ["Eukaryota Xenomorph"] + assert object_tag.object_id == "biology101" + + def test_tag_object_no_multiple(self): + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.taxonomy, + ["A", "B"], + "biology101", + ) + assert "only allows one tag per object" in str(exc.exception) + + def test_tag_object_required(self): + self.taxonomy.required = True + self.taxonomy.save() + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.taxonomy, + [], + "biology101", + ) + assert "requires at least one tag per object" in str(exc.exception) + + def test_tag_object_invalid_tag(self): + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.taxonomy, + ["Eukaryota Xenomorph"], + "biology101", + ) + assert "Invalid object tag for taxonomy (1): Eukaryota Xenomorph" in str( + exc.exception + ) + + def test_get_object_tags(self): + # Alpha tag has no taxonomy + alpha = ObjectTag(object_id="abc") + alpha.name = self.taxonomy.name + alpha.value = self.mammalia.value + alpha.save() + # Beta tag has a closed taxonomy + beta = ObjectTag.objects.create( + object_id="abc", + taxonomy=self.taxonomy, + ) + + # Fetch all the tags for a given object ID + assert list( + tagging_api.get_object_tags( + object_id="abc", + valid_only=False, + ) + ) == [ + alpha, + beta, + ] + + # No valid tags for this object yet.. + assert not list( + tagging_api.get_object_tags( + object_id="abc", + valid_only=True, + ) + ) + beta.tag = self.mammalia + beta.save() + assert list( + tagging_api.get_object_tags( + object_id="abc", + valid_only=True, + ) + ) == [ + beta, + ] + + # Fetch all the tags for a given object ID + taxonomy + assert list( + tagging_api.get_object_tags( + object_id="abc", + taxonomy=self.taxonomy, + valid_only=False, + ) + ) == [ + beta, + ] diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index d74c041db..65af41eaa 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -23,6 +23,7 @@ class TestTagTaxonomyMixin: def setUp(self): super().setUp() self.taxonomy = Taxonomy.objects.get(name="Life on Earth") + self.system_taxonomy = Taxonomy.objects.get(name="System Languages") self.archaea = get_tag("Archaea") self.archaebacteria = get_tag("Archaebacteria") self.bacteria = get_tag("Bacteria") @@ -77,6 +78,39 @@ def setup_tag_depths(self): tag.depth = 2 +class TestTaxonomySubclassA(Taxonomy): + """ + Model A for testing the taxonomy subclass casting. + """ + + class Meta: + managed = False + proxy = True + app_label = "oel_tagging" + + +class TestTaxonomySubclassB(TestTaxonomySubclassA): + """ + Model B for testing the taxonomy subclass casting. + """ + + class Meta: + managed = False + proxy = True + app_label = "oel_tagging" + + +class TestObjectTagSubclass(ObjectTag): + """ + Model for testing the ObjectTag copy. + """ + + class Meta: + managed = False + proxy = True + app_label = "oel_tagging" + + @ddt.ddt class TestModelTagTaxonomy(TestTagTaxonomyMixin, TestCase): """ @@ -85,10 +119,57 @@ class TestModelTagTaxonomy(TestTagTaxonomyMixin, TestCase): def test_system_defined(self): assert not self.taxonomy.system_defined + assert self.system_taxonomy.system_defined def test_representations(self): - assert str(self.bacteria) == "Tag (1) Bacteria" - assert repr(self.bacteria) == "Tag (1) Bacteria" + assert ( + str(self.taxonomy) == repr(self.taxonomy) == " (1) Life on Earth" + ) + assert ( + str(self.system_taxonomy) + == repr(self.system_taxonomy) + == " (2) System Languages" + ) + assert str(self.bacteria) == repr(self.bacteria) == " (1) Bacteria" + + def test_taxonomy_cast(self): + for subclass in ( + TestTaxonomySubclassA, + # Ensure that casting to a sub-subclass works as expected + TestTaxonomySubclassB, + # and that we can un-set the subclass + None, + ): + self.taxonomy.taxonomy_class = subclass + cast_taxonomy = self.taxonomy.cast() + if subclass: + expected_class = subclass.__name__ + else: + expected_class = "Taxonomy" + assert self.taxonomy == cast_taxonomy + assert ( + str(cast_taxonomy) + == repr(cast_taxonomy) + == f"<{expected_class}> (1) Life on Earth" + ) + + def test_taxonomy_cast_import_error(self): + taxonomy = Taxonomy.objects.create( + name="Invalid cast", _taxonomy_class="not.a.class" + ) + # Error is logged, but ignored. + cast_taxonomy = taxonomy.cast() + assert cast_taxonomy == taxonomy + assert ( + str(cast_taxonomy) + == repr(cast_taxonomy) + == f" ({taxonomy.id}) Invalid cast" + ) + + def test_taxonomy_cast_bad_value(self): + with self.assertRaises(ValueError) as exc: + self.taxonomy.taxonomy_class = str + assert " must be a subclass of Taxonomy" in str(exc.exception) @ddt.data( # Root tags just return their own value @@ -136,32 +217,46 @@ class TestModelObjectTag(TestTagTaxonomyMixin, TestCase): def setUp(self): super().setUp() self.tag = self.bacteria + self.object_tag = ObjectTag.objects.create( + object_id="object:id:1", + taxonomy=self.taxonomy, + tag=self.tag, + ) + + def test_representations(self): + assert ( + str(self.object_tag) + == repr(self.object_tag) + == " object:id:1: Life on Earth=Bacteria" + ) + + def test_cast(self): + copy_tag = TestObjectTagSubclass.cast(self.object_tag) + assert ( + str(copy_tag) + == repr(copy_tag) + == " object:id:1: Life on Earth=Bacteria" + ) def test_object_tag_name(self): # ObjectTag's name defaults to its taxonomy's name - object_tag = ObjectTag.objects.create( - object_id="object:id", - object_type="any_old_object", - taxonomy=self.taxonomy, - ) - assert object_tag.name == self.taxonomy.name + assert self.object_tag.name == self.taxonomy.name # Even if we overwrite the name, it still uses the taxonomy's name - object_tag.name = "Another tag" - assert object_tag.name == self.taxonomy.name - object_tag.save() - assert object_tag.name == self.taxonomy.name + self.object_tag.name = "Another tag" + assert self.object_tag.name == self.taxonomy.name + self.object_tag.save() + assert self.object_tag.name == self.taxonomy.name # But if the taxonomy is deleted, then the object_tag's name reverts to our cached name self.taxonomy.delete() - object_tag.refresh_from_db() - assert object_tag.name == "Another tag" + self.object_tag.refresh_from_db() + assert self.object_tag.name == "Another tag" def test_object_tag_value(self): # ObjectTag's value defaults to its tag's value object_tag = ObjectTag.objects.create( object_id="object:id", - object_type="any_old_object", taxonomy=self.taxonomy, tag=self.tag, ) @@ -182,7 +277,6 @@ def test_object_tag_lineage(self): # ObjectTag's value defaults to its tag's lineage object_tag = ObjectTag.objects.create( object_id="object:id", - object_type="any_old_object", taxonomy=self.taxonomy, tag=self.tag, ) @@ -200,41 +294,34 @@ def test_object_tag_lineage(self): assert object_tag.get_lineage() == ["Another tag"] def test_object_tag_is_valid(self): - object_tag = ObjectTag( - object_id="object:id", - object_type="any_old_object", + open_taxonomy = Taxonomy.objects.create( + name="Freetext Life", + allow_free_text=True, ) - assert not object_tag.is_valid - - object_tag.taxonomy = self.taxonomy - assert not object_tag.is_valid - object_tag.tag = self.tag - assert object_tag.is_valid - - # or, we can have no tag, and a free-text taxonomy - object_tag.tag = None - self.taxonomy.allow_free_text = True - assert object_tag.is_valid - - def test_validate_object_tag_invalid(self): - taxonomy = Taxonomy.objects.create( - name="Another taxonomy", - ) object_tag = ObjectTag( taxonomy=self.taxonomy, ) - assert not taxonomy.validate_object_tag(object_tag) - - object_tag.taxonomy = taxonomy - assert not taxonomy.validate_object_tag(object_tag) - - taxonomy.allow_free_text = True - assert not taxonomy.validate_object_tag(object_tag) - + # ObjectTag will only be valid for its taxonomy + assert not open_taxonomy.validate_object_tag(object_tag) + + # ObjectTags in a free-text taxonomy are valid with a value + assert not object_tag.is_valid() + object_tag.value = "Any text we want" + object_tag.taxonomy = open_taxonomy + assert not object_tag.is_valid() object_tag.object_id = "object:id" - object_tag.object_type = "object:type" - assert taxonomy.validate_object_tag(object_tag) + assert object_tag.is_valid() + + # ObjectTags in a closed taxonomy require a tag in that taxonomy + object_tag.taxonomy = self.taxonomy + object_tag.tag = Tag.objects.create( + taxonomy=self.system_taxonomy, + value="PT", + ) + assert not object_tag.is_valid() + object_tag.tag = self.tag + assert object_tag.is_valid() def test_tag_object(self): self.taxonomy.allow_multiple = True @@ -260,14 +347,12 @@ def test_tag_object(self): object_tags = self.taxonomy.tag_object( tag_list, "biology101", - "course", ) # Ensure the expected number of tags exist in the database assert ObjectTag.objects.filter( taxonomy=self.taxonomy, object_id="biology101", - object_type="course", ).count() == len(tag_list) # And the expected number of tags were returned assert len(object_tags) == len(tag_list) @@ -277,14 +362,12 @@ def test_tag_object(self): assert object_tag.taxonomy == self.taxonomy assert object_tag.name == self.taxonomy.name assert object_tag.object_id == "biology101" - assert object_tag.object_type == "course" def test_tag_object_free_text(self): self.taxonomy.allow_free_text = True object_tags = self.taxonomy.tag_object( ["Eukaryota Xenomorph"], "biology101", - "course", ) assert len(object_tags) == 1 object_tag = object_tags[0] @@ -294,14 +377,12 @@ def test_tag_object_free_text(self): assert object_tag.tag_ref == "Eukaryota Xenomorph" assert object_tag.get_lineage() == ["Eukaryota Xenomorph"] assert object_tag.object_id == "biology101" - assert object_tag.object_type == "course" def test_tag_object_no_multiple(self): with self.assertRaises(ValueError) as exc: self.taxonomy.tag_object( ["A", "B"], "biology101", - "course", ) assert "only allows one tag per object" in str(exc.exception) @@ -311,7 +392,6 @@ def test_tag_object_required(self): self.taxonomy.tag_object( [], "biology101", - "course", ) assert "requires at least one tag per object" in str(exc.exception) @@ -320,6 +400,5 @@ def test_tag_object_invalid_tag(self): self.taxonomy.tag_object( ["Eukaryota Xenomorph"], "biology101", - "course", ) assert "Invalid object tag for taxonomy" in str(exc.exception) diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py index 2c2b4e748..577c594f3 100644 --- a/tests/openedx_tagging/core/tagging/test_rules.py +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -3,9 +3,8 @@ import ddt from django.contrib.auth import get_user_model from django.test.testcases import TestCase -from mock import Mock -from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy +from openedx_tagging.core.tagging.models import ObjectTag, Tag from .test_models import TestTagTaxonomyMixin @@ -34,12 +33,10 @@ def setUp(self): username="learner", email="learner@example.com", ) - - self.object_tag = ObjectTag( + self.object_tag = ObjectTag.objects.create( taxonomy=self.taxonomy, tag=self.bacteria, ) - self.object_tag.resync() self.object_tag.save() # Taxonomy @@ -64,12 +61,9 @@ def test_add_change_taxonomy(self, perm): ) def test_system_taxonomy(self, perm): """Taxonomy administrators cannot edit system taxonomies""" - # TODO: use SystemTaxonomy when available - system_taxonomy = Mock(spec=Taxonomy) - system_taxonomy.system_defined.return_value = True - assert self.superuser.has_perm(perm, system_taxonomy) - assert not self.staff.has_perm(perm, system_taxonomy) - assert not self.learner.has_perm(perm, system_taxonomy) + assert self.superuser.has_perm(perm, self.system_taxonomy) + assert not self.staff.has_perm(perm, self.system_taxonomy) + assert not self.learner.has_perm(perm, self.system_taxonomy) @ddt.data( True, From 09a8fec0d8b9bc4536a0cc57ae4d149d08e7cfb6 Mon Sep 17 00:00:00 2001 From: Jillian Date: Thu, 20 Jul 2023 16:26:43 +0930 Subject: [PATCH 029/282] fix: removes COMPRESSED row format from openedx_learning core (#66) AWS Aurora does not support COMPRESSED in its MySQL implementation: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/AuroraMySQL.Migrating.RDSMySQL.Import.html --- .../core/contents/migrations/0001_initial.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/openedx_learning/core/contents/migrations/0001_initial.py b/openedx_learning/core/contents/migrations/0001_initial.py index 768730389..507056fce 100644 --- a/openedx_learning/core/contents/migrations/0001_initial.py +++ b/openedx_learning/core/contents/migrations/0001_initial.py @@ -7,20 +7,6 @@ import openedx_learning.lib.validators -def use_compressed_table_format(apps, schema_editor): - """ - Use the COMPRESSED row format for TextContent if we're using MySQL. - - This table will hold a lot of OLX, which compresses very well using MySQL's - built-in zlib compression. This is especially important because we're - keeping so much version history. - """ - if schema_editor.connection.vendor == 'mysql': - table_name = apps.get_model("oel_contents", "TextContent")._meta.db_table - sql = f"ALTER TABLE {table_name} ROW_FORMAT=COMPRESSED;" - schema_editor.execute(sql) - - class Migration(migrations.Migration): initial = True @@ -55,7 +41,6 @@ class Migration(migrations.Migration): ], ), # Call out to custom code here to change row format for TextContent - migrations.RunPython(use_compressed_table_format, reverse_code=migrations.RunPython.noop, atomic=False), migrations.AddIndex( model_name='rawcontent', index=models.Index(fields=['learning_package', 'mime_type'], name='oel_content_idx_lp_mime_type'), From 0b75bda5454b99baf9e22bc9e6e5e41932826358 Mon Sep 17 00:00:00 2001 From: Jillian Date: Mon, 24 Jul 2023 12:51:41 +0930 Subject: [PATCH 030/282] build: publish openedx-learning to pypi (#65) build: publish openedx-learning to pypi --- .github/workflows/ci.yml | 2 +- .github/workflows/pypi-publish.yml | 30 ++++++++++++++++++++++++++++++ tox.ini | 8 ++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/pypi-publish.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 432111450..953124f45 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: matrix: os: [ubuntu-latest] # Add macos-latest later? python-version: ['3.8'] - toxenv: ["py38-django32", "py38-django42"] + toxenv: ["py38-django32", "py38-django42", "package"] # We're only testing against MySQL 8 right now because 5.7 is # incompatible with Djagno 4.2. We'd have to make the tox.ini file more # complicated than it's worth given the short expected shelf-life of diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml new file mode 100644 index 000000000..325ea8ced --- /dev/null +++ b/.github/workflows/pypi-publish.yml @@ -0,0 +1,30 @@ +name: Publish package to PyPI + +on: + push: + tags: + - '*' + +jobs: + push: + runs-on: ubuntu-20.04 + + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: setup python + uses: actions/setup-python@v4 + with: + python-version: 3.8 + + - name: Install pip + run: pip install -r requirements/pip.txt + + - name: Build package + run: python setup.py sdist bdist_wheel + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + user: __token__ + password: ${{ secrets.PYPI_UPLOAD_TOKEN }} diff --git a/tox.ini b/tox.ini index ad8e2cb5e..4a0100cb2 100644 --- a/tox.ini +++ b/tox.ini @@ -88,3 +88,11 @@ deps = -r{toxinidir}/requirements/test.txt commands = code_annotations django_find_annotations --config_file .pii_annotations.yml --lint --report --coverage + +[testenv:package] +deps = + build + twine +commands = + python -m build + twine check dist/* From 66c8742557d3c7f34bf949529defc8386c4f77f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Sun, 30 Jul 2023 21:01:42 -0300 Subject: [PATCH 031/282] Taxonomy view/management REST APIs (#63) * feat: add taxonomy view/management apis and tests --- openedx_tagging/core/tagging/rest_api/urls.py | 9 + .../core/tagging/rest_api/v1/__init__.py | 0 .../core/tagging/rest_api/v1/permissions.py | 18 + .../core/tagging/rest_api/v1/serializers.py | 30 ++ .../core/tagging/rest_api/v1/urls.py | 16 + .../core/tagging/rest_api/v1/views.py | 147 +++++++ openedx_tagging/core/tagging/rules.py | 6 +- openedx_tagging/core/tagging/urls.py | 10 + projects/urls.py | 1 + requirements/dev.txt | 4 +- requirements/doc.txt | 4 + requirements/quality.txt | 4 + requirements/test.in | 1 + requirements/test.txt | 4 + test_settings.py | 6 +- .../core/tagging/test_rules.py | 2 +- .../core/tagging/test_views.py | 370 ++++++++++++++++++ tox.ini | 2 +- 18 files changed, 626 insertions(+), 8 deletions(-) create mode 100644 openedx_tagging/core/tagging/rest_api/urls.py create mode 100644 openedx_tagging/core/tagging/rest_api/v1/__init__.py create mode 100644 openedx_tagging/core/tagging/rest_api/v1/permissions.py create mode 100644 openedx_tagging/core/tagging/rest_api/v1/serializers.py create mode 100644 openedx_tagging/core/tagging/rest_api/v1/urls.py create mode 100644 openedx_tagging/core/tagging/rest_api/v1/views.py create mode 100644 openedx_tagging/core/tagging/urls.py create mode 100644 tests/openedx_tagging/core/tagging/test_views.py diff --git a/openedx_tagging/core/tagging/rest_api/urls.py b/openedx_tagging/core/tagging/rest_api/urls.py new file mode 100644 index 000000000..d7f012bb7 --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/urls.py @@ -0,0 +1,9 @@ +""" +Taxonomies API URLs. +""" + +from django.urls import path, include + +from .v1 import urls as v1_urls + +urlpatterns = [path("v1/", include(v1_urls))] diff --git a/openedx_tagging/core/tagging/rest_api/v1/__init__.py b/openedx_tagging/core/tagging/rest_api/v1/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_tagging/core/tagging/rest_api/v1/permissions.py b/openedx_tagging/core/tagging/rest_api/v1/permissions.py new file mode 100644 index 000000000..b75c5ff79 --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/v1/permissions.py @@ -0,0 +1,18 @@ +""" +Taxonomy permissions +""" + +from rest_framework.permissions import DjangoObjectPermissions + + +class TaxonomyObjectPermissions(DjangoObjectPermissions): + perms_map = { + "GET": ["%(app_label)s.view_%(model_name)s"], + "OPTIONS": [], + "HEAD": ["%(app_label)s.view_%(model_name)s"], + "POST": ["%(app_label)s.add_%(model_name)s"], + "PUT": ["%(app_label)s.change_%(model_name)s"], + "PATCH": ["%(app_label)s.change_%(model_name)s"], + "DELETE": ["%(app_label)s.delete_%(model_name)s"], + } + diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py new file mode 100644 index 000000000..a46350a9a --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -0,0 +1,30 @@ +""" +API Serializers for taxonomies +""" + +from rest_framework import serializers + +from openedx_tagging.core.tagging.models import Taxonomy + +class TaxonomyListQueryParamsSerializer(serializers.Serializer): + """ + Serializer for the query params for the GET view + """ + + enabled = serializers.BooleanField(required=False) + +class TaxonomySerializer(serializers.ModelSerializer): + class Meta: + model = Taxonomy + fields = [ + "id", + "name", + "description", + "enabled", + "required", + "allow_multiple", + "allow_free_text", + "system_defined", + "visible_to_authors", + ] + diff --git a/openedx_tagging/core/tagging/rest_api/v1/urls.py b/openedx_tagging/core/tagging/rest_api/v1/urls.py new file mode 100644 index 000000000..97d653a28 --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/v1/urls.py @@ -0,0 +1,16 @@ +""" +Taxonomies API v1 URLs. +""" + +from rest_framework.routers import DefaultRouter + +from django.urls.conf import path, include + +from . import views + +router = DefaultRouter() +router.register("taxonomies", views.TaxonomyView, basename="taxonomy") + +urlpatterns = [ + path('', include(router.urls)) +] diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py new file mode 100644 index 000000000..1ea02aca9 --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -0,0 +1,147 @@ +""" +Tagging API Views +""" +from django.http import Http404 +from rest_framework.viewsets import ModelViewSet + +from ...api import ( + create_taxonomy, + get_taxonomy, + get_taxonomies, +) +from .serializers import TaxonomyListQueryParamsSerializer, TaxonomySerializer +from .permissions import TaxonomyObjectPermissions + + +class TaxonomyView(ModelViewSet): + """ + View to list, create, retrieve, update, or delete Taxonomies. + + **List Query Parameters** + * enabled (optional) - Filter by enabled status. Valid values: true, false, 1, 0, "true", "false", "1" + + **List Example Requests** + GET api/tagging/v1/taxonomy - Get all taxonomies + GET api/tagging/v1/taxonomy?enabled=true - Get all enabled taxonomies + GET api/tagging/v1/taxonomy?enabled=false - Get all disabled taxonomies + + **List Query Returns** + * 200 - Success + * 400 - Invalid query parameter + * 403 - Permission denied + + **Retrieve Parameters** + * pk (required): - The pk of the taxonomy to retrieve + + **Retrieve Example Requests** + GET api/tagging/v1/taxonomy/:pk - Get a specific taxonomy + + **Retrieve Query Returns** + * 200 - Success + * 404 - Taxonomy not found or User does not have permission to access the taxonomy + + **Create Parameters** + * name (required): User-facing label used when applying tags from this taxonomy to Open edX objects. + * description (optional): Provides extra information for the user when applying tags from this taxonomy to an object. + * enabled (optional): Only enabled taxonomies will be shown to authors (default: true). + * required (optional): Indicates that one or more tags from this taxonomy must be added to an object (default: False). + * allow_multiple (optional): Indicates that multiple tags from this taxonomy may be added to an object (default: False). + * allow_free_text (optional): Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values (default: False). + + **Create Example Requests** + POST api/tagging/v1/taxonomy - Create a taxonomy + { + "name": "Taxonomy Name", - User-facing label used when applying tags from this taxonomy to Open edX objects." + "description": "This is a description", + "enabled": True, + "required": True, + "allow_multiple": True, + "allow_free_text": True, + } + + + **Create Query Returns** + * 201 - Success + * 403 - Permission denied + + **Update Parameters** + * pk (required): - The pk of the taxonomy to update + + **Update Request Body** + * name (optional): User-facing label used when applying tags from this taxonomy to Open edX objects. + * description (optional): Provides extra information for the user when applying tags from this taxonomy to an object. + * enabled (optional): Only enabled taxonomies will be shown to authors. + * required (optional): Indicates that one or more tags from this taxonomy must be added to an object. + * allow_multiple (optional): Indicates that multiple tags from this taxonomy may be added to an object. + * allow_free_text (optional): Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values. + + **Update Example Requests** + PUT api/tagging/v1/taxonomy/:pk - Update a taxonomy + { + "name": "Taxonomy New Name", + "description": "This is a new description", + "enabled": False, + "required": False, + "allow_multiple": False, + "allow_free_text": True, + } + PATCH api/tagging/v1/taxonomy/:pk - Partially update a taxonomy + { + "name": "Taxonomy New Name", + } + + **Update Query Returns** + * 200 - Success + * 403 - Permission denied + + **Delete Parameters** + * pk (required): - The pk of the taxonomy to delete + + **Delete Example Requests** + DELETE api/tagging/v1/taxonomy/:pk - Delete a taxonomy + + **Delete Query Returns** + * 200 - Success + * 404 - Taxonomy not found + * 403 - Permission denied + + """ + + + serializer_class = TaxonomySerializer + permission_classes = [TaxonomyObjectPermissions] + + def get_object(self): + """ + Return the requested taxonomy object, if the user has appropriate + permissions. + """ + pk = self.kwargs.get("pk") + taxonomy = get_taxonomy(pk) + if not taxonomy: + raise Http404("Taxonomy not found") + self.check_object_permissions(self.request, taxonomy) + + return taxonomy + + def get_queryset(self): + """ + Return a list of taxonomies. + + Returns all taxonomies by default. + If you want the disabled taxonomies, pass enabled=False. + If you want the enabled taxonomies, pass enabled=True. + """ + query_params = TaxonomyListQueryParamsSerializer( + data=self.request.query_params.dict() + ) + query_params.is_valid(raise_exception=True) + enabled = query_params.data.get("enabled", None) + + return get_taxonomies(enabled) + + def perform_create(self, serializer): + """ + Create a new taxonomy. + """ + serializer.instance = create_taxonomy(**serializer.validated_data) diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 72b9fa6b4..2cc94d0dc 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -16,10 +16,10 @@ @rules.predicate def can_view_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: """ - Anyone can view an enabled taxonomy, + Anyone can view an enabled taxonomy or list all taxonomies, but only taxonomy admins can view a disabled taxonomy. """ - return (taxonomy and taxonomy.enabled) or is_taxonomy_admin(user) + return not taxonomy or taxonomy.enabled or is_taxonomy_admin(user) @rules.predicate @@ -28,7 +28,7 @@ def can_change_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: Even taxonomy admins cannot change system taxonomies. """ return is_taxonomy_admin(user) and ( - not taxonomy or not taxonomy or (taxonomy and not taxonomy.system_defined) + not taxonomy or (taxonomy and not taxonomy.system_defined) ) diff --git a/openedx_tagging/core/tagging/urls.py b/openedx_tagging/core/tagging/urls.py new file mode 100644 index 000000000..da2c52081 --- /dev/null +++ b/openedx_tagging/core/tagging/urls.py @@ -0,0 +1,10 @@ +""" +Tagging API URLs. +""" + +from django.urls import path, include + +from .rest_api import urls + +app_name = "oel_tagging" +urlpatterns = [path("", include(urls))] diff --git a/projects/urls.py b/projects/urls.py index e27da7532..cf95abb13 100644 --- a/projects/urls.py +++ b/projects/urls.py @@ -9,5 +9,6 @@ path("admin/", admin.site.urls), path("media_server/", include("openedx_learning.contrib.media_server.urls")), path("rest_api/", include("openedx_learning.rest_api.urls")), + path("tagging/rest_api/", include("openedx_tagging.core.tagging.urls")), path('__debug__/', include('debug_toolbar.urls')), ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) diff --git a/requirements/dev.txt b/requirements/dev.txt index add9cf1de..48cf4fb40 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -81,7 +81,9 @@ django==3.2.19 # djangorestframework # edx-i18n-tools django-debug-toolbar==4.1.0 - # via -r requirements/dev.in + # via + # -r requirements/dev.in + # -r requirements/quality.txt djangorestframework==3.14.0 # via -r requirements/quality.txt docutils==0.20.1 diff --git a/requirements/doc.txt b/requirements/doc.txt index d9a4c3c2a..f63eb9113 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -41,8 +41,11 @@ django==3.2.19 # via # -c requirements/constraints.txt # -r requirements/test.txt + # django-debug-toolbar # djangorestframework # sphinxcontrib-django +django-debug-toolbar==4.1.0 + # via -r requirements/test.txt djangorestframework==3.14.0 # via -r requirements/test.txt doc8==1.1.1 @@ -175,6 +178,7 @@ sqlparse==0.4.4 # via # -r requirements/test.txt # django + # django-debug-toolbar stevedore==5.1.0 # via # -r requirements/test.txt diff --git a/requirements/quality.txt b/requirements/quality.txt index c580bd7e1..425cfa514 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -47,7 +47,10 @@ django==3.2.19 # via # -c requirements/constraints.txt # -r requirements/test.txt + # django-debug-toolbar # djangorestframework +django-debug-toolbar==4.1.0 + # via -r requirements/test.txt djangorestframework==3.14.0 # via -r requirements/test.txt docutils==0.20.1 @@ -198,6 +201,7 @@ sqlparse==0.4.4 # via # -r requirements/test.txt # django + # django-debug-toolbar stevedore==5.1.0 # via # -r requirements/test.txt diff --git a/requirements/test.in b/requirements/test.in index 1d521351d..c050dd241 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -14,3 +14,4 @@ pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. ddt # supports data driven tests mock # supports overriding classes and methods in tests +django-debug-toolbar # provides a debug toolbar for Django diff --git a/requirements/test.txt b/requirements/test.txt index e64d8190b..22c43aede 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -23,7 +23,10 @@ ddt==1.6.0 # via # -c requirements/constraints.txt # -r requirements/base.txt + # django-debug-toolbar # djangorestframework +django-debug-toolbar==4.1.0 + # via -r requirements/test.in djangorestframework==3.14.0 # via -r requirements/base.txt exceptiongroup==1.1.1 @@ -72,6 +75,7 @@ sqlparse==0.4.4 # via # -r requirements/base.txt # django + # django-debug-toolbar stevedore==5.1.0 # via code-annotations text-unidecode==1.3 diff --git a/test_settings.py b/test_settings.py index 2e41694e4..63290bc66 100644 --- a/test_settings.py +++ b/test_settings.py @@ -33,8 +33,10 @@ def root(*args): "django.contrib.sessions", "django.contrib.staticfiles", # Admin - # 'django.contrib.admin', - # 'django.contrib.admindocs', + 'django.contrib.admin', + 'django.contrib.admindocs', + # Debugging + "debug_toolbar", # django-rules based authorization 'rules.apps.AutodiscoverRulesConfig', # Our own apps diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py index 577c594f3..415ae193a 100644 --- a/tests/openedx_tagging/core/tagging/test_rules.py +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -90,7 +90,7 @@ def test_view_taxonomy_enabled(self, enabled): assert self.superuser.has_perm("oel_tagging.view_taxonomy", self.taxonomy) assert self.staff.has_perm("oel_tagging.view_taxonomy") assert self.staff.has_perm("oel_tagging.view_taxonomy", self.taxonomy) - assert not self.learner.has_perm("oel_tagging.view_taxonomy") + assert self.learner.has_perm("oel_tagging.view_taxonomy") assert ( self.learner.has_perm("oel_tagging.view_taxonomy", self.taxonomy) == enabled ) diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py new file mode 100644 index 000000000..0fd0d6840 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -0,0 +1,370 @@ +""" +Tests tagging rest api views +""" + +import ddt +from django.contrib.auth import get_user_model +from rest_framework import status +from rest_framework.test import APITestCase + +from openedx_tagging.core.tagging.models import Taxonomy + +User = get_user_model() + +TAXONOMY_LIST_URL = '/tagging/rest_api/v1/taxonomies/' +TAXONOMY_DETAIL_URL = '/tagging/rest_api/v1/taxonomies/{pk}/' + +def check_taxonomy( + data, + id, + name, + description=None, + enabled=True, + required=False, + allow_multiple=False, + allow_free_text=False, + system_defined=False, + visible_to_authors=True, +): + assert data["id"] == id + assert data["name"] == name + assert data["description"] == description + assert data["enabled"] == enabled + assert data["required"] == required + assert data["allow_multiple"] == allow_multiple + assert data["allow_free_text"] == allow_free_text + assert data["system_defined"] == system_defined + assert data["visible_to_authors"] == visible_to_authors + + +@ddt.ddt +class TestTaxonomyViewSet(APITestCase): + def setUp(self): + super().setUp() + + self.user = User.objects.create( + username="user", + email="user@example.com", + ) + + self.staff = User.objects.create( + username="staff", + email="staff@example.com", + is_staff=True, + ) + + @ddt.data( + (None, status.HTTP_200_OK, 3), + (1, status.HTTP_200_OK, 2), + (0, status.HTTP_200_OK, 1), + (True, status.HTTP_200_OK, 2), + (False, status.HTTP_200_OK, 1), + ("True", status.HTTP_200_OK, 2), + ("False", status.HTTP_200_OK, 1), + ("1", status.HTTP_200_OK, 2), + ("0", status.HTTP_200_OK, 1), + (2, status.HTTP_400_BAD_REQUEST, None), + ("invalid", status.HTTP_400_BAD_REQUEST, None), + ) + @ddt.unpack + def test_list_taxonomy_queryparams(self, enabled, expected_status, expected_count): + Taxonomy.objects.create(name="Taxonomy enabled 1", enabled=True).save() + Taxonomy.objects.create(name="Taxonomy enabled 2", enabled=True).save() + Taxonomy.objects.create(name="Taxonomy disabled", enabled=False).save() + + url = TAXONOMY_LIST_URL + + self.client.force_authenticate(user=self.staff) + if enabled is not None: + response = self.client.get(url, {"enabled": enabled}) + else: + response = self.client.get(url) + assert response.status_code == expected_status + + # If we were able to list the taxonomies, check that we got the expected number back + if status.is_success(expected_status): + assert len(response.data) == expected_count + + @ddt.data( + (None, status.HTTP_403_FORBIDDEN), + ("user", status.HTTP_200_OK), + ("staff", status.HTTP_200_OK), + ) + @ddt.unpack + def test_list_taxonomy(self, user_attr, expected_status): + url = TAXONOMY_LIST_URL + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + response = self.client.get(url) + assert response.status_code == expected_status + + @ddt.data( + (None, {"enabled": True}, status.HTTP_403_FORBIDDEN), + (None, {"enabled": False}, status.HTTP_403_FORBIDDEN), + ( + "user", + {"enabled": True}, + status.HTTP_200_OK, + ), + ("user", {"enabled": False}, status.HTTP_404_NOT_FOUND), + ("staff", {"enabled": True}, status.HTTP_200_OK), + ("staff", {"enabled": False}, status.HTTP_200_OK), + ) + @ddt.unpack + def test_detail_taxonomy(self, user_attr, taxonomy_data, expected_status): + create_data = {**{"name": "taxonomy detail test"}, **taxonomy_data} + taxonomy = Taxonomy.objects.create(**create_data) + url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + response = self.client.get(url) + assert response.status_code == expected_status + + if status.is_success(expected_status): + check_taxonomy(response.data, taxonomy.pk, **create_data) + + def test_detail_taxonomy_404(self): + url = TAXONOMY_DETAIL_URL.format(pk=123123) + + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + assert response.status_code == status.HTTP_404_NOT_FOUND + + @ddt.data( + (None, status.HTTP_403_FORBIDDEN), + ("user", status.HTTP_403_FORBIDDEN), + ("staff", status.HTTP_201_CREATED), + ) + @ddt.unpack + def test_create_taxonomy(self, user_attr, expected_status): + url = TAXONOMY_LIST_URL + + create_data = { + "name": "taxonomy_data_2", + "description": "This is a description", + "enabled": False, + "required": True, + "allow_multiple": True, + } + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + response = self.client.post(url, create_data, format="json") + assert response.status_code == expected_status + + # If we were able to create the taxonomy, check if it was created + if status.is_success(expected_status): + check_taxonomy(response.data, response.data["id"], **create_data) + url = TAXONOMY_DETAIL_URL.format(pk=response.data["id"]) + + response = self.client.get(url) + check_taxonomy(response.data, response.data["id"], **create_data) + + @ddt.data( + {}, + {"name": "Error taxonomy 2", "required": "Invalid value"}, + {"name": "Error taxonomy 3", "enabled": "Invalid value"}, + ) + def test_create_taxonomy_error(self, create_data): + url = TAXONOMY_LIST_URL + + self.client.force_authenticate(user=self.staff) + response = self.client.post(url, create_data, format="json") + assert response.status_code == status.HTTP_400_BAD_REQUEST + + @ddt.data({"name": "System defined taxonomy", "system_defined": True}) + def test_create_taxonomy_system_defined(self, create_data): + """ + Cannont create a taxonomy with system_defined=true + """ + url = TAXONOMY_LIST_URL + + self.client.force_authenticate(user=self.staff) + response = self.client.post(url, create_data, format="json") + assert response.status_code == status.HTTP_201_CREATED + assert response.data["system_defined"] == False + + @ddt.data( + (None, status.HTTP_403_FORBIDDEN), + ("user", status.HTTP_403_FORBIDDEN), + ("staff", status.HTTP_200_OK), + ) + @ddt.unpack + def test_update_taxonomy(self, user_attr, expected_status): + taxonomy = Taxonomy.objects.create( + name="test update taxonomy", + description="taxonomy description", + enabled=True, + required=False, + ) + taxonomy.save() + + url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + response = self.client.put(url, {"name": "new name"}, format="json") + assert response.status_code == expected_status + + # If we were able to update the taxonomy, check if the name changed + if status.is_success(expected_status): + response = self.client.get(url) + check_taxonomy( + response.data, + response.data["id"], + **{ + "name": "new name", + "description": "taxonomy description", + "enabled": True, + "required": False, + }, + ) + + @ddt.data( + (False, False, status.HTTP_200_OK), + (False, True, status.HTTP_200_OK), + (True, False, status.HTTP_403_FORBIDDEN), + (True, True, status.HTTP_403_FORBIDDEN), + ) + @ddt.unpack + def test_update_taxonomy_system_defined( + self, create_value, update_value, expected_status + ): + ''' + Test that we can't update system_defined field + ''' + taxonomy = Taxonomy.objects.create( + name="test system taxonomy", system_defined=create_value + ) + taxonomy.save() + url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) + + self.client.force_authenticate(user=self.staff) + response = self.client.put( + url, {"name": "new name", "system_defined": update_value}, format="json" + ) + assert response.status_code == expected_status + + # Verify that system_defined has not changed + response = self.client.get(url) + assert response.data["system_defined"] == create_value + + def test_update_taxonomy_404(self): + url = TAXONOMY_DETAIL_URL.format(pk=123123) + + self.client.force_authenticate(user=self.staff) + response = self.client.put(url, {"name": "new name"}, format="json") + assert response.status_code == status.HTTP_404_NOT_FOUND + + @ddt.data( + (None, status.HTTP_403_FORBIDDEN), + ("user", status.HTTP_403_FORBIDDEN), + ("staff", status.HTTP_200_OK), + ) + @ddt.unpack + def test_patch_taxonomy(self, user_attr, expected_status): + taxonomy = Taxonomy.objects.create( + name="test patch taxonomy", enabled=False, required=True + ) + taxonomy.save() + + url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + response = self.client.patch( + url, {"name": "new name", "required": False}, format="json" + ) + assert response.status_code == expected_status + + # If we were able to update the taxonomy, check if the name changed + if status.is_success(expected_status): + response = self.client.get(url) + check_taxonomy( + response.data, + response.data["id"], + **{ + "name": "new name", + "enabled": False, + "required": False, + }, + ) + + @ddt.data( + (False, False, status.HTTP_200_OK), + (False, True, status.HTTP_200_OK), + (True, False, status.HTTP_403_FORBIDDEN), + (True, True, status.HTTP_403_FORBIDDEN), + ) + @ddt.unpack + def test_patch_taxonomy_system_defined( + self, create_value, update_value, expected_status + ): + ''' + Test that we can't patch system_defined field + ''' + taxonomy = Taxonomy.objects.create( + name="test system taxonomy", system_defined=create_value + ) + taxonomy.save() + url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) + + self.client.force_authenticate(user=self.staff) + response = self.client.patch( + url, {"system_defined": update_value}, format="json" + ) + assert response.status_code == expected_status + + # Verify that system_defined has not changed + response = self.client.get(url) + assert response.data["system_defined"] == create_value + + def test_patch_taxonomy_404(self): + url = TAXONOMY_DETAIL_URL.format(pk=123123) + + self.client.force_authenticate(user=self.staff) + response = self.client.patch(url, {"name": "new name"}, format="json") + assert response.status_code == status.HTTP_404_NOT_FOUND + + @ddt.data( + (None, status.HTTP_403_FORBIDDEN), + ("user", status.HTTP_403_FORBIDDEN), + ("staff", status.HTTP_204_NO_CONTENT), + ) + @ddt.unpack + def test_delete_taxonomy(self, user_attr, expected_status): + taxonomy = Taxonomy.objects.create(name="test delete taxonomy") + taxonomy.save() + + url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + response = self.client.delete(url) + assert response.status_code == expected_status + + # If we were able to delete the taxonomy, check that it's really gone + if status.is_success(expected_status): + response = self.client.get(url) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_delete_taxonomy_404(self): + url = TAXONOMY_DETAIL_URL.format(pk=123123) + + self.client.force_authenticate(user=self.staff) + response = self.client.delete(url) + assert response.status_code, status.HTTP_404_NOT_FOUND diff --git a/tox.ini b/tox.ini index 4a0100cb2..15f4bb8bb 100644 --- a/tox.ini +++ b/tox.ini @@ -31,7 +31,7 @@ match-dir = (?!migrations) [pytest] DJANGO_SETTINGS_MODULE = test_settings -addopts = --cov openedx_learning --cov openedx_tagging --cov-report term-missing --cov-report xml +addopts = --cov openedx_learning --cov openedx_tagging --cov tests --cov-report term-missing --cov-report xml norecursedirs = .* docs requirements site-packages [testenv] From 04225e7a041d117fa06efe2a1ed20c2f352d8238 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Tue, 1 Aug 2023 19:16:10 -0500 Subject: [PATCH 032/282] System defined taxonomies (#67) * feat: Adds SystemDefinedTaxonomy These taxonomies are created using fixtures and model migrations, and use negative numbers for IDs. They are not editable by any user once created, but can be modified by updating the fixtures. * feat: Adds LanguageTaxonomy This taxonomy type uses the django config settings.LANGUAGES to dynamically determine which languages are valid tags in this taxonomy. * feat: adds ModelSystemDefinedTaxonomy Abstract taxonomy class which pulls its tags list from a associated django model. Subclasses of ModelSystemDefinedTaxonomy must specify a ModelObjectTag subclass which specifies: * the associated django model class * which field on this model to use for the user-facing tag value (default is pk) * feat: Adds Author system taxonomy Instance of ModelSystemDefinedTaxonomy which is associated with the User table, so that existing users constitute valid tags in the taxonomy. * fix: adds unique_together and updates indexes on ObjectTag * chore: Package version update to 0.1.1 --- MANIFEST.in | 2 +- .../0012-system-taxonomy-creation.rst | 27 +- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 3 +- .../tagging/fixtures/language_taxonomy.yaml | 1298 +++++++++++++++++ .../tagging/management/commands/__init__.py | 0 .../commands/build_language_fixture.py | 48 + .../migrations/0003_auto_20230721_1238.py | 76 + .../migrations/0004_auto_20230723_2001.py | 34 + .../migrations/0005_language_taxonomy.py | 29 + .../core/tagging/models/__init__.py | 11 + .../tagging/{models.py => models/base.py} | 89 +- .../core/tagging/models/system_defined.py | 269 ++++ .../core/tagging/rest_api/v1/permissions.py | 1 - .../core/tagging/rest_api/v1/serializers.py | 3 +- .../core/tagging/rest_api/v1/urls.py | 4 +- .../core/tagging/rest_api/v1/views.py | 1 - openedx_tagging/core/tagging/rules.py | 27 +- projects/dev.py | 1 + .../core/fixtures/tagging.yaml | 47 +- .../openedx_tagging/core/tagging/test_api.py | 133 +- .../core/tagging/test_models.py | 52 +- .../tagging/test_system_defined_models.py | 239 +++ .../core/tagging/test_views.py | 50 +- 24 files changed, 2339 insertions(+), 107 deletions(-) create mode 100644 openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml create mode 100644 openedx_tagging/core/tagging/management/commands/__init__.py create mode 100644 openedx_tagging/core/tagging/management/commands/build_language_fixture.py create mode 100644 openedx_tagging/core/tagging/migrations/0003_auto_20230721_1238.py create mode 100644 openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py create mode 100644 openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py create mode 100644 openedx_tagging/core/tagging/models/__init__.py rename openedx_tagging/core/tagging/{models.py => models/base.py} (90%) create mode 100644 openedx_tagging/core/tagging/models/system_defined.py create mode 100644 tests/openedx_tagging/core/tagging/test_system_defined_models.py diff --git a/MANIFEST.in b/MANIFEST.in index 8cbdc3a65..0a2abd084 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ include LICENSE.txt include README.rst include requirements/base.in recursive-include openedx_learning *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py -recursive-include openedx_tagging *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py +recursive-include openedx_tagging *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py *.yaml diff --git a/docs/decisions/0012-system-taxonomy-creation.rst b/docs/decisions/0012-system-taxonomy-creation.rst index 2ed17886f..0036432bf 100644 --- a/docs/decisions/0012-system-taxonomy-creation.rst +++ b/docs/decisions/0012-system-taxonomy-creation.rst @@ -18,7 +18,8 @@ System Tag lists and validation Each System-defined Taxonomy will have its own ``ObjectTag`` subclass which is used for tag validation (e.g. ``LanguageObjectTag``, ``OrganizationObjectTag``). Each subclass can overwrite ``get_tags``; to configure the valid tags, and ``is_valid``; to check if a list of tags are valid. Both functions are implemented on the ``ObjectTag`` base class, but can be overwritten to handle special cases. -We need to create an instance of each System-defined Taxonomy's ObjectTag in a fixture. This instances will be used on different APIs. +We need to create an instance of each System-defined Taxonomy in a fixture. With their respective characteristics and subclasses. +The ``pk`` of these instances must be negative so as not to affect the auto-incremented ``pk`` of Taxonomies. Later, we need to create content-side ObjectTags that live on ``openedx.features.content_tagging`` for each content and taxonomy to be used (eg. ``CourseLanguageObjectTag``, ``CourseOrganizationObjectTag``). This new class is used to configure the automatic content tagging. You can read the `document number 0013`_ to see this configuration. @@ -30,26 +31,28 @@ We have two ways to handle Tags creation and validation for System-defined Taxon **Hardcoded by fixtures/migrations** -#. If the tags don't change over the time, you can create all on a fixture (e.g Languages). +#. If the tags don't change over the time, you can create all on a fixture (e.g Languages). + The ``pk`` of these instances must be negative. #. If the tags change over the time, you can create all on a migration. If you edit, delete, or add new tags, you should also do it in a migration. -**Free-form tags** +**Dynamic tags** -This taxonomy depends on a core data model, but simplifies the creation of Tags by allowing free-form tags, -but we can validate the tags using the ``validate_object_tag`` method. For example we can put the ``AuthorSystemTaxonomy`` associated with -the ``User`` model and use the ``ID`` field as tags. Also we can validate if an ``User`` still exists or has been deleted over time. +Closed Taxonomies that depends on a core data model. Ex. AuthorTaxonomy with Users as Tags + +#. Tags are created on the fly when new ObjectTags are added. +#. Tag.external_id we store an identifier from the instance (eg. User.pk). +#. Tag.value we store a human readable representation of the instance (eg. User.username). +#. Resync the tags to re-fetch the value. Rejected Options ----------------- -Tags created by Auto-generated from the codebase +Free-form tags ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Taxonomies that depend on a core data model could create a Tag for each eligible object. -Maintaining this dynamic list of available Tags is cumbersome: we'd need triggers for creation, editing, and deletion. -And if it's a large list of objects (e.g. Users), then copying that list into the Tag table is overkill. -It is better to dynamically generate the list of available Tags, and/or dynamically validate a submitted object tag than -to store the options in the database. +Open Taxonomy that depends on a core data model, but simplifies the creation of Tags by allowing free-form tags, + +Rejected because it has been seen that using dynamic tags provides more functionality and more advantages. .. _document number 0013: https://github.com/openedx/openedx-learning/blob/main/docs/decisions/0013-system-taxonomy-auto-tagging.rst diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 3dc1f76bc..485f44ac2 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1 +1 @@ -__version__ = "0.1.0" +__version__ = "0.1.1" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 5ca23c485..5d8ca1158 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -106,8 +106,9 @@ def get_object_tags( Pass valid_only=False when displaying tags to content authors, so they can see invalid tags too. Invalid tags will (probably) be hidden from learners. """ + ObjectTagClass = taxonomy.object_tag_class if taxonomy else ObjectTag tags = ( - ObjectTag.objects.filter( + ObjectTagClass.objects.filter( object_id=object_id, ) .select_related("tag", "taxonomy") diff --git a/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml b/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml new file mode 100644 index 000000000..38355b99b --- /dev/null +++ b/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml @@ -0,0 +1,1298 @@ +- model: oel_tagging.tag + pk: -1 + fields: + taxonomy: -1 + parent: null + value: Afar + external_id: aa +- model: oel_tagging.tag + pk: -2 + fields: + taxonomy: -1 + parent: null + value: Abkhazian + external_id: ab +- model: oel_tagging.tag + pk: -3 + fields: + taxonomy: -1 + parent: null + value: Avestan + external_id: ae +- model: oel_tagging.tag + pk: -4 + fields: + taxonomy: -1 + parent: null + value: Afrikaans + external_id: af +- model: oel_tagging.tag + pk: -5 + fields: + taxonomy: -1 + parent: null + value: Akan + external_id: ak +- model: oel_tagging.tag + pk: -6 + fields: + taxonomy: -1 + parent: null + value: Amharic + external_id: am +- model: oel_tagging.tag + pk: -7 + fields: + taxonomy: -1 + parent: null + value: Aragonese + external_id: an +- model: oel_tagging.tag + pk: -8 + fields: + taxonomy: -1 + parent: null + value: Arabic + external_id: ar +- model: oel_tagging.tag + pk: -9 + fields: + taxonomy: -1 + parent: null + value: Assamese + external_id: as +- model: oel_tagging.tag + pk: -10 + fields: + taxonomy: -1 + parent: null + value: Avaric + external_id: av +- model: oel_tagging.tag + pk: -11 + fields: + taxonomy: -1 + parent: null + value: Aymara + external_id: ay +- model: oel_tagging.tag + pk: -12 + fields: + taxonomy: -1 + parent: null + value: Azerbaijani + external_id: az +- model: oel_tagging.tag + pk: -13 + fields: + taxonomy: -1 + parent: null + value: Bashkir + external_id: ba +- model: oel_tagging.tag + pk: -14 + fields: + taxonomy: -1 + parent: null + value: Belarusian + external_id: be +- model: oel_tagging.tag + pk: -15 + fields: + taxonomy: -1 + parent: null + value: Bulgarian + external_id: bg +- model: oel_tagging.tag + pk: -16 + fields: + taxonomy: -1 + parent: null + value: Bihari languages + external_id: bh +- model: oel_tagging.tag + pk: -17 + fields: + taxonomy: -1 + parent: null + value: Bislama + external_id: bi +- model: oel_tagging.tag + pk: -18 + fields: + taxonomy: -1 + parent: null + value: Bambara + external_id: bm +- model: oel_tagging.tag + pk: -19 + fields: + taxonomy: -1 + parent: null + value: Bengali + external_id: bn +- model: oel_tagging.tag + pk: -20 + fields: + taxonomy: -1 + parent: null + value: Tibetan + external_id: bo +- model: oel_tagging.tag + pk: -21 + fields: + taxonomy: -1 + parent: null + value: Breton + external_id: br +- model: oel_tagging.tag + pk: -22 + fields: + taxonomy: -1 + parent: null + value: Bosnian + external_id: bs +- model: oel_tagging.tag + pk: -23 + fields: + taxonomy: -1 + parent: null + value: Catalan + external_id: ca +- model: oel_tagging.tag + pk: -24 + fields: + taxonomy: -1 + parent: null + value: Chechen + external_id: ce +- model: oel_tagging.tag + pk: -25 + fields: + taxonomy: -1 + parent: null + value: Chamorro + external_id: ch +- model: oel_tagging.tag + pk: -26 + fields: + taxonomy: -1 + parent: null + value: Corsican + external_id: co +- model: oel_tagging.tag + pk: -27 + fields: + taxonomy: -1 + parent: null + value: Cree + external_id: cr +- model: oel_tagging.tag + pk: -28 + fields: + taxonomy: -1 + parent: null + value: Czech + external_id: cs +- model: oel_tagging.tag + pk: -29 + fields: + taxonomy: -1 + parent: null + value: Church Slavic + external_id: cu +- model: oel_tagging.tag + pk: -30 + fields: + taxonomy: -1 + parent: null + value: Chuvash + external_id: cv +- model: oel_tagging.tag + pk: -31 + fields: + taxonomy: -1 + parent: null + value: Welsh + external_id: cy +- model: oel_tagging.tag + pk: -32 + fields: + taxonomy: -1 + parent: null + value: Danish + external_id: da +- model: oel_tagging.tag + pk: -33 + fields: + taxonomy: -1 + parent: null + value: German + external_id: de +- model: oel_tagging.tag + pk: -34 + fields: + taxonomy: -1 + parent: null + value: Divehi + external_id: dv +- model: oel_tagging.tag + pk: -35 + fields: + taxonomy: -1 + parent: null + value: Dzongkha + external_id: dz +- model: oel_tagging.tag + pk: -36 + fields: + taxonomy: -1 + parent: null + value: Ewe + external_id: ee +- model: oel_tagging.tag + pk: -37 + fields: + taxonomy: -1 + parent: null + value: Greek, Modern (1453-) + external_id: el +- model: oel_tagging.tag + pk: -38 + fields: + taxonomy: -1 + parent: null + value: English + external_id: en +- model: oel_tagging.tag + pk: -39 + fields: + taxonomy: -1 + parent: null + value: Esperanto + external_id: eo +- model: oel_tagging.tag + pk: -40 + fields: + taxonomy: -1 + parent: null + value: Spanish + external_id: es +- model: oel_tagging.tag + pk: -41 + fields: + taxonomy: -1 + parent: null + value: Estonian + external_id: et +- model: oel_tagging.tag + pk: -42 + fields: + taxonomy: -1 + parent: null + value: Basque + external_id: eu +- model: oel_tagging.tag + pk: -43 + fields: + taxonomy: -1 + parent: null + value: Persian + external_id: fa +- model: oel_tagging.tag + pk: -44 + fields: + taxonomy: -1 + parent: null + value: Fulah + external_id: ff +- model: oel_tagging.tag + pk: -45 + fields: + taxonomy: -1 + parent: null + value: Finnish + external_id: fi +- model: oel_tagging.tag + pk: -46 + fields: + taxonomy: -1 + parent: null + value: Fijian + external_id: fj +- model: oel_tagging.tag + pk: -47 + fields: + taxonomy: -1 + parent: null + value: Faroese + external_id: fo +- model: oel_tagging.tag + pk: -48 + fields: + taxonomy: -1 + parent: null + value: French + external_id: fr +- model: oel_tagging.tag + pk: -49 + fields: + taxonomy: -1 + parent: null + value: Western Frisian + external_id: fy +- model: oel_tagging.tag + pk: -50 + fields: + taxonomy: -1 + parent: null + value: Irish + external_id: ga +- model: oel_tagging.tag + pk: -51 + fields: + taxonomy: -1 + parent: null + value: Gaelic + external_id: gd +- model: oel_tagging.tag + pk: -52 + fields: + taxonomy: -1 + parent: null + value: Galician + external_id: gl +- model: oel_tagging.tag + pk: -53 + fields: + taxonomy: -1 + parent: null + value: Guarani + external_id: gn +- model: oel_tagging.tag + pk: -54 + fields: + taxonomy: -1 + parent: null + value: Gujarati + external_id: gu +- model: oel_tagging.tag + pk: -55 + fields: + taxonomy: -1 + parent: null + value: Manx + external_id: gv +- model: oel_tagging.tag + pk: -56 + fields: + taxonomy: -1 + parent: null + value: Hausa + external_id: ha +- model: oel_tagging.tag + pk: -57 + fields: + taxonomy: -1 + parent: null + value: Hebrew + external_id: he +- model: oel_tagging.tag + pk: -58 + fields: + taxonomy: -1 + parent: null + value: Hindi + external_id: hi +- model: oel_tagging.tag + pk: -59 + fields: + taxonomy: -1 + parent: null + value: Hiri Motu + external_id: ho +- model: oel_tagging.tag + pk: -60 + fields: + taxonomy: -1 + parent: null + value: Croatian + external_id: hr +- model: oel_tagging.tag + pk: -61 + fields: + taxonomy: -1 + parent: null + value: Haitian + external_id: ht +- model: oel_tagging.tag + pk: -62 + fields: + taxonomy: -1 + parent: null + value: Hungarian + external_id: hu +- model: oel_tagging.tag + pk: -63 + fields: + taxonomy: -1 + parent: null + value: Armenian + external_id: hy +- model: oel_tagging.tag + pk: -64 + fields: + taxonomy: -1 + parent: null + value: Herero + external_id: hz +- model: oel_tagging.tag + pk: -65 + fields: + taxonomy: -1 + parent: null + value: Interlingua (International Auxiliary Language Association) + external_id: ia +- model: oel_tagging.tag + pk: -66 + fields: + taxonomy: -1 + parent: null + value: Indonesian + external_id: id +- model: oel_tagging.tag + pk: -67 + fields: + taxonomy: -1 + parent: null + value: Interlingue + external_id: ie +- model: oel_tagging.tag + pk: -68 + fields: + taxonomy: -1 + parent: null + value: Igbo + external_id: ig +- model: oel_tagging.tag + pk: -69 + fields: + taxonomy: -1 + parent: null + value: Sichuan Yi + external_id: ii +- model: oel_tagging.tag + pk: -70 + fields: + taxonomy: -1 + parent: null + value: Inupiaq + external_id: ik +- model: oel_tagging.tag + pk: -71 + fields: + taxonomy: -1 + parent: null + value: Ido + external_id: io +- model: oel_tagging.tag + pk: -72 + fields: + taxonomy: -1 + parent: null + value: Icelandic + external_id: is +- model: oel_tagging.tag + pk: -73 + fields: + taxonomy: -1 + parent: null + value: Italian + external_id: it +- model: oel_tagging.tag + pk: -74 + fields: + taxonomy: -1 + parent: null + value: Inuktitut + external_id: iu +- model: oel_tagging.tag + pk: -75 + fields: + taxonomy: -1 + parent: null + value: Japanese + external_id: ja +- model: oel_tagging.tag + pk: -76 + fields: + taxonomy: -1 + parent: null + value: Javanese + external_id: jv +- model: oel_tagging.tag + pk: -77 + fields: + taxonomy: -1 + parent: null + value: Georgian + external_id: ka +- model: oel_tagging.tag + pk: -78 + fields: + taxonomy: -1 + parent: null + value: Kongo + external_id: kg +- model: oel_tagging.tag + pk: -79 + fields: + taxonomy: -1 + parent: null + value: Kikuyu + external_id: ki +- model: oel_tagging.tag + pk: -80 + fields: + taxonomy: -1 + parent: null + value: Kuanyama + external_id: kj +- model: oel_tagging.tag + pk: -81 + fields: + taxonomy: -1 + parent: null + value: Kazakh + external_id: kk +- model: oel_tagging.tag + pk: -82 + fields: + taxonomy: -1 + parent: null + value: Kalaallisut + external_id: kl +- model: oel_tagging.tag + pk: -83 + fields: + taxonomy: -1 + parent: null + value: Central Khmer + external_id: km +- model: oel_tagging.tag + pk: -84 + fields: + taxonomy: -1 + parent: null + value: Kannada + external_id: kn +- model: oel_tagging.tag + pk: -85 + fields: + taxonomy: -1 + parent: null + value: Korean + external_id: ko +- model: oel_tagging.tag + pk: -86 + fields: + taxonomy: -1 + parent: null + value: Kanuri + external_id: kr +- model: oel_tagging.tag + pk: -87 + fields: + taxonomy: -1 + parent: null + value: Kashmiri + external_id: ks +- model: oel_tagging.tag + pk: -88 + fields: + taxonomy: -1 + parent: null + value: Kurdish + external_id: ku +- model: oel_tagging.tag + pk: -89 + fields: + taxonomy: -1 + parent: null + value: Komi + external_id: kv +- model: oel_tagging.tag + pk: -90 + fields: + taxonomy: -1 + parent: null + value: Cornish + external_id: kw +- model: oel_tagging.tag + pk: -91 + fields: + taxonomy: -1 + parent: null + value: Kirghiz + external_id: ky +- model: oel_tagging.tag + pk: -92 + fields: + taxonomy: -1 + parent: null + value: Latin + external_id: la +- model: oel_tagging.tag + pk: -93 + fields: + taxonomy: -1 + parent: null + value: Luxembourgish + external_id: lb +- model: oel_tagging.tag + pk: -94 + fields: + taxonomy: -1 + parent: null + value: Ganda + external_id: lg +- model: oel_tagging.tag + pk: -95 + fields: + taxonomy: -1 + parent: null + value: Limburgan + external_id: li +- model: oel_tagging.tag + pk: -96 + fields: + taxonomy: -1 + parent: null + value: Lingala + external_id: ln +- model: oel_tagging.tag + pk: -97 + fields: + taxonomy: -1 + parent: null + value: Lao + external_id: lo +- model: oel_tagging.tag + pk: -98 + fields: + taxonomy: -1 + parent: null + value: Lithuanian + external_id: lt +- model: oel_tagging.tag + pk: -99 + fields: + taxonomy: -1 + parent: null + value: Luba-Katanga + external_id: lu +- model: oel_tagging.tag + pk: -100 + fields: + taxonomy: -1 + parent: null + value: Latvian + external_id: lv +- model: oel_tagging.tag + pk: -101 + fields: + taxonomy: -1 + parent: null + value: Malagasy + external_id: mg +- model: oel_tagging.tag + pk: -102 + fields: + taxonomy: -1 + parent: null + value: Marshallese + external_id: mh +- model: oel_tagging.tag + pk: -103 + fields: + taxonomy: -1 + parent: null + value: Maori + external_id: mi +- model: oel_tagging.tag + pk: -104 + fields: + taxonomy: -1 + parent: null + value: Macedonian + external_id: mk +- model: oel_tagging.tag + pk: -105 + fields: + taxonomy: -1 + parent: null + value: Malayalam + external_id: ml +- model: oel_tagging.tag + pk: -106 + fields: + taxonomy: -1 + parent: null + value: Mongolian + external_id: mn +- model: oel_tagging.tag + pk: -107 + fields: + taxonomy: -1 + parent: null + value: Marathi + external_id: mr +- model: oel_tagging.tag + pk: -108 + fields: + taxonomy: -1 + parent: null + value: Malay + external_id: ms +- model: oel_tagging.tag + pk: -109 + fields: + taxonomy: -1 + parent: null + value: Maltese + external_id: mt +- model: oel_tagging.tag + pk: -110 + fields: + taxonomy: -1 + parent: null + value: Burmese + external_id: my +- model: oel_tagging.tag + pk: -111 + fields: + taxonomy: -1 + parent: null + value: Nauru + external_id: na +- model: oel_tagging.tag + pk: -112 + fields: + taxonomy: -1 + parent: null + value: Bokmål, Norwegian + external_id: nb +- model: oel_tagging.tag + pk: -113 + fields: + taxonomy: -1 + parent: null + value: Ndebele, North + external_id: nd +- model: oel_tagging.tag + pk: -114 + fields: + taxonomy: -1 + parent: null + value: Nepali + external_id: ne +- model: oel_tagging.tag + pk: -115 + fields: + taxonomy: -1 + parent: null + value: Ndonga + external_id: ng +- model: oel_tagging.tag + pk: -116 + fields: + taxonomy: -1 + parent: null + value: Dutch + external_id: nl +- model: oel_tagging.tag + pk: -117 + fields: + taxonomy: -1 + parent: null + value: Norwegian Nynorsk + external_id: nn +- model: oel_tagging.tag + pk: -118 + fields: + taxonomy: -1 + parent: null + value: Norwegian + external_id: no +- model: oel_tagging.tag + pk: -119 + fields: + taxonomy: -1 + parent: null + value: Ndebele, South + external_id: nr +- model: oel_tagging.tag + pk: -120 + fields: + taxonomy: -1 + parent: null + value: Navajo + external_id: nv +- model: oel_tagging.tag + pk: -121 + fields: + taxonomy: -1 + parent: null + value: Chichewa + external_id: ny +- model: oel_tagging.tag + pk: -122 + fields: + taxonomy: -1 + parent: null + value: Occitan (post 1500) + external_id: oc +- model: oel_tagging.tag + pk: -123 + fields: + taxonomy: -1 + parent: null + value: Ojibwa + external_id: oj +- model: oel_tagging.tag + pk: -124 + fields: + taxonomy: -1 + parent: null + value: Oromo + external_id: om +- model: oel_tagging.tag + pk: -125 + fields: + taxonomy: -1 + parent: null + value: Oriya + external_id: or +- model: oel_tagging.tag + pk: -126 + fields: + taxonomy: -1 + parent: null + value: Ossetian + external_id: os +- model: oel_tagging.tag + pk: -127 + fields: + taxonomy: -1 + parent: null + value: Panjabi + external_id: pa +- model: oel_tagging.tag + pk: -128 + fields: + taxonomy: -1 + parent: null + value: Pali + external_id: pi +- model: oel_tagging.tag + pk: -129 + fields: + taxonomy: -1 + parent: null + value: Polish + external_id: pl +- model: oel_tagging.tag + pk: -130 + fields: + taxonomy: -1 + parent: null + value: Pushto + external_id: ps +- model: oel_tagging.tag + pk: -131 + fields: + taxonomy: -1 + parent: null + value: Portuguese + external_id: pt +- model: oel_tagging.tag + pk: -132 + fields: + taxonomy: -1 + parent: null + value: Quechua + external_id: qu +- model: oel_tagging.tag + pk: -133 + fields: + taxonomy: -1 + parent: null + value: Romansh + external_id: rm +- model: oel_tagging.tag + pk: -134 + fields: + taxonomy: -1 + parent: null + value: Rundi + external_id: rn +- model: oel_tagging.tag + pk: -135 + fields: + taxonomy: -1 + parent: null + value: Romanian + external_id: ro +- model: oel_tagging.tag + pk: -136 + fields: + taxonomy: -1 + parent: null + value: Russian + external_id: ru +- model: oel_tagging.tag + pk: -137 + fields: + taxonomy: -1 + parent: null + value: Kinyarwanda + external_id: rw +- model: oel_tagging.tag + pk: -138 + fields: + taxonomy: -1 + parent: null + value: Sanskrit + external_id: sa +- model: oel_tagging.tag + pk: -139 + fields: + taxonomy: -1 + parent: null + value: Sardinian + external_id: sc +- model: oel_tagging.tag + pk: -140 + fields: + taxonomy: -1 + parent: null + value: Sindhi + external_id: sd +- model: oel_tagging.tag + pk: -141 + fields: + taxonomy: -1 + parent: null + value: Northern Sami + external_id: se +- model: oel_tagging.tag + pk: -142 + fields: + taxonomy: -1 + parent: null + value: Sango + external_id: sg +- model: oel_tagging.tag + pk: -143 + fields: + taxonomy: -1 + parent: null + value: Sinhala + external_id: si +- model: oel_tagging.tag + pk: -144 + fields: + taxonomy: -1 + parent: null + value: Slovak + external_id: sk +- model: oel_tagging.tag + pk: -145 + fields: + taxonomy: -1 + parent: null + value: Slovenian + external_id: sl +- model: oel_tagging.tag + pk: -146 + fields: + taxonomy: -1 + parent: null + value: Samoan + external_id: sm +- model: oel_tagging.tag + pk: -147 + fields: + taxonomy: -1 + parent: null + value: Shona + external_id: sn +- model: oel_tagging.tag + pk: -148 + fields: + taxonomy: -1 + parent: null + value: Somali + external_id: so +- model: oel_tagging.tag + pk: -149 + fields: + taxonomy: -1 + parent: null + value: Albanian + external_id: sq +- model: oel_tagging.tag + pk: -150 + fields: + taxonomy: -1 + parent: null + value: Serbian + external_id: sr +- model: oel_tagging.tag + pk: -151 + fields: + taxonomy: -1 + parent: null + value: Swati + external_id: ss +- model: oel_tagging.tag + pk: -152 + fields: + taxonomy: -1 + parent: null + value: Sotho, Southern + external_id: st +- model: oel_tagging.tag + pk: -153 + fields: + taxonomy: -1 + parent: null + value: Sundanese + external_id: su +- model: oel_tagging.tag + pk: -154 + fields: + taxonomy: -1 + parent: null + value: Swedish + external_id: sv +- model: oel_tagging.tag + pk: -155 + fields: + taxonomy: -1 + parent: null + value: Swahili + external_id: sw +- model: oel_tagging.tag + pk: -156 + fields: + taxonomy: -1 + parent: null + value: Tamil + external_id: ta +- model: oel_tagging.tag + pk: -157 + fields: + taxonomy: -1 + parent: null + value: Telugu + external_id: te +- model: oel_tagging.tag + pk: -158 + fields: + taxonomy: -1 + parent: null + value: Tajik + external_id: tg +- model: oel_tagging.tag + pk: -159 + fields: + taxonomy: -1 + parent: null + value: Thai + external_id: th +- model: oel_tagging.tag + pk: -160 + fields: + taxonomy: -1 + parent: null + value: Tigrinya + external_id: ti +- model: oel_tagging.tag + pk: -161 + fields: + taxonomy: -1 + parent: null + value: Turkmen + external_id: tk +- model: oel_tagging.tag + pk: -162 + fields: + taxonomy: -1 + parent: null + value: Tagalog + external_id: tl +- model: oel_tagging.tag + pk: -163 + fields: + taxonomy: -1 + parent: null + value: Tswana + external_id: tn +- model: oel_tagging.tag + pk: -164 + fields: + taxonomy: -1 + parent: null + value: Tonga (Tonga Islands) + external_id: to +- model: oel_tagging.tag + pk: -165 + fields: + taxonomy: -1 + parent: null + value: Turkish + external_id: tr +- model: oel_tagging.tag + pk: -166 + fields: + taxonomy: -1 + parent: null + value: Tsonga + external_id: ts +- model: oel_tagging.tag + pk: -167 + fields: + taxonomy: -1 + parent: null + value: Tatar + external_id: tt +- model: oel_tagging.tag + pk: -168 + fields: + taxonomy: -1 + parent: null + value: Twi + external_id: tw +- model: oel_tagging.tag + pk: -169 + fields: + taxonomy: -1 + parent: null + value: Tahitian + external_id: ty +- model: oel_tagging.tag + pk: -170 + fields: + taxonomy: -1 + parent: null + value: Uighur + external_id: ug +- model: oel_tagging.tag + pk: -171 + fields: + taxonomy: -1 + parent: null + value: Ukrainian + external_id: uk +- model: oel_tagging.tag + pk: -172 + fields: + taxonomy: -1 + parent: null + value: Urdu + external_id: ur +- model: oel_tagging.tag + pk: -173 + fields: + taxonomy: -1 + parent: null + value: Uzbek + external_id: uz +- model: oel_tagging.tag + pk: -174 + fields: + taxonomy: -1 + parent: null + value: Venda + external_id: ve +- model: oel_tagging.tag + pk: -175 + fields: + taxonomy: -1 + parent: null + value: Vietnamese + external_id: vi +- model: oel_tagging.tag + pk: -176 + fields: + taxonomy: -1 + parent: null + value: Volapük + external_id: vo +- model: oel_tagging.tag + pk: -177 + fields: + taxonomy: -1 + parent: null + value: Walloon + external_id: wa +- model: oel_tagging.tag + pk: -178 + fields: + taxonomy: -1 + parent: null + value: Wolof + external_id: wo +- model: oel_tagging.tag + pk: -179 + fields: + taxonomy: -1 + parent: null + value: Xhosa + external_id: xh +- model: oel_tagging.tag + pk: -180 + fields: + taxonomy: -1 + parent: null + value: Yiddish + external_id: yi +- model: oel_tagging.tag + pk: -181 + fields: + taxonomy: -1 + parent: null + value: Yoruba + external_id: yo +- model: oel_tagging.tag + pk: -182 + fields: + taxonomy: -1 + parent: null + value: Zhuang + external_id: za +- model: oel_tagging.tag + pk: -183 + fields: + taxonomy: -1 + parent: null + value: Chinese + external_id: zh +- model: oel_tagging.tag + pk: -184 + fields: + taxonomy: -1 + parent: null + value: Zulu + external_id: zu +- model: oel_tagging.taxonomy + pk: -1 + fields: + name: Languages + description: ISO 639-1 Languages. Allows tags for any language configured for use on the instance + enabled: true + required: true + allow_multiple: false + allow_free_text: false + visible_to_authors: true diff --git a/openedx_tagging/core/tagging/management/commands/__init__.py b/openedx_tagging/core/tagging/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_tagging/core/tagging/management/commands/build_language_fixture.py b/openedx_tagging/core/tagging/management/commands/build_language_fixture.py new file mode 100644 index 000000000..9c1e0a85e --- /dev/null +++ b/openedx_tagging/core/tagging/management/commands/build_language_fixture.py @@ -0,0 +1,48 @@ +""" +Script that downloads all the ISO 639-1 languages and processes them +to write the fixture for the Language system-defined taxonomy. + +This function is intended to be used only once, +but can be edited in the future if more data needs to be added to the fixture. +""" +import json +import urllib.request + +from django.core.management.base import BaseCommand + +endpoint = "https://pkgstore.datahub.io/core/language-codes/language-codes_json/data/97607046542b532c395cf83df5185246/language-codes_json.json" +output = "./openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml" + + +class Command(BaseCommand): + def handle(self, **options): + json_data = self.download_json() + self.build_fixture(json_data) + + def download_json(self): + with urllib.request.urlopen(endpoint) as response: + json_data = response.read() + return json.loads(json_data) + + def build_fixture(self, json_data): + tag_pk = -1 + with open(output, "w") as output_file: + for lang_data in json_data: + lang_value = self.get_lang_value(lang_data) + lang_code = lang_data["alpha2"] + output_file.write("- model: oel_tagging.tag\n") + output_file.write(f" pk: {tag_pk}\n") + output_file.write(" fields:\n") + output_file.write(" taxonomy: -1\n") + output_file.write(" parent: null\n") + output_file.write(f" value: {lang_value}\n") + output_file.write(f" external_id: {lang_code}\n") + # System tags are identified with negative numbers to avoid clashing with user-created tags. + tag_pk -= 1 + + def get_lang_value(self, lang_data): + """ + Gets the lang value. Some languages has many values. + """ + lang_list = lang_data["English"].split(";") + return lang_list[0] diff --git a/openedx_tagging/core/tagging/migrations/0003_auto_20230721_1238.py b/openedx_tagging/core/tagging/migrations/0003_auto_20230721_1238.py new file mode 100644 index 000000000..e44f21cf8 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0003_auto_20230721_1238.py @@ -0,0 +1,76 @@ +# Generated by Django 3.2.19 on 2023-07-21 17:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0002_auto_20230718_2026"), + ] + + operations = [ + migrations.CreateModel( + name="ModelObjectTag", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("oel_tagging.objecttag",), + ), + migrations.CreateModel( + name="SystemDefinedTaxonomy", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("oel_tagging.taxonomy",), + ), + migrations.RemoveField( + model_name="taxonomy", + name="system_defined", + ), + migrations.CreateModel( + name="LanguageTaxonomy", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("oel_tagging.systemdefinedtaxonomy",), + ), + migrations.CreateModel( + name="ModelSystemDefinedTaxonomy", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("oel_tagging.systemdefinedtaxonomy",), + ), + migrations.CreateModel( + name="UserModelObjectTag", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("oel_tagging.modelobjecttag",), + ), + migrations.CreateModel( + name="UserSystemDefinedTaxonomy", + fields=[], + options={ + "proxy": True, + "indexes": [], + "constraints": [], + }, + bases=("oel_tagging.modelsystemdefinedtaxonomy",), + ), + ] diff --git a/openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py b/openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py new file mode 100644 index 000000000..4cca2c417 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.19 on 2023-07-24 06:25 + +from django.db import migrations, models +import openedx_learning.lib.fields + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0003_auto_20230721_1238"), + ] + + operations = [ + migrations.AlterField( + model_name="objecttag", + name="object_id", + field=openedx_learning.lib.fields.MultiCollationCharField( + db_collations={"mysql": "utf8mb4_unicode_ci", "sqlite": "NOCASE"}, + db_index=True, + editable=False, + help_text="Identifier for the object being tagged", + max_length=255, + ), + ), + migrations.AlterUniqueTogether( + name="tag", + unique_together={("taxonomy", "external_id"), ("taxonomy", "value")}, + ), + migrations.AddIndex( + model_name="objecttag", + index=models.Index( + fields=["taxonomy", "object_id"], name="oel_tagging_taxonom_aa24e6_idx" + ), + ), + ] diff --git a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py new file mode 100644 index 000000000..48bab478a --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.19 on 2023-07-28 13:33 + +from django.db import migrations +from django.core.management import call_command + + +def load_language_taxonomy(apps, schema_editor): + """ + Load language taxonomy and tags + """ + call_command("loaddata", "--app=oel_tagging", "language_taxonomy.yaml") + + +def revert(apps, schema_editor): + """ + Deletes language taxonomy an tags + """ + Taxonomy = apps.get_model("oel_tagging", "Taxonomy") + Taxonomy.objects.filter(id=-1).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0004_auto_20230723_2001"), + ] + + operations = [ + migrations.RunPython(load_language_taxonomy, revert), + ] diff --git a/openedx_tagging/core/tagging/models/__init__.py b/openedx_tagging/core/tagging/models/__init__.py new file mode 100644 index 000000000..90640ddfc --- /dev/null +++ b/openedx_tagging/core/tagging/models/__init__.py @@ -0,0 +1,11 @@ +from .base import ( + Tag, + Taxonomy, + ObjectTag, +) +from .system_defined import ( + ModelObjectTag, + ModelSystemDefinedTaxonomy, + UserSystemDefinedTaxonomy, + LanguageTaxonomy, +) diff --git a/openedx_tagging/core/tagging/models.py b/openedx_tagging/core/tagging/models/base.py similarity index 90% rename from openedx_tagging/core/tagging/models.py rename to openedx_tagging/core/tagging/models/base.py index 3c8222bcd..499149845 100644 --- a/openedx_tagging/core/tagging/models.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -1,4 +1,4 @@ -""" Tagging app data models """ +""" Tagging app base data models """ import logging from typing import List, Type, Union @@ -6,7 +6,10 @@ from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ -from openedx_learning.lib.fields import MultiCollationTextField, case_insensitive_char_field +from openedx_learning.lib.fields import ( + MultiCollationTextField, + case_insensitive_char_field, +) log = logging.getLogger(__name__) @@ -66,6 +69,10 @@ class Meta: models.Index(fields=["taxonomy", "value"]), models.Index(fields=["taxonomy", "external_id"]), ] + unique_together = [ + ["taxonomy", "external_id"], + ["taxonomy", "value"], + ] def __repr__(self): """ @@ -140,14 +147,6 @@ class Taxonomy(models.Model): "Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values." ), ) - system_defined = models.BooleanField( - default=False, - editable=False, - help_text=_( - "Indicates that tags and metadata for this taxonomy are maintained by the system;" - " taxonomy admins will not be permitted to modify them.", - ), - ) visible_to_authors = models.BooleanField( default=True, editable=False, @@ -177,8 +176,25 @@ def __str__(self): """ User-facing string representation of a Taxonomy. """ + try: + if self._taxonomy_class: + return f"<{self.taxonomy_class.__name__}> ({self.id}) {self.name}" + except ImportError: + # Log error and continue + log.exception( + f"Unable to import taxonomy_class for {self.id}: {self._taxonomy_class}" + ) return f"<{self.__class__.__name__}> ({self.id}) {self.name}" + @property + def object_tag_class(self) -> Type: + """ + Returns the ObjectTag subclass associated with this taxonomy, which is ObjectTag by default. + + Taxonomy subclasses may override this method to use different subclasses of ObjectTag. + """ + return ObjectTag + @property def taxonomy_class(self) -> Type: """ @@ -190,6 +206,14 @@ def taxonomy_class(self) -> Type: return import_string(self._taxonomy_class) return None + @property + def system_defined(self) -> bool: + """ + Indicates that tags and metadata for this taxonomy are maintained by the system; + taxonomy admins will not be permitted to modify them. + """ + return False + @taxonomy_class.setter def taxonomy_class(self, taxonomy_class: Union[Type, None]): """ @@ -239,15 +263,16 @@ def copy(self, taxonomy: "Taxonomy") -> "Taxonomy": self.required = taxonomy.required self.allow_multiple = taxonomy.allow_multiple self.allow_free_text = taxonomy.allow_free_text - self.system_defined = taxonomy.system_defined self.visible_to_authors = taxonomy.visible_to_authors self._taxonomy_class = taxonomy._taxonomy_class return self - def get_tags(self) -> List[Tag]: + def get_tags(self, tag_set: models.QuerySet = None) -> List[Tag]: """ Returns a list of all Tags in the current taxonomy, from the root(s) down to TAXONOMY_MAX_DEPTH tags, in tree order. + Use `tag_set` to do an initial filtering of the tags. + Annotates each returned Tag with its ``depth`` in the tree (starting at 0). Performance note: may perform as many as TAXONOMY_MAX_DEPTH select queries. @@ -256,9 +281,12 @@ def get_tags(self) -> List[Tag]: if self.allow_free_text: return tags + if tag_set is None: + tag_set = self.tag_set + parents = None for depth in range(TAXONOMY_MAX_DEPTH): - filtered_tags = self.tag_set.prefetch_related("parent") + filtered_tags = tag_set.prefetch_related("parent") if parents is None: filtered_tags = filtered_tags.filter(parent=None) else: @@ -366,9 +394,10 @@ def tag_object( _(f"Taxonomy ({self.id}) requires at least one tag per object.") ) + ObjectTagClass = self.object_tag_class current_tags = { tag.tag_ref: tag - for tag in ObjectTag.objects.filter( + for tag in ObjectTagClass.objects.filter( taxonomy=self, object_id=object_id, ) @@ -378,20 +407,12 @@ def tag_object( if tag_ref in current_tags: object_tag = current_tags.pop(tag_ref) else: - object_tag = ObjectTag( + object_tag = ObjectTagClass( taxonomy=self, object_id=object_id, ) - try: - object_tag.tag = self.tag_set.get( - id=tag_ref, - ) - except (ValueError, Tag.DoesNotExist): - # This might be ok, e.g. if self.allow_free_text. - # We'll validate below before saving. - object_tag.value = tag_ref - + object_tag.tag_ref = tag_ref object_tag.resync() if not self.validate_object_tag(object_tag): raise ValueError( @@ -431,6 +452,7 @@ class ObjectTag(models.Model): id = models.BigAutoField(primary_key=True) object_id = case_insensitive_char_field( max_length=255, + db_index=True, editable=False, help_text=_("Identifier for the object being tagged"), ) @@ -472,6 +494,7 @@ class ObjectTag(models.Model): class Meta: indexes = [ + models.Index(fields=["taxonomy", "object_id"]), models.Index(fields=["taxonomy", "_value"]), ] @@ -531,6 +554,24 @@ def tag_ref(self) -> str: """ return self.tag.id if self.tag_id else self._value + @tag_ref.setter + def tag_ref(self, tag_ref: str): + """ + Sets the ObjectTag's Tag and/or value, depending on whether a valid Tag is found. + + Subclasses may override this method to dynamically create Tags. + """ + self.value = tag_ref + + if self.taxonomy_id: + try: + self.tag = self.taxonomy.tag_set.get(pk=tag_ref) + self.value = self.tag.value + except (ValueError, Tag.DoesNotExist): + # This might be ok, e.g. if our taxonomy.allow_free_text, so we just pass through here. + # We rely on the caller to validate before saving. + pass + def is_valid(self) -> bool: """ Returns True if this ObjectTag represents a valid taxonomy tag. diff --git a/openedx_tagging/core/tagging/models/system_defined.py b/openedx_tagging/core/tagging/models/system_defined.py new file mode 100644 index 000000000..7b39b9c2a --- /dev/null +++ b/openedx_tagging/core/tagging/models/system_defined.py @@ -0,0 +1,269 @@ +""" Tagging app system-defined taxonomies data models """ +import logging +from typing import Any, List, Type, Union + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.db import models + +from openedx_tagging.core.tagging.models.base import ObjectTag + +from .base import Tag, Taxonomy, ObjectTag + +log = logging.getLogger(__name__) + + +class SystemDefinedTaxonomy(Taxonomy): + """ + Simple subclass of Taxonomy which requires the system_defined flag to be set. + """ + + class Meta: + proxy = True + + @property + def system_defined(self) -> bool: + """ + Indicates that tags and metadata for this taxonomy are maintained by the system; + taxonomy admins will not be permitted to modify them. + """ + return True + + +class ModelObjectTag(ObjectTag): + """ + Model-based ObjectTag, abstract class. + + Used by ModelSystemDefinedTaxonomy to maintain dynamic Tags which are associated with a configured Model instance. + """ + + class Meta: + proxy = True + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """ + Checks if the `tag_class_model` is correct + """ + assert issubclass(self.tag_class_model, models.Model) + super().__init__(*args, **kwargs) + + @property + def tag_class_model(self) -> Type: + """ + Subclasses must implement this method to return the Django.model + class referenced by these object tags. + """ + raise NotImplementedError + + @property + def tag_class_value(self) -> str: + """ + Returns the name of the tag_class_model field to use as the Tag.value when creating Tags for this taxonomy. + + Subclasses may override this method to use different fields. + """ + return "pk" + + def get_instance(self) -> Union[models.Model, None]: + """ + Returns the instance of tag_class_model associated with this object tag, or None if not found. + """ + instance_id = self.tag.external_id if self.tag else None + if instance_id: + try: + return self.tag_class_model.objects.get(pk=instance_id) + except ValueError as e: + log.exception(f"{self}: {str(e)}") + except self.tag_class_model.DoesNotExist: + log.exception( + f"{self}: {self.tag_class_model.__name__} pk={instance_id} does not exist." + ) + + return None + + def _resync_tag(self) -> bool: + """ + Resync our tag's value with the value from the instance. + + If the instance associated with the tag no longer exists, we unset our tag, because it's no longer valid. + + Returns True if the given tag was changed, False otherwise. + """ + instance = self.get_instance() + if instance: + value = getattr(instance, self.tag_class_value) + self.value = value + if self.tag and self.tag.value != value: + self.tag.value = value + self.tag.save() + return True + else: + self.tag = None + + return False + + @property + def tag_ref(self) -> str: + return (self.tag.external_id or self.tag.id) if self.tag_id else self._value + + @tag_ref.setter + def tag_ref(self, tag_ref: str): + """ + Sets the ObjectTag's Tag and/or value, depending on whether a valid Tag is found, or can be created. + + Creates a Tag for the given tag_ref value, if one containing that external_id not already exist. + """ + self.value = tag_ref + + if self.taxonomy_id: + try: + self.tag = self.taxonomy.tag_set.get( + external_id=tag_ref, + ) + except (ValueError, Tag.DoesNotExist): + # Creates a new Tag for this instance + self.tag = Tag( + taxonomy=self.taxonomy, + external_id=tag_ref, + ) + + self._resync_tag() + + +class ModelSystemDefinedTaxonomy(SystemDefinedTaxonomy): + """ + Model based system taxonomy abstract class. + + This type of taxonomy has an associated Django model in ModelObjectTag.tag_class_model(). + They are designed to create Tags when required for new ObjectTags, to maintain + their status as "closed" taxonomies. + The Tags are representations of the instances of the associated model. + + Tag.external_id stores an identifier from the instance (`pk` as default) + and Tag.value stores a human readable representation of the instance + (e.g. `username`). + The subclasses can override this behavior, to choose the right field. + + When an ObjectTag is created with an existing Tag, + the Tag is re-synchronized with its instance. + """ + + class Meta: + proxy = True + + def __init__(self, *args: Any, **kwargs: Any) -> None: + """ + Checks if the `object_tag_class` is a subclass of ModelObjectTag. + """ + assert issubclass(self.object_tag_class, ModelObjectTag) + super().__init__(*args, **kwargs) + + @property + def object_tag_class(self) -> Type: + """ + Returns the ObjectTag subclass associated with this taxonomy. + + Model Taxonomy subclasses must implement this to provide a ModelObjectTag subclass. + """ + raise NotImplementedError + + def _check_instance(self, object_tag: ObjectTag) -> bool: + """ + Returns True if the instance exists + + Subclasses can override this method to perform their own instance validation checks. + """ + object_tag = self.object_tag_class.cast(object_tag) + return bool(object_tag.get_instance()) + + def _check_tag(self, object_tag: ObjectTag) -> bool: + """ + Returns True if the instance is valid + """ + return super()._check_tag(object_tag) and self._check_instance(object_tag) + + +class UserModelObjectTag(ModelObjectTag): + """ + ObjectTags for the UserSystemDefinedTaxonomy. + """ + + class Meta: + proxy = True + + @property + def tag_class_model(self) -> Type: + """ + Associate the user model + """ + return get_user_model() + + @property + def tag_class_value(self) -> str: + """ + Returns the name of the tag_class_model field to use as the Tag.value when creating Tags for this taxonomy. + + Subclasses may override this method to use different fields. + """ + return "username" + + +class UserSystemDefinedTaxonomy(ModelSystemDefinedTaxonomy): + """ + User based system taxonomy class. + """ + + class Meta: + proxy = True + + @property + def object_tag_class(self) -> Type: + """ + Returns the ObjectTag subclass associated with this taxonomy, which is ModelObjectTag by default. + + Model Taxonomy subclasses must implement this to provide a ModelObjectTag subclass. + """ + return UserModelObjectTag + + +class LanguageTaxonomy(SystemDefinedTaxonomy): + """ + Language System-defined taxonomy + + The tags are filtered and validated taking into account the + languages available in Django LANGUAGES settings var + """ + + class Meta: + proxy = True + + def get_tags(self, tag_set: models.QuerySet = None) -> List[Tag]: + """ + Returns a list of all the available Language Tags, annotated with ``depth`` = 0. + """ + available_langs = self._get_available_languages() + tag_set = self.tag_set.filter(external_id__in=available_langs) + return super().get_tags(tag_set=tag_set) + + def _get_available_languages(cls) -> List[str]: + """ + Get available languages from Django LANGUAGE. + """ + langs = set() + for django_lang in settings.LANGUAGES: + # Split to get the language part + langs.add(django_lang[0].split("-")[0]) + return langs + + def _check_valid_language(self, object_tag: ObjectTag) -> bool: + """ + Returns True if the tag is on the available languages + """ + available_langs = self._get_available_languages() + return object_tag.tag.external_id in available_langs + + def _check_tag(self, object_tag: ObjectTag) -> bool: + """ + Returns True if the tag is on the available languages + """ + return super()._check_tag(object_tag) and self._check_valid_language(object_tag) diff --git a/openedx_tagging/core/tagging/rest_api/v1/permissions.py b/openedx_tagging/core/tagging/rest_api/v1/permissions.py index b75c5ff79..e8915a62f 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/permissions.py +++ b/openedx_tagging/core/tagging/rest_api/v1/permissions.py @@ -15,4 +15,3 @@ class TaxonomyObjectPermissions(DjangoObjectPermissions): "PATCH": ["%(app_label)s.change_%(model_name)s"], "DELETE": ["%(app_label)s.delete_%(model_name)s"], } - diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index a46350a9a..53f647424 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -6,6 +6,7 @@ from openedx_tagging.core.tagging.models import Taxonomy + class TaxonomyListQueryParamsSerializer(serializers.Serializer): """ Serializer for the query params for the GET view @@ -13,6 +14,7 @@ class TaxonomyListQueryParamsSerializer(serializers.Serializer): enabled = serializers.BooleanField(required=False) + class TaxonomySerializer(serializers.ModelSerializer): class Meta: model = Taxonomy @@ -27,4 +29,3 @@ class Meta: "system_defined", "visible_to_authors", ] - diff --git a/openedx_tagging/core/tagging/rest_api/v1/urls.py b/openedx_tagging/core/tagging/rest_api/v1/urls.py index 97d653a28..80500990c 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/urls.py +++ b/openedx_tagging/core/tagging/rest_api/v1/urls.py @@ -11,6 +11,4 @@ router = DefaultRouter() router.register("taxonomies", views.TaxonomyView, basename="taxonomy") -urlpatterns = [ - path('', include(router.urls)) -] +urlpatterns = [path("", include(router.urls))] diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 1ea02aca9..37c3b9689 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -107,7 +107,6 @@ class TaxonomyView(ModelViewSet): """ - serializer_class = TaxonomySerializer permission_classes = [TaxonomyObjectPermissions] diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 2cc94d0dc..364178c70 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -19,7 +19,7 @@ def can_view_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: Anyone can view an enabled taxonomy or list all taxonomies, but only taxonomy admins can view a disabled taxonomy. """ - return not taxonomy or taxonomy.enabled or is_taxonomy_admin(user) + return not taxonomy or taxonomy.cast().enabled or is_taxonomy_admin(user) @rules.predicate @@ -28,24 +28,21 @@ def can_change_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: Even taxonomy admins cannot change system taxonomies. """ return is_taxonomy_admin(user) and ( - not taxonomy or (taxonomy and not taxonomy.system_defined) + not taxonomy or (taxonomy and not taxonomy.cast().system_defined) ) @rules.predicate -def can_change_taxonomy_tag(user: User, tag: Tag = None) -> bool: +def can_change_tag(user: User, tag: Tag = None) -> bool: """ Even taxonomy admins cannot add tags to system taxonomies (their tags are system-defined), or free-text taxonomies (these don't have predefined tags). """ + taxonomy = tag.taxonomy.cast() if (tag and tag.taxonomy_id) else None return is_taxonomy_admin(user) and ( not tag - or not tag.taxonomy - or ( - tag.taxonomy - and not tag.taxonomy.allow_free_text - and not tag.taxonomy.system_defined - ) + or not taxonomy + or (taxonomy and not taxonomy.allow_free_text and not taxonomy.system_defined) ) @@ -54,10 +51,12 @@ def can_change_object_tag(user: User, object_tag: ObjectTag = None) -> bool: """ Taxonomy admins can create or modify object tags on enabled taxonomies. """ + taxonomy = ( + object_tag.taxonomy.cast() if (object_tag and object_tag.taxonomy_id) else None + ) + object_tag = taxonomy.object_tag_class.cast(object_tag) if taxonomy else object_tag return is_taxonomy_admin(user) and ( - not object_tag - or not object_tag.taxonomy - or (object_tag.taxonomy and object_tag.taxonomy.enabled) + not object_tag or not taxonomy or (taxonomy and taxonomy.enabled) ) @@ -68,8 +67,8 @@ def can_change_object_tag(user: User, object_tag: ObjectTag = None) -> bool: rules.add_perm("oel_tagging.view_taxonomy", can_view_taxonomy) # Tag -rules.add_perm("oel_tagging.add_tag", can_change_taxonomy_tag) -rules.add_perm("oel_tagging.change_tag", can_change_taxonomy_tag) +rules.add_perm("oel_tagging.add_tag", can_change_tag) +rules.add_perm("oel_tagging.change_tag", can_change_tag) rules.add_perm("oel_tagging.delete_tag", is_taxonomy_admin) rules.add_perm("oel_tagging.view_tag", rules.always_allow) diff --git a/projects/dev.py b/projects/dev.py index a96473d4b..c65e2c34f 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -52,6 +52,7 @@ AUTHENTICATION_BACKENDS = [ 'rules.permissions.ObjectPermissionBackend', + 'django.contrib.auth.backends.ModelBackend', ] MIDDLEWARE = [ diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index a56ece81a..5bb3936c9 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -145,6 +145,34 @@ parent: 15 value: Mammalia external_id: null +- model: oel_tagging.tag + pk: 22 + fields: + taxonomy: 4 + parent: null + value: System Tag 1 + external_id: 'tag_1' +- model: oel_tagging.tag + pk: 23 + fields: + taxonomy: 4 + parent: null + value: System Tag 2 + external_id: 'tag_2' +- model: oel_tagging.tag + pk: 24 + fields: + taxonomy: 4 + parent: null + value: System Tag 3 + external_id: 'tag_3' +- model: oel_tagging.tag + pk: 25 + fields: + taxonomy: 4 + parent: null + value: System Tag 4 + external_id: 'tag_4' - model: oel_tagging.taxonomy pk: 1 fields: @@ -154,14 +182,23 @@ required: false allow_multiple: false allow_free_text: false - system_defined: false - model: oel_tagging.taxonomy - pk: 2 + pk: 3 + fields: + name: User Authors + description: Allows tags for any User on the instance. + enabled: true + required: false + allow_multiple: false + allow_free_text: false + _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.UserSystemDefinedTaxonomy +- model: oel_tagging.taxonomy + pk: 4 fields: - name: System Languages - description: Allows tags for any language configured for use on the instance. + name: System defined taxonomy + description: Generic System defined taxonomy enabled: true required: false allow_multiple: false allow_free_text: false - system_defined: true + _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.SystemDefinedTaxonomy diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 798b4826b..a75aebb61 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -1,11 +1,19 @@ """ Test the tagging APIs """ -from django.test.testcases import TestCase +from django.test.testcases import TestCase, override_settings import openedx_tagging.core.tagging.api as tagging_api from openedx_tagging.core.tagging.models import ObjectTag, Tag -from .test_models import TestTagTaxonomyMixin +from .test_models import TestTagTaxonomyMixin, get_tag + +test_languages = [ + ("az", "Azerbaijani"), + ("en", "English"), + ("id", "Indonesian"), + ("qu", "Quechua"), + ("zu", "Zulu"), +] class TestApiTagging(TestTagTaxonomyMixin, TestCase): @@ -47,10 +55,18 @@ def test_get_taxonomies(self): tax2 = tagging_api.create_taxonomy("Disabled", enabled=False) with self.assertNumQueries(1): enabled = list(tagging_api.get_taxonomies()) - assert enabled == [tax1, self.taxonomy, self.system_taxonomy] + + assert enabled == [ + tax1, + self.language_taxonomy, + self.taxonomy, + self.system_taxonomy, + self.user_taxonomy, + ] assert str(enabled[0]) == f" ({tax1.id}) Enabled" - assert str(enabled[1]) == " (1) Life on Earth" - assert str(enabled[2]) == " (2) System Languages" + assert str(enabled[1]) == " (-1) Languages" + assert str(enabled[2]) == " (1) Life on Earth" + assert str(enabled[3]) == " (4) System defined taxonomy" with self.assertNumQueries(1): disabled = list(tagging_api.get_taxonomies(enabled=False)) @@ -59,8 +75,16 @@ def test_get_taxonomies(self): with self.assertNumQueries(1): both = list(tagging_api.get_taxonomies(enabled=None)) - assert both == [tax2, tax1, self.taxonomy, self.system_taxonomy] + assert both == [ + tax2, + tax1, + self.language_taxonomy, + self.taxonomy, + self.system_taxonomy, + self.user_taxonomy, + ] + @override_settings(LANGUAGES=test_languages) def test_get_tags(self): self.setup_tag_depths() assert tagging_api.get_tags(self.taxonomy) == [ @@ -68,6 +92,11 @@ def test_get_tags(self): *self.kingdom_tags, *self.phylum_tags, ] + assert tagging_api.get_tags(self.system_taxonomy) == self.system_tags + tags = tagging_api.get_tags(self.language_taxonomy) + langs = [tag.external_id for tag in tags] + expected_langs = [lang[0] for lang in test_languages] + assert langs == expected_langs def check_object_tag(self, object_tag, taxonomy, tag, name, value): """ @@ -288,6 +317,98 @@ def test_tag_object_invalid_tag(self): exc.exception ) + @override_settings(LANGUAGES=test_languages) + def test_tag_object_language_taxonomy(self): + tags_list = [ + [get_tag("Azerbaijani").id], + [get_tag("English").id], + ] + + for tags in tags_list: + object_tags = tagging_api.tag_object( + self.language_taxonomy, + tags, + "biology101", + ) + + # Ensure the expected number of tags exist in the database + assert ( + list( + tagging_api.get_object_tags( + taxonomy=self.language_taxonomy, + object_id="biology101", + ) + ) + == object_tags + ) + # And the expected number of tags were returned + assert len(object_tags) == len(tags) + for index, object_tag in enumerate(object_tags): + assert object_tag.tag_id == tags[index] + assert object_tag.is_valid() + assert object_tag.taxonomy == self.language_taxonomy + assert object_tag.name == self.language_taxonomy.name + assert object_tag.object_id == "biology101" + + @override_settings(LANGUAGES=test_languages) + def test_tag_object_language_taxonomy_ivalid(self): + tags = [get_tag("Spanish").id] + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.language_taxonomy, + tags, + "biology101", + ) + assert "Invalid object tag for taxonomy (-1): -40" in str( + exc.exception + ) + + def test_tag_object_model_system_taxonomy(self): + users = [ + self.user_1, + self.user_2, + ] + + for user in users: + tags = [user.id] + object_tags = tagging_api.tag_object( + self.user_taxonomy, + tags, + "biology101", + ) + + # Ensure the expected number of tags exist in the database + assert ( + list( + tagging_api.get_object_tags( + taxonomy=self.user_taxonomy, + object_id="biology101", + ) + ) + == object_tags + ) + # And the expected number of tags were returned + assert len(object_tags) == len(tags) + for object_tag in object_tags: + assert object_tag.tag.external_id == str(user.id) + assert object_tag.tag.value == user.username + assert object_tag.is_valid() + assert object_tag.taxonomy == self.user_taxonomy + assert object_tag.name == self.user_taxonomy.name + assert object_tag.object_id == "biology101" + + def test_tag_object_model_system_taxonomy_invalid(self): + tags = ["Invalid id"] + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.user_taxonomy, + tags, + "biology101", + ) + assert "Invalid object tag for taxonomy (3): Invalid id" in str( + exc.exception + ) + def test_get_object_tags(self): # Alpha tag has no taxonomy alpha = ObjectTag(object_id="abc") diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 65af41eaa..2db7911af 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -1,9 +1,14 @@ -""" Test the tagging models """ - +""" Test the tagging base models """ import ddt +from django.contrib.auth import get_user_model from django.test.testcases import TestCase -from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy +from openedx_tagging.core.tagging.models import ( + ObjectTag, + Tag, + Taxonomy, + LanguageTaxonomy, +) def get_tag(value): @@ -23,13 +28,30 @@ class TestTagTaxonomyMixin: def setUp(self): super().setUp() self.taxonomy = Taxonomy.objects.get(name="Life on Earth") - self.system_taxonomy = Taxonomy.objects.get(name="System Languages") + self.system_taxonomy = Taxonomy.objects.get( + name="System defined taxonomy" + ) + self.language_taxonomy = Taxonomy.objects.get(name="Languages") + self.language_taxonomy.taxonomy_class = LanguageTaxonomy + self.language_taxonomy = self.language_taxonomy.cast() + self.user_taxonomy = Taxonomy.objects.get(name="User Authors").cast() self.archaea = get_tag("Archaea") self.archaebacteria = get_tag("Archaebacteria") self.bacteria = get_tag("Bacteria") self.eubacteria = get_tag("Eubacteria") self.chordata = get_tag("Chordata") self.mammalia = get_tag("Mammalia") + self.system_taxonomy_tag = get_tag("System Tag 1") + self.user_1 = get_user_model()( + id=1, + username="test_user_1", + ) + self.user_1.save() + self.user_2 = get_user_model()( + id=2, + username="test_user_2", + ) + self.user_2.save() # Domain tags (depth=0) # https://en.wikipedia.org/wiki/Domain_(biology) @@ -66,6 +88,13 @@ def setUp(self): get_tag("Porifera"), ] + self.system_tags = [ + get_tag("System Tag 1"), + get_tag("System Tag 2"), + get_tag("System Tag 3"), + get_tag("System Tag 4"), + ] + def setup_tag_depths(self): """ Annotate our tags with depth so we can compare them. @@ -119,16 +148,16 @@ class TestModelTagTaxonomy(TestTagTaxonomyMixin, TestCase): def test_system_defined(self): assert not self.taxonomy.system_defined - assert self.system_taxonomy.system_defined + assert self.system_taxonomy.cast().system_defined def test_representations(self): assert ( str(self.taxonomy) == repr(self.taxonomy) == " (1) Life on Earth" ) assert ( - str(self.system_taxonomy) - == repr(self.system_taxonomy) - == " (2) System Languages" + str(self.language_taxonomy) + == repr(self.language_taxonomy) + == " (-1) Languages" ) assert str(self.bacteria) == repr(self.bacteria) == " (1) Bacteria" @@ -293,6 +322,13 @@ def test_object_tag_lineage(self): object_tag.refresh_from_db() assert object_tag.get_lineage() == ["Another tag"] + def test_tag_ref(self): + object_tag = ObjectTag() + object_tag.tag_ref = 1 + object_tag.save() + assert object_tag.tag is None + assert object_tag.value == 1 + def test_object_tag_is_valid(self): open_taxonomy = Taxonomy.objects.create( name="Freetext Life", diff --git a/tests/openedx_tagging/core/tagging/test_system_defined_models.py b/tests/openedx_tagging/core/tagging/test_system_defined_models.py new file mode 100644 index 000000000..432d07110 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/test_system_defined_models.py @@ -0,0 +1,239 @@ +""" Test the tagging system-defined taxonomy models """ +import ddt + +from django.test.testcases import TestCase, override_settings +from django.contrib.auth import get_user_model + +from openedx_tagging.core.tagging.models import ( + ObjectTag, + Tag, +) +from openedx_tagging.core.tagging.models.system_defined import ( + ModelObjectTag, + ModelSystemDefinedTaxonomy, + UserSystemDefinedTaxonomy, +) + +from .test_models import TestTagTaxonomyMixin + + +test_languages = [ + ("en", "English"), + ("az", "Azerbaijani"), + ("id", "Indonesian"), + ("qu", "Quechua"), + ("zu", "Zulu"), +] + + +class EmptyTestClass: + """ + Empty class used for testing + """ + + +class InvalidModelTaxonomy(ModelSystemDefinedTaxonomy): + """ + Model used for testing + """ + + @property + def object_tag_class(self): + return EmptyTestClass + + class Meta: + proxy = True + managed = False + app_label = "oel_tagging" + + +class TestModelTag(ModelObjectTag): + """ + Model used for testing + """ + + @property + def tag_class_model(self): + return get_user_model() + + class Meta: + proxy = True + managed = False + app_label = "oel_tagging" + + +class TestModelTaxonomy(ModelSystemDefinedTaxonomy): + """ + Model used for testing + """ + + @property + def object_tag_class(self): + return TestModelTag + + class Meta: + proxy = True + managed = False + app_label = "oel_tagging" + + +@ddt.ddt +class TestModelSystemDefinedTaxonomy(TestTagTaxonomyMixin, TestCase): + """ + Test for Model Model System defined taxonomy + """ + + @ddt.data( + (ModelSystemDefinedTaxonomy, NotImplementedError), + (ModelObjectTag, NotImplementedError), + (InvalidModelTaxonomy, AssertionError), + (UserSystemDefinedTaxonomy, None), + ) + @ddt.unpack + def test_implementation_error(self, taxonomy_cls, expected_exception): + if not expected_exception: + assert taxonomy_cls() + else: + with self.assertRaises(expected_exception): + taxonomy_cls() + + @ddt.data( + (1, "tag_id", True), # Valid + (0, "tag_id", False), # Invalid user + ("test_id", "tag_id", False), # Invalid user id + (1, None, False), # Testing parent validations + ) + @ddt.unpack + def test_validations(self, tag_external_id, tag_id, expected): + tag = Tag( + id=tag_id, + taxonomy=self.user_taxonomy, + value="_val", + external_id=tag_external_id, + ) + object_tag = ObjectTag( + object_id="id", + tag=tag, + ) + + assert self.user_taxonomy.validate_object_tag( + object_tag=object_tag, + check_object=False, + check_taxonomy=False, + check_tag=True, + ) == expected + + def test_tag_object_invalid_user(self): + # Test user that doesn't exist + with self.assertRaises(ValueError): + self.user_taxonomy.tag_object(tags=[4], object_id="object_id") + + def _tag_object(self): + return self.user_taxonomy.tag_object( + tags=[self.user_1.id], object_id="object_id" + ) + + def test_tag_object_tag_creation(self): + # Test creation of a new Tag with user taxonomy + assert self.user_taxonomy.tag_set.count() == 0 + updated_tags = self._tag_object() + assert self.user_taxonomy.tag_set.count() == 1 + assert len(updated_tags) == 1 + assert updated_tags[0].tag.external_id == str(self.user_1.id) + assert updated_tags[0].tag.value == self.user_1.get_username() + + # Test parent functions + taxonomy = TestModelTaxonomy( + name="Test", + description="Test", + ) + taxonomy.save() + assert taxonomy.tag_set.count() == 0 + updated_tags = taxonomy.tag_object(tags=[self.user_1.id], object_id="object_id") + assert taxonomy.tag_set.count() == 1 + assert taxonomy.tag_set.count() == 1 + assert len(updated_tags) == 1 + assert updated_tags[0].tag.external_id == str(self.user_1.id) + assert updated_tags[0].tag.value == str(self.user_1.id) + + def test_tag_object_existing_tag(self): + # Test add an existing Tag + self._tag_object() + assert self.user_taxonomy.tag_set.count() == 1 + updated_tags = self._tag_object() + assert self.user_taxonomy.tag_set.count() == 1 + assert len(updated_tags) == 1 + assert updated_tags[0].tag.external_id == str(self.user_1.id) + assert updated_tags[0].tag.value == self.user_1.get_username() + + def test_tag_object_resync(self): + self._tag_object() + + self.user_1.username = "new_username" + self.user_1.save() + updated_tags = self._tag_object() + assert self.user_taxonomy.tag_set.count() == 1 + assert len(updated_tags) == 1 + assert updated_tags[0].tag.external_id == str(self.user_1.id) + assert updated_tags[0].tag.value == self.user_1.get_username() + + def test_tag_object_delete_user(self): + # Test after delete user + self._tag_object() + user_1_id = self.user_1.id + self.user_1.delete() + with self.assertRaises(ValueError): + self.user_taxonomy.tag_object( + tags=[user_1_id], + object_id="object_id", + ) + + def test_tag_ref(self): + object_tag = TestModelTag() + object_tag.tag_ref = 1 + object_tag.save() + assert object_tag.tag is None + assert object_tag.value == 1 + + def test_get_instance(self): + object_tag = TestModelTag() + assert object_tag.get_instance() is None + + +@ddt.ddt +@override_settings(LANGUAGES=test_languages) +class TestLanguageTaxonomy(TestTagTaxonomyMixin, TestCase): + """ + Test for Language taxonomy + """ + + @ddt.data( + ("en", "tag_id"), # Valid + ("es", "tag_id"), # Not available lang + ("en", None), # Test parent validations + ) + @ddt.unpack + def test_validations(self, lang, tag_id): + tag = Tag( + id=tag_id, + taxonomy=self.language_taxonomy, + value="_val", + external_id=lang, + ) + object_tag = ObjectTag( + object_id="id", + tag=tag, + ) + self.language_taxonomy.validate_object_tag( + object_tag=object_tag, + check_object=False, + check_taxonomy=False, + check_tag=True, + ) + + def test_get_tags(self): + tags = self.language_taxonomy.get_tags() + expected_langs = [lang[0] for lang in test_languages] + for tag in tags: + assert tag.external_id in expected_langs + assert tag.annotated_field == 0 diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 0fd0d6840..6ed24246c 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -8,6 +8,7 @@ from rest_framework.test import APITestCase from openedx_tagging.core.tagging.models import Taxonomy +from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy User = get_user_model() @@ -54,14 +55,14 @@ def setUp(self): ) @ddt.data( - (None, status.HTTP_200_OK, 3), - (1, status.HTTP_200_OK, 2), + (None, status.HTTP_200_OK, 4), + (1, status.HTTP_200_OK, 3), (0, status.HTTP_200_OK, 1), - (True, status.HTTP_200_OK, 2), + (True, status.HTTP_200_OK, 3), (False, status.HTTP_200_OK, 1), - ("True", status.HTTP_200_OK, 2), + ("True", status.HTTP_200_OK, 3), ("False", status.HTTP_200_OK, 1), - ("1", status.HTTP_200_OK, 2), + ("1", status.HTTP_200_OK, 3), ("0", status.HTTP_200_OK, 1), (2, status.HTTP_400_BAD_REQUEST, None), ("invalid", status.HTTP_400_BAD_REQUEST, None), @@ -82,6 +83,7 @@ def test_list_taxonomy_queryparams(self, enabled, expected_status, expected_coun assert response.status_code == expected_status # If we were able to list the taxonomies, check that we got the expected number back + # We take into account the Language Taxonomy that is created by the system in a migration if status.is_success(expected_status): assert len(response.data) == expected_count @@ -231,34 +233,28 @@ def test_update_taxonomy(self, user_attr, expected_status): ) @ddt.data( - (False, False, status.HTTP_200_OK), - (False, True, status.HTTP_200_OK), - (True, False, status.HTTP_403_FORBIDDEN), - (True, True, status.HTTP_403_FORBIDDEN), + (False, status.HTTP_200_OK), + (True, status.HTTP_403_FORBIDDEN), ) @ddt.unpack def test_update_taxonomy_system_defined( - self, create_value, update_value, expected_status + self, system_defined, expected_status ): ''' Test that we can't update system_defined field ''' - taxonomy = Taxonomy.objects.create( - name="test system taxonomy", system_defined=create_value - ) + taxonomy = Taxonomy.objects.create(name="test system taxonomy") + if system_defined: + taxonomy.taxonomy_class = SystemDefinedTaxonomy taxonomy.save() url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) self.client.force_authenticate(user=self.staff) response = self.client.put( - url, {"name": "new name", "system_defined": update_value}, format="json" + url, {"name": "new name"}, format="json" ) assert response.status_code == expected_status - # Verify that system_defined has not changed - response = self.client.get(url) - assert response.data["system_defined"] == create_value - def test_update_taxonomy_404(self): url = TAXONOMY_DETAIL_URL.format(pk=123123) @@ -303,34 +299,30 @@ def test_patch_taxonomy(self, user_attr, expected_status): ) @ddt.data( - (False, False, status.HTTP_200_OK), - (False, True, status.HTTP_200_OK), - (True, False, status.HTTP_403_FORBIDDEN), - (True, True, status.HTTP_403_FORBIDDEN), + (False, status.HTTP_200_OK), + (True, status.HTTP_403_FORBIDDEN), ) @ddt.unpack def test_patch_taxonomy_system_defined( - self, create_value, update_value, expected_status + self, system_defined, expected_status ): ''' Test that we can't patch system_defined field ''' taxonomy = Taxonomy.objects.create( - name="test system taxonomy", system_defined=create_value + name="test system taxonomy" ) + if system_defined: + taxonomy.taxonomy_class = SystemDefinedTaxonomy taxonomy.save() url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) self.client.force_authenticate(user=self.staff) response = self.client.patch( - url, {"system_defined": update_value}, format="json" + url, {"name": "New name"}, format="json" ) assert response.status_code == expected_status - # Verify that system_defined has not changed - response = self.client.get(url) - assert response.data["system_defined"] == create_value - def test_patch_taxonomy_404(self): url = TAXONOMY_DETAIL_URL.format(pk=123123) From fb1fd0860f4873aae9250901a6af75a5f86384cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Sun, 6 Aug 2023 18:13:49 -0500 Subject: [PATCH 033/282] Autocomplete tags API function (#59) * feat: Autocomplete tags Returns an ObjectTags queryset of where the query text matches anywhere in the tag value. * chore: updates relevant Indexes * chore: bumps version --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 43 ++++++ .../0006_alter_objecttag_unique_together.py | 16 +++ openedx_tagging/core/tagging/models/base.py | 47 +++++++ .../openedx_tagging/core/tagging/test_api.py | 126 ++++++++++++++++++ .../core/tagging/test_models.py | 15 +++ .../tagging/test_system_defined_models.py | 8 +- 7 files changed, 251 insertions(+), 6 deletions(-) create mode 100644 openedx_tagging/core/tagging/migrations/0006_alter_objecttag_unique_together.py diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 485f44ac2..b3f475621 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 5d8ca1158..55aa0d376 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -137,3 +137,46 @@ def tag_object( Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. """ return taxonomy.cast().tag_object(tags, object_id) + + +def autocomplete_tags( + taxonomy: Taxonomy, + search: str, + object_id: str = None, + object_tags_only=True, +) -> QuerySet: + """ + Provides auto-complete suggestions by matching the `search` string against existing + ObjectTags linked to the given taxonomy. A case-insensitive search is used in order + to return the highest number of relevant tags. + + If `object_id` is provided, then object tag values already linked to this object + are omitted from the returned suggestions. (ObjectTag values must be unique for a + given object + taxonomy, and so omitting these suggestions helps users avoid + duplication errors.). + + Returns a QuerySet of dictionaries containing distinct `value` (string) and + `tag` (numeric ID) values, sorted alphabetically by `value`. + The `value` is what should be shown as a suggestion to users, + and if it's a free-text taxonomy, `tag` will be `None`: we include the `tag` ID + in anticipation of the second use case listed below. + + Use cases: + * This method is useful for reducing tag variation in free-text taxonomies by showing + users tags that are similar to what they're typing. E.g., if the `search` string "dn" + shows that other objects have been tagged with "DNA", "DNA electrophoresis", and "DNA fingerprinting", + this encourages users to use those existing tags if relevant, instead of creating new ones that + look similar (e.g. "dna finger-printing"). + * It could also be used to assist tagging for closed taxonomies with a list of possible tags which is too + large to return all at once, e.g. a user model taxonomy that dynamically creates tags on request for any + registered user in the database. (Note that this is not implemented yet, but may be as part of a future change.) + """ + if not object_tags_only: + raise NotImplementedError( + _( + "Using this would return a query set of tags instead of object tags." + "For now we recommend fetching all of the taxonomy's tags " + "using get_tags() and filtering them on the frontend." + ) + ) + return taxonomy.cast().autocomplete_tags(search, object_id) diff --git a/openedx_tagging/core/tagging/migrations/0006_alter_objecttag_unique_together.py b/openedx_tagging/core/tagging/migrations/0006_alter_objecttag_unique_together.py new file mode 100644 index 000000000..0ffe5f7c6 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0006_alter_objecttag_unique_together.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.19 on 2023-08-02 16:20 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0005_language_taxonomy"), + ] + + operations = [ + migrations.AlterUniqueTogether( + name="objecttag", + unique_together={("taxonomy", "_value", "object_id")}, + ), + ] diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index 499149845..efdbda55a 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -430,6 +430,52 @@ def tag_object( return updated_tags + def autocomplete_tags( + self, + search: str, + object_id: str = None, + ) -> models.QuerySet: + """ + Provides auto-complete suggestions by matching the `search` string against existing + ObjectTags linked to the given taxonomy. A case-insensitive search is used in order + to return the highest number of relevant tags. + + If `object_id` is provided, then object tag values already linked to this object + are omitted from the returned suggestions. (ObjectTag values must be unique for a + given object + taxonomy, and so omitting these suggestions helps users avoid + duplication errors.). + + Returns a QuerySet of dictionaries containing distinct `value` (string) and `tag` + (numeric ID) values, sorted alphabetically by `value`. + + Subclasses can override this method to perform their own autocomplete process. + Subclass use cases: + * Large taxonomy associated with a model. It can be overridden to get + the suggestions directly from the model by doing own filtering. + * Taxonomy with a list of available tags: It can be overridden to only + search the suggestions on a list of available tags. + """ + # Fetch tags that the object already has to exclude them from the result + excluded_tags = [] + if object_id: + excluded_tags = self.objecttag_set.filter(object_id=object_id).values_list( + "_value", flat=True + ) + return ( + # Fetch object tags from this taxonomy whose value contains the search + self.objecttag_set.filter(_value__icontains=search) + # omit any tags whose values match the tags on the given object + .exclude(_value__in=excluded_tags) + # alphabetical ordering + .order_by("_value") + # Alias the `_value` field to `value` to make it nicer for users + .annotate(value=models.F("_value")) + # obtain tag values + .values("value", "tag_id") + # remove repeats + .distinct() + ) + class ObjectTag(models.Model): """ @@ -497,6 +543,7 @@ class Meta: models.Index(fields=["taxonomy", "object_id"]), models.Index(fields=["taxonomy", "_value"]), ] + unique_together = ("taxonomy", "_value", "object_id") def __repr__(self): """ diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index a75aebb61..b6affec4a 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -1,4 +1,5 @@ """ Test the tagging APIs """ +import ddt from django.test.testcases import TestCase, override_settings @@ -16,6 +17,7 @@ ] +@ddt.ddt class TestApiTagging(TestTagTaxonomyMixin, TestCase): """ Test the Tagging API methods. @@ -460,3 +462,127 @@ def test_get_object_tags(self): ) == [ beta, ] + + @ddt.data( + ("ChA", ["Archaea", "Archaebacteria"], [2,5]), + ("ar", ['Archaea', 'Archaebacteria', 'Arthropoda'], [2,5,14]), + ("aE", ['Archaea', 'Archaebacteria', 'Plantae'], [2,5,10]), + ( + "a", + [ + 'Animalia', + 'Archaea', + 'Archaebacteria', + 'Arthropoda', + 'Gastrotrich', + 'Monera', + 'Placozoa', + 'Plantae', + ], + [9,2,5,14,16,13,19,10], + ), + ) + @ddt.unpack + def test_autocomplete_tags(self, search, expected_values, expected_ids): + tags = [ + 'Archaea', + 'Archaebacteria', + 'Animalia', + 'Arthropoda', + 'Plantae', + 'Monera', + 'Gastrotrich', + 'Placozoa', + ] + expected_values # To create repeats + closed_taxonomy = self.taxonomy + open_taxonomy = tagging_api.create_taxonomy( + "Free_Text_Taxonomy", + allow_free_text=True, + ) + + for index, value in enumerate(tags): + # Creating ObjectTags for open taxonomy + ObjectTag( + object_id=f"object_id_{index}", + taxonomy=open_taxonomy, + _value=value, + ).save() + + # Creating ObjectTags for closed taxonomy + tag = get_tag(value) + ObjectTag( + object_id=f"object_id_{index}", + taxonomy=closed_taxonomy, + tag=tag, + _value=value, + ).save() + + # Test for open taxonomy + self._validate_autocomplete_tags( + open_taxonomy, + search, + expected_values, + [None] * len(expected_ids), + ) + + # Test for closed taxonomy + self._validate_autocomplete_tags( + closed_taxonomy, + search, + expected_values, + expected_ids, + ) + + def test_autocompleate_not_implemented(self): + with self.assertRaises(NotImplementedError): + tagging_api.autocomplete_tags(self.taxonomy, 'test', None, object_tags_only=False) + + def _get_tag_values(self, tags): + """ + Get tag values from tagging_api.autocomplete_tags() result + """ + return [tag.get("value") for tag in tags] + + def _get_tag_ids(self, tags): + """ + Get tag ids from tagging_api.autocomplete_tags() result + """ + return [tag.get("tag_id") for tag in tags] + + def _validate_autocomplete_tags( + self, + taxonomy, + search, + expected_values, + expected_ids, + ): + """ + Validate autocomplete tags + """ + + # Normal search + result = tagging_api.autocomplete_tags(taxonomy, search) + tag_values = self._get_tag_values(result) + for value in tag_values: + assert search.lower() in value.lower() + + assert tag_values == expected_values + assert self._get_tag_ids(result) == expected_ids + + # Create ObjectTag to simulate the content tagging + tag_model = None + if not taxonomy.allow_free_text: + tag_model = get_tag(tag_values[0]) + + object_id = 'new_object_id' + ObjectTag( + object_id=object_id, + taxonomy=taxonomy, + tag=tag_model, + _value=tag_values[0], + ).save() + + # Search with object + result = tagging_api.autocomplete_tags(taxonomy, search, object_id) + assert self._get_tag_values(result) == expected_values[1:] + assert self._get_tag_ids(result) == expected_ids[1:] diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 2db7911af..a30093bd4 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -2,6 +2,7 @@ import ddt from django.contrib.auth import get_user_model from django.test.testcases import TestCase +from django.db.utils import IntegrityError from openedx_tagging.core.tagging.models import ( ObjectTag, @@ -237,6 +238,20 @@ def test_get_tags_shallow_taxonomy(self): with self.assertNumQueries(2): assert taxonomy.get_tags() == tags + def test_unique_tags(self): + # Creating new tag + Tag( + taxonomy=self.taxonomy, + value='New value' + ).save() + + # Creating repeated tag + with self.assertRaises(IntegrityError): + Tag( + taxonomy=self.taxonomy, + value=self.archaea.value, + ).save() + class TestModelObjectTag(TestTagTaxonomyMixin, TestCase): """ diff --git a/tests/openedx_tagging/core/tagging/test_system_defined_models.py b/tests/openedx_tagging/core/tagging/test_system_defined_models.py index 432d07110..783a2604e 100644 --- a/tests/openedx_tagging/core/tagging/test_system_defined_models.py +++ b/tests/openedx_tagging/core/tagging/test_system_defined_models.py @@ -1,6 +1,7 @@ """ Test the tagging system-defined taxonomy models """ import ddt +from django.db.utils import IntegrityError from django.test.testcases import TestCase, override_settings from django.contrib.auth import get_user_model @@ -160,11 +161,8 @@ def test_tag_object_existing_tag(self): # Test add an existing Tag self._tag_object() assert self.user_taxonomy.tag_set.count() == 1 - updated_tags = self._tag_object() - assert self.user_taxonomy.tag_set.count() == 1 - assert len(updated_tags) == 1 - assert updated_tags[0].tag.external_id == str(self.user_1.id) - assert updated_tags[0].tag.value == self.user_1.get_username() + with self.assertRaises(IntegrityError): + self._tag_object() def test_tag_object_resync(self): self._tag_object() From 199285028fc2adf0517e9b701b4259c8ad4af639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Sun, 13 Aug 2023 21:19:11 -0300 Subject: [PATCH 034/282] feat: add taxonomy api pagination (#69) * feat: set default pagination to edx_rest_framework_extensions.paginators.DefaultPagination * docs: add page and page_size parameters in docstring * style: fix formatting --- .../core/tagging/rest_api/v1/views.py | 5 +- projects/dev.py | 7 ++ requirements/base.in | 1 + requirements/base.txt | 68 +++++++++++ requirements/ci.txt | 2 +- requirements/dev.txt | 80 ++++++++++++- requirements/doc.txt | 113 ++++++++++++++++-- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 107 +++++++++++++++-- requirements/test.in | 2 +- requirements/test.txt | 112 ++++++++++++++++- test_settings.py | 7 ++ .../core/tagging/test_views.py | 82 ++++++++----- 13 files changed, 518 insertions(+), 70 deletions(-) diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 37c3b9689..a86e3c278 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -9,8 +9,8 @@ get_taxonomy, get_taxonomies, ) -from .serializers import TaxonomyListQueryParamsSerializer, TaxonomySerializer from .permissions import TaxonomyObjectPermissions +from .serializers import TaxonomyListQueryParamsSerializer, TaxonomySerializer class TaxonomyView(ModelViewSet): @@ -19,6 +19,8 @@ class TaxonomyView(ModelViewSet): **List Query Parameters** * enabled (optional) - Filter by enabled status. Valid values: true, false, 1, 0, "true", "false", "1" + * page (optional) - Page number (default: 1) + * page_size (optional) - Number of items per page (default: 10) **List Example Requests** GET api/tagging/v1/taxonomy - Get all taxonomies @@ -59,7 +61,6 @@ class TaxonomyView(ModelViewSet): "allow_free_text": True, } - **Create Query Returns** * 201 - Success * 403 - Permission denied diff --git a/projects/dev.py b/projects/dev.py index c65e2c34f..9735b35de 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -113,3 +113,10 @@ INTERNAL_IPS = [ "127.0.0.1", ] + +######################### Django Rest Framework ######################## + +REST_FRAMEWORK = { + 'DEFAULT_PAGINATION_CLASS': 'edx_rest_framework_extensions.paginators.DefaultPagination', + 'PAGE_SIZE': 10, +} diff --git a/requirements/base.in b/requirements/base.in index d17e06397..94723505f 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -4,5 +4,6 @@ Django<5.0 # Web application framework djangorestframework<4.0 # REST API +edx-drf-extensions # Extensions to the Django REST Framework used by Open edX rules<4.0 # Django extension for rules-based authorization checks diff --git a/requirements/base.txt b/requirements/base.txt index 8c0f00b63..dc2d4c77f 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -6,20 +6,88 @@ # asgiref==3.7.2 # via django +certifi==2023.7.22 + # via requests +cffi==1.15.1 + # via + # cryptography + # pynacl +charset-normalizer==3.2.0 + # via requests +click==8.1.6 + # via edx-django-utils +cryptography==41.0.3 + # via pyjwt django==3.2.19 # via # -c requirements/constraints.txt # -r requirements/base.in + # django-crum + # django-waffle # djangorestframework + # drf-jwt + # edx-django-utils + # edx-drf-extensions +django-crum==0.7.9 + # via edx-django-utils +django-waffle==4.0.0 + # via + # edx-django-utils + # edx-drf-extensions djangorestframework==3.14.0 + # via + # -r requirements/base.in + # drf-jwt + # edx-drf-extensions +drf-jwt==1.19.2 + # via edx-drf-extensions +edx-django-utils==5.7.0 + # via edx-drf-extensions +edx-drf-extensions==8.8.0 # via -r requirements/base.in +edx-opaque-keys==2.4.0 + # via edx-drf-extensions +idna==3.4 + # via requests +newrelic==8.9.0 + # via edx-django-utils +pbr==5.11.1 + # via stevedore +psutil==5.9.5 + # via edx-django-utils +pycparser==2.21 + # via cffi +pyjwt[crypto]==2.8.0 + # via + # drf-jwt + # edx-drf-extensions +pymongo==3.13.0 + # via edx-opaque-keys +pynacl==1.5.0 + # via edx-django-utils +python-dateutil==2.8.2 + # via edx-drf-extensions pytz==2023.3 # via # django # djangorestframework +requests==2.31.0 + # via edx-drf-extensions rules==3.3 # via -r requirements/base.in +semantic-version==2.10.0 + # via edx-drf-extensions +six==1.16.0 + # via + # edx-drf-extensions + # python-dateutil sqlparse==0.4.4 # via django +stevedore==5.1.0 + # via + # edx-django-utils + # edx-opaque-keys typing-extensions==4.6.3 # via asgiref +urllib3==2.0.4 + # via requests diff --git a/requirements/ci.txt b/requirements/ci.txt index 6e1dd7f92..c03598b3e 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -4,7 +4,7 @@ # # make upgrade # -click==8.1.3 +click==8.1.6 # via import-linter distlib==0.3.6 # via virtualenv diff --git a/requirements/dev.txt b/requirements/dev.txt index 48cf4fb40..7c85fde2e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -21,7 +21,7 @@ build==0.10.0 # via # -r requirements/pip-tools.txt # pip-tools -certifi==2023.5.7 +certifi==2023.7.22 # via # -r requirements/quality.txt # requests @@ -29,19 +29,21 @@ cffi==1.15.1 # via # -r requirements/quality.txt # cryptography + # pynacl chardet==5.1.0 # via diff-cover -charset-normalizer==3.1.0 +charset-normalizer==3.2.0 # via # -r requirements/quality.txt # requests -click==8.1.3 +click==8.1.6 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt # click-log # code-annotations + # edx-django-utils # edx-lint # import-linter # pip-tools @@ -57,9 +59,10 @@ coverage[toml]==7.2.7 # via # -r requirements/quality.txt # pytest-cov -cryptography==41.0.1 +cryptography==41.0.3 # via # -r requirements/quality.txt + # pyjwt # secretstorage ddt==1.6.0 # via -r requirements/quality.txt @@ -77,23 +80,54 @@ django==3.2.19 # via # -c requirements/constraints.txt # -r requirements/quality.txt + # django-crum # django-debug-toolbar + # django-waffle # djangorestframework + # drf-jwt + # edx-django-utils + # edx-drf-extensions # edx-i18n-tools +django-crum==0.7.9 + # via + # -r requirements/quality.txt + # edx-django-utils django-debug-toolbar==4.1.0 # via # -r requirements/dev.in # -r requirements/quality.txt +django-waffle==4.0.0 + # via + # -r requirements/quality.txt + # edx-django-utils + # edx-drf-extensions djangorestframework==3.14.0 - # via -r requirements/quality.txt + # via + # -r requirements/quality.txt + # drf-jwt + # edx-drf-extensions docutils==0.20.1 # via # -r requirements/quality.txt # readme-renderer +drf-jwt==1.19.2 + # via + # -r requirements/quality.txt + # edx-drf-extensions +edx-django-utils==5.7.0 + # via + # -r requirements/quality.txt + # edx-drf-extensions +edx-drf-extensions==8.8.0 + # via -r requirements/quality.txt edx-i18n-tools==0.9.2 # via -r requirements/dev.in edx-lint==5.3.4 # via -r requirements/quality.txt +edx-opaque-keys==2.4.0 + # via + # -r requirements/quality.txt + # edx-drf-extensions exceptiongroup==1.1.1 # via # -r requirements/quality.txt @@ -179,6 +213,10 @@ more-itertools==9.1.0 # jaraco-classes mysqlclient==2.1.1 # via -r requirements/quality.txt +newrelic==8.9.0 + # via + # -r requirements/quality.txt + # edx-django-utils packaging==23.1 # via # -r requirements/ci.txt @@ -214,6 +252,10 @@ pluggy==1.0.0 # tox polib==1.2.0 # via edx-i18n-tools +psutil==5.9.5 + # via + # -r requirements/quality.txt + # edx-django-utils py==1.11.0 # via # -r requirements/ci.txt @@ -232,6 +274,11 @@ pygments==2.15.1 # diff-cover # readme-renderer # rich +pyjwt[crypto]==2.8.0 + # via + # -r requirements/quality.txt + # drf-jwt + # edx-drf-extensions pylint==2.17.4 # via # -r requirements/quality.txt @@ -252,6 +299,14 @@ pylint-plugin-utils==0.8.2 # -r requirements/quality.txt # pylint-celery # pylint-django +pymongo==3.13.0 + # via + # -r requirements/quality.txt + # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/quality.txt + # edx-django-utils pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt @@ -265,6 +320,10 @@ pytest-cov==4.1.0 # via -r requirements/quality.txt pytest-django==4.5.2 # via -r requirements/quality.txt +python-dateutil==2.8.2 + # via + # -r requirements/quality.txt + # edx-drf-extensions python-slugify==8.0.1 # via # -r requirements/quality.txt @@ -286,6 +345,7 @@ readme-renderer==40.0 requests==2.31.0 # via # -r requirements/quality.txt + # edx-drf-extensions # requests-toolbelt # twine requests-toolbelt==1.0.0 @@ -306,12 +366,18 @@ secretstorage==3.3.3 # via # -r requirements/quality.txt # keyring +semantic-version==2.10.0 + # via + # -r requirements/quality.txt + # edx-drf-extensions six==1.16.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # bleach + # edx-drf-extensions # edx-lint + # python-dateutil # tox snowballstemmer==2.2.0 # via @@ -326,6 +392,8 @@ stevedore==5.1.0 # via # -r requirements/quality.txt # code-annotations + # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via # -r requirements/quality.txt @@ -365,7 +433,7 @@ typing-extensions==4.6.3 # import-linter # pylint # rich -urllib3==2.0.3 +urllib3==2.0.4 # via # -r requirements/quality.txt # requests diff --git a/requirements/doc.txt b/requirements/doc.txt index f63eb9113..f997b86a7 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -20,14 +20,24 @@ beautifulsoup4==4.12.2 # via pydata-sphinx-theme bleach==6.0.0 # via readme-renderer -certifi==2023.5.7 - # via requests -charset-normalizer==3.1.0 - # via requests -click==8.1.3 +certifi==2023.7.22 + # via + # -r requirements/test.txt + # requests +cffi==1.15.1 + # via + # -r requirements/test.txt + # cryptography + # pynacl +charset-normalizer==3.2.0 + # via + # -r requirements/test.txt + # requests +click==8.1.6 # via # -r requirements/test.txt # code-annotations + # edx-django-utils # import-linter code-annotations==1.3.0 # via -r requirements/test.txt @@ -35,19 +45,40 @@ coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov +cryptography==41.0.3 + # via + # -r requirements/test.txt + # pyjwt ddt==1.6.0 # via -r requirements/test.txt django==3.2.19 # via # -c requirements/constraints.txt # -r requirements/test.txt + # django-crum # django-debug-toolbar + # django-waffle # djangorestframework + # drf-jwt + # edx-django-utils + # edx-drf-extensions # sphinxcontrib-django +django-crum==0.7.9 + # via + # -r requirements/test.txt + # edx-django-utils django-debug-toolbar==4.1.0 # via -r requirements/test.txt +django-waffle==4.0.0 + # via + # -r requirements/test.txt + # edx-django-utils + # edx-drf-extensions djangorestframework==3.14.0 - # via -r requirements/test.txt + # via + # -r requirements/test.txt + # drf-jwt + # edx-drf-extensions doc8==1.1.1 # via -r requirements/doc.in docutils==0.19 @@ -57,6 +88,20 @@ docutils==0.19 # readme-renderer # restructuredtext-lint # sphinx +drf-jwt==1.19.2 + # via + # -r requirements/test.txt + # edx-drf-extensions +edx-django-utils==5.7.0 + # via + # -r requirements/test.txt + # edx-drf-extensions +edx-drf-extensions==8.8.0 + # via -r requirements/test.txt +edx-opaque-keys==2.4.0 + # via + # -r requirements/test.txt + # edx-drf-extensions exceptiongroup==1.1.1 # via # -r requirements/test.txt @@ -66,7 +111,9 @@ grimp==2.4 # -r requirements/test.txt # import-linter idna==3.4 - # via requests + # via + # -r requirements/test.txt + # requests imagesize==1.4.1 # via sphinx import-linter==1.9.0 @@ -90,6 +137,10 @@ mock==5.0.2 # via -r requirements/test.txt mysqlclient==2.1.1 # via -r requirements/test.txt +newrelic==8.9.0 + # via + # -r requirements/test.txt + # edx-django-utils packaging==23.1 # via # -r requirements/test.txt @@ -106,6 +157,14 @@ pluggy==1.0.0 # pytest pprintpp==0.4.0 # via sphinxcontrib-django +psutil==5.9.5 + # via + # -r requirements/test.txt + # edx-django-utils +pycparser==2.21 + # via + # -r requirements/test.txt + # cffi pydata-sphinx-theme==0.13.3 # via sphinx-book-theme pygments==2.15.1 @@ -115,6 +174,19 @@ pygments==2.15.1 # pydata-sphinx-theme # readme-renderer # sphinx +pyjwt[crypto]==2.8.0 + # via + # -r requirements/test.txt + # drf-jwt + # edx-drf-extensions +pymongo==3.13.0 + # via + # -r requirements/test.txt + # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/test.txt + # edx-django-utils pytest==7.3.2 # via # -r requirements/test.txt @@ -124,6 +196,10 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # edx-drf-extensions python-slugify==8.0.1 # via # -r requirements/test.txt @@ -141,13 +217,24 @@ pyyaml==6.0 readme-renderer==40.0 # via -r requirements/doc.in requests==2.31.0 - # via sphinx + # via + # -r requirements/test.txt + # edx-drf-extensions + # sphinx restructuredtext-lint==1.4.0 # via doc8 rules==3.3 # via -r requirements/test.txt +semantic-version==2.10.0 + # via + # -r requirements/test.txt + # edx-drf-extensions six==1.16.0 - # via bleach + # via + # -r requirements/test.txt + # bleach + # edx-drf-extensions + # python-dateutil snowballstemmer==2.2.0 # via sphinx soupsieve==2.4.1 @@ -184,6 +271,8 @@ stevedore==5.1.0 # -r requirements/test.txt # code-annotations # doc8 + # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via # -r requirements/test.txt @@ -202,8 +291,10 @@ typing-extensions==4.6.3 # grimp # import-linter # pydata-sphinx-theme -urllib3==2.0.3 - # via requests +urllib3==2.0.4 + # via + # -r requirements/test.txt + # requests webencodings==0.5.1 # via bleach zipp==3.15.0 diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index fd0cc1c78..4b7fb4e53 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -6,7 +6,7 @@ # build==0.10.0 # via pip-tools -click==8.1.3 +click==8.1.6 # via pip-tools packaging==23.1 # via build diff --git a/requirements/quality.txt b/requirements/quality.txt index 425cfa514..0509905b3 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -14,17 +14,25 @@ astroid==2.15.5 # pylint-celery bleach==6.0.0 # via readme-renderer -certifi==2023.5.7 - # via requests +certifi==2023.7.22 + # via + # -r requirements/test.txt + # requests cffi==1.15.1 - # via cryptography -charset-normalizer==3.1.0 - # via requests -click==8.1.3 + # via + # -r requirements/test.txt + # cryptography + # pynacl +charset-normalizer==3.2.0 + # via + # -r requirements/test.txt + # requests +click==8.1.6 # via # -r requirements/test.txt # click-log # code-annotations + # edx-django-utils # edx-lint # import-linter click-log==0.4.0 @@ -37,8 +45,11 @@ coverage[toml]==7.2.7 # via # -r requirements/test.txt # pytest-cov -cryptography==41.0.1 - # via secretstorage +cryptography==41.0.3 + # via + # -r requirements/test.txt + # pyjwt + # secretstorage ddt==1.6.0 # via -r requirements/test.txt dill==0.3.6 @@ -47,16 +58,47 @@ django==3.2.19 # via # -c requirements/constraints.txt # -r requirements/test.txt + # django-crum # django-debug-toolbar + # django-waffle # djangorestframework + # drf-jwt + # edx-django-utils + # edx-drf-extensions +django-crum==0.7.9 + # via + # -r requirements/test.txt + # edx-django-utils django-debug-toolbar==4.1.0 # via -r requirements/test.txt +django-waffle==4.0.0 + # via + # -r requirements/test.txt + # edx-django-utils + # edx-drf-extensions djangorestframework==3.14.0 - # via -r requirements/test.txt + # via + # -r requirements/test.txt + # drf-jwt + # edx-drf-extensions docutils==0.20.1 # via readme-renderer +drf-jwt==1.19.2 + # via + # -r requirements/test.txt + # edx-drf-extensions +edx-django-utils==5.7.0 + # via + # -r requirements/test.txt + # edx-drf-extensions +edx-drf-extensions==8.8.0 + # via -r requirements/test.txt edx-lint==5.3.4 # via -r requirements/quality.in +edx-opaque-keys==2.4.0 + # via + # -r requirements/test.txt + # edx-drf-extensions exceptiongroup==1.1.1 # via # -r requirements/test.txt @@ -66,7 +108,9 @@ grimp==2.4 # -r requirements/test.txt # import-linter idna==3.4 - # via requests + # via + # -r requirements/test.txt + # requests import-linter==1.9.0 # via -r requirements/test.txt importlib-metadata==6.7.0 @@ -113,6 +157,10 @@ more-itertools==9.1.0 # via jaraco-classes mysqlclient==2.1.1 # via -r requirements/test.txt +newrelic==8.9.0 + # via + # -r requirements/test.txt + # edx-django-utils packaging==23.1 # via # -r requirements/test.txt @@ -129,16 +177,27 @@ pluggy==1.0.0 # via # -r requirements/test.txt # pytest +psutil==5.9.5 + # via + # -r requirements/test.txt + # edx-django-utils pycodestyle==2.10.0 # via -r requirements/quality.in pycparser==2.21 - # via cffi + # via + # -r requirements/test.txt + # cffi pydocstyle==6.3.0 # via -r requirements/quality.in pygments==2.15.1 # via # readme-renderer # rich +pyjwt[crypto]==2.8.0 + # via + # -r requirements/test.txt + # drf-jwt + # edx-drf-extensions pylint==2.17.4 # via # edx-lint @@ -153,6 +212,14 @@ pylint-plugin-utils==0.8.2 # via # pylint-celery # pylint-django +pymongo==3.13.0 + # via + # -r requirements/test.txt + # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/test.txt + # edx-django-utils pytest==7.3.2 # via # -r requirements/test.txt @@ -162,6 +229,10 @@ pytest-cov==4.1.0 # via -r requirements/test.txt pytest-django==4.5.2 # via -r requirements/test.txt +python-dateutil==2.8.2 + # via + # -r requirements/test.txt + # edx-drf-extensions python-slugify==8.0.1 # via # -r requirements/test.txt @@ -179,6 +250,8 @@ readme-renderer==40.0 # via twine requests==2.31.0 # via + # -r requirements/test.txt + # edx-drf-extensions # requests-toolbelt # twine requests-toolbelt==1.0.0 @@ -191,10 +264,17 @@ rules==3.3 # via -r requirements/test.txt secretstorage==3.3.3 # via keyring +semantic-version==2.10.0 + # via + # -r requirements/test.txt + # edx-drf-extensions six==1.16.0 # via + # -r requirements/test.txt # bleach + # edx-drf-extensions # edx-lint + # python-dateutil snowballstemmer==2.2.0 # via pydocstyle sqlparse==0.4.4 @@ -206,6 +286,8 @@ stevedore==5.1.0 # via # -r requirements/test.txt # code-annotations + # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via # -r requirements/test.txt @@ -230,8 +312,9 @@ typing-extensions==4.6.3 # import-linter # pylint # rich -urllib3==2.0.3 +urllib3==2.0.4 # via + # -r requirements/test.txt # requests # twine webencodings==0.5.1 diff --git a/requirements/test.in b/requirements/test.in index c050dd241..50feee663 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -14,4 +14,4 @@ pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. ddt # supports data driven tests mock # supports overriding classes and methods in tests -django-debug-toolbar # provides a debug toolbar for Django +django-debug-toolbar # provides a debug toolbar for Django diff --git a/requirements/test.txt b/requirements/test.txt index 22c43aede..e1240b21f 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -8,9 +8,24 @@ asgiref==3.7.2 # via # -r requirements/base.txt # django -click==8.1.3 +certifi==2023.7.22 # via + # -r requirements/base.txt + # requests +cffi==1.15.1 + # via + # -r requirements/base.txt + # cryptography + # pynacl +charset-normalizer==3.2.0 + # via + # -r requirements/base.txt + # requests +click==8.1.6 + # via + # -r requirements/base.txt # code-annotations + # edx-django-utils # import-linter code-annotations==1.3.0 # via -r requirements/test.in @@ -18,21 +33,60 @@ coverage[toml]==7.2.7 # via # -r requirements/test.in # pytest-cov +cryptography==41.0.3 + # via + # -r requirements/base.txt + # pyjwt ddt==1.6.0 # via -r requirements/test.in # via # -c requirements/constraints.txt # -r requirements/base.txt + # django-crum # django-debug-toolbar + # django-waffle # djangorestframework + # drf-jwt + # edx-django-utils + # edx-drf-extensions +django-crum==0.7.9 + # via + # -r requirements/base.txt + # edx-django-utils django-debug-toolbar==4.1.0 # via -r requirements/test.in +django-waffle==4.0.0 + # via + # -r requirements/base.txt + # edx-django-utils + # edx-drf-extensions djangorestframework==3.14.0 + # via + # -r requirements/base.txt + # drf-jwt + # edx-drf-extensions +drf-jwt==1.19.2 + # via + # -r requirements/base.txt + # edx-drf-extensions +edx-django-utils==5.7.0 + # via + # -r requirements/base.txt + # edx-drf-extensions +edx-drf-extensions==8.8.0 # via -r requirements/base.txt +edx-opaque-keys==2.4.0 + # via + # -r requirements/base.txt + # edx-drf-extensions exceptiongroup==1.1.1 # via pytest grimp==2.4 # via import-linter +idna==3.4 + # via + # -r requirements/base.txt + # requests import-linter==1.9.0 # via -r requirements/test.in iniconfig==2.0.0 @@ -45,12 +99,39 @@ mock==5.0.2 # via -r requirements/test.in mysqlclient==2.1.1 # via -r requirements/test.in +newrelic==8.9.0 + # via + # -r requirements/base.txt + # edx-django-utils packaging==23.1 # via pytest pbr==5.11.1 - # via stevedore + # via + # -r requirements/base.txt + # stevedore pluggy==1.0.0 # via pytest +psutil==5.9.5 + # via + # -r requirements/base.txt + # edx-django-utils +pycparser==2.21 + # via + # -r requirements/base.txt + # cffi +pyjwt[crypto]==2.8.0 + # via + # -r requirements/base.txt + # drf-jwt + # edx-drf-extensions +pymongo==3.13.0 + # via + # -r requirements/base.txt + # edx-opaque-keys +pynacl==1.5.0 + # via + # -r requirements/base.txt + # edx-django-utils pytest==7.3.2 # via # -r requirements/test.in @@ -60,6 +141,10 @@ pytest-cov==4.1.0 # via -r requirements/test.in pytest-django==4.5.2 # via -r requirements/test.in +python-dateutil==2.8.2 + # via + # -r requirements/base.txt + # edx-drf-extensions python-slugify==8.0.1 # via code-annotations pytz==2023.3 @@ -69,15 +154,32 @@ pytz==2023.3 # djangorestframework pyyaml==6.0 # via code-annotations +requests==2.31.0 + # via + # -r requirements/base.txt + # edx-drf-extensions rules==3.3 # via -r requirements/base.txt +semantic-version==2.10.0 + # via + # -r requirements/base.txt + # edx-drf-extensions +six==1.16.0 + # via + # -r requirements/base.txt + # edx-drf-extensions + # python-dateutil sqlparse==0.4.4 # via # -r requirements/base.txt # django # django-debug-toolbar stevedore==5.1.0 - # via code-annotations + # via + # -r requirements/base.txt + # code-annotations + # edx-django-utils + # edx-opaque-keys text-unidecode==1.3 # via python-slugify tomli==2.0.1 @@ -91,3 +193,7 @@ typing-extensions==4.6.3 # asgiref # grimp # import-linter +urllib3==2.0.4 + # via + # -r requirements/base.txt + # requests diff --git a/test_settings.py b/test_settings.py index 63290bc66..28676e13e 100644 --- a/test_settings.py +++ b/test_settings.py @@ -66,3 +66,10 @@ def root(*args): # STORAGES setting in Django >= 4.2 "STORAGE": None, } + +######################### Django Rest Framework ######################## + +REST_FRAMEWORK = { + 'DEFAULT_PAGINATION_CLASS': 'edx_rest_framework_extensions.paginators.DefaultPagination', + 'PAGE_SIZE': 10, +} diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 6ed24246c..69805e234 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -6,14 +6,16 @@ from django.contrib.auth import get_user_model from rest_framework import status from rest_framework.test import APITestCase +from urllib.parse import urlparse, parse_qs from openedx_tagging.core.tagging.models import Taxonomy from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy User = get_user_model() -TAXONOMY_LIST_URL = '/tagging/rest_api/v1/taxonomies/' -TAXONOMY_DETAIL_URL = '/tagging/rest_api/v1/taxonomies/{pk}/' +TAXONOMY_LIST_URL = "/tagging/rest_api/v1/taxonomies/" +TAXONOMY_DETAIL_URL = "/tagging/rest_api/v1/taxonomies/{pk}/" + def check_taxonomy( data, @@ -85,7 +87,7 @@ def test_list_taxonomy_queryparams(self, enabled, expected_status, expected_coun # If we were able to list the taxonomies, check that we got the expected number back # We take into account the Language Taxonomy that is created by the system in a migration if status.is_success(expected_status): - assert len(response.data) == expected_count + assert len(response.data["results"]) == expected_count @ddt.data( (None, status.HTTP_403_FORBIDDEN), @@ -103,14 +105,42 @@ def test_list_taxonomy(self, user_attr, expected_status): response = self.client.get(url) assert response.status_code == expected_status + def test_list_taxonomy_pagination(self): + url = TAXONOMY_LIST_URL + Taxonomy.objects.create(name="T1", enabled=True).save() + Taxonomy.objects.create(name="T2", enabled=True).save() + Taxonomy.objects.create(name="T3", enabled=False).save() + Taxonomy.objects.create(name="T4", enabled=False).save() + Taxonomy.objects.create(name="T5", enabled=False).save() + + self.client.force_authenticate(user=self.staff) + + query_params = {"page_size": 2, "page": 2} + response = self.client.get(url, query_params, format="json") + + assert response.status_code == status.HTTP_200_OK + + self.assertEqual(set(t["name"] for t in response.data["results"]), set(("T2", "T3"))) + parsed_url = urlparse(response.data["next"]) + + next_page = parse_qs(parsed_url.query).get("page", [None])[0] + assert next_page == "3" + + def test_list_invalid_page(self): + url = TAXONOMY_LIST_URL + + self.client.force_authenticate(user=self.user) + + query_params = {"page": 123123} + + response = self.client.get(url, query_params, format="json") + + assert response.status_code == status.HTTP_404_NOT_FOUND + @ddt.data( (None, {"enabled": True}, status.HTTP_403_FORBIDDEN), (None, {"enabled": False}, status.HTTP_403_FORBIDDEN), - ( - "user", - {"enabled": True}, - status.HTTP_200_OK, - ), + ("user", {"enabled": True}, status.HTTP_200_OK), ("user", {"enabled": False}, status.HTTP_404_NOT_FOUND), ("staff", {"enabled": True}, status.HTTP_200_OK), ("staff", {"enabled": False}, status.HTTP_200_OK), @@ -237,12 +267,10 @@ def test_update_taxonomy(self, user_attr, expected_status): (True, status.HTTP_403_FORBIDDEN), ) @ddt.unpack - def test_update_taxonomy_system_defined( - self, system_defined, expected_status - ): - ''' + def test_update_taxonomy_system_defined(self, system_defined, expected_status): + """ Test that we can't update system_defined field - ''' + """ taxonomy = Taxonomy.objects.create(name="test system taxonomy") if system_defined: taxonomy.taxonomy_class = SystemDefinedTaxonomy @@ -250,9 +278,7 @@ def test_update_taxonomy_system_defined( url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) self.client.force_authenticate(user=self.staff) - response = self.client.put( - url, {"name": "new name"}, format="json" - ) + response = self.client.put(url, {"name": "new name"}, format="json") assert response.status_code == expected_status def test_update_taxonomy_404(self): @@ -269,9 +295,7 @@ def test_update_taxonomy_404(self): ) @ddt.unpack def test_patch_taxonomy(self, user_attr, expected_status): - taxonomy = Taxonomy.objects.create( - name="test patch taxonomy", enabled=False, required=True - ) + taxonomy = Taxonomy.objects.create(name="test patch taxonomy", enabled=False, required=True) taxonomy.save() url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) @@ -280,9 +304,7 @@ def test_patch_taxonomy(self, user_attr, expected_status): user = getattr(self, user_attr) self.client.force_authenticate(user=user) - response = self.client.patch( - url, {"name": "new name", "required": False}, format="json" - ) + response = self.client.patch(url, {"name": "new name", "required": False}, format="json") assert response.status_code == expected_status # If we were able to update the taxonomy, check if the name changed @@ -303,24 +325,18 @@ def test_patch_taxonomy(self, user_attr, expected_status): (True, status.HTTP_403_FORBIDDEN), ) @ddt.unpack - def test_patch_taxonomy_system_defined( - self, system_defined, expected_status - ): - ''' + def test_patch_taxonomy_system_defined(self, system_defined, expected_status): + """ Test that we can't patch system_defined field - ''' - taxonomy = Taxonomy.objects.create( - name="test system taxonomy" - ) + """ + taxonomy = Taxonomy.objects.create(name="test system taxonomy") if system_defined: taxonomy.taxonomy_class = SystemDefinedTaxonomy taxonomy.save() url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) self.client.force_authenticate(user=self.staff) - response = self.client.patch( - url, {"name": "New name"}, format="json" - ) + response = self.client.patch(url, {"name": "New name"}, format="json") assert response.status_code == expected_status def test_patch_taxonomy_404(self): From a1db0fe9e631de8dc727da170ab749b9f70fbc21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Wed, 16 Aug 2023 00:17:05 -0500 Subject: [PATCH 035/282] Modular import/export taxonomy tags (#64) * feat: Initial import/export configuration and Parsers - Added Parser, JSONParser and CSVParser classes to validate file and tags format - Tests added * feat: TagImportDSL and action implementation * test: Import actions tests & DSL tests * feat: plan() function * style: Rename exceptions and running black * feat: Implemented execute() func in actions * feat: execute() function on DSL * feat: Added TagImportTask feature * feat: Creating TagImportTask model migration * test: Testing import task creation, states and logs * feat: export functions * docs: Added docstrings * chore: Added attrs dependency * style: typos, docs and new error handler * style: Updated valid_for to applies_for * docs: Updating docstrings and logs * style: nits and docstring * fix: action in TagItem removed * fix: Migration conflicts * chore: bump version to 0.1.3 --- openedx_learning/__init__.py | 2 +- .../core/tagging/import_export/__init__.py | 1 + .../core/tagging/import_export/actions.py | 443 ++++++++++++++++ .../core/tagging/import_export/api.py | 188 +++++++ .../core/tagging/import_export/exceptions.py | 98 ++++ .../core/tagging/import_export/import_plan.py | 198 +++++++ .../core/tagging/import_export/parsers.py | 325 ++++++++++++ .../core/tagging/import_export/tasks.py | 38 ++ .../migrations/0005_language_taxonomy.py | 2 +- .../migrations/0006_auto_20230802_1631.py | 81 +++ .../core/tagging/models/__init__.py | 4 + .../core/tagging/models/import_export.py | 107 ++++ requirements/base.in | 4 + requirements/base.txt | 40 +- requirements/ci.txt | 2 +- requirements/dev.txt | 60 ++- requirements/doc.txt | 58 +- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 61 ++- requirements/test.txt | 53 +- .../core/fixtures/tagging.yaml | 38 ++ .../core/tagging/import_export/__init__.py | 0 .../core/tagging/import_export/mixins.py | 16 + .../tagging/import_export/test_actions.py | 500 ++++++++++++++++++ .../core/tagging/import_export/test_api.py | 219 ++++++++ .../tagging/import_export/test_import_plan.py | 418 +++++++++++++++ .../tagging/import_export/test_parsers.py | 310 +++++++++++ .../core/tagging/import_export/test_tasks.py | 42 ++ .../openedx_tagging/core/tagging/test_api.py | 13 +- 29 files changed, 3288 insertions(+), 35 deletions(-) create mode 100644 openedx_tagging/core/tagging/import_export/__init__.py create mode 100644 openedx_tagging/core/tagging/import_export/actions.py create mode 100644 openedx_tagging/core/tagging/import_export/api.py create mode 100644 openedx_tagging/core/tagging/import_export/exceptions.py create mode 100644 openedx_tagging/core/tagging/import_export/import_plan.py create mode 100644 openedx_tagging/core/tagging/import_export/parsers.py create mode 100644 openedx_tagging/core/tagging/import_export/tasks.py create mode 100644 openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py create mode 100644 openedx_tagging/core/tagging/models/import_export.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/__init__.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/mixins.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/test_actions.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/test_api.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/test_import_plan.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/test_parsers.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/test_tasks.py diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index b3f475621..ae7362549 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1 +1 @@ -__version__ = "0.1.2" +__version__ = "0.1.3" diff --git a/openedx_tagging/core/tagging/import_export/__init__.py b/openedx_tagging/core/tagging/import_export/__init__.py new file mode 100644 index 000000000..55bcdaebd --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/__init__.py @@ -0,0 +1 @@ +from .parsers import ParserFormat diff --git a/openedx_tagging/core/tagging/import_export/actions.py b/openedx_tagging/core/tagging/import_export/actions.py new file mode 100644 index 000000000..da757cf4b --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/actions.py @@ -0,0 +1,443 @@ +""" +Actions for import tags +""" +from typing import List + +from django.utils.translation import gettext_lazy as _ + +from ..models import Taxonomy, Tag +from .exceptions import ImportActionError, ImportActionConflict + + +class ImportAction: + """ + Base class to create actions + + Each action is a simple operation to be performed on the database. + There are no compound actions or actions that have to do with each other. + + To create an Action you need to implement the following: + + Given a TagItem, the actions to be performed must be deduced + by comparing with the tag on the database. + Ex. The create action is inferred if the tag does not exist in the database. + This check is done in `applies_for` + + Then each action validates if the change is consistent with the database + or with previous actions. + Ex. Verify that when creating a tag, there is not a previous creation action + that has the same tag_id. + This checks is done in `validate` + + Then the actions are executed. Ex. Create the tag on the database + This is done in `execute` + """ + + name = "import_action" + + def __init__(self, taxonomy: Taxonomy, tag, index: int): + self.taxonomy = taxonomy + self.tag = tag + self.index = index + + def __repr__(self): + return str(_(f"Action {self.name} (index={self.index},id={self.tag.id})")) + + def __str__(self): + return self.__repr__() + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + Implement this to meet the conditions that a `TagItem` needs + to have for this action. If this function returns `True` for `tag` + then the action is created. + """ + raise NotImplementedError + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Implement this to find inconsistencies with tags in the + database or with previous actions. + """ + raise NotImplementedError + + def execute(self): + """ + Implement this to execute the action. + """ + raise NotImplementedError + + def _get_tag(self): + """ + Returns the respective tag of this actions + """ + return self.taxonomy.tag_set.get(external_id=self.tag.id) + + def _search_action( + self, + indexed_actions: dict, + action_name: str, + attr: str, + search_value: str, + ): + """ + Use this function to find and action using an `attr` of `TagItem` + """ + for action in indexed_actions[action_name]: + if search_value == getattr(action.tag, attr): + return action + + return None + + def _validate_parent(self, indexed_actions) -> ImportActionError: + """ + Helper method to validate that the parent tag has already been defined. + """ + try: + # Validates that the parent exists on the taxonomy + self.taxonomy.tag_set.get(external_id=self.tag.parent_id) + except Tag.DoesNotExist: + # Or if the parent is created on previous actions + if not self._search_action( + indexed_actions, CreateTag.name, "id", self.tag.parent_id + ): + return ImportActionError( + action=self, + tag_id=self.tag.id, + message=_( + f"Unknown parent tag ({self.tag.parent_id}). " + "You need to add parent before the child in your file." + ), + ) + + def _validate_value(self, indexed_actions): + """ + Check for value duplicates in the models and in previous create/rename + actions + """ + try: + # Validates if exists a tag with the same value on the Taxonomy + taxonomy_tag = self.taxonomy.tag_set.get(value=self.tag.value) + return ImportActionError( + action=self, + tag_id=self.tag.id, + message=_( + f"Duplicated tag value with tag in database (external_id={taxonomy_tag.external_id})." + ), + ) + except Tag.DoesNotExist: + # Validates value duplication on create actions + action = self._search_action( + indexed_actions, + CreateTag.name, + "value", + self.tag.value, + ) + + if not action: + # Validates value duplication on rename actions + action = self._search_action( + indexed_actions, + RenameTag.name, + "value", + self.tag.value, + ) + + if action: + return ImportActionConflict( + action=self, + tag_id=self.tag.id, + conflict_action_index=action.index, + message=_("Duplicated tag value."), + ) + + +class CreateTag(ImportAction): + """ + Action for create a Tag + + Action created if the tag doesn't exist on the database + + Validations: + - Id duplicates with previous create actions. + - Value duplicates with tags on the database. + - Value duplicates with previous create and rename actions. + - Parent validation. If the parent is in the database or created + in previous actions. + """ + + name = "create" + + def __str__(self): + return str( + _( + "Create a new tag with values " + f"(external_id={self.tag.id}, value={self.tag.value}, " + f"parent_id={self.tag.parent_id})." + ) + ) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action applies whenever the tag does not exist + """ + try: + taxonomy.tag_set.get(external_id=tag.id) + return False + except Tag.DoesNotExist: + return True + + def _validate_id(self, indexed_actions): + """ + Check for id duplicates in previous create actions + """ + action = self._search_action(indexed_actions, self.name, "id", self.tag.id) + if action: + return ImportActionConflict( + action=self, + tag_id=self.tag.id, + conflict_action_index=action.index, + message=_("Duplicated external_id tag."), + ) + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Validates the creation action + """ + errors = [] + + # Duplicate id validation with previous create actions + error = self._validate_id(indexed_actions) + if error: + errors.append(error) + + # Duplicate value validation + error = self._validate_value(indexed_actions) + if error: + errors.append(error) + + # Parent validation + if self.tag.parent_id: + error = self._validate_parent(indexed_actions) + if error: + errors.append(error) + + return errors + + def execute(self): + """ + Creates a Tag + """ + parent = None + if self.tag.parent_id: + parent = self.taxonomy.tag_set.get(external_id=self.tag.parent_id) + taxonomy_tag = Tag( + taxonomy=self.taxonomy, + parent=parent, + value=self.tag.value, + external_id=self.tag.id, + ) + taxonomy_tag.save() + + +class UpdateParentTag(ImportAction): + """ + Action for update the parent of a Tag + + Action created if there is a change on the parent + + Validations: + - Parent validation. If the parent is in the database + or created in previous actions. + """ + + name = "update_parent" + + def __str__(self): + taxonomy_tag = self._get_tag() + if not taxonomy_tag.parent: + from_str = _("from empty parent") + else: + from_str = _(f"from parent (external_id={taxonomy_tag.parent.external_id})") + + return str( + _( + f"Update the parent of tag (external_id={taxonomy_tag.external_id}) " + f"{from_str} to parent (external_id={self.tag.parent_id})." + ) + ) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action applies whenever there is a change on the parent + """ + try: + taxonomy_tag = taxonomy.tag_set.get(external_id=tag.id) + return ( + taxonomy_tag.parent is not None + and taxonomy_tag.parent.external_id != tag.parent_id + ) or (taxonomy_tag.parent is None and tag.parent_id is not None) + except Tag.DoesNotExist: + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Validates the update parent action + """ + errors = [] + + # Parent validation + if self.tag.parent_id: + error = self._validate_parent(indexed_actions) + if error: + errors.append(error) + + return errors + + def execute(self): + """ + Updates the parent of a tag + """ + taxonomy_tag = self._get_tag() + parent = None + if self.tag.parent_id: + parent = self.taxonomy.tag_set.get(external_id=self.tag.parent_id) + taxonomy_tag.parent = parent + taxonomy_tag.save() + + +class RenameTag(ImportAction): + """ + Action for rename a Tag + + Action created if there is a change on the tag value + + Validations: + - Value duplicates with tags on the database. + - Value duplicates with previous create and rename actions. + """ + + name = "rename" + + def __str__(self): + taxonomy_tag = self._get_tag() + return str( + _( + f"Rename tag value of tag (external_id={taxonomy_tag.external_id}) " + f"from '{taxonomy_tag.value}' to '{self.tag.value}'" + ) + ) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action applies whenever there is a change on the tag value + """ + try: + taxonomy_tag = taxonomy.tag_set.get(external_id=tag.id) + return taxonomy_tag.value != tag.value + except Tag.DoesNotExist: + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + Validates the rename action + """ + errors = [] + + # Duplicate value validation + error = self._validate_value(indexed_actions) + if error: + errors.append(error) + + return errors + + def execute(self): + """ + Rename a tag + """ + taxonomy_tag = self._get_tag() + taxonomy_tag.value = self.tag.value + taxonomy_tag.save() + + +class DeleteTag(ImportAction): + """ + Action for delete a Tag + + Action created if the action of the tag is 'delete' + + Does not require validations + """ + + def __str__(self): + taxonomy_tag = self._get_tag() + return str(_(f"Delete tag (external_id={taxonomy_tag.external_id})")) + + name = "delete" + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + This action is an exception. + These actions are created in `TagImportPlan.generate_actions` if `replace=True` + """ + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + No validations necessary + """ + # TODO: Will it be necessary to check if this tag has children? + return [] + + def execute(self): + """ + Delete a tag + """ + taxonomy_tag = self._get_tag() + taxonomy_tag.delete() + + +class WithoutChanges(ImportAction): + """ + Action when there is no change on the Tag + + Does not require validations + """ + + name = "without_changes" + + def __str__(self): + return str(_(f"No changes needed for tag (external_id={self.tag.id})")) + + @classmethod + def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: + """ + No validations necessary + """ + return False + + def validate(self, indexed_actions) -> List[ImportActionError]: + """ + No validations necessary + """ + return [] + + def execute(self): + """ + Do nothing + """ + + +# Register actions here in the order in which you want to check. +available_actions = [ + UpdateParentTag, + RenameTag, + CreateTag, + DeleteTag, + WithoutChanges, +] diff --git a/openedx_tagging/core/tagging/import_export/api.py b/openedx_tagging/core/tagging/import_export/api.py new file mode 100644 index 000000000..4e26854ed --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/api.py @@ -0,0 +1,188 @@ +""" +Import/export API functions + +A modular implementation has been followed for both functionalities. + +Import +------------ + +In this functionality we have the following pipeline with the following classes: + +Parser.parse_import() -> TagImportPlan.generate_actions() -> [ImportActions] +-> TagImportPlan.plan() -> TagImportPlan.execute() + +Parsers are in charge of reading the input file, +making the respective verifications of its format and returning a list of TagItems. +You need to create parser for each format that the system will accept. +For more information see parsers.py + +TagImportPlan receives a list of TagItems. With this, it generates each +action that will be executed in the import. +Each Action are in charge of verifying and executing specific +and simple operations in the database, such as creating or rename tag. +For more information see actions.py + +In each action it is verified if there are no errors or inconsistencies +with taxonomy tags or with previous actions. +In the end, TagImportPlan contains all actions and possible errors. +You can run `plan()` to see the actions and errors or you can run `execute()` +to execute each action. + +Export +---------- + +The export only uses Parsers. Calls the respective function and +returns a string with the data. + + +TODO for next versions +--------- +- Function to force clean the status of an import task, or a way to avoid + a lock: a task not in SUCCESS or ERROR due to something unexpected + (ex. server crash) +- Join/reduce actions on TagImportPlan. See `generate_actions()` +""" +from io import BytesIO + +from django.utils.translation import gettext_lazy as _ + +from ..models import Taxonomy, TagImportTask, TagImportTaskState +from .parsers import get_parser, ParserFormat +from .import_plan import TagImportPlan, TagImportTask + + +def import_tags( + taxonomy: Taxonomy, + file: BytesIO, + parser_format: ParserFormat, + replace=False, +) -> bool: + """ + Execute the necessary actions to import the tags from `file` + + You can read the docstring of the top for more info about the + modular architecture. + + It creates an TagImportTask to keep logs of the execution + of each import step and the current status. + There can only be one task in progress at a time per taxonomy + + Set `replace` to True to delete all not readed Tag of the given taxonomy. + Ex. Given a taxonomy with `tag_1`, `tag_2` and `tag_3`. If there is only `tag_1` + in the file (regardless of action), then `tag_2` and `tag_3` will be deleted + if `replace=True` + """ + _import_export_validations(taxonomy) + + # Checks that exists only one task import in progress at a time per taxonomy + if not _check_unique_import_task(taxonomy): + raise ValueError( + _( + "There is an import task running. " + "Only one task per taxonomy can be created at a time." + ) + ) + + # Creating import task + task = TagImportTask.create(taxonomy) + + try: + # Get the parser and parse the file + task.log_parser_start() + parser = get_parser(parser_format) + tags, errors = parser.parse_import(file) + + # Check if there are errors in the parse + if errors: + task.handle_parser_errors(errors) + return False + + task.log_parser_end() + + # Generate actions + task.log_start_planning() + tag_import_plan = TagImportPlan(taxonomy) + tag_import_plan.generate_actions(tags, replace) + task.log_plan(tag_import_plan) + + if tag_import_plan.errors: + task.handle_plan_errors() + return False + + task.log_start_execute() + tag_import_plan.execute(task) + task.end_success() + return True + except Exception as exception: + # Log any exception + task.log_exception(exception) + return False + + +def get_last_import_status(taxonomy: Taxonomy) -> TagImportTaskState: + """ + Get status of the last import task of the given taxonomy + """ + task = _get_last_import_task(taxonomy) + return task.status + + +def get_last_import_log(taxonomy: Taxonomy) -> str: + """ + Get logs of the last import task of the given taxonomy + """ + task = _get_last_import_task(taxonomy) + return task.log + + +def export_tags(taxonomy: Taxonomy, output_format: ParserFormat) -> str: + """ + Returns a string with all tag data of the given taxonomy + """ + _import_export_validations(taxonomy) + parser = get_parser(output_format) + return parser.export(taxonomy) + + +def _check_unique_import_task(taxonomy: Taxonomy) -> bool: + """ + Verifies if there is another in progress import task for the + given taxonomy + """ + last_task = _get_last_import_task(taxonomy) + if not last_task: + return True + return ( + last_task.status == TagImportTaskState.SUCCESS.value + or last_task.status == TagImportTaskState.ERROR.value + ) + + +def _get_last_import_task(taxonomy: Taxonomy) -> TagImportTask: + """ + Get the last import task for the given taxonomy + """ + return ( + TagImportTask.objects.filter(taxonomy=taxonomy) + .order_by("-creation_date") + .first() + ) + + +def _import_export_validations(taxonomy: Taxonomy): + """ + Validates if the taxonomy is allowed to import or export tags + """ + taxonomy = taxonomy.cast() + if taxonomy.allow_free_text: + raise NotImplementedError( + _( + f"Import/export for free-form taxonomies will be implemented in the future." + ) + ) + if taxonomy.system_defined: + raise ValueError( + _( + f"Invalid taxonomy ({taxonomy.id}): You cannot import/export a system-defined taxonomy." + ) + ) diff --git a/openedx_tagging/core/tagging/import_export/exceptions.py b/openedx_tagging/core/tagging/import_export/exceptions.py new file mode 100644 index 000000000..2330cae6a --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/exceptions.py @@ -0,0 +1,98 @@ +""" +Exceptions for tag import/export actions +""" +from django.utils.translation import gettext_lazy as _ + + +class TagImportError(Exception): + """ + Base exception for import + """ + + def __init__(self, message: str = "", **kargs): + self.message = message + + def __str__(self): + return str(self.message) + + def __repr__(self): + return f"{self.__class__.__name__}({str(self)})" + + +class TagParserError(TagImportError): + """ + Base exception for parsers + """ + + def __init__(self, tag, **kargs): + self.message = _(f"Import parser error on {tag}") + + +class ImportActionError(TagImportError): + """ + Base exception for actions + """ + + def __init__(self, action: str, tag_id: str, message: str, **kargs): + self.message = _( + f"Action error in '{action.name}' (#{action.index}): {message}" + ) + + +class ImportActionConflict(ImportActionError): + """ + Exception used when exists a conflict between actions + """ + + def __init__( + self, + action: str, + tag_id: str, + conflict_action_index: int, + message: str, + **kargs, + ): + self.message = _( + f"Conflict with '{action.name}' (#{action.index}) " + f"and action #{conflict_action_index}: {message}" + ) + + +class InvalidFormat(TagParserError): + """ + Exception used when there is an error with the format + """ + + def __init__(self, tag: dict, format: str, message: str, **kargs): + self.tag = tag + self.message = _(f"Invalid '{format}' format: {message}") + + +class FieldJSONError(TagParserError): + """ + Exception used when missing a required field on the .json + """ + + def __init__(self, tag, field, **kargs): + self.tag = tag + self.message = _(f"Missing '{field}' field on {tag}") + + +class EmptyJSONField(TagParserError): + """ + Exception used when a required field is empty on the .json + """ + + def __init__(self, tag, field, **kargs): + self.tag = tag + self.message = _(f"Empty '{field}' field on {tag}") + + +class EmptyCSVField(TagParserError): + """ + Exception used when a required field is empty on the .csv + """ + + def __init__(self, tag, field, row, **kargs): + self.tag = tag + self.message = _(f"Empty '{field}' field on the row {row}") diff --git a/openedx_tagging/core/tagging/import_export/import_plan.py b/openedx_tagging/core/tagging/import_export/import_plan.py new file mode 100644 index 000000000..3af3597d6 --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/import_plan.py @@ -0,0 +1,198 @@ +""" +Classes and functions to create an import plan and execution. +""" +from attrs import define +from typing import List, Optional + +from django.db import transaction + +from ..models import Taxonomy, TagImportTask +from .actions import ( + DeleteTag, + ImportAction, + UpdateParentTag, + WithoutChanges, + available_actions, +) +from .exceptions import ImportActionError + + +@define +class TagItem: + """ + Tag representation on the tag import plan + """ + + id: str + value: str + index: Optional[int] = 0 + parent_id: Optional[str] = None + + +class TagImportPlan: + """ + Class with functions to build an import plan and excute the plan + """ + + actions: List[ImportAction] + errors: List[ImportActionError] + indexed_actions: dict + actions_dict: dict + taxonomy: Taxonomy + + def __init__(self, taxonomy: Taxonomy): + self.actions = [] + self.errors = [] + self.taxonomy = taxonomy + self.actions_dict = {} + self._init_indexed_actions() + + def _init_indexed_actions(self): + """ + Initialize the `indexed_actions` dict + """ + self.indexed_actions = {} + for action in available_actions: + self.indexed_actions[action.name] = [] + + def _build_action(self, action_cls, tag: TagItem): + """ + Build an action with `tag`. + + Run action validation and adds the errors to the errors lists + Add to the action list and the indexed actions + """ + action = action_cls(self.taxonomy, tag, len(self.actions) + 1) + + # We validate if there are no inconsistencies when executing this action + self.errors.extend(action.validate(self.indexed_actions)) + + # Add action + self.actions.append(action) + + # Index the actions for search + self.indexed_actions[action.name].append(action) + + def _search_parent_update( + self, + child_external_id, + parent_external_id, + ): + """ + Checks if there is a parent update in a child + """ + for action in self.indexed_actions["update_parent"]: + if ( + child_external_id == action.tag.id + and parent_external_id != action.tag.parent_id + ): + return True + + return False + + def _build_delete_actions(self, tags: dict): + """ + Adds delete actions for `tags` + """ + for tag in tags.values(): + for child in tag.children.all(): + # Verify if there is not a parent update before + if not self._search_parent_update(child.external_id, tag.external_id): + # Change parent to avoid delete childs + self._build_action( + UpdateParentTag, + TagItem( + id=child.external_id, + value=child.value, + parent_id=None, + ), + ) + + # Delete action + self._build_action( + DeleteTag, + TagItem( + id=tag.external_id, + value=tag.value, + ), + ) + + def generate_actions( + self, + tags: List[TagItem], + replace=False, + ): + """ + Reads each tag and generates the corresponding actions. + + Validates each action and create respective errors + If `replace` is True, then creates the delete action for tags + that are in the existing taxonomy but not the new tags list. + + TODO: Join/reduce actions. Ex. A tag may have no changes, + but then its parent needs to be updated because its parent is deleted. + Those two actions should be merged. + """ + self.actions.clear() + self.errors.clear() + self._init_indexed_actions() + tags_for_delete = {} + + if replace: + tags_for_delete = { + tag.external_id: tag for tag in self.taxonomy.tag_set.all() + } + + for tag in tags: + has_action = False + + # Check all available actions and add which ones should be executed + for action_cls in available_actions: + if action_cls.applies_for(self.taxonomy, tag): + self._build_action(action_cls, tag) + has_action = True + + if not has_action: + # If it doesn't find an action, a "without changes" is added + self._build_action(WithoutChanges, tag) + + if replace and tag.id in tags_for_delete: + tags_for_delete.pop(tag.id) + + if replace: + # Delete all not readed tags + self._build_delete_actions(tags_for_delete) + + def plan(self) -> str: + """ + Returns an string with the plan and errors + """ + result = ( + f"Import plan for {self.taxonomy.name}\n" + "--------------------------------\n" + ) + for action in self.actions: + result += f"#{action.index}: {str(action)}\n" + + if self.errors: + result += "\nOutput errors\n" "--------------------------------\n" + for error in self.errors: + result += f"{str(error)}\n" + + return result + + @transaction.atomic() + def execute(self, task: TagImportTask = None): + """ + Executes each action + + If task is set, creates logs for each action + """ + if self.errors: + return + for action in self.actions: + if task: + task.add_log(f"#{action.index}: {str(action)} [Started]") + action.execute() + if task: + task.add_log("Success") diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py new file mode 100644 index 000000000..79171e7e5 --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -0,0 +1,325 @@ +""" +Parsers to import and export tags +""" +import csv +import json +from enum import Enum +from io import BytesIO, TextIOWrapper, StringIO +from typing import List, Tuple + +from django.utils.translation import gettext_lazy as _ + +from .import_plan import TagItem +from .exceptions import ( + TagParserError, + InvalidFormat, + FieldJSONError, + EmptyJSONField, + EmptyCSVField, +) +from ..models import Taxonomy +from ..api import get_tags + + +class ParserFormat(Enum): + """ + Format of import tags to taxonomies + """ + + JSON = ".json" + CSV = ".csv" + + +class Parser: + """ + Base class to create a parser + + This contains the base functions to convert between + a simple file format like CSV/JSON and a list of TagItems. + It can convert in both directions, for use during import or export. + + If you want to add a new field, you can add it to + `required_fields` or `optional_fields` depending on the field type + + To create a new Parser you need to implement `_load_data` and `_export_data` + """ + + required_fields = ["id", "value"] + optional_fields = ["parent_id"] + + # Set the format associated to the parser + format = None + # We can change the error when is missing a required field + missing_field_error = TagParserError + # We can change the error when a required field is empty + empty_field_error = TagParserError + # We can change the initial row/index + inital_row = 1 + + @classmethod + def parse_import(cls, file: BytesIO) -> Tuple[List[TagItem], List[TagParserError]]: + """ + Parse tags in file an returns tags ready for use in TagImportPlan + + Top function that calls `_load_data` and `_parse_tags`. + Handle errors returned by both functions. + """ + try: + tags_data, load_errors = cls._load_data(file) + if load_errors: + return [], load_errors + except Exception as error: + raise error + finally: + file.close() + + return cls._parse_tags(tags_data) + + @classmethod + def export(cls, taxonomy: Taxonomy) -> str: + """ + Returns all tags in taxonomy. + The output file can be used to recreate the taxonomy with `parse_import` + """ + tags = cls._load_tags_for_export(taxonomy) + return cls._export_data(tags, taxonomy) + + @classmethod + def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + """ + Each parser implements this function according to its format. + This function reads the file and returns a list with the values of each tag. + + This function does not do field validations, it only does validations of the + file structure in the parser format. Field validations are done in `_parse_tags` + """ + raise NotImplementedError + + @classmethod + def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + """ + Each parser implements this function according to its format. + Returns a string with tags data in the parser format. + Can use `taxonomy` to export taxonomy metadata. + + It must be implemented in such a way that the output of + this function works with _load_data + """ + raise NotImplementedError + + @classmethod + def _parse_tags(cls, tags_data: dict) -> Tuple[List[TagItem], List[TagParserError]]: + """ + Validate the required fields of each tag. + + Return a list of TagItems + and a list of validation errors. + """ + tags = [] + errors = [] + row = cls.inital_row + for tag in tags_data: + has_error = False + + # Verify the required fields + for req_field in cls.required_fields: + if req_field not in tag: + # Verify if the field exists + errors.append( + cls.missing_field_error( + tag, + field=req_field, + row=row, + ) + ) + has_error = True + elif not tag.get(req_field): + # Verify if the value of the field is not empty + errors.append( + cls.empty_field_error( + tag, + field=req_field, + row=row, + ) + ) + has_error = True + + tag["index"] = row + row += 1 + + # Skip parse if there is an error + if has_error: + continue + + # Updating any empty optional field to None + for opt_field in cls.optional_fields: + if opt_field in tag and not tag.get(opt_field): + tag[opt_field] = None + + tags.append(TagItem(**tag)) + + return tags, errors + + @classmethod + def _load_tags_for_export(cls, taxonomy: Taxonomy) -> List[dict]: + """ + Returns a list of taxonomy's tags in the form of a dictionary + with required and optional fields + + The tags are ordered by hierarchy, first, parents and then children. + `get_tags` is in charge of returning this in a hierarchical way. + """ + tags = get_tags(taxonomy) + result = [] + for tag in tags: + result_tag = { + "id": tag.external_id or tag.id, + "value": tag.value, + } + if tag.parent: + result_tag["parent_id"] = tag.parent.external_id or tag.parent.id + result.append(result_tag) + return result + + +class JSONParser(Parser): + """ + Parser used with .json files + + Valid file: + ``` + { + "tags": [ + { + "id": "tag_1", + "value": "tag 1", + "parent_id": "tag_2", + } + ] + } + ``` + """ + + format = ParserFormat.JSON + missing_field_error = FieldJSONError + empty_field_error = EmptyJSONField + inital_row = 0 + + @classmethod + def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + """ + Read a .json file and validates the root structure of the json + """ + file.seek(0) + try: + tags_data = json.load(file) + except json.JSONDecodeError as error: + return None, [ + InvalidFormat(tag=None, format=cls.format.value, message=str(error)) + ] + if "tags" not in tags_data: + return None, [ + InvalidFormat( + tag=None, + format=cls.format.value, + message=_("Missing 'tags' field on the .json file"), + ) + ] + + tags_data = tags_data.get("tags") + return tags_data, [] + + @classmethod + def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + """ + Export tags and taxonomy metadata in JSON format + """ + json_result = { + "name": taxonomy.name, + "description": taxonomy.description, + "tags": tags, + } + return json.dumps(json_result) + + +class CSVParser(Parser): + """ + Parser used with .csv files + + Valid file: + ``` + id,value,parent_id + tag_1,tag 1, + tag_2,tag 2,tag_1 + ``` + """ + + format = ParserFormat.CSV + empty_field_error = EmptyCSVField + inital_row = 2 + + @classmethod + def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + """ + Read a .csv file and validates the header fields + """ + file.seek(0) + text_tags = TextIOWrapper(file, encoding="utf-8") + csv_reader = csv.DictReader(text_tags) + header_fields = csv_reader.fieldnames + errors = cls._verify_header(header_fields) + if errors: + return None, errors + return list(csv_reader), [] + + @classmethod + def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + """ + Export tags in CSV format + + """ + fields = cls.required_fields + cls.optional_fields + + with StringIO() as csv_buffer: + csv_writer = csv.DictWriter(csv_buffer, fieldnames=fields) + csv_writer.writeheader() + + for tag in tags: + csv_writer.writerow(tag) + + csv_string = csv_buffer.getvalue() + return csv_string + + @classmethod + def _verify_header(cls, header_fields: List[str]) -> List[TagParserError]: + """ + Verify that the header contains the required fields + """ + errors = [] + for req_field in cls.required_fields: + if req_field not in header_fields: + errors.append( + InvalidFormat( + tag=None, + format=cls.format.value, + message=_(f"Missing '{req_field}' field on CSV headers"), + ) + ) + return errors + + +# Add parsers here +_parsers = [JSONParser, CSVParser] + + +def get_parser(parser_format: ParserFormat) -> Parser: + """ + Get the parser for the respective `format` + + Raise `ValueError` if no parser found + """ + for parser in _parsers: + if parser_format == parser.format: + return parser + + raise ValueError(_(f"Parser not found for format {parser_format}")) diff --git a/openedx_tagging/core/tagging/import_export/tasks.py b/openedx_tagging/core/tagging/import_export/tasks.py new file mode 100644 index 000000000..99d3cd49d --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/tasks.py @@ -0,0 +1,38 @@ +""" +Import and export celery tasks +""" +from io import BytesIO +from celery import shared_task + +import openedx_tagging.core.tagging.import_export.api as import_export_api +from ..models import Taxonomy +from .parsers import ParserFormat + + +@shared_task +def import_tags_task( + taxonomy: Taxonomy, + file: BytesIO, + parser_format: ParserFormat, + replace=False, +) -> bool: + """ + Runs import on a celery task + """ + return import_export_api.import_tags( + taxonomy, + file, + parser_format, + replace, + ) + + +@shared_task +def export_tags_task( + taxonomy: Taxonomy, + output_format: ParserFormat, +) -> str: + """ + Runs export on a celery task + """ + return import_export_api.export_tags(taxonomy, output_format) diff --git a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py index 48bab478a..a6d4fd0cf 100644 --- a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py +++ b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py @@ -11,7 +11,7 @@ def load_language_taxonomy(apps, schema_editor): call_command("loaddata", "--app=oel_tagging", "language_taxonomy.yaml") -def revert(apps, schema_editor): +def revert(apps, schema_editor): # pragma: no cover """ Deletes language taxonomy an tags """ diff --git a/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py b/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py new file mode 100644 index 000000000..87bab9cd7 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py @@ -0,0 +1,81 @@ +# Generated by Django 3.2.19 on 2023-08-02 21:31 + +from django.db import migrations, models +import django.db.models.deletion +import openedx_tagging.core.tagging.models.import_export + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0006_alter_objecttag_unique_together"), + ] + + operations = [ + migrations.CreateModel( + name="TagImportTask", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ( + "log", + models.TextField( + default=None, help_text="Action execution logs", null=True + ), + ), + ( + "status", + models.CharField( + choices=[ + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "LOADING_DATA" + ], + "loading_data", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "PLANNING" + ], + "planning", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "EXECUTING" + ], + "executing", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "SUCCESS" + ], + "success", + ), + ( + openedx_tagging.core.tagging.models.import_export.TagImportTaskState[ + "ERROR" + ], + "error", + ), + ], + help_text="Task status", + max_length=20, + ), + ), + ("creation_date", models.DateTimeField(auto_now_add=True)), + ( + "taxonomy", + models.ForeignKey( + help_text="Taxonomy associated with this import", + on_delete=django.db.models.deletion.CASCADE, + to="oel_tagging.taxonomy", + ), + ), + ], + ), + migrations.AddIndex( + model_name="tagimporttask", + index=models.Index( + fields=["taxonomy", "-creation_date"], + name="oel_tagging_taxonom_5e948c_idx", + ), + ), + ] diff --git a/openedx_tagging/core/tagging/models/__init__.py b/openedx_tagging/core/tagging/models/__init__.py index 90640ddfc..295e38bdb 100644 --- a/openedx_tagging/core/tagging/models/__init__.py +++ b/openedx_tagging/core/tagging/models/__init__.py @@ -9,3 +9,7 @@ UserSystemDefinedTaxonomy, LanguageTaxonomy, ) +from .import_export import ( + TagImportTask, + TagImportTaskState, +) diff --git a/openedx_tagging/core/tagging/models/import_export.py b/openedx_tagging/core/tagging/models/import_export.py new file mode 100644 index 000000000..1deaf9c9d --- /dev/null +++ b/openedx_tagging/core/tagging/models/import_export.py @@ -0,0 +1,107 @@ +from datetime import datetime +from enum import Enum +from django.db import models + +from django.utils.translation import gettext_lazy as _ + +from .base import Taxonomy + + +class TagImportTaskState(Enum): + LOADING_DATA = "loading_data" + PLANNING = "planning" + EXECUTING = "executing" + SUCCESS = "success" + ERROR = "error" + + +class TagImportTask(models.Model): + """ + Stores the state, plan and logs of a tag import task + """ + + id = models.BigAutoField(primary_key=True) + + taxonomy = models.ForeignKey( + "Taxonomy", + on_delete=models.CASCADE, + help_text=_("Taxonomy associated with this import"), + ) + + log = models.TextField( + null=True, default=None, help_text=_("Action execution logs") + ) + + status = models.CharField( + max_length=20, + choices=[(status, status.value) for status in TagImportTaskState], + help_text=_("Task status"), + ) + + creation_date = models.DateTimeField(auto_now_add=True) + + class Meta: + indexes = [ + models.Index(fields=["taxonomy", "-creation_date"]), + ] + + @classmethod + def create(cls, taxonomy: Taxonomy): + task = cls( + taxonomy=taxonomy, + status=TagImportTaskState.LOADING_DATA.value, + log="", + ) + task.add_log(_("Import task created"), save=False) + task.save() + return task + + def add_log(self, message: str, save=True): + timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + log_message = f"[{timestamp}] {message}\n" + self.log += log_message + if save: + self.save() + + def log_exception(self, exception: Exception): + self.add_log(repr(exception), save=False) + self.status = TagImportTaskState.ERROR.value + self.save() + + def log_parser_start(self): + self.add_log(_("Starting to load data from file")) + + def log_parser_end(self): + self.add_log(_("Load data finished")) + + def handle_parser_errors(self, errors): + for error in errors: + self.add_log(f"{str(error)}", save=False) + self.status = TagImportTaskState.ERROR.value + self.save() + + def log_start_planning(self): + self.add_log(_("Starting plan actions"), save=False) + self.status = TagImportTaskState.PLANNING.value + self.save() + + def log_plan(self, plan): + self.add_log(_("Plan finished")) + plan_str = plan.plan() + self.log += f"\n{plan_str}\n" + self.save() + + def handle_plan_errors(self): + # Error are logged with plan + self.status = TagImportTaskState.ERROR.value + self.save() + + def log_start_execute(self): + self.add_log(_("Starting execute actions"), save=False) + self.status = TagImportTaskState.EXECUTING.value + self.save() + + def end_success(self): + self.add_log(_("Execution finished"), save=False) + self.status = TagImportTaskState.SUCCESS.value + self.save() diff --git a/requirements/base.in b/requirements/base.in index 94723505f..33b66281a 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,6 +1,10 @@ # Core requirements for using this application -c constraints.txt +attrs # Reduces boilerplate code involving class attributes + +celery # Asynchronous task execution library + Django<5.0 # Web application framework djangorestframework<4.0 # REST API diff --git a/requirements/base.txt b/requirements/base.txt index dc2d4c77f..3ad003afb 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,11 +1,19 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via kombu asgiref==3.7.2 # via django +attrs==23.1.0 + # via -r requirements/base.in +billiard==4.1.0 + # via celery +celery==5.3.1 + # via -r requirements/base.in certifi==2023.7.22 # via requests cffi==1.15.1 @@ -15,7 +23,18 @@ cffi==1.15.1 charset-normalizer==3.2.0 # via requests click==8.1.6 - # via edx-django-utils + # via + # celery + # click-didyoumean + # click-plugins + # click-repl + # edx-django-utils +click-didyoumean==0.3.0 + # via celery +click-plugins==1.1.1 + # via celery +click-repl==0.3.0 + # via celery cryptography==41.0.3 # via pyjwt django==3.2.19 @@ -49,10 +68,14 @@ edx-opaque-keys==2.4.0 # via edx-drf-extensions idna==3.4 # via requests +kombu==5.3.1 + # via celery newrelic==8.9.0 # via edx-django-utils pbr==5.11.1 # via stevedore +prompt-toolkit==3.0.39 + # via click-repl psutil==5.9.5 # via edx-django-utils pycparser==2.21 @@ -66,7 +89,9 @@ pymongo==3.13.0 pynacl==1.5.0 # via edx-django-utils python-dateutil==2.8.2 - # via edx-drf-extensions + # via + # celery + # edx-drf-extensions pytz==2023.3 # via # django @@ -89,5 +114,14 @@ stevedore==5.1.0 # edx-opaque-keys typing-extensions==4.6.3 # via asgiref +tzdata==2023.3 + # via celery urllib3==2.0.4 # via requests +vine==5.0.0 + # via + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index c03598b3e..f47e116e0 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade diff --git a/requirements/dev.txt b/requirements/dev.txt index 7c85fde2e..d5ad00f9d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via + # -r requirements/quality.txt + # kombu asgiref==3.7.2 # via # -r requirements/quality.txt @@ -13,6 +17,12 @@ astroid==2.15.5 # -r requirements/quality.txt # pylint # pylint-celery +attrs==23.1.0 + # via -r requirements/quality.txt +billiard==4.1.0 + # via + # -r requirements/quality.txt + # celery bleach==6.0.0 # via # -r requirements/quality.txt @@ -21,6 +31,8 @@ build==0.10.0 # via # -r requirements/pip-tools.txt # pip-tools +celery==5.3.1 + # via -r requirements/quality.txt certifi==2023.7.22 # via # -r requirements/quality.txt @@ -41,16 +53,32 @@ click==8.1.6 # -r requirements/ci.txt # -r requirements/pip-tools.txt # -r requirements/quality.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint # import-linter # pip-tools +click-didyoumean==0.3.0 + # via + # -r requirements/quality.txt + # celery click-log==0.4.0 # via # -r requirements/quality.txt # edx-lint +click-plugins==1.1.1 + # via + # -r requirements/quality.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/quality.txt + # celery code-annotations==1.3.0 # via # -r requirements/quality.txt @@ -155,10 +183,6 @@ importlib-metadata==6.7.0 # -r requirements/quality.txt # keyring # twine -importlib-resources==5.12.0 - # via - # -r requirements/quality.txt - # keyring iniconfig==2.0.0 # via # -r requirements/quality.txt @@ -185,6 +209,10 @@ keyring==24.0.0 # via # -r requirements/quality.txt # twine +kombu==5.3.1 + # via + # -r requirements/quality.txt + # celery lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt @@ -252,6 +280,10 @@ pluggy==1.0.0 # tox polib==1.2.0 # via edx-i18n-tools +prompt-toolkit==3.0.39 + # via + # -r requirements/quality.txt + # click-repl psutil==5.9.5 # via # -r requirements/quality.txt @@ -323,6 +355,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/quality.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via @@ -431,17 +464,29 @@ typing-extensions==4.6.3 # astroid # grimp # import-linter - # pylint - # rich +tzdata==2023.3 + # via + # -r requirements/quality.txt + # celery urllib3==2.0.4 # via # -r requirements/quality.txt # requests # twine +vine==5.0.0 + # via + # -r requirements/quality.txt + # amqp + # celery + # kombu virtualenv==20.23.1 # via # -r requirements/ci.txt # tox +wcwidth==0.2.6 + # via + # -r requirements/quality.txt + # prompt-toolkit webencodings==0.5.1 # via # -r requirements/quality.txt @@ -458,7 +503,6 @@ zipp==3.15.0 # via # -r requirements/quality.txt # importlib-metadata - # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/doc.txt b/requirements/doc.txt index f997b86a7..06ae86ff7 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade @@ -8,18 +8,30 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx +amqp==5.1.1 + # via + # -r requirements/test.txt + # kombu asgiref==3.7.2 # via # -r requirements/test.txt # django +attrs==23.1.0 + # via -r requirements/test.txt babel==2.12.1 # via # pydata-sphinx-theme # sphinx beautifulsoup4==4.12.2 # via pydata-sphinx-theme +billiard==4.1.0 + # via + # -r requirements/test.txt + # celery bleach==6.0.0 # via readme-renderer +celery==5.3.1 + # via -r requirements/test.txt certifi==2023.7.22 # via # -r requirements/test.txt @@ -36,9 +48,25 @@ charset-normalizer==3.2.0 click==8.1.6 # via # -r requirements/test.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils # import-linter +click-didyoumean==0.3.0 + # via + # -r requirements/test.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.3.0 # via -r requirements/test.txt coverage[toml]==7.2.7 @@ -118,8 +146,6 @@ imagesize==1.4.1 # via sphinx import-linter==1.9.0 # via -r requirements/test.txt -importlib-metadata==6.7.0 - # via sphinx iniconfig==2.0.0 # via # -r requirements/test.txt @@ -129,6 +155,10 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx +kombu==5.3.1 + # via + # -r requirements/test.txt + # celery markupsafe==2.1.3 # via # -r requirements/test.txt @@ -157,6 +187,10 @@ pluggy==1.0.0 # pytest pprintpp==0.4.0 # via sphinxcontrib-django +prompt-toolkit==3.0.39 + # via + # -r requirements/test.txt + # click-repl psutil==5.9.5 # via # -r requirements/test.txt @@ -199,6 +233,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/test.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via @@ -207,7 +242,6 @@ python-slugify==8.0.1 pytz==2023.3 # via # -r requirements/test.txt - # babel # django # djangorestframework pyyaml==6.0 @@ -291,11 +325,23 @@ typing-extensions==4.6.3 # grimp # import-linter # pydata-sphinx-theme +tzdata==2023.3 + # via + # -r requirements/test.txt + # celery urllib3==2.0.4 # via # -r requirements/test.txt # requests +vine==5.0.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via + # -r requirements/test.txt + # prompt-toolkit webencodings==0.5.1 # via bleach -zipp==3.15.0 - # via importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 4b7fb4e53..9ce8ce227 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade diff --git a/requirements/quality.txt b/requirements/quality.txt index 0509905b3..22fb452e9 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via + # -r requirements/test.txt + # kombu asgiref==3.7.2 # via # -r requirements/test.txt @@ -12,8 +16,16 @@ astroid==2.15.5 # via # pylint # pylint-celery +attrs==23.1.0 + # via -r requirements/test.txt +billiard==4.1.0 + # via + # -r requirements/test.txt + # celery bleach==6.0.0 # via readme-renderer +celery==5.3.1 + # via -r requirements/test.txt certifi==2023.7.22 # via # -r requirements/test.txt @@ -30,13 +42,29 @@ charset-normalizer==3.2.0 click==8.1.6 # via # -r requirements/test.txt + # celery + # click-didyoumean # click-log + # click-plugins + # click-repl # code-annotations # edx-django-utils # edx-lint # import-linter +click-didyoumean==0.3.0 + # via + # -r requirements/test.txt + # celery click-log==0.4.0 # via edx-lint +click-plugins==1.1.1 + # via + # -r requirements/test.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/test.txt + # celery code-annotations==1.3.0 # via # -r requirements/test.txt @@ -117,8 +145,6 @@ importlib-metadata==6.7.0 # via # keyring # twine -importlib-resources==5.12.0 - # via keyring iniconfig==2.0.0 # via # -r requirements/test.txt @@ -139,6 +165,10 @@ jinja2==3.1.2 # code-annotations keyring==24.0.0 # via twine +kombu==5.3.1 + # via + # -r requirements/test.txt + # celery lazy-object-proxy==1.9.0 # via astroid markdown-it-py==3.0.0 @@ -177,6 +207,10 @@ pluggy==1.0.0 # via # -r requirements/test.txt # pytest +prompt-toolkit==3.0.39 + # via + # -r requirements/test.txt + # click-repl psutil==5.9.5 # via # -r requirements/test.txt @@ -232,6 +266,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/test.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via @@ -310,18 +345,28 @@ typing-extensions==4.6.3 # astroid # grimp # import-linter - # pylint - # rich +tzdata==2023.3 + # via + # -r requirements/test.txt + # celery urllib3==2.0.4 # via # -r requirements/test.txt # requests # twine +vine==5.0.0 + # via + # -r requirements/test.txt + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via + # -r requirements/test.txt + # prompt-toolkit webencodings==0.5.1 # via bleach wrapt==1.15.0 # via astroid zipp==3.15.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata diff --git a/requirements/test.txt b/requirements/test.txt index e1240b21f..ba50193f0 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,13 +1,25 @@ # -# This file is autogenerated by pip-compile with Python 3.8 +# This file is autogenerated by pip-compile with Python 3.10 # by the following command: # # make upgrade # +amqp==5.1.1 + # via + # -r requirements/base.txt + # kombu asgiref==3.7.2 # via # -r requirements/base.txt # django +attrs==23.1.0 + # via -r requirements/base.txt +billiard==4.1.0 + # via + # -r requirements/base.txt + # celery +celery==5.3.1 + # via -r requirements/base.txt certifi==2023.7.22 # via # -r requirements/base.txt @@ -24,9 +36,25 @@ charset-normalizer==3.2.0 click==8.1.6 # via # -r requirements/base.txt + # celery + # click-didyoumean + # click-plugins + # click-repl # code-annotations # edx-django-utils # import-linter +click-didyoumean==0.3.0 + # via + # -r requirements/base.txt + # celery +click-plugins==1.1.1 + # via + # -r requirements/base.txt + # celery +click-repl==0.3.0 + # via + # -r requirements/base.txt + # celery code-annotations==1.3.0 # via -r requirements/test.in coverage[toml]==7.2.7 @@ -93,6 +121,10 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations +kombu==5.3.1 + # via + # -r requirements/base.txt + # celery markupsafe==2.1.3 # via jinja2 mock==5.0.2 @@ -111,6 +143,10 @@ pbr==5.11.1 # stevedore pluggy==1.0.0 # via pytest +prompt-toolkit==3.0.39 + # via + # -r requirements/base.txt + # click-repl psutil==5.9.5 # via # -r requirements/base.txt @@ -144,6 +180,7 @@ pytest-django==4.5.2 python-dateutil==2.8.2 # via # -r requirements/base.txt + # celery # edx-drf-extensions python-slugify==8.0.1 # via code-annotations @@ -193,7 +230,21 @@ typing-extensions==4.6.3 # asgiref # grimp # import-linter +tzdata==2023.3 + # via + # -r requirements/base.txt + # celery urllib3==2.0.4 # via # -r requirements/base.txt # requests +vine==5.0.0 + # via + # -r requirements/base.txt + # amqp + # celery + # kombu +wcwidth==0.2.6 + # via + # -r requirements/base.txt + # prompt-toolkit diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index 5bb3936c9..3593a095c 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -173,6 +173,34 @@ parent: null value: System Tag 4 external_id: 'tag_4' +- model: oel_tagging.tag + pk: 26 + fields: + taxonomy: 5 + parent: null + value: Tag 1 + external_id: tag_1 +- model: oel_tagging.tag + pk: 27 + fields: + taxonomy: 5 + parent: 26 + value: Tag 2 + external_id: tag_2 +- model: oel_tagging.tag + pk: 28 + fields: + taxonomy: 5 + parent: null + value: Tag 3 + external_id: tag_3 +- model: oel_tagging.tag + pk: 29 + fields: + taxonomy: 5 + parent: 28 + value: Tag 4 + external_id: tag_4 - model: oel_tagging.taxonomy pk: 1 fields: @@ -202,3 +230,13 @@ allow_multiple: false allow_free_text: false _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.SystemDefinedTaxonomy +- model: oel_tagging.taxonomy + pk: 5 + fields: + name: Import Taxonomy Test + description: null + enabled: true + required: false + allow_multiple: false + allow_free_text: false + diff --git a/tests/openedx_tagging/core/tagging/import_export/__init__.py b/tests/openedx_tagging/core/tagging/import_export/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/openedx_tagging/core/tagging/import_export/mixins.py b/tests/openedx_tagging/core/tagging/import_export/mixins.py new file mode 100644 index 000000000..045d3c180 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/mixins.py @@ -0,0 +1,16 @@ +""" +Mixins for ImportExport tests +""" +from openedx_tagging.core.tagging.models import Taxonomy + + +class TestImportExportMixin: + """ + Mixin that loads the base data for import/export tests + """ + + fixtures = ["tests/openedx_tagging/core/fixtures/tagging.yaml"] + + def setUp(self): + self.taxonomy = Taxonomy.objects.get(name="Import Taxonomy Test") + return super().setUp() diff --git a/tests/openedx_tagging/core/tagging/import_export/test_actions.py b/tests/openedx_tagging/core/tagging/import_export/test_actions.py new file mode 100644 index 000000000..1c0b14852 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_actions.py @@ -0,0 +1,500 @@ +""" +Tests for actions +""" +import ddt + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.models import Tag +from openedx_tagging.core.tagging.import_export.import_plan import TagItem +from openedx_tagging.core.tagging.import_export.actions import ( + ImportAction, + CreateTag, + UpdateParentTag, + RenameTag, + DeleteTag, + WithoutChanges, +) +from .mixins import TestImportExportMixin + + +class TestImportActionMixin(TestImportExportMixin): + """ + Mixin for import action tests + """ + def setUp(self): + super().setUp() + self.indexed_actions = { + 'create': [ + CreateTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_10', + value='Tag 10', + index=0 + ), + index=0, + ) + ], + 'rename': [ + RenameTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_11', + value='Tag 11', + index=1 + ), + index=1, + ) + ] + } + + +@ddt.ddt +class TestImportAction(TestImportActionMixin, TestCase): + """ + Test for general function of the ImportAction class + """ + + def test_not_implemented_functions(self): + with self.assertRaises(NotImplementedError): + ImportAction.applies_for(None, None) + action = ImportAction(None, None, None) + with self.assertRaises(NotImplementedError): + action.validate(None) + with self.assertRaises(NotImplementedError): + action.execute() + + def test_str(self): + expected = "Action import_action (index=100,id=tag_1)" + action = ImportAction( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_1', + value='value', + ), + index=100, + ) + assert str(action) == expected + + @ddt.data( + ('create', 'id', 'tag_10', True), + ('rename', 'value', 'Tag 11', True), + ('rename', 'id', 'tag_10', False), + ('create', 'value', 'Tag 11', False), + ) + @ddt.unpack + def test_search_action(self, action_name, attr, search_value, expected): + import_action = ImportAction(self.taxonomy, None, None) + action = import_action._search_action( # pylint: disable=protected-access + self.indexed_actions, + action_name, + attr, + search_value, + ) + if expected: + self.assertEqual(getattr(action.tag, attr), search_value) + else: + self.assertIsNone(action) + + @ddt.data( + ('tag_1', True), + ('tag_10', True), + ('tag_100', False), + ) + @ddt.unpack + def test_validate_parent(self, parent_id, expected): + action = ImportAction( + self.taxonomy, + TagItem( + id='tag_110', + value='_', + parent_id=parent_id, + index=100 + ), + index=100, + ) + error = action._validate_parent(self.indexed_actions) # pylint: disable=protected-access + if expected: + self.assertIsNone(error) + else: + self.assertEqual( + str(error), + ( + "Action error in 'import_action' (#100): " + "Unknown parent tag (tag_100). " + "You need to add parent before the child in your file." + ) + ) + + @ddt.data( + ( + 'Tag 1', + ( + "Action error in 'import_action' (#100): " + "Duplicated tag value with tag in database (external_id=tag_1)." + ) + ), + ( + 'Tag 10', + ( + "Conflict with 'import_action' (#100) " + "and action #0: Duplicated tag value." + ) + ), + ( + 'Tag 11', + ( + "Conflict with 'import_action' (#100) " + "and action #1: Duplicated tag value." + ) + ), + ('Tag 20', None) + ) + @ddt.unpack + def test_validate_value(self, value, expected): + action = ImportAction( + self.taxonomy, + TagItem( + id='tag_110', + value=value, + index=100 + ), + index=100, + ) + error = action._validate_value(self.indexed_actions) # pylint: disable=protected-access + if not expected: + self.assertIsNone(error) + else: + self.assertEqual(str(error), expected) + + +@ddt.ddt +class TestCreateTag(TestImportActionMixin, TestCase): + """ + Test for 'create' action + """ + + @ddt.data( + ('tag_1', False), + ('tag_100', True), + ) + @ddt.unpack + def test_applies_for(self, tag_id, expected): + result = CreateTag.applies_for( + self.taxonomy, + TagItem( + id=tag_id, + value='_', + index=100, + ) + ) + self.assertEqual(result, expected) + + @ddt.data( + ('tag_10', False), + ('tag_100', True), + ) + @ddt.unpack + def test_validate_id(self, tag_id, expected): + action = CreateTag( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value='_', + index=100, + ), + index=100 + ) + error = action._validate_id(self.indexed_actions) # pylint: disable=protected-access + if expected: + self.assertIsNone(error) + else: + self.assertEqual( + str(error), + ( + "Conflict with 'create' (#100) " + "and action #0: Duplicated external_id tag." + ) + ) + + @ddt.data( + ('tag_10', "Tag 20", None, 1), # Invalid tag id + ('tag_20', "Tag 10", None, 1), # Invalid value, + ('tag_20', "Tag 20", 'tag_100', 1), # Invalid parent id, + ('tag_10', "Tag 10", None, 2), # Invalid tag id and value, + ('tag_10', "Tag 10", 'tag_100', 3), # Invalid tag id, value and parent, + ('tag_20', "Tag 20", 'tag_1', 0), # Valid + ) + @ddt.unpack + def test_validate(self, tag_id, tag_value, parent_id, expected): + action = CreateTag( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value=tag_value, + index=100, + parent_id=parent_id + ), + index=100 + ) + errors = action.validate(self.indexed_actions) + self.assertEqual(len(errors), expected) + + @ddt.data( + ('tag_30', 'Tag 30', None), # No parent + ('tag_31', 'Tag 31', 'tag_3'), # With parent + ) + @ddt.unpack + def test_execute(self, tag_id, value, parent_id): + tag = TagItem( + id=tag_id, + value=value, + parent_id=parent_id, + ) + action = CreateTag( + self.taxonomy, + tag, + index=100, + ) + with self.assertRaises(Tag.DoesNotExist): + self.taxonomy.tag_set.get(external_id=tag_id) + action.execute() + tag = self.taxonomy.tag_set.get(external_id=tag_id) + assert tag.value == value + if parent_id: + assert tag.parent.external_id == parent_id + else: + assert tag.parent is None + + +@ddt.ddt +class TestUpdateParentTag(TestImportActionMixin, TestCase): + """ + Test for 'update_parent' action + """ + + @ddt.data( + ( + "tag_4", + "tag_3", + ( + "Update the parent of tag (external_id=tag_4) from parent " + "(external_id=tag_3) to parent (external_id=tag_3)." + ) + ), + ( + "tag_3", + "tag_2", + ( + "Update the parent of tag (external_id=tag_3) from empty parent " + "to parent (external_id=tag_2)." + ) + ), + ) + @ddt.unpack + def test_str(self, tag_id, parent_id, expected): + tag_item = TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + ) + action = UpdateParentTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + assert str(action) == expected + + @ddt.data( + ('tag_100', None, False), # Tag doesn't exist on database + ('tag_2', 'tag_1', False), # Parent don't change + ('tag_2', 'tag_3', True), # Valid + ('tag_1', None, False), # Both parent id are None + ('tag_1', 'tag_3', True), # Valid + ) + @ddt.unpack + def test_applies_for(self, tag_id, parent_id, expected): + result = UpdateParentTag.applies_for( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + index=100 + ) + ) + self.assertEqual(result, expected) + + @ddt.data( + ('tag_2', 'tag_30', 1), # Invalid parent + ('tag_2', None, 0), # Without parent + ('tag_2', 'tag_10', 0), # Valid + ) + @ddt.unpack + def test_validate(self, tag_id, parent_id, expected): + action = UpdateParentTag( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + ), + index=100 + ) + errors = action.validate(self.indexed_actions) + self.assertEqual(len(errors), expected) + + @ddt.data( + ('tag_4', 'tag_2'), # Change parent + ('tag_4', None), # Set parent as None + ('tag_3', 'tag_1'), # Add parent + ) + @ddt.unpack + def test_execute(self, tag_id, parent_id): + tag_item = TagItem( + id=tag_id, + value='_', + parent_id=parent_id, + ) + action = UpdateParentTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + tag = self.taxonomy.tag_set.get(external_id=tag_id) + if tag.parent: + assert tag.parent.external_id != parent_id + action.execute() + tag = self.taxonomy.tag_set.get(external_id=tag_id) + if not parent_id: + assert tag.parent is None + else: + assert tag.parent.external_id == parent_id + + +@ddt.ddt +class TestRenameTag(TestImportActionMixin, TestCase): + """ + Test for 'rename' action + """ + + @ddt.data( + ('tag_10', 'value', False), # Tag doesn't exist on database + ('tag_1', 'Tag 1', False), # Same value + ('tag_1', 'Tag 1 v2', True), # Valid + ) + @ddt.unpack + def test_applies_for(self, tag_id, value, expected): + result = RenameTag.applies_for( + taxonomy=self.taxonomy, + tag=TagItem( + id=tag_id, + value=value, + index=100, + ) + ) + self.assertEqual(result, expected) + + @ddt.data( + ('Tag 2', 1), # There is a tag with the same value on database + ('Tag 10', 1), # There is a tag with the same value on create action + ('Tag 11', 1), # There is a tag with the same value on rename action + ('Tag 12', 0), # Valid + ) + @ddt.unpack + def test_validate(self, value, expected): + action = RenameTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='tag_1', + value=value, + index=100, + ), + index=100, + ) + errors = action.validate(self.indexed_actions) + self.assertEqual(len(errors), expected) + + def test_execute(self): + tag_id = 'tag_1' + value = 'Tag 1 V2' + tag_item = TagItem( + id=tag_id, + value=value, + ) + action = RenameTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + tag = self.taxonomy.tag_set.get(external_id=tag_id) + assert tag.value != value + action.execute() + tag = self.taxonomy.tag_set.get(external_id=tag_id) + assert tag.value == value + + +class TestDeleteTag(TestImportActionMixin, TestCase): + """ + Test for 'delete' action + """ + + def test_applies_for(self): + assert not DeleteTag.applies_for(self.taxonomy, None) + + def test_validate(self): + action = DeleteTag( + taxonomy=self.taxonomy, + tag=TagItem( + id='_', + value='_', + index=100, + ), + index=100, + ) + assert not action.validate(self.indexed_actions) + + def test_execute(self): + tag_id = 'tag_3' + tag_item = TagItem( + id=tag_id, + value='_', + ) + action = DeleteTag( + taxonomy=self.taxonomy, + tag=tag_item, + index=100, + ) + assert self.taxonomy.tag_set.filter(external_id=tag_id).exists() + action.execute() + assert not self.taxonomy.tag_set.filter(external_id=tag_id).exists() + + +class TestWithoutChanges(TestImportActionMixin, TestCase): + """ + Test for 'without_changes' action + """ + def test_applies_for(self): + result = WithoutChanges.applies_for( + self.taxonomy, + tag=TagItem( + id='_', + value='_', + index=100, + ), + ) + self.assertFalse(result) + + def test_validate(self): + action = WithoutChanges( + taxonomy=self.taxonomy, + tag=TagItem( + id='_', + value='_', + index=100, + ), + index=100, + ) + result = action.validate(self.indexed_actions) + self.assertEqual(result, []) diff --git a/tests/openedx_tagging/core/tagging/import_export/test_api.py b/tests/openedx_tagging/core/tagging/import_export/test_api.py new file mode 100644 index 000000000..453979e01 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_api.py @@ -0,0 +1,219 @@ +""" +Test for import/export API +""" +import json +from io import BytesIO + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.models import ( + TagImportTask, + TagImportTaskState, + Taxonomy, + LanguageTaxonomy, +) +from openedx_tagging.core.tagging.import_export import ParserFormat +import openedx_tagging.core.tagging.import_export.api as import_export_api + +from .mixins import TestImportExportMixin + + +class TestImportExportApi(TestImportExportMixin, TestCase): + """ + Test import/export API functions + """ + + def setUp(self): + self.tags = [ + {"id": "tag_31", "value": "Tag 31"}, + {"id": "tag_32", "value": "Tag 32"}, + {"id": "tag_33", "value": "Tag 33", "parent_id": "tag_31"}, + {"id": "tag_1", "value": "Tag 1 V2"}, + {"id": "tag_4", "value": "Tag 4", "parent_id": "tag_32"}, + ] + json_data = {"tags": self.tags} + self.file = BytesIO(json.dumps(json_data).encode()) + + json_data = {"invalid": [ + {"id": "tag_1", "name": "Tag 1"}, + ]} + self.invalid_parser_file = BytesIO(json.dumps(json_data).encode()) + json_data = {"tags": [ + {'id': 'tag_31', 'value': 'Tag 31',}, + {'id': 'tag_31', 'value': 'Tag 32',}, + ]} + self.invalid_plan_file = BytesIO(json.dumps(json_data).encode()) + + self.parser_format = ParserFormat.JSON + + self.open_taxonomy = Taxonomy( + name="Open taxonomy", + allow_free_text=True + ) + self.system_taxonomy = Taxonomy( + name="System taxonomy", + ) + self.system_taxonomy.taxonomy_class = LanguageTaxonomy + self.system_taxonomy = self.system_taxonomy.cast() + return super().setUp() + + def test_check_status(self): + TagImportTask.create(self.taxonomy) + status = import_export_api.get_last_import_status(self.taxonomy) + assert status == TagImportTaskState.LOADING_DATA.value + + def test_check_log(self): + TagImportTask.create(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert "Import task created" in log + + def test_invalid_import_tags(self): + TagImportTask.create(self.taxonomy) + with self.assertRaises(ValueError): + # Raise error if there is a current in progress task + import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + def test_import_export_validations(self): + # Check that import is invalid with open taxonomy + with self.assertRaises(NotImplementedError): + import_export_api.import_tags( + self.open_taxonomy, + self.file, + self.parser_format, + ) + + # Check that import is invalid with system taxonomy + with self.assertRaises(ValueError): + import_export_api.import_tags( + self.system_taxonomy, + self.file, + self.parser_format, + ) + + def test_with_python_error(self): + self.file.close() + assert not import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.ERROR.value + assert "ValueError('I/O operation on closed file.')" in log + + def test_with_parser_error(self): + assert not import_export_api.import_tags( + self.taxonomy, + self.invalid_parser_file, + self.parser_format, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.ERROR.value + assert "Starting to load data from file" in log + assert "Invalid '.json' format" in log + + def test_with_plan_errors(self): + assert not import_export_api.import_tags( + self.taxonomy, + self.invalid_plan_file, + self.parser_format, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.ERROR.value + assert "Starting to load data from file" in log + assert "Load data finished" in log + assert "Starting plan actions" in log + assert "Plan finished" in log + assert "Conflict with 'create'" in log + + def test_valid(self): + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + replace=True, + ) + status = import_export_api.get_last_import_status(self.taxonomy) + log = import_export_api.get_last_import_log(self.taxonomy) + assert status == TagImportTaskState.SUCCESS.value + assert "Starting to load data from file" in log + assert "Load data finished" in log + assert "Starting plan actions" in log + assert "Plan finished" in log + assert "Starting execute actions" in log + assert "Execution finished" in log + + def test_start_task_after_error(self): + assert not import_export_api.import_tags( + self.taxonomy, + self.invalid_parser_file, + self.parser_format, + ) + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + def test_start_task_after_success(self): + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + # Opening again the file + json_data = {"tags": self.tags} + self.file = BytesIO(json.dumps(json_data).encode()) + + assert import_export_api.import_tags( + self.taxonomy, + self.file, + self.parser_format, + ) + + def test_export_validations(self): + # Check that import is invalid with open taxonomy + with self.assertRaises(NotImplementedError): + import_export_api.export_tags( + self.open_taxonomy, + self.parser_format, + ) + + # Check that import is invalid with system taxonomy + with self.assertRaises(ValueError): + import_export_api.export_tags( + self.system_taxonomy, + self.parser_format, + ) + + def test_import_with_export_output(self): + for parser_format in ParserFormat: + output = import_export_api.export_tags( + self.taxonomy, + parser_format, + ) + file = BytesIO(output.encode()) + new_taxonomy = Taxonomy(name="New taxonomy") + new_taxonomy.save() + assert import_export_api.import_tags( + new_taxonomy, + file, + parser_format, + ) + old_tags = self.taxonomy.tag_set.all() + assert len(old_tags) == new_taxonomy.tag_set.count() + + for tag in old_tags: + new_tag = new_taxonomy.tag_set.get(external_id=tag.external_id) + assert new_tag.value == tag.value + if tag.parent: + assert tag.parent.external_id == new_tag.parent.external_id + \ No newline at end of file diff --git a/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py new file mode 100644 index 000000000..af1e55f30 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py @@ -0,0 +1,418 @@ +""" +Test for import_plan functions +""" +import ddt + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.import_export.import_plan import TagItem, TagImportPlan +from openedx_tagging.core.tagging.import_export.actions import CreateTag +from openedx_tagging.core.tagging.import_export.exceptions import TagImportError +from .test_actions import TestImportActionMixin + +@ddt.ddt +class TestTagImportPlan(TestImportActionMixin, TestCase): + """ + Test for import plan functions + """ + + def setUp(self): + super().setUp() + self.import_plan = TagImportPlan(self.taxonomy) + + def test_tag_import_error(self): + message = "Error message" + expected_repr = f"TagImportError({message})" + error = TagImportError(message) + assert str(error) == message + assert repr(error) == expected_repr + + + @ddt.data( + ('tag_10', 1), # Test invalid + ('tag_30', 0), # Test valid + ) + @ddt.unpack + def test_build_action(self, tag_id, errors_expected): + self.import_plan.indexed_actions = self.indexed_actions + self.import_plan._build_action( # pylint: disable=protected-access + CreateTag, + TagItem( + id=tag_id, + value='_', + index=100 + ) + ) + assert len(self.import_plan.errors) == errors_expected + assert len(self.import_plan.actions) == 1 + assert self.import_plan.actions[0].name == 'create' + assert self.import_plan.indexed_actions['create'][1].tag.id == tag_id + + def test_build_delete_actions(self): + tags = { + tag.external_id: tag + for tag in self.taxonomy.tag_set.exclude(pk=25) + } + # Clear other actions to only have the delete ones + self.import_plan.actions.clear() + + self.import_plan._build_delete_actions(tags) # pylint: disable=protected-access + assert len(self.import_plan.errors) == 0 + + # Check actions in order + # #1 Update parent of 'tag_2' + assert self.import_plan.actions[0].name == 'update_parent' + assert self.import_plan.actions[0].tag.id == 'tag_2' + assert self.import_plan.actions[0].tag.parent_id is None + # #2 Delete 'tag_1' + assert self.import_plan.actions[1].name == 'delete' + assert self.import_plan.actions[1].tag.id == 'tag_1' + # #3 Delete 'tag_2' + assert self.import_plan.actions[2].name == 'delete' + assert self.import_plan.actions[2].tag.id == 'tag_2' + # #4 Update parent of 'tag_4' + assert self.import_plan.actions[3].name == 'update_parent' + assert self.import_plan.actions[3].tag.id == 'tag_4' + assert self.import_plan.actions[3].tag.parent_id is None + # #5 Delete 'tag_3' + assert self.import_plan.actions[4].name == 'delete' + assert self.import_plan.actions[4].tag.id == 'tag_3' + + @ddt.data( + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], + False, + 0, + [ + { + 'name': 'create', + 'id': 'tag_31' + }, + { + 'name': 'create', + 'id': 'tag_32' + }, + { + 'name': 'rename', + 'id': 'tag_2' + }, + { + 'name': 'update_parent', + 'id': 'tag_4' + }, + { + 'name': 'rename', + 'id': 'tag_4' + }, + { + 'name': 'without_changes', + 'id': 'tag_1' + }, + ]), # Test valid actions + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_31', + 'value': 'Tag 32', + }, + { + 'id': 'tag_1', + 'value': 'Tag 2', + }, + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_100', + }, + ], + False, + 3, + [ + { + 'name': 'create', + 'id': 'tag_31', + }, + { + 'name': 'create', + 'id': 'tag_31', + }, + { + 'name': 'rename', + 'id': 'tag_1', + }, + { + 'name': 'update_parent', + 'id': 'tag_4', + } + ]), # Test with errors in actions + ([ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], + True, + 0, + [ + { + 'name': 'without_changes', + 'id': 'tag_4', + }, + { + 'name': 'update_parent', + 'id': 'tag_2', + }, + { + 'name': 'delete', + 'id': 'tag_1', + }, + { + 'name': 'delete', + 'id': 'tag_2', + }, + { + 'name': 'update_parent', + 'id': 'tag_4', + }, + { + 'name': 'delete', + 'id': 'tag_3', + }, + ]) # Test with deletes (replace=True) + ) + @ddt.unpack + def test_generate_actions(self, tags, replace, expected_errors, expected_actions): + tags = [TagItem(**tag) for tag in tags] + self.import_plan.generate_actions(tags=tags, replace=replace) + assert len(self.import_plan.errors) == expected_errors + assert len(self.import_plan.actions) == len(expected_actions) + + for index, action in enumerate(expected_actions): + assert self.import_plan.actions[index].name == action['name'] + assert self.import_plan.actions[index].tag.id == action['id'] + assert self.import_plan.actions[index].index == index + 1 + + @ddt.data( + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_31', + 'value': 'Tag 32', + }, + { + 'id': 'tag_1', + 'value': 'Tag 2', + }, + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_100', + }, + { + 'id': 'tag_33', + 'value': 'Tag 32', + }, + { + 'id': 'tag_2', + 'value': 'Tag 31', + }, + ], + False, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" + "#2: Create a new tag with values (external_id=tag_31, value=Tag 32, parent_id=None).\n" + "#3: Rename tag value of tag (external_id=tag_1) from 'Tag 1' to 'Tag 2'\n" + "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=tag_100).\n" + "#5: Create a new tag with values (external_id=tag_33, value=Tag 32, parent_id=None).\n" + "#6: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " + "to parent (external_id=None).\n" + "#7: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 31'\n" + "\nOutput errors\n" + "--------------------------------\n" + "Conflict with 'create' (#2) and action #1: Duplicated external_id tag.\n" + "Action error in 'rename' (#3): Duplicated tag value with tag in database (external_id=tag_2).\n" + "Action error in 'update_parent' (#4): Unknown parent tag (tag_100). " + "You need to add parent before the child in your file.\n" + "Conflict with 'create' (#5) and action #2: Duplicated tag value.\n" + "Conflict with 'rename' (#7) and action #1: Duplicated tag value.\n" + ), # Testing plan with errors + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], + False, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" + "#2: Create a new tag with values (external_id=tag_32, value=Tag 32, parent_id=tag_1).\n" + "#3: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 2 v2'\n" + "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=tag_1).\n" + "#5: Rename tag value of tag (external_id=tag_4) from 'Tag 4' to 'Tag 4 v2'\n" + "#6: No changes needed for tag (external_id=tag_1)\n" + ), # Testing valid plan + ([ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], + True, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: No changes needed for tag (external_id=tag_4)\n" + "#2: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " + "to parent (external_id=None).\n" + "#3: Delete tag (external_id=tag_1)\n" + "#4: Delete tag (external_id=tag_2)\n" + "#5: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=None).\n" + "#6: Delete tag (external_id=tag_3)\n" + ) # Testing deletes (replace=True) + ) + @ddt.unpack + def test_plan(self, tags, replace, expected): + """ + Test the output of plan() function + + It has been decided to verify the output exactly to detect + any error when printing this information that the user is going to read. + """ + tags = [TagItem(**tag) for tag in tags] + self.import_plan.generate_actions(tags=tags, replace=replace) + plan = self.import_plan.plan() + print(plan) + assert plan == expected + + @ddt.data( + ([ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], False), # Testing all actions + ([ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], True), # Testing deletes (replace=True) + ) + @ddt.unpack + def test_execute(self, tags, replace): + tags = [TagItem(**tag) for tag in tags] + self.import_plan.generate_actions(tags=tags, replace=replace) + self.import_plan.execute() + tag_external_ids = [] + for tag_item in tags: + # This checks any creation + tag = self.taxonomy.tag_set.get(external_id=tag_item.id) + + # Checks any rename + assert tag.value == tag_item.value + + # Checks any parent update + if not replace: + if not tag_item.parent_id: + assert tag.parent is None + else: + assert tag.parent.external_id == tag_item.parent_id + + tag_external_ids.append(tag_item.id) + + if replace: + # Checks deletions checking that exists the updated tags + external_ids = list(self.taxonomy.tag_set.values_list("external_id", flat=True)) + assert tag_external_ids == external_ids + + def test_error_in_execute(self): + created_tag = 'tag_31' + tags = [ + TagItem( + id=created_tag, + value='Tag 31' + ), # Valid tag (creation) + TagItem( + id='tag_32', + value='Tag 31' + ), # Invalid + ] + self.import_plan.generate_actions(tags=tags) + assert not self.taxonomy.tag_set.filter(external_id=created_tag).exists() + assert not self.import_plan.execute() + assert not self.taxonomy.tag_set.filter(external_id=created_tag).exists() + \ No newline at end of file diff --git a/tests/openedx_tagging/core/tagging/import_export/test_parsers.py b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py new file mode 100644 index 000000000..5b77e3a54 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py @@ -0,0 +1,310 @@ +""" +Test for import/export parsers +""" +from io import BytesIO +import json +import ddt + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.import_export.parsers import ( + Parser, + get_parser, + JSONParser, + CSVParser, + ParserFormat, +) +from openedx_tagging.core.tagging.import_export.exceptions import ( + TagParserError, +) +from openedx_tagging.core.tagging.models import Taxonomy +from .mixins import TestImportExportMixin + + +class TestParser(TestCase): + """ + Test for general parser functions + """ + + def test_get_parser(self): + for parser_format in ParserFormat: + parser = get_parser(parser_format) + self.assertEqual(parser.format, parser_format) + + def test_parser_not_found(self): + with self.assertRaises(ValueError): + get_parser(None) + + def test_not_implemented(self): + taxonomy = Taxonomy(name="Test taxonomy") + taxonomy.save() + with self.assertRaises(NotImplementedError): + Parser.parse_import(BytesIO()) + with self.assertRaises(NotImplementedError): + Parser.export(taxonomy) + + def test_tag_parser_error(self): + tag = {"id": 'tag_id', "value": "tag_value"} + expected_str = f"Import parser error on {tag}" + expected_repr = f"TagParserError(Import parser error on {tag})" + error = TagParserError(tag) + assert str(error) == expected_str + assert repr(error) == expected_repr + + +@ddt.ddt +class TestJSONParser(TestImportExportMixin, TestCase): + """ + Test for .json parser + """ + + def test_invalid_json(self): + json_data = "{This is an invalid json}" + json_file = BytesIO(json_data.encode()) + tags, errors = JSONParser.parse_import(json_file) + assert len(tags) == 0 + assert len(errors) == 1 + assert "Expecting property name enclosed in double quotes" in str(errors[0]) + + def test_load_data_errors(self): + json_data = {"invalid": [ + {"id": "tag_1", "name": "Tag 1"}, + ]} + + json_file = BytesIO(json.dumps(json_data).encode()) + + tags, errors = JSONParser.parse_import(json_file) + self.assertEqual(len(tags), 0) + self.assertEqual(len(errors), 1) + self.assertEqual( + str(errors[0]), + "Invalid '.json' format: Missing 'tags' field on the .json file" + ) + + @ddt.data( + ( + {"tags": [ + {"id": "tag_1", "value": "Tag 1"}, # Valid + ]}, + [] + ), + ( + {"tags": [ + {"id": "tag_1"}, + {"value": "tag_1"}, + {}, + ]}, + [ + "Missing 'value' field on {'id': 'tag_1'}", + "Missing 'id' field on {'value': 'tag_1'}", + "Missing 'value' field on {}", + "Missing 'id' field on {}", + ] + ), + ( + {"tags": [ + {"id": "", "value": "tag 1"}, + {"id": "tag_2", "value": ""}, + {"id": "tag_3", "value": "tag 3", "parent_id": ""}, # Valid + ]}, + [ + "Empty 'id' field on {'id': '', 'value': 'tag 1'}", + "Empty 'value' field on {'id': 'tag_2', 'value': ''}", + ] + ) + ) + @ddt.unpack + def test_parse_tags_errors(self, json_data, expected_errors): + json_file = BytesIO(json.dumps(json_data).encode()) + + _, errors = JSONParser.parse_import(json_file) + self.assertEqual(len(errors), len(expected_errors)) + + for error in errors: + self.assertIn(str(error), expected_errors) + + def test_parse_tags(self): + expected_tags = [ + {"id": "tag_1", "value": "tag 1"}, + {"id": "tag_2", "value": "tag 2"}, + {"id": "tag_3", "value": "tag 3", "parent_id": "tag_1"}, + {"id": "tag_4", "value": "tag 4"}, + ] + json_data = {"tags": expected_tags} + + json_file = BytesIO(json.dumps(json_data).encode()) + + tags, errors = JSONParser.parse_import(json_file) + self.assertEqual(len(errors), 0) + self.assertEqual(len(tags), 4) + + # Result tags must be in the same order of the file + for index, expected_tag in enumerate(expected_tags): + self.assertEqual( + tags[index].id, + expected_tag.get('id') + ) + self.assertEqual( + tags[index].value, + expected_tag.get('value') + ) + self.assertEqual( + tags[index].parent_id, + expected_tag.get('parent_id') + ) + self.assertEqual( + tags[index].index, + index + JSONParser.inital_row + ) + + def test_export_data(self): + result = JSONParser.export(self.taxonomy) + tags = json.loads(result).get("tags") + assert len(tags) == self.taxonomy.tag_set.count() + for tag in tags: + taxonomy_tag = self.taxonomy.tag_set.get(external_id=tag.get("id")) + assert tag.get("value") == taxonomy_tag.value + if tag.get("parent_id"): + assert tag.get("parent_id") == taxonomy_tag.parent.external_id + + def test_import_with_export_output(self): + output = JSONParser.export(self.taxonomy) + json_file = BytesIO(output.encode()) + tags, errors = JSONParser.parse_import(json_file) + output_tags = json.loads(output).get("tags") + + self.assertEqual(len(errors), 0) + self.assertEqual(len(tags), len(output_tags)) + + + for tag in tags: + output_tag = None + for out_tag in output_tags: + if out_tag.get("id") == tag.id: + output_tag = out_tag + # Don't break because test coverage + assert output_tag + assert output_tag.get("value") == tag.value + if output_tag.get("parent_id"): + assert output_tag.get("parent_id") == tag.parent_id + + +@ddt.ddt +class TestCSVParser(TestImportExportMixin, TestCase): + """ + Test for .csv parser + """ + + @ddt.data( + ( + "value\n", + ["Invalid '.csv' format: Missing 'id' field on CSV headers"], + ), + ( + "id\n", + ["Invalid '.csv' format: Missing 'value' field on CSV headers"], + ), + ( + "id_name,value_name\n", + [ + "Invalid '.csv' format: Missing 'id' field on CSV headers", + "Invalid '.csv' format: Missing 'value' field on CSV headers" + ], + ), + ( + # Valid + "id,value\n", + [] + ) + ) + @ddt.unpack + def test_load_data_errors(self, csv_data, expected_errors): + csv_file = BytesIO(csv_data.encode()) + + tags, errors = CSVParser.parse_import(csv_file) + self.assertEqual(len(tags), 0) + self.assertEqual(len(errors), len(expected_errors)) + + for error in errors: + self.assertIn(str(error), expected_errors) + + @ddt.data( + ( + "id,value\ntag_1\ntag_2,\n", + [ + "Empty 'value' field on the row 2", + "Empty 'value' field on the row 3", + ] + ), + ( + "id,value\ntag_1,tag 1\n", # Valid + [] + ) + ) + @ddt.unpack + def test_parse_tags_errors(self, csv_data, expected_errors): + csv_file = BytesIO(csv_data.encode()) + + _, errors = CSVParser.parse_import(csv_file) + self.assertEqual(len(errors), len(expected_errors)) + + for error in errors: + self.assertIn(str(error), expected_errors) + + def _build_csv(self, tags): + """ + Builds a csv from 'tags' dict + """ + csv = "id,value,parent_id\n" + for tag in tags: + csv += ( + f"{tag.get('id')},{tag.get('value')}," + f"{tag.get('parent_id') or ''}\n" + ) + return csv + + def test_parse_tags(self): + expected_tags = [ + {"id": "tag_1", "value": "tag 1"}, + {"id": "tag_2", "value": "tag 2"}, + {"id": "tag_3", "value": "tag 3", "parent_id": "tag_1"}, + {"id": "tag_4", "value": "tag 4"}, + ] + csv_data = self._build_csv(expected_tags) + csv_file = BytesIO(csv_data.encode()) + tags, errors = CSVParser.parse_import(csv_file) + + self.assertEqual(len(errors), 0) + self.assertEqual(len(tags), 4) + + # Result tags must be in the same order of the file + for index, expected_tag in enumerate(expected_tags): + self.assertEqual( + tags[index].id, + expected_tag.get('id') + ) + self.assertEqual( + tags[index].value, + expected_tag.get('value') + ) + self.assertEqual( + tags[index].parent_id, + expected_tag.get('parent_id') + ) + self.assertEqual( + tags[index].index, + index + CSVParser.inital_row + ) + + def test_import_with_export_output(self): + output = CSVParser.export(self.taxonomy) + csv_file = BytesIO(output.encode()) + tags, errors = CSVParser.parse_import(csv_file) + self.assertEqual(len(errors), 0) + assert len(tags) == self.taxonomy.tag_set.count() + + for tag in tags: + taxonomy_tag = self.taxonomy.tag_set.get(external_id=tag.id) + assert tag.value == taxonomy_tag.value + if tag.parent_id: + assert tag.parent_id == taxonomy_tag.parent.external_id diff --git a/tests/openedx_tagging/core/tagging/import_export/test_tasks.py b/tests/openedx_tagging/core/tagging/import_export/test_tasks.py new file mode 100644 index 000000000..4b88fc6d6 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_tasks.py @@ -0,0 +1,42 @@ +""" +Test import/export celery tasks +""" +from io import BytesIO +from unittest.mock import patch + +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.import_export import ParserFormat +import openedx_tagging.core.tagging.import_export.tasks as import_export_tasks + +from .mixins import TestImportExportMixin + + +class TestImportExportCeleryTasks(TestImportExportMixin, TestCase): + """ + Test import/export celery tasks + """ + + def test_import_tags_task(self): + file = BytesIO(b"some_data") + parser_format = ParserFormat.CSV + replace = True + + with patch('openedx_tagging.core.tagging.import_export.api.import_tags') as mock_import_tags: + mock_import_tags.return_value = True + + result = import_export_tasks.import_tags_task(self.taxonomy, file, parser_format, replace) + + self.assertTrue(result) + mock_import_tags.assert_called_once_with(self.taxonomy, file, parser_format, replace) + + def test_export_tags_task(self): + output_format = ParserFormat.JSON + + with patch('openedx_tagging.core.tagging.import_export.api.export_tags') as mock_export_tags: + mock_export_tags.return_value = "exported_data" + + result = import_export_tasks.export_tags_task(self.taxonomy, output_format) + + self.assertEqual(result, "exported_data") + mock_export_tags.assert_called_once_with(self.taxonomy, output_format) diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index b6affec4a..5d6d4d0c6 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -4,7 +4,7 @@ from django.test.testcases import TestCase, override_settings import openedx_tagging.core.tagging.api as tagging_api -from openedx_tagging.core.tagging.models import ObjectTag, Tag +from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy from .test_models import TestTagTaxonomyMixin, get_tag @@ -55,20 +55,22 @@ def test_get_taxonomy(self): def test_get_taxonomies(self): tax1 = tagging_api.create_taxonomy("Enabled") tax2 = tagging_api.create_taxonomy("Disabled", enabled=False) + tax3 = Taxonomy.objects.get(name="Import Taxonomy Test") with self.assertNumQueries(1): enabled = list(tagging_api.get_taxonomies()) - assert enabled == [ tax1, + tax3, self.language_taxonomy, self.taxonomy, self.system_taxonomy, self.user_taxonomy, ] assert str(enabled[0]) == f" ({tax1.id}) Enabled" - assert str(enabled[1]) == " (-1) Languages" - assert str(enabled[2]) == " (1) Life on Earth" - assert str(enabled[3]) == " (4) System defined taxonomy" + assert str(enabled[1]) == " (5) Import Taxonomy Test" + assert str(enabled[2]) == " (-1) Languages" + assert str(enabled[3]) == " (1) Life on Earth" + assert str(enabled[4]) == " (4) System defined taxonomy" with self.assertNumQueries(1): disabled = list(tagging_api.get_taxonomies(enabled=False)) @@ -80,6 +82,7 @@ def test_get_taxonomies(self): assert both == [ tax2, tax1, + tax3, self.language_taxonomy, self.taxonomy, self.system_taxonomy, From 79c16a08b30aefaf5ae0991b6dbd06d25087cf02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Fri, 18 Aug 2023 02:33:10 -0300 Subject: [PATCH 036/282] feat: add delete_object_tags function to oel_tagging api (#71) --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 13 +++++++++++++ tests/openedx_tagging/core/tagging/test_api.py | 15 +++++++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index ae7362549..bbab0242f 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1 +1 @@ -__version__ = "0.1.3" +__version__ = "0.1.4" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 55aa0d376..ab1fef91d 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -122,6 +122,19 @@ def get_object_tags( yield object_tag +def delete_object_tags(object_id: str): + """ + Delete all ObjectTag entries for a given object. + """ + tags = ( + ObjectTag.objects.filter( + object_id=object_id, + ) + ) + + tags.delete() + + def tag_object( taxonomy: Taxonomy, tags: List, diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 5d6d4d0c6..42639219c 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -274,6 +274,21 @@ def test_tag_object(self): assert object_tag.name == self.taxonomy.name assert object_tag.object_id == "biology101" + # Delete the tags + tagging_api.delete_object_tags("biology101") + + # Ensure the tags were deleted + assert ( + len( + list( + tagging_api.get_object_tags( + object_id="biology101", + ) + ) + ) + == 0 + ) + def test_tag_object_free_text(self): self.taxonomy.allow_free_text = True self.taxonomy.save() From 593d63ccbaf394c9a1ea5f81d8c1a9429053472f Mon Sep 17 00:00:00 2001 From: Yusuf Musleh Date: Fri, 18 Aug 2023 07:24:17 +0100 Subject: [PATCH 037/282] feat: Implement ObjectTag retrieve REST API (#68) * chore: Remove is_valid checks from get_object_tags * fix: Rename ObjectTag perms to match model name * feat: Implement ObjectTag retrieve REST API Retrieve ObjectTags for given Object IDs, and optionally filter by taxonomy. * chore: bumped version --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 14 +- .../core/tagging/rest_api/v1/permissions.py | 14 +- .../core/tagging/rest_api/v1/serializers.py | 28 +- .../core/tagging/rest_api/v1/urls.py | 1 + .../core/tagging/rest_api/v1/views.py | 87 +++++- openedx_tagging/core/tagging/rules.py | 8 +- .../openedx_tagging/core/tagging/test_api.py | 28 +- .../core/tagging/test_rules.py | 40 +-- .../core/tagging/test_views.py | 273 +++++++++++++++++- 10 files changed, 431 insertions(+), 64 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index bbab0242f..1276d0254 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1 +1 @@ -__version__ = "0.1.4" +__version__ = "0.1.5" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index ab1fef91d..0ccb46883 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -96,16 +96,14 @@ def resync_object_tags(object_tags: QuerySet = None) -> int: def get_object_tags( - object_id: str, taxonomy: Taxonomy = None, valid_only=True -) -> Iterator[ObjectTag]: + object_id: str, taxonomy_id: str = None +) -> QuerySet: """ - Generates a list of object tags for a given object. + Returns a Queryset of object tags for a given object. Pass taxonomy to limit the returned object_tags to a specific taxonomy. - - Pass valid_only=False when displaying tags to content authors, so they can see invalid tags too. - Invalid tags will (probably) be hidden from learners. """ + taxonomy = get_taxonomy(taxonomy_id) ObjectTagClass = taxonomy.object_tag_class if taxonomy else ObjectTag tags = ( ObjectTagClass.objects.filter( @@ -117,9 +115,7 @@ def get_object_tags( if taxonomy: tags = tags.filter(taxonomy=taxonomy) - for object_tag in tags: - if not valid_only or object_tag.is_valid(): - yield object_tag + return tags def delete_object_tags(object_id: str): diff --git a/openedx_tagging/core/tagging/rest_api/v1/permissions.py b/openedx_tagging/core/tagging/rest_api/v1/permissions.py index e8915a62f..245fc3cb2 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/permissions.py +++ b/openedx_tagging/core/tagging/rest_api/v1/permissions.py @@ -1,5 +1,5 @@ """ -Taxonomy permissions +Tagging permissions """ from rest_framework.permissions import DjangoObjectPermissions @@ -15,3 +15,15 @@ class TaxonomyObjectPermissions(DjangoObjectPermissions): "PATCH": ["%(app_label)s.change_%(model_name)s"], "DELETE": ["%(app_label)s.delete_%(model_name)s"], } + + +class ObjectTagObjectPermissions(DjangoObjectPermissions): + perms_map = { + "GET": ["%(app_label)s.view_%(model_name)s"], + "OPTIONS": [], + "HEAD": ["%(app_label)s.view_%(model_name)s"], + "POST": ["%(app_label)s.add_%(model_name)s"], + "PUT": ["%(app_label)s.change_%(model_name)s"], + "PATCH": ["%(app_label)s.change_%(model_name)s"], + "DELETE": ["%(app_label)s.delete_%(model_name)s"], + } diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 53f647424..593b72989 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -4,7 +4,7 @@ from rest_framework import serializers -from openedx_tagging.core.tagging.models import Taxonomy +from openedx_tagging.core.tagging.models import Taxonomy, ObjectTag class TaxonomyListQueryParamsSerializer(serializers.Serializer): @@ -29,3 +29,29 @@ class Meta: "system_defined", "visible_to_authors", ] + + +class ObjectTagListQueryParamsSerializer(serializers.Serializer): + """ + Serializer for the query params for the ObjectTag GET view + """ + + taxonomy = serializers.PrimaryKeyRelatedField( + queryset=Taxonomy.objects.all(), required=False + ) + + +class ObjectTagSerializer(serializers.ModelSerializer): + """ + Serializer for the ObjectTag model. + """ + + class Meta: + model = ObjectTag + fields = [ + "name", + "value", + "taxonomy_id", + "tag_ref", + "is_valid", + ] diff --git a/openedx_tagging/core/tagging/rest_api/v1/urls.py b/openedx_tagging/core/tagging/rest_api/v1/urls.py index 80500990c..c449eb5c2 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/urls.py +++ b/openedx_tagging/core/tagging/rest_api/v1/urls.py @@ -10,5 +10,6 @@ router = DefaultRouter() router.register("taxonomies", views.TaxonomyView, basename="taxonomy") +router.register("object_tags", views.ObjectTagView, basename="object_tag") urlpatterns = [path("", include(router.urls))] diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index a86e3c278..13e00acd6 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -2,15 +2,23 @@ Tagging API Views """ from django.http import Http404 -from rest_framework.viewsets import ModelViewSet +from rest_framework import status +from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet +from rest_framework.response import Response from ...api import ( create_taxonomy, get_taxonomy, get_taxonomies, + get_object_tags, +) +from .permissions import TaxonomyObjectPermissions, ObjectTagObjectPermissions +from .serializers import ( + TaxonomyListQueryParamsSerializer, + TaxonomySerializer, + ObjectTagListQueryParamsSerializer, + ObjectTagSerializer, ) -from .permissions import TaxonomyObjectPermissions -from .serializers import TaxonomyListQueryParamsSerializer, TaxonomySerializer class TaxonomyView(ModelViewSet): @@ -145,3 +153,76 @@ def perform_create(self, serializer): Create a new taxonomy. """ serializer.instance = create_taxonomy(**serializer.validated_data) + + +class ObjectTagView(ReadOnlyModelViewSet): + """ + View to retrieve paginated ObjectTags for an Object, given its Object ID. + (What tags does this object have?) + + **Retrieve Parameters** + * object_id (required): - The Object ID to retrieve ObjectTags for. + + **Retrieve Query Parameters** + * taxonomy (optional) - PK of taxonomy to filter ObjectTags for. + * page (optional) - Page number of paginated results. + * page_size (optional) - Number of results included in each page. + + **Retrieve Example Requests** + GET api/tagging/v1/object_tags/:object_id + GET api/tagging/v1/object_tags/:object_id?taxonomy=1 + GET api/tagging/v1/object_tags/:object_id?taxonomy=1&page=2 + GET api/tagging/v1/object_tags/:object_id?taxonomy=1&page=2&page_size=10 + + **Retrieve Query Returns** + * 200 - Success + * 400 - Invalid query parameter + * 403 - Permission denied + + **Create Query Returns** + * 403 - Permission denied + * 405 - Method not allowed + + **Update Query Returns** + * 403 - Permission denied + * 405 - Method not allowed + + **Delete Query Returns** + * 403 - Permission denied + * 405 - Method not allowed + """ + + serializer_class = ObjectTagSerializer + permission_classes = [ObjectTagObjectPermissions] + lookup_field = "object_id" + + def get_queryset(self): + """ + Return a queryset of object tags for a given object. + + If a taxonomy is passed in, object tags are limited to that taxonomy. + """ + object_id = self.kwargs.get("object_id") + query_params = ObjectTagListQueryParamsSerializer( + data=self.request.query_params.dict() + ) + query_params.is_valid(raise_exception=True) + taxonomy_id = query_params.data.get("taxonomy", None) + return get_object_tags(object_id, taxonomy_id) + + def retrieve(self, request, object_id=None): + """ + Retrieve ObjectTags that belong to a given Object given its + object_id and return paginated results. + + Note: We override `retrieve` here instead of `list` because we are + passing in the Object ID (object_id) in the path (as opposed to passing + it in as a query_param) to retrieve the related ObjectTags. + By default retrieve would expect an ObjectTag ID to be passed in the + path and returns a it as a single result however that is not + behavior we want. + """ + object_tags = self.get_queryset() + paginated_object_tags = self.paginate_queryset(object_tags) + serializer = ObjectTagSerializer(paginated_object_tags, many=True) + return self.get_paginated_response(serializer.data) diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 364178c70..b79a56344 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -73,7 +73,7 @@ def can_change_object_tag(user: User, object_tag: ObjectTag = None) -> bool: rules.add_perm("oel_tagging.view_tag", rules.always_allow) # ObjectTag -rules.add_perm("oel_tagging.add_object_tag", can_change_object_tag) -rules.add_perm("oel_tagging.change_object_tag", can_change_object_tag) -rules.add_perm("oel_tagging.delete_object_tag", is_taxonomy_admin) -rules.add_perm("oel_tagging.view_object_tag", rules.always_allow) +rules.add_perm("oel_tagging.add_objecttag", can_change_object_tag) +rules.add_perm("oel_tagging.change_objecttag", can_change_object_tag) +rules.add_perm("oel_tagging.delete_objecttag", is_taxonomy_admin) +rules.add_perm("oel_tagging.view_objecttag", rules.always_allow) diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 42639219c..ae77d715b 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -259,7 +259,7 @@ def test_tag_object(self): assert ( list( tagging_api.get_object_tags( - taxonomy=self.taxonomy, + taxonomy_id=self.taxonomy.pk, object_id="biology101", ) ) @@ -355,7 +355,7 @@ def test_tag_object_language_taxonomy(self): assert ( list( tagging_api.get_object_tags( - taxonomy=self.language_taxonomy, + taxonomy_id=self.language_taxonomy.pk, object_id="biology101", ) ) @@ -401,7 +401,7 @@ def test_tag_object_model_system_taxonomy(self): assert ( list( tagging_api.get_object_tags( - taxonomy=self.user_taxonomy, + taxonomy_id=self.user_taxonomy.pk, object_id="biology101", ) ) @@ -445,37 +445,17 @@ def test_get_object_tags(self): assert list( tagging_api.get_object_tags( object_id="abc", - valid_only=False, ) ) == [ alpha, beta, ] - # No valid tags for this object yet.. - assert not list( - tagging_api.get_object_tags( - object_id="abc", - valid_only=True, - ) - ) - beta.tag = self.mammalia - beta.save() - assert list( - tagging_api.get_object_tags( - object_id="abc", - valid_only=True, - ) - ) == [ - beta, - ] - # Fetch all the tags for a given object ID + taxonomy assert list( tagging_api.get_object_tags( object_id="abc", - taxonomy=self.taxonomy, - valid_only=False, + taxonomy_id=self.taxonomy.pk, ) ) == [ beta, diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py index 415ae193a..cc4f50403 100644 --- a/tests/openedx_tagging/core/tagging/test_rules.py +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -161,8 +161,8 @@ def test_view_tag(self): # ObjectTag @ddt.data( - "oel_tagging.add_object_tag", - "oel_tagging.change_object_tag", + "oel_tagging.add_objecttag", + "oel_tagging.change_objecttag", ) def test_add_change_object_tag(self, perm): """Taxonomy administrators can create/edit an ObjectTag with an enabled Taxonomy""" @@ -174,8 +174,8 @@ def test_add_change_object_tag(self, perm): assert not self.learner.has_perm(perm, self.object_tag) @ddt.data( - "oel_tagging.add_object_tag", - "oel_tagging.change_object_tag", + "oel_tagging.add_objecttag", + "oel_tagging.change_objecttag", ) def test_object_tag_disabled_taxonomy(self, perm): """Taxonomy administrators cannot create/edit an ObjectTag with a disabled Taxonomy""" @@ -189,23 +189,23 @@ def test_object_tag_disabled_taxonomy(self, perm): True, False, ) - def test_delete_object_tag(self, enabled): + def test_delete_objecttag(self, enabled): """Taxonomy administrators can delete any ObjectTag, even those associated with a disabled Taxonomy.""" self.taxonomy.enabled = enabled self.taxonomy.save() - assert self.superuser.has_perm("oel_tagging.delete_object_tag") - assert self.superuser.has_perm("oel_tagging.delete_object_tag", self.object_tag) - assert self.staff.has_perm("oel_tagging.delete_object_tag") - assert self.staff.has_perm("oel_tagging.delete_object_tag", self.object_tag) - assert not self.learner.has_perm("oel_tagging.delete_object_tag") + assert self.superuser.has_perm("oel_tagging.delete_objecttag") + assert self.superuser.has_perm("oel_tagging.delete_objecttag", self.object_tag) + assert self.staff.has_perm("oel_tagging.delete_objecttag") + assert self.staff.has_perm("oel_tagging.delete_objecttag", self.object_tag) + assert not self.learner.has_perm("oel_tagging.delete_objecttag") assert not self.learner.has_perm( - "oel_tagging.delete_object_tag", self.object_tag + "oel_tagging.delete_objecttag", self.object_tag ) @ddt.data( - "oel_tagging.add_object_tag", - "oel_tagging.change_object_tag", - "oel_tagging.delete_object_tag", + "oel_tagging.add_objecttag", + "oel_tagging.change_objecttag", + "oel_tagging.delete_objecttag", ) def test_object_tag_no_taxonomy(self, perm): """Taxonomy administrators can modify an ObjectTag with no Taxonomy""" @@ -216,9 +216,9 @@ def test_object_tag_no_taxonomy(self, perm): def test_view_object_tag(self): """Anyone can view any ObjectTag""" - assert self.superuser.has_perm("oel_tagging.view_object_tag") - assert self.superuser.has_perm("oel_tagging.view_object_tag", self.object_tag) - assert self.staff.has_perm("oel_tagging.view_object_tag") - assert self.staff.has_perm("oel_tagging.view_object_tag", self.object_tag) - assert self.learner.has_perm("oel_tagging.view_object_tag") - assert self.learner.has_perm("oel_tagging.view_object_tag", self.object_tag) + assert self.superuser.has_perm("oel_tagging.view_objecttag") + assert self.superuser.has_perm("oel_tagging.view_objecttag", self.object_tag) + assert self.staff.has_perm("oel_tagging.view_objecttag") + assert self.staff.has_perm("oel_tagging.view_objecttag", self.object_tag) + assert self.learner.has_perm("oel_tagging.view_objecttag") + assert self.learner.has_perm("oel_tagging.view_objecttag", self.object_tag) diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 69805e234..a0b946cbf 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -8,7 +8,7 @@ from rest_framework.test import APITestCase from urllib.parse import urlparse, parse_qs -from openedx_tagging.core.tagging.models import Taxonomy +from openedx_tagging.core.tagging.models import Taxonomy, ObjectTag, Tag from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy User = get_user_model() @@ -17,6 +17,9 @@ TAXONOMY_DETAIL_URL = "/tagging/rest_api/v1/taxonomies/{pk}/" +OBJECT_TAGS_RETRIEVE_URL = '/tagging/rest_api/v1/object_tags/{object_id}/' + + def check_taxonomy( data, id, @@ -376,3 +379,271 @@ def test_delete_taxonomy_404(self): self.client.force_authenticate(user=self.staff) response = self.client.delete(url) assert response.status_code, status.HTTP_404_NOT_FOUND + + +@ddt.ddt +class TestObjectTagViewSet(APITestCase): + """ + Testing various cases for the ObjectTagView. + """ + + def setUp(self): + super().setUp() + + self.user = User.objects.create( + username="user", + email="user@example.com", + ) + + self.staff = User.objects.create( + username="staff", + email="staff@example.com", + is_staff=True, + ) + + # System-defined language taxonomy with valid ObjectTag + self.system_taxonomy = SystemDefinedTaxonomy.objects.create( + name="System Taxonomy" + ) + self.tag1 = Tag.objects.create( + taxonomy=self.system_taxonomy, value="Tag 1" + ) + ObjectTag.objects.create( + object_id="abc", + taxonomy=self.system_taxonomy, + tag=self.tag1 + ) + + # Closed Taxonomies created by taxonomy admins, each with 20 ObjectTags + self.enabled_taxonomy = Taxonomy.objects.create(name="Enabled Taxonomy") + for i in range(20): + # Valid ObjectTags + tag = Tag.objects.create( + taxonomy=self.enabled_taxonomy, value=f"Tag {i}" + ) + ObjectTag.objects.create( + object_id="abc", taxonomy=self.enabled_taxonomy, + tag=tag, _value=tag.value + ) + + # Taxonomy with invalid ObjectTag + self.taxonomy_with_invalid_object_tag = Taxonomy.objects.create() + self.to_be_deleted_tag = Tag.objects.create( + taxonomy=self.enabled_taxonomy, value="Deleted Tag" + ) + ObjectTag.objects.create( + object_id="abc", taxonomy=self.taxonomy_with_invalid_object_tag, + tag=self.to_be_deleted_tag, _value=self.to_be_deleted_tag.value + ) + self.to_be_deleted_tag.delete() # Delete tag so ObjectTag is invalid + + # Free-Text Taxonomies created by taxonomy admins, each linked + # to 200 ObjectTags + self.open_taxonomy_enabled = Taxonomy.objects.create( + name="Enabled Free-Text Taxonomy", allow_free_text=True + ) + self.open_taxonomy_disabled = Taxonomy.objects.create( + name="Disabled Free-Text Taxonomy", enabled=False, allow_free_text=True + ) + for i in range(200): + ObjectTag.objects.create( + object_id="abc", taxonomy=self.open_taxonomy_enabled, _value=f"Free Text {i}" + ) + ObjectTag.objects.create( + object_id="abc", taxonomy=self.open_taxonomy_disabled, _value=f"Free Text {i}" + ) + + @ddt.data( + (None, "abc", status.HTTP_403_FORBIDDEN, None, None), + ("user", "abc", status.HTTP_200_OK, 422, 10), + ("staff", "abc", status.HTTP_200_OK, 422, 10), + (None, "non-existing-id", status.HTTP_403_FORBIDDEN, None, None), + ("user", "non-existing-id", status.HTTP_200_OK, 0, 0), + ("staff", "non-existing-id", status.HTTP_200_OK, 0, 0), + ) + @ddt.unpack + def test_retrieve_object_tags( + self, user_attr, object_id, expected_status, expected_count, expected_results + + ): + """ + Test retrieving object tags + """ + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + response = self.client.get(url) + assert response.status_code == expected_status + + if status.is_success(expected_status): + assert response.data.get("count") == expected_count + assert response.data.get("results") is not None + assert len(response.data.get("results")) == expected_results + + @ddt.data( + (None, "abc", status.HTTP_403_FORBIDDEN, None, None, None, None), + ("user", "abc", status.HTTP_200_OK, 20, 10, 1, 1), + ("staff", "abc", status.HTTP_200_OK, 20, 10, 1, 1), + ) + @ddt.unpack + def test_retrieve_object_tags_taxonomy_queryparam( + self, user_attr, object_id, expected_status, + expected_count, expected_results, + expected_invalid_count, expected_invalid_results + ): + """ + Test retrieving object tags for specific taxonomies provided + """ + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + # Check valid object tags + response = self.client.get(url, {"taxonomy": self.enabled_taxonomy.pk}) + assert response.status_code == expected_status + if status.is_success(expected_status): + assert response.data.get("count") == expected_count + assert response.data.get("results") is not None + assert len(response.data.get("results")) == expected_results + object_tags = response.data.get("results") + for object_tag in object_tags: + assert object_tag.get("is_valid") is True + assert object_tag.get("taxonomy_id") == self.enabled_taxonomy.pk + + # Check invalid object tags + response = self.client.get( + url, {"taxonomy": self.taxonomy_with_invalid_object_tag.pk} + ) + assert response.status_code == expected_status + if status.is_success(expected_status): + assert response.data.get("count") == expected_invalid_count + assert response.data.get("results") is not None + assert len(response.data.get("results")) == expected_invalid_results + object_tags = response.data.get("results") + for object_tag in object_tags: + assert object_tag.get("is_valid") is False + assert object_tag.get("taxonomy_id") == \ + self.taxonomy_with_invalid_object_tag.pk + + @ddt.data( + (None, "abc", status.HTTP_403_FORBIDDEN), + ("user", "abc", status.HTTP_400_BAD_REQUEST), + ("staff", "abc", status.HTTP_400_BAD_REQUEST), + ) + @ddt.unpack + def test_retrieve_object_tags_invalid_taxonomy_queryparam( + self, user_attr, object_id, expected_status + ): + """ + Test retrieving object tags for invalid taxonomy + """ + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + # Invalid Taxonomy + response = self.client.get(url, {"taxonomy": 123123}) + assert response.status_code == expected_status + + @ddt.data( + # Page 1, default page size 10, total count 200, returns 10 results + (None, 1, None, status.HTTP_403_FORBIDDEN, None, None), + ("user", 1, None, status.HTTP_200_OK, 200, 10), + ("staff", 1, None, status.HTTP_200_OK, 200, 10), + # Page 2, default page size 10, total count 200, returns 10 results + (None, 2, None, status.HTTP_403_FORBIDDEN, None, None), + ("user", 2, None, status.HTTP_200_OK, 200, 10), + ("staff", 2, None, status.HTTP_200_OK, 200, 10), + # Page 21, default page size 10, total count 200, no more results + (None, 21, None, status.HTTP_403_FORBIDDEN, None, None), + ("user", 21, None, status.HTTP_404_NOT_FOUND, None, None), + ("staff", 21, None, status.HTTP_404_NOT_FOUND, None, None), + # Page 3, page size 2, total count 200, returns 2 results + (None, 3, 2, status.HTTP_403_FORBIDDEN, 200, 2), + ("user", 3, 2, status.HTTP_200_OK, 200, 2), + ("staff", 3, 2, status.HTTP_200_OK, 200, 2), + ) + @ddt.unpack + def test_retrieve_object_tags_pagination( + self, user_attr, page, page_size, expected_status, expected_count, expected_results + ): + """ + Test pagination for retrieve object tags + """ + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id="abc") + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + query_params = { + "taxonomy": self.open_taxonomy_enabled.pk, + "page": page + } + if page_size: + query_params["page_size"] = page_size + + response = self.client.get(url, query_params) + assert response.status_code == expected_status + if status.is_success(expected_status): + assert response.data.get("count") == expected_count + assert response.data.get("results") is not None + assert len(response.data.get("results")) == expected_results + object_tags = response.data.get("results") + for object_tag in object_tags: + assert object_tag.get("taxonomy_id") == self.open_taxonomy_enabled.pk + + @ddt.data( + (None, "POST", status.HTTP_403_FORBIDDEN), + (None, "PUT", status.HTTP_403_FORBIDDEN), + (None, "PATCH", status.HTTP_403_FORBIDDEN), + (None, "DELETE", status.HTTP_403_FORBIDDEN), + ("user", "POST", status.HTTP_403_FORBIDDEN), + ("user", "PUT", status.HTTP_403_FORBIDDEN), + ("user", "PATCH", status.HTTP_403_FORBIDDEN), + ("user", "DELETE", status.HTTP_403_FORBIDDEN), + ("staff", "POST", status.HTTP_405_METHOD_NOT_ALLOWED), + ("staff", "PUT", status.HTTP_405_METHOD_NOT_ALLOWED), + ("staff", "PATCH", status.HTTP_405_METHOD_NOT_ALLOWED), + ("staff", "DELETE", status.HTTP_405_METHOD_NOT_ALLOWED), + ) + @ddt.unpack + def test_object_tags_remaining_http_methods( + self, user_attr, http_method, expected_status, + + ): + """ + Test POST/PUT/PATCH/DELETE method for ObjectTagView + + Only staff users should have permissions to perform the actions, + however the methods are currently not allowed. + """ + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id="abc") + + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + if http_method == "POST": + response = self.client.post( + url, {"test": "payload"}, format="json" + ) + elif http_method == "PUT": + response = self.client.put( + url, {"test": "payload"}, format="json" + ) + elif http_method == "PATCH": + response = self.client.patch( + url, {"test": "payload"}, format="json" + ) + elif http_method == "DELETE": + response = self.client.delete(url) + + assert response.status_code == expected_status From 01185f3d4bc0b9b5da2231064ed8bf1dc36e08b5 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 24 Aug 2023 09:48:17 -0700 Subject: [PATCH 038/282] feat: enforce quality checks and type hints, improve quality and typing (#73) --- .github/workflows/ci.yml | 2 +- .gitignore | 1 + docs/conf.py | 4 +- manage.py | 2 +- mypy.ini | 15 + .../management/commands/load_components.py | 4 +- openedx_learning/__init__.py | 3 + .../contrib/media_server/__init__.py | 3 + openedx_learning/contrib/media_server/apps.py | 5 +- openedx_learning/contrib/media_server/urls.py | 3 + .../contrib/media_server/views.py | 10 +- openedx_learning/core/components/admin.py | 69 +-- openedx_learning/core/components/api.py | 54 +- openedx_learning/core/components/apps.py | 7 +- .../components/migrations/0001_initial.py | 6 +- openedx_learning/core/components/models.py | 26 +- openedx_learning/core/contents/admin.py | 10 +- openedx_learning/core/contents/api.py | 60 +- openedx_learning/core/contents/apps.py | 3 + .../core/contents/migrations/0001_initial.py | 3 +- openedx_learning/core/contents/models.py | 9 +- openedx_learning/core/publishing/admin.py | 47 +- openedx_learning/core/publishing/api.py | 47 +- .../publishing/migrations/0001_initial.py | 8 +- .../core/publishing/model_mixins.py | 18 +- openedx_learning/core/publishing/models.py | 6 +- openedx_learning/lib/collations.py | 3 +- openedx_learning/lib/fields.py | 24 +- openedx_learning/lib/validators.py | 3 + openedx_learning/py.typed | 0 openedx_learning/rest_api/apps.py | 3 + openedx_learning/rest_api/urls.py | 3 + openedx_learning/rest_api/v1/components.py | 5 +- openedx_learning/rest_api/v1/urls.py | 3 + openedx_tagging/__init__.py | 4 +- openedx_tagging/core/tagging/admin.py | 4 +- openedx_tagging/core/tagging/api.py | 44 +- .../core/tagging/import_export/actions.py | 57 +- .../core/tagging/import_export/api.py | 14 +- .../core/tagging/import_export/exceptions.py | 26 +- .../core/tagging/import_export/import_plan.py | 27 +- .../core/tagging/import_export/parsers.py | 59 +- .../core/tagging/import_export/tasks.py | 4 +- .../commands/build_language_fixture.py | 2 +- .../migrations/0004_auto_20230723_2001.py | 1 + .../migrations/0005_language_taxonomy.py | 2 +- .../migrations/0006_auto_20230802_1631.py | 3 +- .../0007_tag_import_task_log_null_fix.py | 18 + .../0008_taxonomy_description_not_null.py | 21 + .../core/tagging/models/__init__.py | 18 +- openedx_tagging/core/tagging/models/base.py | 101 ++-- .../core/tagging/models/import_export.py | 9 +- .../core/tagging/models/system_defined.py | 32 +- openedx_tagging/core/tagging/rest_api/urls.py | 2 +- .../core/tagging/rest_api/v1/serializers.py | 2 +- .../core/tagging/rest_api/v1/urls.py | 3 +- .../core/tagging/rest_api/v1/views.py | 88 +-- openedx_tagging/core/tagging/rules.py | 30 +- openedx_tagging/core/tagging/urls.py | 2 +- openedx_tagging/py.typed | 0 projects/dev.py | 5 +- requirements/dev.txt | 46 ++ requirements/doc.txt | 45 +- requirements/quality.txt | 48 +- requirements/test.in | 3 + requirements/test.txt | 36 +- tests/__init__.py | 3 + .../core/components/test_models.py | 14 +- .../core/publishing/test_api.py | 32 +- .../core/fixtures/tagging.yaml | 4 +- .../tagging/import_export/test_actions.py | 70 ++- .../core/tagging/import_export/test_api.py | 51 +- .../tagging/import_export/test_import_plan.py | 568 +++++++++--------- .../tagging/import_export/test_parsers.py | 54 +- .../core/tagging/import_export/test_tasks.py | 2 +- .../openedx_tagging/core/tagging/test_api.py | 82 +-- .../core/tagging/test_models.py | 15 +- .../core/tagging/test_rules.py | 63 +- .../tagging/test_system_defined_models.py | 120 ++-- .../core/tagging/test_views.py | 44 +- tox.ini | 3 +- 81 files changed, 1423 insertions(+), 927 deletions(-) create mode 100644 mypy.ini create mode 100644 openedx_learning/py.typed create mode 100644 openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py create mode 100644 openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py create mode 100644 openedx_tagging/py.typed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 953124f45..7d7624f60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -16,7 +16,7 @@ jobs: matrix: os: [ubuntu-latest] # Add macos-latest later? python-version: ['3.8'] - toxenv: ["py38-django32", "py38-django42", "package"] + toxenv: ["py38-django32", "py38-django42", "package", "quality"] # We're only testing against MySQL 8 right now because 5.7 is # incompatible with Djagno 4.2. We'd have to make the tox.ini file more # complicated than it's worth given the short expected shelf-life of diff --git a/.gitignore b/.gitignore index 0fc03c7df..6b5c4fed5 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ pip-log.txt # Unit test / coverage reports .cache/ .pytest_cache/ +.mypy_cache/ .coverage .coverage.* .tox diff --git a/docs/conf.py b/docs/conf.py index 763a5ffb6..6308e2c23 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -524,6 +524,8 @@ def on_init(app): # pylint: disable=unused-argument def setup(app): - """Sphinx extension: run sphinx-apidoc.""" + """ + Sphinx extension: run sphinx-apidoc. + """ event = 'builder-inited' app.connect(event, on_init) diff --git a/manage.py b/manage.py index 0bf631df6..640c5043f 100644 --- a/manage.py +++ b/manage.py @@ -18,7 +18,7 @@ # issue is really that Django is missing to avoid masking other # exceptions on Python 2. try: - import django # pylint: disable=unused-import, wrong-import-position + import django # pylint: disable=unused-import except ImportError as import_error: raise ImportError( "Couldn't import Django. Are you sure it's installed and " diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 000000000..d3bb5b9f8 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,15 @@ +[mypy] +follow_imports = normal +ignore_missing_imports = False +allow_untyped_globals = False +plugins = + mypy_django_plugin.main, + mypy_drf_plugin.main +files = + openedx_learning, + openedx_tagging, + tests + +[mypy.plugins.django-stubs] +# content_staging only works with CMS; others work with either, so we run mypy with CMS settings. +django_settings_module = "projects.dev" diff --git a/olx_importer/management/commands/load_components.py b/olx_importer/management/commands/load_components.py index d987751b8..666eeeedc 100644 --- a/olx_importer/management/commands/load_components.py +++ b/olx_importer/management/commands/load_components.py @@ -46,7 +46,9 @@ def __init__(self, *args, **kwargs): self.init_known_types() def init_known_types(self): - """Intialize mimetypes with some custom mappings we want to use.""" + """ + Intialize mimetypes with some custom mappings we want to use. + """ # This is our own hacky video transcripts related format. mimetypes.add_type("application/vnd.openedx.srt+json", ".sjson") diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 1276d0254..5f1cea0b2 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1 +1,4 @@ +""" +Open edX Learning ("Learning Core"). +""" __version__ = "0.1.5" diff --git a/openedx_learning/contrib/media_server/__init__.py b/openedx_learning/contrib/media_server/__init__.py index e69de29bb..623258f2e 100644 --- a/openedx_learning/contrib/media_server/__init__.py +++ b/openedx_learning/contrib/media_server/__init__.py @@ -0,0 +1,3 @@ +""" +Media server (for dev or low-traffic instances) +""" diff --git a/openedx_learning/contrib/media_server/apps.py b/openedx_learning/contrib/media_server/apps.py index 2612be027..77ac37b34 100644 --- a/openedx_learning/contrib/media_server/apps.py +++ b/openedx_learning/contrib/media_server/apps.py @@ -1,6 +1,9 @@ +""" +Django app metadata for the Media Server application. +""" from django.apps import AppConfig -from django.core.exceptions import ImproperlyConfigured from django.conf import settings +from django.core.exceptions import ImproperlyConfigured class MediaServerConfig(AppConfig): diff --git a/openedx_learning/contrib/media_server/urls.py b/openedx_learning/contrib/media_server/urls.py index a76155dec..8e7604fdd 100644 --- a/openedx_learning/contrib/media_server/urls.py +++ b/openedx_learning/contrib/media_server/urls.py @@ -1,3 +1,6 @@ +""" +URLs for the media server application +""" from django.urls import path from .views import component_asset diff --git a/openedx_learning/contrib/media_server/views.py b/openedx_learning/contrib/media_server/views.py index d376283eb..752c915cc 100644 --- a/openedx_learning/contrib/media_server/views.py +++ b/openedx_learning/contrib/media_server/views.py @@ -1,8 +1,12 @@ +""" +Views for the media server application + +(serves media files in dev or low-traffic instances). +""" from pathlib import Path -from django.http import FileResponse -from django.http import Http404 from django.core.exceptions import ObjectDoesNotExist, PermissionDenied +from django.http import FileResponse, Http404 from openedx_learning.core.components.api import get_component_version_content @@ -28,7 +32,7 @@ def component_asset( learning_package_key, component_key, version_num, asset_path ) except ObjectDoesNotExist: - raise Http404("File not found") + raise Http404("File not found") # pylint: disable=raise-missing-from if not cvc.learner_downloadable and not ( request.user and request.user.is_superuser diff --git a/openedx_learning/core/components/admin.py b/openedx_learning/core/components/admin.py index 34f091d92..5710303ac 100644 --- a/openedx_learning/core/components/admin.py +++ b/openedx_learning/core/components/admin.py @@ -1,22 +1,27 @@ +""" +Django admin for components models +""" from django.contrib import admin from django.template.defaultfilters import filesizeformat from django.urls import reverse from django.utils.html import format_html +from django.utils.safestring import SafeText -from .models import ( - Component, - ComponentVersion, - ComponentVersionRawContent, -) from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin +from .models import Component, ComponentVersion, ComponentVersionRawContent + class ComponentVersionInline(admin.TabularInline): + """ + Inline admin view of ComponentVersion from the Component Admin + """ model = ComponentVersion fields = ["version_num", "created", "title", "format_uuid"] readonly_fields = ["version_num", "created", "title", "uuid", "format_uuid"] extra = 0 + @admin.display(description="UUID") def format_uuid(self, cv_obj): return format_html( '{}', @@ -24,11 +29,12 @@ def format_uuid(self, cv_obj): cv_obj.uuid, ) - format_uuid.short_description = "UUID" - @admin.register(Component) class ComponentAdmin(ReadOnlyModelAdmin): + """ + Django admin configuration for Component + """ list_display = ("key", "uuid", "namespace", "type", "created") readonly_fields = [ "learning_package", @@ -44,6 +50,9 @@ class ComponentAdmin(ReadOnlyModelAdmin): class RawContentInline(admin.TabularInline): + """ + Django admin configuration for RawContent + """ model = ComponentVersion.raw_contents.through def get_queryset(self, request): @@ -75,11 +84,11 @@ def get_queryset(self, request): def rendered_data(self, cvc_obj): return content_preview(cvc_obj) + @admin.display(description="Size") def format_size(self, cvc_obj): return filesizeformat(cvc_obj.raw_content.size) - format_size.short_description = "Size" - + @admin.display(description="Key") def format_key(self, cvc_obj): return format_html( '{}', @@ -88,11 +97,12 @@ def format_key(self, cvc_obj): cvc_obj.key, ) - format_key.short_description = "Key" - @admin.register(ComponentVersion) class ComponentVersionAdmin(ReadOnlyModelAdmin): + """ + Django admin configuration for ComponentVersion + """ readonly_fields = [ "component", "uuid", @@ -120,29 +130,10 @@ def get_queryset(self, request): ) -def is_displayable_text(mime_type): - # Our usual text files, including things like text/markdown, text/html - media_type, media_subtype = mime_type.split("/") - - if media_type == "text": - return True - - # Our OLX goes here, but so do some other things like - if media_subtype.endswith("+xml"): - return True - - # Other application/* types that we know we can display. - if media_subtype in ["json", "x-subrip"]: - return True - - # Other formats that are really specific types of JSON - if media_subtype.endswith("+json"): - return True - - return False - - -def link_for_cvc(cvc_obj: ComponentVersionRawContent): +def link_for_cvc(cvc_obj: ComponentVersionRawContent) -> str: + """ + Get the download URL for the given ComponentVersionRawContent instance + """ return "/media_server/component_asset/{}/{}/{}/{}".format( cvc_obj.raw_content.learning_package.key, cvc_obj.component_version.component.key, @@ -151,14 +142,20 @@ def link_for_cvc(cvc_obj: ComponentVersionRawContent): ) -def format_text_for_admin_display(text): +def format_text_for_admin_display(text: str) -> SafeText: + """ + Get the HTML to display the given plain text (preserving formatting) + """ return format_html( '
\n{}\n
', text, ) -def content_preview(cvc_obj: ComponentVersionRawContent): +def content_preview(cvc_obj: ComponentVersionRawContent) -> SafeText: + """ + Get the HTML to display a preview of the given ComponentVersionRawContent + """ raw_content_obj = cvc_obj.raw_content if raw_content_obj.mime_type.startswith("image/"): diff --git a/openedx_learning/core/components/api.py b/openedx_learning/core/components/api.py index 1a3972dec..61e7e574f 100644 --- a/openedx_learning/core/components/api.py +++ b/openedx_learning/core/components/api.py @@ -10,21 +10,29 @@ Please look at the models.py file for more information about the kinds of data are stored in this app. """ +from __future__ import annotations + +from datetime import datetime from pathlib import Path from django.db.models import Q from django.db.transaction import atomic -from ..publishing.api import ( - create_publishable_entity, - create_publishable_entity_version, -) -from .models import ComponentVersionRawContent, Component, ComponentVersion +from ..publishing.api import create_publishable_entity, create_publishable_entity_version +from .models import Component, ComponentVersion, ComponentVersionRawContent def create_component( - learning_package_id, namespace, type, local_key, created, created_by + learning_package_id: int, + namespace: str, + type: str, # pylint: disable=redefined-builtin + local_key: str, + created: datetime, + created_by: int | None, ): + """ + Create a new Component (an entity like a Problem or Video) + """ key = f"{namespace}:{type}@{local_key}" with atomic(): publishable_entity = create_publishable_entity( @@ -40,7 +48,16 @@ def create_component( return component -def create_component_version(component_pk, version_num, title, created, created_by): +def create_component_version( + component_pk: int, + version_num: int, + title: str, + created: datetime, + created_by: int | None, +) -> ComponentVersion: + """ + Create a new ComponentVersion + """ with atomic(): publishable_entity_version = create_publishable_entity_version( entity_id=component_pk, @@ -57,8 +74,17 @@ def create_component_version(component_pk, version_num, title, created, created_ def create_component_and_version( - learning_package_id, namespace, type, local_key, title, created, created_by + learning_package_id: int, + namespace: str, + type: str, # pylint: disable=redefined-builtin + local_key: str, + title: str, + created: datetime, + created_by: int | None, ): + """ + Create a Component and associated ComponentVersion atomically + """ with atomic(): component = create_component( learning_package_id, namespace, type, local_key, created, created_by @@ -73,7 +99,7 @@ def create_component_and_version( return (component, component_version) -def get_component_by_pk(component_pk): +def get_component_by_pk(component_pk: int) -> Component: return Component.objects.get(pk=component_pk) @@ -103,8 +129,14 @@ def get_component_version_content( def add_content_to_component_version( - component_version, raw_content_id, key, learner_downloadable=False -): + component_version: ComponentVersion, + raw_content_id: int, + key: str, + learner_downloadable=False, +) -> ComponentVersionRawContent: + """ + Add a RawContent to the given ComponentVersion + """ cvrc, _created = ComponentVersionRawContent.objects.get_or_create( component_version=component_version, raw_content_id=raw_content_id, diff --git a/openedx_learning/core/components/apps.py b/openedx_learning/core/components/apps.py index 9b25954ac..60ff42ad4 100644 --- a/openedx_learning/core/components/apps.py +++ b/openedx_learning/core/components/apps.py @@ -1,3 +1,6 @@ +""" +Django metadata for the Components Django application. +""" from django.apps import AppConfig @@ -15,7 +18,7 @@ def ready(self): """ Register Component and ComponentVersion. """ - from ..publishing.api import register_content_models - from .models import Component, ComponentVersion + from ..publishing.api import register_content_models # pylint: disable=import-outside-toplevel + from .models import Component, ComponentVersion # pylint: disable=import-outside-toplevel register_content_models(Component, ComponentVersion) diff --git a/openedx_learning/core/components/migrations/0001_initial.py b/openedx_learning/core/components/migrations/0001_initial.py index 1d3707476..0ca82b95c 100644 --- a/openedx_learning/core/components/migrations/0001_initial.py +++ b/openedx_learning/core/components/migrations/0001_initial.py @@ -1,9 +1,11 @@ # Generated by Django 3.2.19 on 2023-06-15 14:43 -from django.db import migrations, models +import uuid + import django.db.models.deletion +from django.db import migrations, models + import openedx_learning.lib.fields -import uuid class Migration(migrations.Migration): diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index 2d2f21391..35910ba21 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -16,22 +16,18 @@ by convention, but it's possible we might want to have special identifiers later. """ +from __future__ import annotations + from django.db import models -from openedx_learning.lib.fields import ( - case_sensitive_char_field, - immutable_uuid_field, - key_field, -) -from ..publishing.models import LearningPackage -from ..publishing.model_mixins import ( - PublishableEntityMixin, - PublishableEntityVersionMixin, -) +from openedx_learning.lib.fields import case_sensitive_char_field, immutable_uuid_field, key_field + from ..contents.models import RawContent +from ..publishing.model_mixins import PublishableEntityMixin, PublishableEntityVersionMixin +from ..publishing.models import LearningPackage -class Component(PublishableEntityMixin): +class Component(PublishableEntityMixin): # type: ignore[django-manager-missing] """ This represents any Component that has ever existed in a LearningPackage. @@ -69,6 +65,10 @@ class Component(PublishableEntityMixin): Make a foreign key to the Component model when you need a stable reference that will exist for as long as the LearningPackage itself exists. """ + # Tell mypy what type our objects manager has. + # It's actually PublishableEntityMixinManager, but that has the exact same + # interface as the base manager class. + objects: models.Manager[Component] # This foreign key is technically redundant because we're already locked to # a single LearningPackage through our publishable_entity relation. However, having @@ -143,6 +143,10 @@ class ComponentVersion(PublishableEntityVersionMixin): This holds the content using a M:M relationship with RawContent via ComponentVersionRawContent. """ + # Tell mypy what type our objects manager has. + # It's actually PublishableEntityVersionMixinManager, but that has the exact + # same interface as the base manager class. + objects: models.Manager[ComponentVersion] # This is technically redundant, since we can get this through # publishable_entity_version.publishable.component, but this is more diff --git a/openedx_learning/core/contents/admin.py b/openedx_learning/core/contents/admin.py index 81230d3d2..0ce7ed617 100644 --- a/openedx_learning/core/contents/admin.py +++ b/openedx_learning/core/contents/admin.py @@ -1,13 +1,19 @@ +""" +Django admin for contents models +""" from django.contrib import admin from django.utils.html import format_html -from .models import RawContent - from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin +from .models import RawContent + @admin.register(RawContent) class RawContentAdmin(ReadOnlyModelAdmin): + """ + Django admin for RawContent model + """ list_display = [ "hash_digest", "file_link", diff --git a/openedx_learning/core/contents/api.py b/openedx_learning/core/contents/api.py index 282bb820f..fd4e069ad 100644 --- a/openedx_learning/core/contents/api.py +++ b/openedx_learning/core/contents/api.py @@ -4,18 +4,29 @@ Please look at the models.py file for more information about the kinds of data are stored in this app. """ +from __future__ import annotations + import codecs +from datetime import datetime from django.core.files.base import ContentFile from django.db.transaction import atomic from openedx_learning.lib.fields import create_hash_digest + from .models import RawContent, TextContent def create_raw_content( - learning_package_id, data_bytes, mime_type, created, hash_digest=None -): + learning_package_id: int, + data_bytes: bytes, + mime_type: str, + created: datetime, + hash_digest: str | None = None, +) -> RawContent: + """ + Create a new RawContent instance and persist it to storage. + """ hash_digest = hash_digest or create_hash_digest(data_bytes) raw_content = RawContent.objects.create( learning_package_id=learning_package_id, @@ -31,8 +42,11 @@ def create_raw_content( return raw_content -def create_text_from_raw_content(raw_content, encoding="utf-8-sig"): - text = codecs.decode(raw_content.file.open().read(), "utf-8-sig") +def create_text_from_raw_content(raw_content: RawContent, encoding="utf-8-sig") -> TextContent: + """ + Create a new TextContent instance for the given RawContent. + """ + text = codecs.decode(raw_content.file.open().read(), encoding) return TextContent.objects.create( raw_content=raw_content, text=text, @@ -41,37 +55,49 @@ def create_text_from_raw_content(raw_content, encoding="utf-8-sig"): def get_or_create_raw_content( - learning_package_id, data_bytes, mime_type, created, hash_digest=None -): + learning_package_id: int, + data_bytes: bytes, + mime_type: str, + created: datetime, + hash_digest: str | None = None, +) -> tuple[RawContent, bool]: + """ + Get the RawContent in the given learning package with the specified data, + or create it if it doesn't exist. + """ hash_digest = hash_digest or create_hash_digest(data_bytes) try: raw_content = RawContent.objects.get( learning_package_id=learning_package_id, hash_digest=hash_digest ) - created = False + was_created = False except RawContent.DoesNotExist: raw_content = create_raw_content( learning_package_id, data_bytes, mime_type, created, hash_digest ) - created = True + was_created = True - return raw_content, created + return raw_content, was_created def get_or_create_text_content_from_bytes( - learning_package_id, - data_bytes, - mime_type, - created, - hash_digest=None, - encoding="utf-8-sig", + learning_package_id: int, + data_bytes: bytes, + mime_type: str, + created: datetime, + hash_digest: str | None = None, + encoding: str = "utf-8-sig", ): + """ + Get the TextContent in the given learning package with the specified data, + or create it if it doesn't exist. + """ with atomic(): raw_content, rc_created = get_or_create_raw_content( - learning_package_id, data_bytes, mime_type, created, hash_digest=None + learning_package_id, data_bytes, mime_type, created, hash_digest ) if rc_created or not hasattr(raw_content, "text_content"): - text = codecs.decode(data_bytes, "utf-8-sig") + text = codecs.decode(data_bytes, encoding) text_content = TextContent.objects.create( raw_content=raw_content, text=text, diff --git a/openedx_learning/core/contents/apps.py b/openedx_learning/core/contents/apps.py index 4c736b812..97cc82733 100644 --- a/openedx_learning/core/contents/apps.py +++ b/openedx_learning/core/contents/apps.py @@ -1,3 +1,6 @@ +""" +Django metadata for the Contents Django application. +""" from django.apps import AppConfig diff --git a/openedx_learning/core/contents/migrations/0001_initial.py b/openedx_learning/core/contents/migrations/0001_initial.py index 507056fce..20b6b01cd 100644 --- a/openedx_learning/core/contents/migrations/0001_initial.py +++ b/openedx_learning/core/contents/migrations/0001_initial.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.19 on 2023-06-15 14:43 import django.core.validators -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + import openedx_learning.lib.fields import openedx_learning.lib.validators diff --git a/openedx_learning/core/contents/models.py b/openedx_learning/core/contents/models.py index 6794f1b23..2da7ebfe3 100644 --- a/openedx_learning/core/contents/models.py +++ b/openedx_learning/core/contents/models.py @@ -3,22 +3,22 @@ the simplest building blocks to store data with. They need to be composed into more intelligent data models to be useful. """ -from django.db import models from django.conf import settings from django.core.files.storage import default_storage from django.core.validators import MaxValueValidator +from django.db import models from openedx_learning.lib.fields import ( + MultiCollationTextField, case_insensitive_char_field, hash_field, manual_date_time_field, - MultiCollationTextField, ) from ..publishing.models import LearningPackage -class RawContent(models.Model): +class RawContent(models.Model): # type: ignore[django-manager-missing] """ This is the most basic piece of raw content data, with no version metadata. @@ -104,7 +104,7 @@ class RawContent(models.Model): # models that offer better latency guarantees. file = models.FileField( null=True, - storage=settings.OPENEDX_LEARNING.get("STORAGE", default_storage), + storage=settings.OPENEDX_LEARNING.get("STORAGE", default_storage), # type: ignore ) class Meta: @@ -180,7 +180,6 @@ class TextContent(models.Model): text = MultiCollationTextField( blank=True, max_length=MAX_TEXT_LENGTH, - # We don't really expect to ever sort by the text column. This is here # primarily to force the column to be created as utf8mb4 on MySQL. I'm # using the binary collation because it's a little cheaper/faster. diff --git a/openedx_learning/core/publishing/admin.py b/openedx_learning/core/publishing/admin.py index 9ce6a9ef6..56c59d28f 100644 --- a/openedx_learning/core/publishing/admin.py +++ b/openedx_learning/core/publishing/admin.py @@ -1,17 +1,20 @@ +""" +Django admin for publishing models +""" +from __future__ import annotations + from django.contrib import admin from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin -from .models import ( - LearningPackage, - PublishableEntity, - Published, - PublishLog, - PublishLogRecord, -) + +from .models import LearningPackage, PublishableEntity, Published, PublishLog, PublishLogRecord @admin.register(LearningPackage) class LearningPackageAdmin(ReadOnlyModelAdmin): + """ + Read-only admin for LearningPackage model + """ fields = ["key", "title", "uuid", "created", "updated"] readonly_fields = ["key", "title", "uuid", "created", "updated"] list_display = ["key", "title", "uuid", "created", "updated"] @@ -19,13 +22,16 @@ class LearningPackageAdmin(ReadOnlyModelAdmin): class PublishLogRecordTabularInline(admin.TabularInline): + """ + Inline read-only tabular view for PublishLogRecords + """ model = PublishLogRecord - fields = [ + fields = ( "entity", "title", "old_version_num", "new_version_num", - ] + ) readonly_fields = fields def get_queryset(self, request): @@ -43,6 +49,9 @@ def new_version_num(self, pl_record: PublishLogRecord): return pl_record.new_version.version_num def title(self, pl_record: PublishLogRecord): + """ + Get the title to display for the PublishLogRecord + """ if pl_record.new_version: return pl_record.new_version.title if pl_record.old_version: @@ -52,9 +61,12 @@ def title(self, pl_record: PublishLogRecord): @admin.register(PublishLog) class PublishLogAdmin(ReadOnlyModelAdmin): + """ + Read-only admin view for PublishLog + """ inlines = [PublishLogRecordTabularInline] - fields = ["uuid", "learning_package", "published_at", "published_by", "message"] + fields = ("uuid", "learning_package", "published_at", "published_by", "message") readonly_fields = fields list_display = fields list_filter = ["learning_package"] @@ -62,7 +74,10 @@ class PublishLogAdmin(ReadOnlyModelAdmin): @admin.register(PublishableEntity) class PublishableEntityAdmin(ReadOnlyModelAdmin): - fields = [ + """ + Read-only admin view for Publishable Entities + """ + fields = ( "key", "draft_version", "published_version", @@ -70,7 +85,7 @@ class PublishableEntityAdmin(ReadOnlyModelAdmin): "learning_package", "created", "created_by", - ] + ) readonly_fields = fields list_display = fields list_filter = ["learning_package"] @@ -91,7 +106,10 @@ def published_version(self, entity): @admin.register(Published) class PublishedAdmin(ReadOnlyModelAdmin): - fields = ["entity", "version_num", "previous", "published_at", "message"] + """ + Read-only admin view for Published model + """ + fields = ("entity", "version_num", "previous", "published_at", "message") list_display = fields def get_queryset(self, request): @@ -108,6 +126,9 @@ def version_num(self, published_obj): return published_obj.version.version_num def previous(self, published_obj): + """ + Determine what to show in the "Previous" field + """ old_version = published_obj.publish_log_record.old_version # if there was no previous old version, old version is None if not old_version: diff --git a/openedx_learning/core/publishing/api.py b/openedx_learning/core/publishing/api.py index 52ac7a90e..6a425b8cc 100644 --- a/openedx_learning/core/publishing/api.py +++ b/openedx_learning/core/publishing/api.py @@ -4,27 +4,28 @@ Please look at the models.py file for more information about the kinds of data are stored in this app. """ +from __future__ import annotations + from datetime import datetime, timezone -from typing import Optional from django.core.exceptions import ObjectDoesNotExist from django.db.models import F, QuerySet from django.db.transaction import atomic +from .model_mixins import PublishableContentModelRegistry, PublishableEntityMixin, PublishableEntityVersionMixin from .models import ( Draft, LearningPackage, + PublishableEntity, + PublishableEntityVersion, Published, PublishLog, PublishLogRecord, - PublishableEntity, - PublishableEntityVersion, ) -from .model_mixins import PublishableContentModelRegistry def create_learning_package( - key: str, title: str, created: Optional[datetime] = None + key: str, title: str, created: datetime | None = None ) -> LearningPackage: """ Create a new LearningPackage. @@ -50,7 +51,13 @@ def create_learning_package( return package -def create_publishable_entity(learning_package_id, key, created, created_by): +def create_publishable_entity( + learning_package_id: int, + key: str, + created: datetime, + # User ID who created this + created_by: int | None, +) -> PublishableEntity: """ Create a PublishableEntity. @@ -61,13 +68,17 @@ def create_publishable_entity(learning_package_id, key, created, created_by): learning_package_id=learning_package_id, key=key, created=created, - created_by=created_by, + created_by_id=created_by, ) def create_publishable_entity_version( - entity_id, version_num, title, created, created_by -): + entity_id: int, + version_num: int, + title: str, + created: datetime, + created_by: int | None, +) -> PublishableEntityVersion: """ Create a PublishableEntityVersion. @@ -93,11 +104,14 @@ def learning_package_exists(key: str) -> bool: """ Check whether a LearningPackage with a particular key exists. """ - LearningPackage.objects.filter(key=key).exists() + return LearningPackage.objects.filter(key=key).exists() def publish_all_drafts( - learning_package_id, message="", published_at=None, published_by=None + learning_package_id: int, + message="", + published_at: datetime | None = None, + published_by: int | None = None ): """ Publish everything that is a Draft and is not already published. @@ -116,8 +130,8 @@ def publish_from_drafts( learning_package_id: int, # LearningPackage.id draft_qset: QuerySet, message: str = "", - published_at: Optional[datetime] = None, - published_by: Optional[int] = None, # User.id + published_at: datetime | None = None, + published_by: int | None = None, # User.id ) -> PublishLog: """ Publish the rows in the ``draft_model_qsets`` args passed in. @@ -131,7 +145,7 @@ def publish_from_drafts( learning_package_id=learning_package_id, message=message, published_at=published_at, - published_by=published_by, + published_by_id=published_by, ) publish_log.full_clean() publish_log.save(force_insert=True) @@ -166,7 +180,10 @@ def publish_from_drafts( return publish_log -def register_content_models(content_model_cls, content_version_model_cls): +def register_content_models( + content_model_cls: type[PublishableEntityMixin], + content_version_model_cls: type[PublishableEntityVersionMixin], +): """ Register what content model maps to what content version model. diff --git a/openedx_learning/core/publishing/migrations/0001_initial.py b/openedx_learning/core/publishing/migrations/0001_initial.py index 3f41cd45a..d7663748a 100644 --- a/openedx_learning/core/publishing/migrations/0001_initial.py +++ b/openedx_learning/core/publishing/migrations/0001_initial.py @@ -1,12 +1,14 @@ # Generated by Django 3.2.19 on 2023-06-15 14:43 -from django.conf import settings +import uuid + import django.core.validators -from django.db import migrations, models import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + import openedx_learning.lib.fields import openedx_learning.lib.validators -import uuid class Migration(migrations.Migration): diff --git a/openedx_learning/core/publishing/model_mixins.py b/openedx_learning/core/publishing/model_mixins.py index 1623bb256..02cc7f200 100644 --- a/openedx_learning/core/publishing/model_mixins.py +++ b/openedx_learning/core/publishing/model_mixins.py @@ -1,13 +1,15 @@ """ Helper mixin classes for content apps that want to use the publishing app. """ +from __future__ import annotations + from functools import cached_property from django.core.exceptions import ImproperlyConfigured from django.db import models from django.db.models.query import QuerySet -from .models import Draft, Published, PublishableEntity, PublishableEntityVersion +from .models import Draft, PublishableEntity, PublishableEntityVersion, Published class PublishableEntityMixin(models.Model): @@ -25,7 +27,7 @@ class PublishableEntityMixinManager(models.Manager): def get_queryset(self) -> QuerySet: return super().get_queryset().select_related("publishable_entity") - objects = PublishableEntityMixinManager() + objects: models.Manager[PublishableEntityMixin] = PublishableEntityMixinManager() publishable_entity = models.OneToOneField( PublishableEntity, on_delete=models.CASCADE, primary_key=True @@ -216,7 +218,7 @@ def get_queryset(self) -> QuerySet: ) ) - objects = PublishableEntityVersionMixinManager() + objects: models.Manager[PublishableEntityVersionMixin] = PublishableEntityVersionMixinManager() publishable_entity_version = models.OneToOneField( PublishableEntityVersion, on_delete=models.CASCADE, primary_key=True @@ -247,11 +249,15 @@ class PublishableContentModelRegistry: This class tracks content models built on PublishableEntity(Version). """ - _unversioned_to_versioned = {} - _versioned_to_unversioned = {} + _unversioned_to_versioned: dict[type[PublishableEntityMixin], type[PublishableEntityVersionMixin]] = {} + _versioned_to_unversioned: dict[type[PublishableEntityVersionMixin], type[PublishableEntityMixin]] = {} @classmethod - def register(cls, content_model_cls, content_version_model_cls): + def register( + cls, + content_model_cls: type[PublishableEntityMixin], + content_version_model_cls: type[PublishableEntityVersionMixin], + ): """ Register what content model maps to what content version model. diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 78f19b4ba..7cbeef87c 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -11,10 +11,9 @@ * Managing reverts. * Storing and querying publish history. """ -from django.db import models - from django.conf import settings from django.core.validators import MinValueValidator +from django.db import models from openedx_learning.lib.fields import ( case_insensitive_char_field, @@ -24,13 +23,12 @@ ) -class LearningPackage(models.Model): +class LearningPackage(models.Model): # type: ignore[django-manager-missing] """ Top level container for a grouping of authored content. Each PublishableEntity belongs to exactly one LearningPackage. """ - uuid = immutable_uuid_field() key = key_field() title = case_insensitive_char_field(max_length=500, blank=False) diff --git a/openedx_learning/lib/collations.py b/openedx_learning/lib/collations.py index 561b1f316..0019f887a 100644 --- a/openedx_learning/lib/collations.py +++ b/openedx_learning/lib/collations.py @@ -15,7 +15,7 @@ class MultiCollationMixin: they're the only Field types that store text data. """ - def __init__(self, *args, db_collations=None, db_collation=None, **kwargs): + def __init__(self, *args, db_collations=None, db_collation=None, **kwargs): # pylint: disable=unused-argument """ Init like any field but add db_collations and disallow db_collation @@ -77,7 +77,6 @@ def db_collation(self, value): This can be removed when we move to Django 4.2. """ - pass def db_parameters(self, connection): """ diff --git a/openedx_learning/lib/fields.py b/openedx_learning/lib/fields.py index 0e231e3a6..b239eb8c0 100644 --- a/openedx_learning/lib/fields.py +++ b/openedx_learning/lib/fields.py @@ -7,9 +7,9 @@ We have helpers to make case sensitivity consistent across backends. MySQL is case-insensitive by default, SQLite and Postgres are case-sensitive. - - """ +from __future__ import annotations + import hashlib import uuid @@ -19,11 +19,11 @@ from .validators import validate_utc_datetime -def create_hash_digest(data_bytes): +def create_hash_digest(data_bytes: bytes) -> str: return hashlib.blake2b(data_bytes, digest_size=20).hexdigest() -def case_insensitive_char_field(**kwargs): +def case_insensitive_char_field(**kwargs) -> MultiCollationCharField: """ Return a case-insensitive ``MultiCollationCharField``. @@ -53,7 +53,7 @@ def case_insensitive_char_field(**kwargs): return MultiCollationCharField(**final_kwargs) -def case_sensitive_char_field(**kwargs): +def case_sensitive_char_field(**kwargs) -> MultiCollationCharField: """ Return a case-sensitive ``MultiCollationCharField``. @@ -79,7 +79,7 @@ def case_sensitive_char_field(**kwargs): return MultiCollationCharField(**final_kwargs) -def immutable_uuid_field(): +def immutable_uuid_field() -> models.UUIDField: """ Stable, randomly-generated UUIDs. @@ -97,7 +97,7 @@ def immutable_uuid_field(): ) -def key_field(): +def key_field() -> MultiCollationCharField: """ Externally created Identifier fields. @@ -111,7 +111,7 @@ def key_field(): return case_sensitive_char_field(max_length=500, blank=False) -def hash_field(): +def hash_field() -> models.CharField: """ Holds a hash digest meant to identify a piece of content. @@ -130,7 +130,7 @@ def hash_field(): ) -def manual_date_time_field(): +def manual_date_time_field() -> models.DateTimeField: """ DateTimeField that does not auto-generate values. @@ -159,7 +159,7 @@ def manual_date_time_field(): ], ) - + class MultiCollationCharField(MultiCollationMixin, models.CharField): """ CharField subclass with per-database-vendor collation settings. @@ -172,7 +172,6 @@ class MultiCollationCharField(MultiCollationMixin, models.CharField): PostgreSQL. Even MariaDB is starting to diverge from MySQL in terms of what collations are supported. """ - pass class MultiCollationTextField(MultiCollationMixin, models.TextField): @@ -181,6 +180,5 @@ class MultiCollationTextField(MultiCollationMixin, models.TextField): We don't ever really want to _sort_ by a TextField, but setting a collation forces the compatible charset to be set in MySQL, and that's the part that - matters for our purposes. So for example, if you set + matters for our purposes. """ - pass diff --git a/openedx_learning/lib/validators.py b/openedx_learning/lib/validators.py index 035a35540..a59cfd1ad 100644 --- a/openedx_learning/lib/validators.py +++ b/openedx_learning/lib/validators.py @@ -1,3 +1,6 @@ +""" +Useful validation methods +""" from datetime import datetime, timezone from django.core.exceptions import ValidationError diff --git a/openedx_learning/py.typed b/openedx_learning/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_learning/rest_api/apps.py b/openedx_learning/rest_api/apps.py index fe7f93f4c..dd3da9955 100644 --- a/openedx_learning/rest_api/apps.py +++ b/openedx_learning/rest_api/apps.py @@ -1,3 +1,6 @@ +""" +Django metadata for the Learning Core REST API app +""" from django.apps import AppConfig diff --git a/openedx_learning/rest_api/urls.py b/openedx_learning/rest_api/urls.py index edc533ff9..4c7fe274f 100644 --- a/openedx_learning/rest_api/urls.py +++ b/openedx_learning/rest_api/urls.py @@ -1,3 +1,6 @@ +""" +URLs for the Learning Core REST API +""" from django.urls import include, path urlpatterns = [path("v1/", include("openedx_learning.rest_api.v1.urls"))] diff --git a/openedx_learning/rest_api/v1/components.py b/openedx_learning/rest_api/v1/components.py index 571753b3f..d522f03ad 100644 --- a/openedx_learning/rest_api/v1/components.py +++ b/openedx_learning/rest_api/v1/components.py @@ -7,8 +7,11 @@ class ComponentViewSet(viewsets.ViewSet): + """ + Example endpoints for a Components REST API. Not implemented. + """ def list(self, request): - items = Component.objects.all() + _items = Component.objects.all() raise NotImplementedError def retrieve(self, request, pk=None): diff --git a/openedx_learning/rest_api/v1/urls.py b/openedx_learning/rest_api/v1/urls.py index 3b1a66c35..b1f5e3768 100644 --- a/openedx_learning/rest_api/v1/urls.py +++ b/openedx_learning/rest_api/v1/urls.py @@ -1,3 +1,6 @@ +""" +URLs for the Learning Core REST API v1 +""" from rest_framework.routers import DefaultRouter from . import components diff --git a/openedx_tagging/__init__.py b/openedx_tagging/__init__.py index 6b99f0677..ea956e000 100644 --- a/openedx_tagging/__init__.py +++ b/openedx_tagging/__init__.py @@ -1 +1,3 @@ -"""Open edX Tagging app.""" +""" +Open edX Tagging app. +""" diff --git a/openedx_tagging/core/tagging/admin.py b/openedx_tagging/core/tagging/admin.py index 91b1d753d..5be444cfc 100644 --- a/openedx_tagging/core/tagging/admin.py +++ b/openedx_tagging/core/tagging/admin.py @@ -1,4 +1,6 @@ -""" Tagging app admin """ +""" +Tagging app admin +""" from django.contrib import admin from .models import ObjectTag, Tag, Taxonomy diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 0ccb46883..e0295df59 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -10,7 +10,9 @@ Please look at the models.py file for more information about the kinds of data are stored in this app. """ -from typing import Iterator, List, Type, Union +from __future__ import annotations + +from typing import Iterator from django.db.models import QuerySet from django.utils.translation import gettext_lazy as _ @@ -20,19 +22,19 @@ def create_taxonomy( name: str, - description: str = None, + description: str | None = None, enabled=True, required=False, allow_multiple=False, allow_free_text=False, - taxonomy_class: Type = None, + taxonomy_class: type[Taxonomy] | None = None, ) -> Taxonomy: """ Creates, saves, and returns a new Taxonomy with the given attributes. """ taxonomy = Taxonomy( name=name, - description=description, + description=description or "", enabled=enabled, required=required, allow_multiple=allow_multiple, @@ -44,7 +46,7 @@ def create_taxonomy( return taxonomy.cast() -def get_taxonomy(id: int) -> Union[Taxonomy, None]: +def get_taxonomy(id: int) -> Taxonomy | None: """ Returns a Taxonomy cast to the appropriate subclass which has the given ID. """ @@ -52,7 +54,7 @@ def get_taxonomy(id: int) -> Union[Taxonomy, None]: return taxonomy.cast() if taxonomy else None -def get_taxonomies(enabled=True) -> QuerySet: +def get_taxonomies(enabled=True) -> QuerySet[Taxonomy]: """ Returns a queryset containing the enabled taxonomies, sorted by name. @@ -68,7 +70,7 @@ def get_taxonomies(enabled=True) -> QuerySet: return queryset.filter(enabled=enabled) -def get_tags(taxonomy: Taxonomy) -> List[Tag]: +def get_tags(taxonomy: Taxonomy) -> list[Tag]: """ Returns a list of predefined tags for the given taxonomy. @@ -77,7 +79,7 @@ def get_tags(taxonomy: Taxonomy) -> List[Tag]: return taxonomy.cast().get_tags() -def resync_object_tags(object_tags: QuerySet = None) -> int: +def resync_object_tags(object_tags: QuerySet | None = None) -> int: """ Reconciles ObjectTag entries with any changes made to their associated taxonomies and tags. @@ -96,25 +98,25 @@ def resync_object_tags(object_tags: QuerySet = None) -> int: def get_object_tags( - object_id: str, taxonomy_id: str = None -) -> QuerySet: + object_id: str, + taxonomy_id: str | None = None +) -> QuerySet[ObjectTag]: """ Returns a Queryset of object tags for a given object. Pass taxonomy to limit the returned object_tags to a specific taxonomy. """ - taxonomy = get_taxonomy(taxonomy_id) - ObjectTagClass = taxonomy.object_tag_class if taxonomy else ObjectTag + ObjectTagClass = ObjectTag + extra_filters = {} + if taxonomy_id is not None: + taxonomy = Taxonomy.objects.get(pk=taxonomy_id) + ObjectTagClass = taxonomy.object_tag_class + extra_filters["taxonomy_id"] = taxonomy_id tags = ( - ObjectTagClass.objects.filter( - object_id=object_id, - ) + ObjectTagClass.objects.filter(object_id=object_id, **extra_filters) .select_related("tag", "taxonomy") .order_by("id") ) - if taxonomy: - tags = tags.filter(taxonomy=taxonomy) - return tags @@ -133,9 +135,9 @@ def delete_object_tags(object_id: str): def tag_object( taxonomy: Taxonomy, - tags: List, + tags: list[str], object_id: str, -) -> List[ObjectTag]: +) -> list[ObjectTag]: """ Replaces the existing ObjectTag entries for the given taxonomy + object_id with the given list of tags. @@ -151,7 +153,7 @@ def tag_object( def autocomplete_tags( taxonomy: Taxonomy, search: str, - object_id: str = None, + object_id: str | None = None, object_tags_only=True, ) -> QuerySet: """ diff --git a/openedx_tagging/core/tagging/import_export/actions.py b/openedx_tagging/core/tagging/import_export/actions.py index da757cf4b..409194277 100644 --- a/openedx_tagging/core/tagging/import_export/actions.py +++ b/openedx_tagging/core/tagging/import_export/actions.py @@ -1,12 +1,12 @@ """ Actions for import tags """ -from typing import List +from __future__ import annotations -from django.utils.translation import gettext_lazy as _ +from django.utils.translation import gettext as _ -from ..models import Taxonomy, Tag -from .exceptions import ImportActionError, ImportActionConflict +from ..models import Tag, Taxonomy +from .exceptions import ImportActionConflict, ImportActionError class ImportAction: @@ -40,10 +40,10 @@ def __init__(self, taxonomy: Taxonomy, tag, index: int): self.tag = tag self.index = index - def __repr__(self): + def __repr__(self) -> str: return str(_(f"Action {self.name} (index={self.index},id={self.tag.id})")) - def __str__(self): + def __str__(self) -> str: return self.__repr__() @classmethod @@ -55,20 +55,20 @@ def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: """ raise NotImplementedError - def validate(self, indexed_actions) -> List[ImportActionError]: + def validate(self, indexed_actions) -> list[ImportActionError]: """ Implement this to find inconsistencies with tags in the database or with previous actions. """ raise NotImplementedError - def execute(self): + def execute(self) -> None: """ Implement this to execute the action. """ raise NotImplementedError - def _get_tag(self): + def _get_tag(self) -> Tag: """ Returns the respective tag of this actions """ @@ -90,7 +90,7 @@ def _search_action( return None - def _validate_parent(self, indexed_actions) -> ImportActionError: + def _validate_parent(self, indexed_actions) -> ImportActionError | None: """ Helper method to validate that the parent tag has already been defined. """ @@ -110,8 +110,9 @@ def _validate_parent(self, indexed_actions) -> ImportActionError: "You need to add parent before the child in your file." ), ) + return None - def _validate_value(self, indexed_actions): + def _validate_value(self, indexed_actions) -> ImportActionError | None: """ Check for value duplicates in the models and in previous create/rename actions @@ -151,6 +152,7 @@ def _validate_value(self, indexed_actions): conflict_action_index=action.index, message=_("Duplicated tag value."), ) + return None class CreateTag(ImportAction): @@ -169,7 +171,7 @@ class CreateTag(ImportAction): name = "create" - def __str__(self): + def __str__(self) -> str: return str( _( "Create a new tag with values " @@ -189,7 +191,7 @@ def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: except Tag.DoesNotExist: return True - def _validate_id(self, indexed_actions): + def _validate_id(self, indexed_actions) -> ImportActionError | None: """ Check for id duplicates in previous create actions """ @@ -201,8 +203,9 @@ def _validate_id(self, indexed_actions): conflict_action_index=action.index, message=_("Duplicated external_id tag."), ) + return None - def validate(self, indexed_actions) -> List[ImportActionError]: + def validate(self, indexed_actions) -> list[ImportActionError]: """ Validates the creation action """ @@ -226,7 +229,7 @@ def validate(self, indexed_actions) -> List[ImportActionError]: return errors - def execute(self): + def execute(self) -> None: """ Creates a Tag """ @@ -255,7 +258,7 @@ class UpdateParentTag(ImportAction): name = "update_parent" - def __str__(self): + def __str__(self) -> str: taxonomy_tag = self._get_tag() if not taxonomy_tag.parent: from_str = _("from empty parent") @@ -283,7 +286,7 @@ def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: except Tag.DoesNotExist: return False - def validate(self, indexed_actions) -> List[ImportActionError]: + def validate(self, indexed_actions) -> list[ImportActionError]: """ Validates the update parent action """ @@ -297,7 +300,7 @@ def validate(self, indexed_actions) -> List[ImportActionError]: return errors - def execute(self): + def execute(self) -> None: """ Updates the parent of a tag """ @@ -322,7 +325,7 @@ class RenameTag(ImportAction): name = "rename" - def __str__(self): + def __str__(self) -> str: taxonomy_tag = self._get_tag() return str( _( @@ -342,7 +345,7 @@ def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: except Tag.DoesNotExist: return False - def validate(self, indexed_actions) -> List[ImportActionError]: + def validate(self, indexed_actions) -> list[ImportActionError]: """ Validates the rename action """ @@ -355,7 +358,7 @@ def validate(self, indexed_actions) -> List[ImportActionError]: return errors - def execute(self): + def execute(self) -> None: """ Rename a tag """ @@ -373,7 +376,7 @@ class DeleteTag(ImportAction): Does not require validations """ - def __str__(self): + def __str__(self) -> str: taxonomy_tag = self._get_tag() return str(_(f"Delete tag (external_id={taxonomy_tag.external_id})")) @@ -387,14 +390,14 @@ def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: """ return False - def validate(self, indexed_actions) -> List[ImportActionError]: + def validate(self, indexed_actions) -> list[ImportActionError]: """ No validations necessary """ # TODO: Will it be necessary to check if this tag has children? return [] - def execute(self): + def execute(self) -> None: """ Delete a tag """ @@ -411,7 +414,7 @@ class WithoutChanges(ImportAction): name = "without_changes" - def __str__(self): + def __str__(self) -> str: return str(_(f"No changes needed for tag (external_id={self.tag.id})")) @classmethod @@ -421,13 +424,13 @@ def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: """ return False - def validate(self, indexed_actions) -> List[ImportActionError]: + def validate(self, indexed_actions) -> list[ImportActionError]: """ No validations necessary """ return [] - def execute(self): + def execute(self) -> None: """ Do nothing """ diff --git a/openedx_tagging/core/tagging/import_export/api.py b/openedx_tagging/core/tagging/import_export/api.py index 4e26854ed..80f01ad64 100644 --- a/openedx_tagging/core/tagging/import_export/api.py +++ b/openedx_tagging/core/tagging/import_export/api.py @@ -42,13 +42,15 @@ (ex. server crash) - Join/reduce actions on TagImportPlan. See `generate_actions()` """ +from __future__ import annotations + from io import BytesIO from django.utils.translation import gettext_lazy as _ -from ..models import Taxonomy, TagImportTask, TagImportTaskState -from .parsers import get_parser, ParserFormat +from ..models import TagImportTask, TagImportTaskState, Taxonomy from .import_plan import TagImportPlan, TagImportTask +from .parsers import ParserFormat, get_parser def import_tags( @@ -124,7 +126,9 @@ def get_last_import_status(taxonomy: Taxonomy) -> TagImportTaskState: Get status of the last import task of the given taxonomy """ task = _get_last_import_task(taxonomy) - return task.status + if task is None: + raise ValueError("No import task was created yet.") + return TagImportTaskState(task.status) def get_last_import_log(taxonomy: Taxonomy) -> str: @@ -132,6 +136,8 @@ def get_last_import_log(taxonomy: Taxonomy) -> str: Get logs of the last import task of the given taxonomy """ task = _get_last_import_task(taxonomy) + if task is None: + raise ValueError("No import task was created yet.") return task.log @@ -158,7 +164,7 @@ def _check_unique_import_task(taxonomy: Taxonomy) -> bool: ) -def _get_last_import_task(taxonomy: Taxonomy) -> TagImportTask: +def _get_last_import_task(taxonomy: Taxonomy) -> TagImportTask | None: """ Get the last import task for the given taxonomy """ diff --git a/openedx_tagging/core/tagging/import_export/exceptions.py b/openedx_tagging/core/tagging/import_export/exceptions.py index 2330cae6a..91a6b6f9b 100644 --- a/openedx_tagging/core/tagging/import_export/exceptions.py +++ b/openedx_tagging/core/tagging/import_export/exceptions.py @@ -1,7 +1,14 @@ """ Exceptions for tag import/export actions """ -from django.utils.translation import gettext_lazy as _ +from __future__ import annotations + +import typing + +from django.utils.translation import gettext as _ + +if typing.TYPE_CHECKING: + from .actions import ImportAction class TagImportError(Exception): @@ -11,6 +18,7 @@ class TagImportError(Exception): def __init__(self, message: str = "", **kargs): self.message = message + super().__init__(message, **kargs) def __str__(self): return str(self.message) @@ -24,7 +32,9 @@ class TagParserError(TagImportError): Base exception for parsers """ - def __init__(self, tag, **kargs): + def __init__(self, tag: dict | None, **kargs): + super().__init__() + self.tag = tag self.message = _(f"Import parser error on {tag}") @@ -33,7 +43,7 @@ class ImportActionError(TagImportError): Base exception for actions """ - def __init__(self, action: str, tag_id: str, message: str, **kargs): + def __init__(self, action: ImportAction, tag_id: str, message: str, **kargs): self.message = _( f"Action error in '{action.name}' (#{action.index}): {message}" ) @@ -46,7 +56,7 @@ class ImportActionConflict(ImportActionError): def __init__( self, - action: str, + action: ImportAction, tag_id: str, conflict_action_index: int, message: str, @@ -63,8 +73,8 @@ class InvalidFormat(TagParserError): Exception used when there is an error with the format """ - def __init__(self, tag: dict, format: str, message: str, **kargs): - self.tag = tag + def __init__(self, tag: dict | None, format: str, message: str, **kargs): + super().__init__(tag) self.message = _(f"Invalid '{format}' format: {message}") @@ -73,8 +83,8 @@ class FieldJSONError(TagParserError): Exception used when missing a required field on the .json """ - def __init__(self, tag, field, **kargs): - self.tag = tag + def __init__(self, tag: dict | None, field, **kargs): + super().__init__(tag) self.message = _(f"Missing '{field}' field on {tag}") diff --git a/openedx_tagging/core/tagging/import_export/import_plan.py b/openedx_tagging/core/tagging/import_export/import_plan.py index 3af3597d6..774afcadb 100644 --- a/openedx_tagging/core/tagging/import_export/import_plan.py +++ b/openedx_tagging/core/tagging/import_export/import_plan.py @@ -1,19 +1,13 @@ """ Classes and functions to create an import plan and execution. """ -from attrs import define -from typing import List, Optional +from __future__ import annotations +from attrs import define from django.db import transaction -from ..models import Taxonomy, TagImportTask -from .actions import ( - DeleteTag, - ImportAction, - UpdateParentTag, - WithoutChanges, - available_actions, -) +from ..models import TagImportTask, Taxonomy +from .actions import DeleteTag, ImportAction, UpdateParentTag, WithoutChanges, available_actions from .exceptions import ImportActionError @@ -22,11 +16,10 @@ class TagItem: """ Tag representation on the tag import plan """ - id: str value: str - index: Optional[int] = 0 - parent_id: Optional[str] = None + index: int | None = 0 + parent_id: str | None = None class TagImportPlan: @@ -34,8 +27,8 @@ class TagImportPlan: Class with functions to build an import plan and excute the plan """ - actions: List[ImportAction] - errors: List[ImportActionError] + actions: list[ImportAction] + errors: list[ImportActionError] indexed_actions: dict actions_dict: dict taxonomy: Taxonomy @@ -119,7 +112,7 @@ def _build_delete_actions(self, tags: dict): def generate_actions( self, - tags: List[TagItem], + tags: list[TagItem], replace=False, ): """ @@ -182,7 +175,7 @@ def plan(self) -> str: return result @transaction.atomic() - def execute(self, task: TagImportTask = None): + def execute(self, task: TagImportTask | None = None): """ Executes each action diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py index 79171e7e5..1fb714735 100644 --- a/openedx_tagging/core/tagging/import_export/parsers.py +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -1,24 +1,19 @@ """ Parsers to import and export tags """ +from __future__ import annotations + import csv import json from enum import Enum -from io import BytesIO, TextIOWrapper, StringIO -from typing import List, Tuple +from io import BytesIO, StringIO, TextIOWrapper -from django.utils.translation import gettext_lazy as _ +from django.utils.translation import gettext as _ -from .import_plan import TagItem -from .exceptions import ( - TagParserError, - InvalidFormat, - FieldJSONError, - EmptyJSONField, - EmptyCSVField, -) -from ..models import Taxonomy from ..api import get_tags +from ..models import Taxonomy +from .exceptions import EmptyCSVField, EmptyJSONField, FieldJSONError, InvalidFormat, TagParserError +from .import_plan import TagItem class ParserFormat(Enum): @@ -48,7 +43,7 @@ class Parser: optional_fields = ["parent_id"] # Set the format associated to the parser - format = None + format: ParserFormat # We can change the error when is missing a required field missing_field_error = TagParserError # We can change the error when a required field is empty @@ -57,7 +52,7 @@ class Parser: inital_row = 1 @classmethod - def parse_import(cls, file: BytesIO) -> Tuple[List[TagItem], List[TagParserError]]: + def parse_import(cls, file: BytesIO) -> tuple[list[TagItem], list[TagParserError]]: """ Parse tags in file an returns tags ready for use in TagImportPlan @@ -85,7 +80,7 @@ def export(cls, taxonomy: Taxonomy) -> str: return cls._export_data(tags, taxonomy) @classmethod - def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + def _load_data(cls, file: BytesIO) -> tuple[list[dict], list[TagParserError]]: """ Each parser implements this function according to its format. This function reads the file and returns a list with the values of each tag. @@ -96,7 +91,7 @@ def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: raise NotImplementedError @classmethod - def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + def _export_data(cls, tags: list[dict], taxonomy: Taxonomy) -> str: """ Each parser implements this function according to its format. Returns a string with tags data in the parser format. @@ -108,7 +103,7 @@ def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: raise NotImplementedError @classmethod - def _parse_tags(cls, tags_data: dict) -> Tuple[List[TagItem], List[TagParserError]]: + def _parse_tags(cls, tags_data: list[dict]) -> tuple[list[TagItem], list[TagParserError]]: """ Validate the required fields of each tag. @@ -161,7 +156,7 @@ def _parse_tags(cls, tags_data: dict) -> Tuple[List[TagItem], List[TagParserErro return tags, errors @classmethod - def _load_tags_for_export(cls, taxonomy: Taxonomy) -> List[dict]: + def _load_tags_for_export(cls, taxonomy: Taxonomy) -> list[dict]: """ Returns a list of taxonomy's tags in the form of a dictionary with required and optional fields @@ -201,12 +196,12 @@ class JSONParser(Parser): """ format = ParserFormat.JSON - missing_field_error = FieldJSONError - empty_field_error = EmptyJSONField + missing_field_error: type[TagParserError] = FieldJSONError + empty_field_error: type[TagParserError] = EmptyJSONField inital_row = 0 @classmethod - def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + def _load_data(cls, file: BytesIO) -> tuple[list[dict], list[TagParserError]]: """ Read a .json file and validates the root structure of the json """ @@ -214,11 +209,11 @@ def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: try: tags_data = json.load(file) except json.JSONDecodeError as error: - return None, [ + return [], [ InvalidFormat(tag=None, format=cls.format.value, message=str(error)) ] if "tags" not in tags_data: - return None, [ + return [], [ InvalidFormat( tag=None, format=cls.format.value, @@ -230,7 +225,7 @@ def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: return tags_data, [] @classmethod - def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + def _export_data(cls, tags: list[dict], taxonomy: Taxonomy) -> str: """ Export tags and taxonomy metadata in JSON format """ @@ -255,11 +250,11 @@ class CSVParser(Parser): """ format = ParserFormat.CSV - empty_field_error = EmptyCSVField + empty_field_error: type[TagParserError] = EmptyCSVField inital_row = 2 @classmethod - def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: + def _load_data(cls, file: BytesIO) -> tuple[list[dict], list[TagParserError]]: """ Read a .csv file and validates the header fields """ @@ -267,13 +262,13 @@ def _load_data(cls, file: BytesIO) -> Tuple[List[dict], List[TagParserError]]: text_tags = TextIOWrapper(file, encoding="utf-8") csv_reader = csv.DictReader(text_tags) header_fields = csv_reader.fieldnames - errors = cls._verify_header(header_fields) + errors = cls._verify_header(list(header_fields or [])) if errors: - return None, errors + return [], errors return list(csv_reader), [] @classmethod - def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: + def _export_data(cls, tags: list[dict], taxonomy: Taxonomy) -> str: """ Export tags in CSV format @@ -291,11 +286,11 @@ def _export_data(cls, tags: List[dict], taxonomy: Taxonomy) -> str: return csv_string @classmethod - def _verify_header(cls, header_fields: List[str]) -> List[TagParserError]: + def _verify_header(cls, header_fields: list[str]) -> list[TagParserError]: """ Verify that the header contains the required fields """ - errors = [] + errors: list[TagParserError] = [] for req_field in cls.required_fields: if req_field not in header_fields: errors.append( @@ -312,7 +307,7 @@ def _verify_header(cls, header_fields: List[str]) -> List[TagParserError]: _parsers = [JSONParser, CSVParser] -def get_parser(parser_format: ParserFormat) -> Parser: +def get_parser(parser_format: ParserFormat) -> type[Parser]: """ Get the parser for the respective `format` diff --git a/openedx_tagging/core/tagging/import_export/tasks.py b/openedx_tagging/core/tagging/import_export/tasks.py index 99d3cd49d..f351ea8ec 100644 --- a/openedx_tagging/core/tagging/import_export/tasks.py +++ b/openedx_tagging/core/tagging/import_export/tasks.py @@ -2,9 +2,11 @@ Import and export celery tasks """ from io import BytesIO -from celery import shared_task + +from celery import shared_task # type: ignore[import] import openedx_tagging.core.tagging.import_export.api as import_export_api + from ..models import Taxonomy from .parsers import ParserFormat diff --git a/openedx_tagging/core/tagging/management/commands/build_language_fixture.py b/openedx_tagging/core/tagging/management/commands/build_language_fixture.py index 9c1e0a85e..4901ca9fe 100644 --- a/openedx_tagging/core/tagging/management/commands/build_language_fixture.py +++ b/openedx_tagging/core/tagging/management/commands/build_language_fixture.py @@ -10,7 +10,7 @@ from django.core.management.base import BaseCommand -endpoint = "https://pkgstore.datahub.io/core/language-codes/language-codes_json/data/97607046542b532c395cf83df5185246/language-codes_json.json" +endpoint = "https://pkgstore.datahub.io/core/language-codes/language-codes_json/data/97607046542b532c395cf83df5185246/language-codes_json.json" # noqa output = "./openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml" diff --git a/openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py b/openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py index 4cca2c417..c96e05209 100644 --- a/openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py +++ b/openedx_tagging/core/tagging/migrations/0004_auto_20230723_2001.py @@ -1,6 +1,7 @@ # Generated by Django 3.2.19 on 2023-07-24 06:25 from django.db import migrations, models + import openedx_learning.lib.fields diff --git a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py index a6d4fd0cf..b6758da11 100644 --- a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py +++ b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.19 on 2023-07-28 13:33 -from django.db import migrations from django.core.management import call_command +from django.db import migrations def load_language_taxonomy(apps, schema_editor): diff --git a/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py b/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py index 87bab9cd7..1bb738545 100644 --- a/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py +++ b/openedx_tagging/core/tagging/migrations/0006_auto_20230802_1631.py @@ -1,7 +1,8 @@ # Generated by Django 3.2.19 on 2023-08-02 21:31 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + import openedx_tagging.core.tagging.models.import_export diff --git a/openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py b/openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py new file mode 100644 index 000000000..c4a4067a2 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.20 on 2023-08-17 21:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_tagging', '0006_auto_20230802_1631'), + ] + + operations = [ + migrations.AlterField( + model_name='tagimporttask', + name='log', + field=models.TextField(blank=True, default=None, help_text='Action execution logs'), + ), + ] diff --git a/openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py b/openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py new file mode 100644 index 000000000..37b352823 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.20 on 2023-08-23 18:14 + +from django.db import migrations + +import openedx_learning.lib.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_tagging', '0007_tag_import_task_log_null_fix'), + ] + + operations = [ + migrations.AlterField( + model_name='taxonomy', + name='description', + field=openedx_learning.lib.fields.MultiCollationTextField(blank=True, default='', help_text='Provides extra information for the user when applying tags from this taxonomy to an object.'), + preserve_default=False, + ), + ] diff --git a/openedx_tagging/core/tagging/models/__init__.py b/openedx_tagging/core/tagging/models/__init__.py index 295e38bdb..226d9991b 100644 --- a/openedx_tagging/core/tagging/models/__init__.py +++ b/openedx_tagging/core/tagging/models/__init__.py @@ -1,15 +1,3 @@ -from .base import ( - Tag, - Taxonomy, - ObjectTag, -) -from .system_defined import ( - ModelObjectTag, - ModelSystemDefinedTaxonomy, - UserSystemDefinedTaxonomy, - LanguageTaxonomy, -) -from .import_export import ( - TagImportTask, - TagImportTaskState, -) +from .base import ObjectTag, Tag, Taxonomy +from .import_export import TagImportTask, TagImportTaskState +from .system_defined import LanguageTaxonomy, ModelObjectTag, ModelSystemDefinedTaxonomy, UserSystemDefinedTaxonomy diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index efdbda55a..a8e0ea718 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -1,15 +1,17 @@ -""" Tagging app base data models """ +""" +Tagging app base data models +""" +from __future__ import annotations + import logging -from typing import List, Type, Union +from typing import List from django.db import models from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ +from typing_extensions import Self # Until we upgrade to python 3.11 -from openedx_learning.lib.fields import ( - MultiCollationTextField, - case_insensitive_char_field, -) +from openedx_learning.lib.fields import MultiCollationTextField, case_insensitive_char_field log = logging.getLogger(__name__) @@ -95,7 +97,7 @@ def get_lineage(self) -> Lineage: Performance note: may perform as many as TAXONOMY_MAX_DEPTH select queries. """ lineage: Lineage = [] - tag = self + tag: Tag | None = self depth = TAXONOMY_MAX_DEPTH while tag and depth > 0: lineage.insert(0, tag.value) @@ -119,7 +121,6 @@ class Taxonomy(models.Model): ), ) description = MultiCollationTextField( - null=True, blank=True, help_text=_( "Provides extra information for the user when applying tags from this taxonomy to an object." @@ -187,7 +188,7 @@ def __str__(self): return f"<{self.__class__.__name__}> ({self.id}) {self.name}" @property - def object_tag_class(self) -> Type: + def object_tag_class(self) -> type[ObjectTag]: """ Returns the ObjectTag subclass associated with this taxonomy, which is ObjectTag by default. @@ -196,7 +197,7 @@ def object_tag_class(self) -> Type: return ObjectTag @property - def taxonomy_class(self) -> Type: + def taxonomy_class(self) -> type[Taxonomy] | None: """ Returns the Taxonomy subclass associated with this instance, or None if none supplied. @@ -206,16 +207,8 @@ def taxonomy_class(self) -> Type: return import_string(self._taxonomy_class) return None - @property - def system_defined(self) -> bool: - """ - Indicates that tags and metadata for this taxonomy are maintained by the system; - taxonomy admins will not be permitted to modify them. - """ - return False - @taxonomy_class.setter - def taxonomy_class(self, taxonomy_class: Union[Type, None]): + def taxonomy_class(self, taxonomy_class: type[Taxonomy] | None): """ Assigns the given taxonomy_class's module path.class to the field. @@ -234,6 +227,14 @@ def taxonomy_class(self, taxonomy_class: Union[Type, None]): else: self._taxonomy_class = None + @property + def system_defined(self) -> bool: + """ + Indicates that tags and metadata for this taxonomy are maintained by the system; + taxonomy admins will not be permitted to modify them. + """ + return False + def cast(self): """ Returns the current Taxonomy instance cast into its taxonomy_class. @@ -252,7 +253,7 @@ def cast(self): return self - def copy(self, taxonomy: "Taxonomy") -> "Taxonomy": + def copy(self, taxonomy: Taxonomy) -> Taxonomy: """ Copy the fields from the given Taxonomy into the current instance. """ @@ -267,22 +268,25 @@ def copy(self, taxonomy: "Taxonomy") -> "Taxonomy": self._taxonomy_class = taxonomy._taxonomy_class return self - def get_tags(self, tag_set: models.QuerySet = None) -> List[Tag]: + def get_tags(self, tag_set: models.QuerySet | None = None) -> list[Tag]: """ - Returns a list of all Tags in the current taxonomy, from the root(s) down to TAXONOMY_MAX_DEPTH tags, in tree order. + Returns a list of all Tags in the current taxonomy, from the root(s) + down to TAXONOMY_MAX_DEPTH tags, in tree order. Use `tag_set` to do an initial filtering of the tags. - Annotates each returned Tag with its ``depth`` in the tree (starting at 0). + Annotates each returned Tag with its ``depth`` in the tree (starting at + 0). - Performance note: may perform as many as TAXONOMY_MAX_DEPTH select queries. + Performance note: may perform as many as TAXONOMY_MAX_DEPTH select + queries. """ - tags = [] + tags: list[Tag] = [] if self.allow_free_text: return tags if tag_set is None: - tag_set = self.tag_set + tag_set = self.tag_set.all() parents = None for depth in range(TAXONOMY_MAX_DEPTH): @@ -336,7 +340,7 @@ def validate_object_tag( def _check_taxonomy( self, - object_tag: "ObjectTag", + object_tag: ObjectTag, ) -> bool: """ Returns True if the given object tag is valid for the current Taxonomy. @@ -344,11 +348,11 @@ def _check_taxonomy( Subclasses can override this method to perform their own taxonomy validation checks. """ # Must be linked to this taxonomy - return object_tag.taxonomy_id and object_tag.taxonomy_id == self.id + return (object_tag.taxonomy_id is not None) and object_tag.taxonomy_id == self.id def _check_tag( self, - object_tag: "ObjectTag", + object_tag: ObjectTag, ) -> bool: """ Returns True if the given object tag's value is valid for the current Taxonomy. @@ -360,11 +364,11 @@ def _check_tag( return bool(object_tag.value) # Closed taxonomies need an associated tag in this taxonomy - return object_tag.tag_id and object_tag.tag.taxonomy_id == self.id + return (object_tag.tag is not None) and object_tag.tag.taxonomy_id == self.id def _check_object( self, - object_tag: "ObjectTag", + object_tag: ObjectTag, ) -> bool: """ Returns True if the given object tag's object is valid for the current Taxonomy. @@ -375,9 +379,9 @@ def _check_object( def tag_object( self, - tags: List, + tags: list[str], object_id: str, - ) -> List["ObjectTag"]: + ) -> list[ObjectTag]: """ Replaces the existing ObjectTag entries for the current taxonomy + object_id with the given list of tags. If self.allows_free_text, then the list should be a list of tag values. @@ -433,7 +437,7 @@ def tag_object( def autocomplete_tags( self, search: str, - object_id: str = None, + object_id: str | None = None, ) -> models.QuerySet: """ Provides auto-complete suggestions by matching the `search` string against existing @@ -456,11 +460,11 @@ def autocomplete_tags( search the suggestions on a list of available tags. """ # Fetch tags that the object already has to exclude them from the result - excluded_tags = [] + excluded_tags: list[str] = [] if object_id: - excluded_tags = self.objecttag_set.filter(object_id=object_id).values_list( + excluded_tags = list(self.objecttag_set.filter(object_id=object_id).values_list( "_value", flat=True - ) + )) return ( # Fetch object tags from this taxonomy whose value contains the search self.objecttag_set.filter(_value__icontains=search) @@ -508,7 +512,8 @@ class ObjectTag(models.Model): default=None, on_delete=models.SET_NULL, help_text=_( - "Taxonomy that this object tag belongs to. Used for validating the tag and provides the tag's 'name' if set." + "Taxonomy that this object tag belongs to. " + "Used for validating the tag and provides the tag's 'name' if set." ), ) tag = models.ForeignKey( @@ -565,7 +570,7 @@ def name(self) -> str: If taxonomy is set, then returns its name. Otherwise, returns the cached _name field. """ - return self.taxonomy.name if self.taxonomy_id else self._name + return self.taxonomy.name if self.taxonomy else self._name @name.setter def name(self, name: str): @@ -582,7 +587,7 @@ def value(self) -> str: If tag is set, then returns its value. Otherwise, returns the cached _value field. """ - return self.tag.value if self.tag_id else self._value + return self.tag.value if self.tag else self._value @value.setter def value(self, value: str): @@ -599,7 +604,7 @@ def tag_ref(self) -> str: If tag is set, then returns its id. Otherwise, returns the cached _value field. """ - return self.tag.id if self.tag_id else self._value + return self.tag.id if self.tag else self._value @tag_ref.setter def tag_ref(self, tag_ref: str): @@ -610,7 +615,7 @@ def tag_ref(self, tag_ref: str): """ self.value = tag_ref - if self.taxonomy_id: + if self.taxonomy: try: self.tag = self.taxonomy.tag_set.get(pk=tag_ref) self.value = self.tag.value @@ -625,7 +630,7 @@ def is_valid(self) -> bool: A valid ObjectTag must be linked to a Taxonomy, and be a valid tag in that taxonomy. """ - return self.taxonomy_id and self.taxonomy.validate_object_tag(self) + return self.taxonomy.validate_object_tag(self) if self.taxonomy else False def get_lineage(self) -> Lineage: """ @@ -634,7 +639,7 @@ def get_lineage(self) -> Lineage: If linked to a Tag, returns its lineage. Otherwise, returns an array containing its value string. """ - return self.tag.get_lineage() if self.tag_id else [self._value] + return self.tag.get_lineage() if self.tag else [self._value] def resync(self) -> bool: """ @@ -652,7 +657,7 @@ def resync(self) -> bool: # Locate an enabled taxonomy matching _name, and maybe a tag matching _value if not self.taxonomy_id: # Use the linked tag's taxonomy if there is one. - if self.tag_id: + if self.tag: self.taxonomy_id = self.tag.taxonomy_id changed = True else: @@ -679,7 +684,7 @@ def resync(self) -> bool: self.tag = None # Sync the stored _name with the taxonomy.name - if self.taxonomy_id and self._name != self.taxonomy.name: + if self.taxonomy and self._name != self.taxonomy.name: self.name = self.taxonomy.name changed = True @@ -698,13 +703,13 @@ def resync(self) -> bool: return changed @classmethod - def cast(cls, object_tag: "ObjectTag") -> "ObjectTag": + def cast(cls, object_tag: ObjectTag) -> Self: """ Returns a cls instance with the same properties as the given ObjectTag. """ return cls().copy(object_tag) - def copy(self, object_tag: "ObjectTag") -> "ObjectTag": + def copy(self, object_tag: ObjectTag) -> Self: """ Copy the fields from the given ObjectTag into the current instance. """ diff --git a/openedx_tagging/core/tagging/models/import_export.py b/openedx_tagging/core/tagging/models/import_export.py index 1deaf9c9d..c1b41b365 100644 --- a/openedx_tagging/core/tagging/models/import_export.py +++ b/openedx_tagging/core/tagging/models/import_export.py @@ -1,8 +1,9 @@ from datetime import datetime from enum import Enum -from django.db import models -from django.utils.translation import gettext_lazy as _ +from django.db import models +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy from .base import Taxonomy @@ -29,13 +30,13 @@ class TagImportTask(models.Model): ) log = models.TextField( - null=True, default=None, help_text=_("Action execution logs") + blank=True, default=None, help_text=gettext_lazy("Action execution logs") ) status = models.CharField( max_length=20, choices=[(status, status.value) for status in TagImportTaskState], - help_text=_("Task status"), + help_text=gettext_lazy("Task status"), ) creation_date = models.DateTimeField(auto_now_add=True) diff --git a/openedx_tagging/core/tagging/models/system_defined.py b/openedx_tagging/core/tagging/models/system_defined.py index 7b39b9c2a..275d6e97d 100644 --- a/openedx_tagging/core/tagging/models/system_defined.py +++ b/openedx_tagging/core/tagging/models/system_defined.py @@ -1,6 +1,10 @@ -""" Tagging app system-defined taxonomies data models """ +""" +Tagging app system-defined taxonomies data models +""" +from __future__ import annotations + import logging -from typing import Any, List, Type, Union +from typing import Any from django.conf import settings from django.contrib.auth import get_user_model @@ -8,7 +12,7 @@ from openedx_tagging.core.tagging.models.base import ObjectTag -from .base import Tag, Taxonomy, ObjectTag +from .base import ObjectTag, Tag, Taxonomy log = logging.getLogger(__name__) @@ -48,7 +52,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @property - def tag_class_model(self) -> Type: + def tag_class_model(self) -> type[models.Model]: """ Subclasses must implement this method to return the Django.model class referenced by these object tags. @@ -64,7 +68,7 @@ def tag_class_value(self) -> str: """ return "pk" - def get_instance(self) -> Union[models.Model, None]: + def get_instance(self) -> models.Model | None: """ Returns the instance of tag_class_model associated with this object tag, or None if not found. """ @@ -104,7 +108,7 @@ def _resync_tag(self) -> bool: @property def tag_ref(self) -> str: - return (self.tag.external_id or self.tag.id) if self.tag_id else self._value + return (self.tag.external_id or self.tag.id) if self.tag else self._value @tag_ref.setter def tag_ref(self, tag_ref: str): @@ -115,7 +119,7 @@ def tag_ref(self, tag_ref: str): """ self.value = tag_ref - if self.taxonomy_id: + if self.taxonomy: try: self.tag = self.taxonomy.tag_set.get( external_id=tag_ref, @@ -159,9 +163,9 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) @property - def object_tag_class(self) -> Type: + def object_tag_class(self) -> type[ModelObjectTag]: """ - Returns the ObjectTag subclass associated with this taxonomy. + Returns the ModelObjectTag subclass associated with this taxonomy. Model Taxonomy subclasses must implement this to provide a ModelObjectTag subclass. """ @@ -192,7 +196,7 @@ class Meta: proxy = True @property - def tag_class_model(self) -> Type: + def tag_class_model(self) -> type[models.Model]: """ Associate the user model """ @@ -217,7 +221,7 @@ class Meta: proxy = True @property - def object_tag_class(self) -> Type: + def object_tag_class(self): """ Returns the ObjectTag subclass associated with this taxonomy, which is ModelObjectTag by default. @@ -237,7 +241,7 @@ class LanguageTaxonomy(SystemDefinedTaxonomy): class Meta: proxy = True - def get_tags(self, tag_set: models.QuerySet = None) -> List[Tag]: + def get_tags(self, tag_set: models.QuerySet | None = None) -> list[Tag]: """ Returns a list of all the available Language Tags, annotated with ``depth`` = 0. """ @@ -245,7 +249,7 @@ def get_tags(self, tag_set: models.QuerySet = None) -> List[Tag]: tag_set = self.tag_set.filter(external_id__in=available_langs) return super().get_tags(tag_set=tag_set) - def _get_available_languages(cls) -> List[str]: + def _get_available_languages(cls) -> set[str]: """ Get available languages from Django LANGUAGE. """ @@ -260,6 +264,8 @@ def _check_valid_language(self, object_tag: ObjectTag) -> bool: Returns True if the tag is on the available languages """ available_langs = self._get_available_languages() + if not object_tag.tag: + raise AttributeError("Expected object_tag.tag to be set") return object_tag.tag.external_id in available_langs def _check_tag(self, object_tag: ObjectTag) -> bool: diff --git a/openedx_tagging/core/tagging/rest_api/urls.py b/openedx_tagging/core/tagging/rest_api/urls.py index d7f012bb7..2cd82d03b 100644 --- a/openedx_tagging/core/tagging/rest_api/urls.py +++ b/openedx_tagging/core/tagging/rest_api/urls.py @@ -2,7 +2,7 @@ Taxonomies API URLs. """ -from django.urls import path, include +from django.urls import include, path from .v1 import urls as v1_urls diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 593b72989..e278b84fe 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -4,7 +4,7 @@ from rest_framework import serializers -from openedx_tagging.core.tagging.models import Taxonomy, ObjectTag +from openedx_tagging.core.tagging.models import ObjectTag, Taxonomy class TaxonomyListQueryParamsSerializer(serializers.Serializer): diff --git a/openedx_tagging/core/tagging/rest_api/v1/urls.py b/openedx_tagging/core/tagging/rest_api/v1/urls.py index c449eb5c2..02cb48e40 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/urls.py +++ b/openedx_tagging/core/tagging/rest_api/v1/urls.py @@ -2,10 +2,9 @@ Taxonomies API v1 URLs. """ +from django.urls.conf import include, path from rest_framework.routers import DefaultRouter -from django.urls.conf import path, include - from . import views router = DefaultRouter() diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 13e00acd6..3e8ae18c2 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -1,23 +1,21 @@ """ Tagging API Views """ +from django.db import models from django.http import Http404 +from django.shortcuts import get_object_or_404 from rest_framework import status -from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet from rest_framework.response import Response +from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet -from ...api import ( - create_taxonomy, - get_taxonomy, - get_taxonomies, - get_object_tags, -) -from .permissions import TaxonomyObjectPermissions, ObjectTagObjectPermissions +from ...api import create_taxonomy, get_object_tags, get_taxonomies, get_taxonomy +from ...models import Taxonomy +from .permissions import ObjectTagObjectPermissions, TaxonomyObjectPermissions from .serializers import ( - TaxonomyListQueryParamsSerializer, - TaxonomySerializer, ObjectTagListQueryParamsSerializer, ObjectTagSerializer, + TaxonomyListQueryParamsSerializer, + TaxonomySerializer, ) @@ -26,14 +24,15 @@ class TaxonomyView(ModelViewSet): View to list, create, retrieve, update, or delete Taxonomies. **List Query Parameters** - * enabled (optional) - Filter by enabled status. Valid values: true, false, 1, 0, "true", "false", "1" + * enabled (optional) - Filter by enabled status. Valid values: true, + false, 1, 0, "true", "false", "1" * page (optional) - Page number (default: 1) * page_size (optional) - Number of items per page (default: 10) **List Example Requests** - GET api/tagging/v1/taxonomy - Get all taxonomies - GET api/tagging/v1/taxonomy?enabled=true - Get all enabled taxonomies - GET api/tagging/v1/taxonomy?enabled=false - Get all disabled taxonomies + GET api/tagging/v1/taxonomy - Get all taxonomies + GET api/tagging/v1/taxonomy?enabled=true - Get all enabled taxonomies + GET api/tagging/v1/taxonomy?enabled=false - Get all disabled taxonomies **List Query Returns** * 200 - Success @@ -44,24 +43,32 @@ class TaxonomyView(ModelViewSet): * pk (required): - The pk of the taxonomy to retrieve **Retrieve Example Requests** - GET api/tagging/v1/taxonomy/:pk - Get a specific taxonomy + GET api/tagging/v1/taxonomy/:pk - Get a specific taxonomy **Retrieve Query Returns** * 200 - Success - * 404 - Taxonomy not found or User does not have permission to access the taxonomy + * 404 - Taxonomy not found or User does not have permission to access + the taxonomy **Create Parameters** - * name (required): User-facing label used when applying tags from this taxonomy to Open edX objects. - * description (optional): Provides extra information for the user when applying tags from this taxonomy to an object. - * enabled (optional): Only enabled taxonomies will be shown to authors (default: true). - * required (optional): Indicates that one or more tags from this taxonomy must be added to an object (default: False). - * allow_multiple (optional): Indicates that multiple tags from this taxonomy may be added to an object (default: False). - * allow_free_text (optional): Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values (default: False). + * name (required): User-facing label used when applying tags from this + taxonomy to Open edX objects. + * description (optional): Provides extra information for the user when + applying tags from this taxonomy to an object. + * enabled (optional): Only enabled taxonomies will be shown to authors + (default: true). + * required (optional): Indicates that one or more tags from this + taxonomy must be added to an object (default: False). + * allow_multiple (optional): Indicates that multiple tags from this + taxonomy may be added to an object (default: False). + * allow_free_text (optional): Indicates that tags in this taxonomy need + not be predefined; authors may enter their own tag values (default: + False). **Create Example Requests** - POST api/tagging/v1/taxonomy - Create a taxonomy + POST api/tagging/v1/taxonomy - Create a taxonomy { - "name": "Taxonomy Name", - User-facing label used when applying tags from this taxonomy to Open edX objects." + "name": "Taxonomy Name", "description": "This is a description", "enabled": True, "required": True, @@ -77,15 +84,20 @@ class TaxonomyView(ModelViewSet): * pk (required): - The pk of the taxonomy to update **Update Request Body** - * name (optional): User-facing label used when applying tags from this taxonomy to Open edX objects. - * description (optional): Provides extra information for the user when applying tags from this taxonomy to an object. + * name (optional): User-facing label used when applying tags from this + taxonomy to Open edX objects. + * description (optional): Provides extra information for the user when + applying tags from this taxonomy to an object. * enabled (optional): Only enabled taxonomies will be shown to authors. - * required (optional): Indicates that one or more tags from this taxonomy must be added to an object. - * allow_multiple (optional): Indicates that multiple tags from this taxonomy may be added to an object. - * allow_free_text (optional): Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values. + * required (optional): Indicates that one or more tags from this + taxonomy must be added to an object. + * allow_multiple (optional): Indicates that multiple tags from this + taxonomy may be added to an object. + * allow_free_text (optional): Indicates that tags in this taxonomy need + not be predefined; authors may enter their own tag values. **Update Example Requests** - PUT api/tagging/v1/taxonomy/:pk - Update a taxonomy + PUT api/tagging/v1/taxonomy/:pk - Update a taxonomy { "name": "Taxonomy New Name", "description": "This is a new description", @@ -94,7 +106,7 @@ class TaxonomyView(ModelViewSet): "allow_multiple": False, "allow_free_text": True, } - PATCH api/tagging/v1/taxonomy/:pk - Partially update a taxonomy + PATCH api/tagging/v1/taxonomy/:pk - Partially update a taxonomy { "name": "Taxonomy New Name", } @@ -107,7 +119,7 @@ class TaxonomyView(ModelViewSet): * pk (required): - The pk of the taxonomy to delete **Delete Example Requests** - DELETE api/tagging/v1/taxonomy/:pk - Delete a taxonomy + DELETE api/tagging/v1/taxonomy/:pk - Delete a taxonomy **Delete Query Returns** * 200 - Success @@ -119,12 +131,12 @@ class TaxonomyView(ModelViewSet): serializer_class = TaxonomySerializer permission_classes = [TaxonomyObjectPermissions] - def get_object(self): + def get_object(self) -> Taxonomy: """ Return the requested taxonomy object, if the user has appropriate permissions. """ - pk = self.kwargs.get("pk") + pk = int(self.kwargs["pk"]) taxonomy = get_taxonomy(pk) if not taxonomy: raise Http404("Taxonomy not found") @@ -132,7 +144,7 @@ def get_object(self): return taxonomy - def get_queryset(self): + def get_queryset(self) -> models.QuerySet: """ Return a list of taxonomies. @@ -148,7 +160,7 @@ def get_queryset(self): return get_taxonomies(enabled) - def perform_create(self, serializer): + def perform_create(self, serializer) -> None: """ Create a new taxonomy. """ @@ -196,13 +208,13 @@ class ObjectTagView(ReadOnlyModelViewSet): permission_classes = [ObjectTagObjectPermissions] lookup_field = "object_id" - def get_queryset(self): + def get_queryset(self) -> models.QuerySet: """ Return a queryset of object tags for a given object. If a taxonomy is passed in, object tags are limited to that taxonomy. """ - object_id = self.kwargs.get("object_id") + object_id: str = self.kwargs["object_id"] query_params = ObjectTagListQueryParamsSerializer( data=self.request.query_params.dict() ) diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index b79a56344..8f0852a10 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -1,20 +1,26 @@ -"""Django rules-based permissions for tagging""" +""" +Django rules-based permissions for tagging +""" +from __future__ import annotations -import rules -from django.contrib.auth import get_user_model +from typing import Callable, Union + +import django.contrib.auth.models +# typing support in rules depends on https://github.com/dfunckt/django-rules/pull/177 +import rules # type: ignore[import] from .models import ObjectTag, Tag, Taxonomy -User = get_user_model() +UserType = Union[django.contrib.auth.models.User, django.contrib.auth.models.AnonymousUser] # Global staff are taxonomy admins. # (Superusers can already do anything) -is_taxonomy_admin = rules.is_staff +is_taxonomy_admin: Callable[[UserType], bool] = rules.is_staff @rules.predicate -def can_view_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: +def can_view_taxonomy(user: UserType, taxonomy: Taxonomy | None = None) -> bool: """ Anyone can view an enabled taxonomy or list all taxonomies, but only taxonomy admins can view a disabled taxonomy. @@ -23,22 +29,22 @@ def can_view_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: @rules.predicate -def can_change_taxonomy(user: User, taxonomy: Taxonomy = None) -> bool: +def can_change_taxonomy(user: UserType, taxonomy: Taxonomy | None = None) -> bool: """ Even taxonomy admins cannot change system taxonomies. """ return is_taxonomy_admin(user) and ( - not taxonomy or (taxonomy and not taxonomy.cast().system_defined) + not taxonomy or bool(taxonomy and not taxonomy.cast().system_defined) ) @rules.predicate -def can_change_tag(user: User, tag: Tag = None) -> bool: +def can_change_tag(user: UserType, tag: Tag | None = None) -> bool: """ Even taxonomy admins cannot add tags to system taxonomies (their tags are system-defined), or free-text taxonomies (these don't have predefined tags). """ - taxonomy = tag.taxonomy.cast() if (tag and tag.taxonomy_id) else None + taxonomy = tag.taxonomy.cast() if (tag and tag.taxonomy) else None return is_taxonomy_admin(user) and ( not tag or not taxonomy @@ -47,12 +53,12 @@ def can_change_tag(user: User, tag: Tag = None) -> bool: @rules.predicate -def can_change_object_tag(user: User, object_tag: ObjectTag = None) -> bool: +def can_change_object_tag(user: UserType, object_tag: ObjectTag | None = None) -> bool: """ Taxonomy admins can create or modify object tags on enabled taxonomies. """ taxonomy = ( - object_tag.taxonomy.cast() if (object_tag and object_tag.taxonomy_id) else None + object_tag.taxonomy.cast() if (object_tag and object_tag.taxonomy) else None ) object_tag = taxonomy.object_tag_class.cast(object_tag) if taxonomy else object_tag return is_taxonomy_admin(user) and ( diff --git a/openedx_tagging/core/tagging/urls.py b/openedx_tagging/core/tagging/urls.py index da2c52081..effb166a7 100644 --- a/openedx_tagging/core/tagging/urls.py +++ b/openedx_tagging/core/tagging/urls.py @@ -2,7 +2,7 @@ Tagging API URLs. """ -from django.urls import path, include +from django.urls import include, path from .rest_api import urls diff --git a/openedx_tagging/py.typed b/openedx_tagging/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/projects/dev.py b/projects/dev.py index 9735b35de..2dacb56bd 100644 --- a/projects/dev.py +++ b/projects/dev.py @@ -1,6 +1,7 @@ """ - +Django settings for testing and development purposes """ +from __future__ import annotations from pathlib import Path # Build paths inside the project like this: BASE_DIR / {dir_name} / @@ -96,7 +97,7 @@ "django.contrib.staticfiles.finders.FileSystemFinder", "django.contrib.staticfiles.finders.AppDirectoriesFinder", ] -STATICFILES_DIRS = [ +STATICFILES_DIRS: list[Path] = [ # BASE_DIR / 'projects' / 'static' ] MEDIA_URL = "/media/" diff --git a/requirements/dev.txt b/requirements/dev.txt index d5ad00f9d..0fe66827f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -110,6 +110,8 @@ django==3.2.19 # -r requirements/quality.txt # django-crum # django-debug-toolbar + # django-stubs + # django-stubs-ext # django-waffle # djangorestframework # drf-jwt @@ -124,6 +126,14 @@ django-debug-toolbar==4.1.0 # via # -r requirements/dev.in # -r requirements/quality.txt +django-stubs==4.2.3 + # via + # -r requirements/quality.txt + # djangorestframework-stubs +django-stubs-ext==4.2.2 + # via + # -r requirements/quality.txt + # django-stubs django-waffle==4.0.0 # via # -r requirements/quality.txt @@ -134,6 +144,8 @@ djangorestframework==3.14.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions +djangorestframework-stubs==3.14.2 + # via -r requirements/quality.txt docutils==0.20.1 # via # -r requirements/quality.txt @@ -239,6 +251,15 @@ more-itertools==9.1.0 # via # -r requirements/quality.txt # jaraco-classes +mypy==1.5.1 + # via + # -r requirements/quality.txt + # django-stubs + # djangorestframework-stubs +mypy-extensions==1.0.0 + # via + # -r requirements/quality.txt + # mypy mysqlclient==2.1.1 # via -r requirements/quality.txt newrelic==8.9.0 @@ -378,6 +399,7 @@ readme-renderer==40.0 requests==2.31.0 # via # -r requirements/quality.txt + # djangorestframework-stubs # edx-drf-extensions # requests-toolbelt # twine @@ -438,7 +460,9 @@ tomli==2.0.1 # -r requirements/quality.txt # build # coverage + # django-stubs # import-linter + # mypy # pylint # pyproject-hooks # pytest @@ -456,14 +480,36 @@ tox-battery==0.6.1 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/quality.txt +types-pytz==2023.3.0.1 + # via + # -r requirements/quality.txt + # django-stubs +types-pyyaml==6.0.12.11 + # via + # -r requirements/quality.txt + # django-stubs + # djangorestframework-stubs +types-requests==2.31.0.2 + # via + # -r requirements/quality.txt + # djangorestframework-stubs +types-urllib3==1.26.25.14 + # via + # -r requirements/quality.txt + # types-requests typing-extensions==4.6.3 # via # -r requirements/ci.txt # -r requirements/quality.txt # asgiref # astroid + # django-stubs + # django-stubs-ext + # djangorestframework-stubs # grimp # import-linter + # mypy + # pylint tzdata==2023.3 # via # -r requirements/quality.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index 06ae86ff7..35ac24a0a 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # make upgrade @@ -97,6 +97,14 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.1.0 # via -r requirements/test.txt +django-stubs==4.2.3 + # via + # -r requirements/test.txt + # djangorestframework-stubs +django-stubs-ext==4.2.2 + # via + # -r requirements/test.txt + # django-stubs django-waffle==4.0.0 # via # -r requirements/test.txt @@ -107,6 +115,8 @@ djangorestframework==3.14.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions +djangorestframework-stubs==3.14.2 + # via -r requirements/test.txt doc8==1.1.1 # via -r requirements/doc.in docutils==0.19 @@ -165,6 +175,15 @@ markupsafe==2.1.3 # jinja2 mock==5.0.2 # via -r requirements/test.txt +mypy==1.5.1 + # via + # -r requirements/test.txt + # django-stubs + # djangorestframework-stubs +mypy-extensions==1.0.0 + # via + # -r requirements/test.txt + # mypy mysqlclient==2.1.1 # via -r requirements/test.txt newrelic==8.9.0 @@ -253,6 +272,7 @@ readme-renderer==40.0 requests==2.31.0 # via # -r requirements/test.txt + # djangorestframework-stubs # edx-drf-extensions # sphinx restructuredtext-lint==1.4.0 @@ -315,15 +335,38 @@ tomli==2.0.1 # via # -r requirements/test.txt # coverage + # django-stubs # doc8 # import-linter + # mypy # pytest +types-pytz==2023.3.0.1 + # via + # -r requirements/test.txt + # django-stubs +types-pyyaml==6.0.12.11 + # via + # -r requirements/test.txt + # django-stubs + # djangorestframework-stubs +types-requests==2.31.0.2 + # via + # -r requirements/test.txt + # djangorestframework-stubs +types-urllib3==1.26.25.14 + # via + # -r requirements/test.txt + # types-requests typing-extensions==4.6.3 # via # -r requirements/test.txt # asgiref + # django-stubs + # django-stubs-ext + # djangorestframework-stubs # grimp # import-linter + # mypy # pydata-sphinx-theme tzdata==2023.3 # via diff --git a/requirements/quality.txt b/requirements/quality.txt index 22fb452e9..a6a9044b3 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # make upgrade @@ -88,6 +88,8 @@ django==3.2.19 # -r requirements/test.txt # django-crum # django-debug-toolbar + # django-stubs + # django-stubs-ext # django-waffle # djangorestframework # drf-jwt @@ -99,6 +101,14 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.1.0 # via -r requirements/test.txt +django-stubs==4.2.3 + # via + # -r requirements/test.txt + # djangorestframework-stubs +django-stubs-ext==4.2.2 + # via + # -r requirements/test.txt + # django-stubs django-waffle==4.0.0 # via # -r requirements/test.txt @@ -109,6 +119,8 @@ djangorestframework==3.14.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions +djangorestframework-stubs==3.14.2 + # via -r requirements/test.txt docutils==0.20.1 # via readme-renderer drf-jwt==1.19.2 @@ -185,6 +197,15 @@ mock==5.0.2 # via -r requirements/test.txt more-itertools==9.1.0 # via jaraco-classes +mypy==1.5.1 + # via + # -r requirements/test.txt + # django-stubs + # djangorestframework-stubs +mypy-extensions==1.0.0 + # via + # -r requirements/test.txt + # mypy mysqlclient==2.1.1 # via -r requirements/test.txt newrelic==8.9.0 @@ -286,6 +307,7 @@ readme-renderer==40.0 requests==2.31.0 # via # -r requirements/test.txt + # djangorestframework-stubs # edx-drf-extensions # requests-toolbelt # twine @@ -331,20 +353,44 @@ tomli==2.0.1 # via # -r requirements/test.txt # coverage + # django-stubs # import-linter + # mypy # pylint # pytest tomlkit==0.11.8 # via pylint twine==4.0.2 # via -r requirements/quality.in +types-pytz==2023.3.0.1 + # via + # -r requirements/test.txt + # django-stubs +types-pyyaml==6.0.12.11 + # via + # -r requirements/test.txt + # django-stubs + # djangorestframework-stubs +types-requests==2.31.0.2 + # via + # -r requirements/test.txt + # djangorestframework-stubs +types-urllib3==1.26.25.14 + # via + # -r requirements/test.txt + # types-requests typing-extensions==4.6.3 # via # -r requirements/test.txt # asgiref # astroid + # django-stubs + # django-stubs-ext + # djangorestframework-stubs # grimp # import-linter + # mypy + # pylint tzdata==2023.3 # via # -r requirements/test.txt diff --git a/requirements/test.in b/requirements/test.in index 50feee663..0d3126aef 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -14,4 +14,7 @@ pytest-django # pytest extension for better Django support code-annotations # provides commands used by the pii_check make target. ddt # supports data driven tests mock # supports overriding classes and methods in tests +mypy # static type checking +django-stubs # Typing stubs for Django, so it works with mypy +djangorestframework-stubs # Typing stubs for DRF django-debug-toolbar # provides a debug toolbar for Django diff --git a/requirements/test.txt b/requirements/test.txt index ba50193f0..51275c8f8 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # make upgrade @@ -72,6 +72,8 @@ ddt==1.6.0 # -r requirements/base.txt # django-crum # django-debug-toolbar + # django-stubs + # django-stubs-ext # django-waffle # djangorestframework # drf-jwt @@ -83,6 +85,12 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.1.0 # via -r requirements/test.in +django-stubs==4.2.3 + # via + # -r requirements/test.in + # djangorestframework-stubs +django-stubs-ext==4.2.2 + # via django-stubs django-waffle==4.0.0 # via # -r requirements/base.txt @@ -93,6 +101,8 @@ djangorestframework==3.14.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions +djangorestframework-stubs==3.14.2 + # via -r requirements/test.in drf-jwt==1.19.2 # via # -r requirements/base.txt @@ -129,6 +139,13 @@ markupsafe==2.1.3 # via jinja2 mock==5.0.2 # via -r requirements/test.in +mypy==1.5.1 + # via + # -r requirements/test.in + # django-stubs + # djangorestframework-stubs +mypy-extensions==1.0.0 + # via mypy mysqlclient==2.1.1 # via -r requirements/test.in newrelic==8.9.0 @@ -194,6 +211,7 @@ pyyaml==6.0 requests==2.31.0 # via # -r requirements/base.txt + # djangorestframework-stubs # edx-drf-extensions rules==3.3 # via -r requirements/base.txt @@ -222,14 +240,30 @@ text-unidecode==1.3 tomli==2.0.1 # via # coverage + # django-stubs # import-linter + # mypy # pytest +types-pytz==2023.3.0.1 + # via django-stubs +types-pyyaml==6.0.12.11 + # via + # django-stubs + # djangorestframework-stubs +types-requests==2.31.0.2 + # via djangorestframework-stubs +types-urllib3==1.26.25.14 + # via types-requests typing-extensions==4.6.3 # via # -r requirements/base.txt # asgiref + # django-stubs + # django-stubs-ext + # djangorestframework-stubs # grimp # import-linter + # mypy tzdata==2023.3 # via # -r requirements/base.txt diff --git a/tests/__init__.py b/tests/__init__.py index e69de29bb..afb65fef5 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1,3 @@ +""" +Tests for Learning Core. +""" diff --git a/tests/openedx_learning/core/components/test_models.py b/tests/openedx_learning/core/components/test_models.py index f41df14d0..54af3de19 100644 --- a/tests/openedx_learning/core/components/test_models.py +++ b/tests/openedx_learning/core/components/test_models.py @@ -1,28 +1,30 @@ +""" +Tests related to the Component models +""" from datetime import datetime, timezone from django.test.testcases import TestCase -from openedx_learning.core.publishing.api import ( - create_learning_package, - publish_all_drafts, -) from openedx_learning.core.components.api import create_component_and_version +from openedx_learning.core.publishing.api import LearningPackage, create_learning_package, publish_all_drafts class TestModelVersioningQueries(TestCase): """ Test that Component/ComponentVersion are registered with the publishing app. """ + learning_package: LearningPackage + now: datetime @classmethod - def setUpTestData(cls): + def setUpTestData(cls) -> None: # Note: we must specify '-> None' to opt in to type checking cls.learning_package = create_learning_package( "components.TestVersioning", "Learning Package for Testing Component Versioning", ) cls.now = datetime(2023, 5, 8, tzinfo=timezone.utc) - def test_latest_version(self): + def test_latest_version(self) -> None: component, component_version = create_component_and_version( learning_package_id=self.learning_package.id, namespace="xblock.v1", diff --git a/tests/openedx_learning/core/publishing/test_api.py b/tests/openedx_learning/core/publishing/test_api.py index ebfdf05e1..2a4fdced3 100644 --- a/tests/openedx_learning/core/publishing/test_api.py +++ b/tests/openedx_learning/core/publishing/test_api.py @@ -1,16 +1,24 @@ +""" +Tests of the Publishing app's python API +""" from datetime import datetime, timezone from uuid import UUID +import pytest from django.core.exceptions import ValidationError from django.test import TestCase -import pytest from openedx_learning.core.publishing.api import create_learning_package class CreateLearningPackageTestCase(TestCase): - def test_normal(self): - """Normal flow with no errors.""" + """ + Test creating a LearningPackage + """ + def test_normal(self) -> None: # Note: we must specify '-> None' to opt in to type checking + """ + Normal flow with no errors. + """ key = "my_key" title = "My Excellent Title with Emoji 🔥" created = datetime(2023, 4, 2, 15, 9, 0, tzinfo=timezone.utc) @@ -27,8 +35,10 @@ def test_normal(self): # Having an actual value here means we were persisted to the database. assert isinstance(package.id, int) - def test_auto_datetime(self): - """Auto-generated created datetime works as expected.""" + def test_auto_datetime(self) -> None: + """ + Auto-generated created datetime works as expected. + """ key = "my_key" title = "My Excellent Title with Emoji 🔥" package = create_learning_package(key, title) @@ -47,8 +57,10 @@ def test_auto_datetime(self): # Having an actual value here means we were persisted to the database. assert isinstance(package.id, int) - def test_non_utc_time(self): - """Require UTC timezone for created.""" + def test_non_utc_time(self) -> None: + """ + Require UTC timezone for created. + """ with pytest.raises(ValidationError) as excinfo: create_learning_package("my_key", "A Title", datetime(2023, 4, 2)) message_dict = excinfo.value.message_dict @@ -57,8 +69,10 @@ def test_non_utc_time(self): assert "created" in message_dict assert "updated" in message_dict - def test_already_exists(self): - """Raises ValidationError for duplicate keys.""" + def test_already_exists(self) -> None: + """ + Raises ValidationError for duplicate keys. + """ create_learning_package("my_key", "Original") with pytest.raises(ValidationError) as excinfo: create_learning_package("my_key", "Duplicate") diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index 3593a095c..b3c070035 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -205,7 +205,7 @@ pk: 1 fields: name: Life on Earth - description: null + description: A taxonomy about life on earth. enabled: true required: false allow_multiple: false @@ -234,7 +234,7 @@ pk: 5 fields: name: Import Taxonomy Test - description: null + description: "" enabled: true required: false allow_multiple: false diff --git a/tests/openedx_tagging/core/tagging/import_export/test_actions.py b/tests/openedx_tagging/core/tagging/import_export/test_actions.py index 1c0b14852..22c85b2a7 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_actions.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_actions.py @@ -1,20 +1,22 @@ """ Tests for actions """ -import ddt +from __future__ import annotations +import ddt # type: ignore[import] from django.test.testcases import TestCase -from openedx_tagging.core.tagging.models import Tag -from openedx_tagging.core.tagging.import_export.import_plan import TagItem from openedx_tagging.core.tagging.import_export.actions import ( - ImportAction, CreateTag, - UpdateParentTag, - RenameTag, DeleteTag, + ImportAction, + RenameTag, + UpdateParentTag, WithoutChanges, ) +from openedx_tagging.core.tagging.import_export.import_plan import TagItem +from openedx_tagging.core.tagging.models import Tag + from .mixins import TestImportExportMixin @@ -22,7 +24,9 @@ class TestImportActionMixin(TestImportExportMixin): """ Mixin for import action tests """ - def setUp(self): + indexed_actions: dict[str, list[ImportAction]] + + def setUp(self) -> None: # Note: we must specify '-> None' to opt in to type checking super().setUp() self.indexed_actions = { 'create': [ @@ -65,7 +69,7 @@ def test_not_implemented_functions(self): with self.assertRaises(NotImplementedError): action.execute() - def test_str(self): + def test_str(self) -> None: expected = "Action import_action (index=100,id=tag_1)" action = ImportAction( taxonomy=self.taxonomy, @@ -103,14 +107,14 @@ def test_search_action(self, action_name, attr, search_value, expected): ('tag_100', False), ) @ddt.unpack - def test_validate_parent(self, parent_id, expected): + def test_validate_parent(self, parent_id: str, expected: bool): action = ImportAction( self.taxonomy, TagItem( id='tag_110', value='_', parent_id=parent_id, - index=100 + index=100, ), index=100, ) @@ -152,7 +156,7 @@ def test_validate_parent(self, parent_id, expected): ('Tag 20', None) ) @ddt.unpack - def test_validate_value(self, value, expected): + def test_validate_value(self, value: str, expected: str | None): action = ImportAction( self.taxonomy, TagItem( @@ -180,7 +184,7 @@ class TestCreateTag(TestImportActionMixin, TestCase): ('tag_100', True), ) @ddt.unpack - def test_applies_for(self, tag_id, expected): + def test_applies_for(self, tag_id: str, expected: bool): result = CreateTag.applies_for( self.taxonomy, TagItem( @@ -196,7 +200,7 @@ def test_applies_for(self, tag_id, expected): ('tag_100', True), ) @ddt.unpack - def test_validate_id(self, tag_id, expected): + def test_validate_id(self, tag_id: str, expected: bool): action = CreateTag( taxonomy=self.taxonomy, tag=TagItem( @@ -227,7 +231,7 @@ def test_validate_id(self, tag_id, expected): ('tag_20', "Tag 20", 'tag_1', 0), # Valid ) @ddt.unpack - def test_validate(self, tag_id, tag_value, parent_id, expected): + def test_validate(self, tag_id: str, tag_value: str, parent_id: str | None, expected: bool): action = CreateTag( taxonomy=self.taxonomy, tag=TagItem( @@ -242,11 +246,11 @@ def test_validate(self, tag_id, tag_value, parent_id, expected): self.assertEqual(len(errors), expected) @ddt.data( - ('tag_30', 'Tag 30', None), # No parent - ('tag_31', 'Tag 31', 'tag_3'), # With parent + ('tag_30', 'Tag 30', None), # No parent + ('tag_31', 'Tag 31', 'tag_3'), # With parent ) @ddt.unpack - def test_execute(self, tag_id, value, parent_id): + def test_execute(self, tag_id: str, value: str, parent_id: str | None): tag = TagItem( id=tag_id, value=value, @@ -260,12 +264,12 @@ def test_execute(self, tag_id, value, parent_id): with self.assertRaises(Tag.DoesNotExist): self.taxonomy.tag_set.get(external_id=tag_id) action.execute() - tag = self.taxonomy.tag_set.get(external_id=tag_id) - assert tag.value == value + tag_obj = self.taxonomy.tag_set.get(external_id=tag_id) + assert tag_obj.value == value if parent_id: - assert tag.parent.external_id == parent_id + assert tag_obj.parent.external_id == parent_id else: - assert tag.parent is None + assert tag_obj.parent is None @ddt.ddt @@ -293,7 +297,7 @@ class TestUpdateParentTag(TestImportActionMixin, TestCase): ), ) @ddt.unpack - def test_str(self, tag_id, parent_id, expected): + def test_str(self, tag_id: str, parent_id: str, expected: str): tag_item = TagItem( id=tag_id, value='_', @@ -311,10 +315,10 @@ def test_str(self, tag_id, parent_id, expected): ('tag_2', 'tag_1', False), # Parent don't change ('tag_2', 'tag_3', True), # Valid ('tag_1', None, False), # Both parent id are None - ('tag_1', 'tag_3', True), # Valid + ('tag_1', 'tag_3', True), # Valid ) @ddt.unpack - def test_applies_for(self, tag_id, parent_id, expected): + def test_applies_for(self, tag_id: str, parent_id: str | None, expected: bool): result = UpdateParentTag.applies_for( taxonomy=self.taxonomy, tag=TagItem( @@ -332,7 +336,7 @@ def test_applies_for(self, tag_id, parent_id, expected): ('tag_2', 'tag_10', 0), # Valid ) @ddt.unpack - def test_validate(self, tag_id, parent_id, expected): + def test_validate(self, tag_id: str, parent_id: str | None, expected: int): action = UpdateParentTag( taxonomy=self.taxonomy, tag=TagItem( @@ -385,7 +389,7 @@ class TestRenameTag(TestImportActionMixin, TestCase): ('tag_1', 'Tag 1 v2', True), # Valid ) @ddt.unpack - def test_applies_for(self, tag_id, value, expected): + def test_applies_for(self, tag_id: str, value: str, expected: bool): result = RenameTag.applies_for( taxonomy=self.taxonomy, tag=TagItem( @@ -403,7 +407,7 @@ def test_applies_for(self, tag_id, value, expected): ('Tag 12', 0), # Valid ) @ddt.unpack - def test_validate(self, value, expected): + def test_validate(self, value: str, expected: int): action = RenameTag( taxonomy=self.taxonomy, tag=TagItem( @@ -416,7 +420,7 @@ def test_validate(self, value, expected): errors = action.validate(self.indexed_actions) self.assertEqual(len(errors), expected) - def test_execute(self): + def test_execute(self) -> None: tag_id = 'tag_1' value = 'Tag 1 V2' tag_item = TagItem( @@ -440,10 +444,10 @@ class TestDeleteTag(TestImportActionMixin, TestCase): Test for 'delete' action """ - def test_applies_for(self): + def test_applies_for(self) -> None: assert not DeleteTag.applies_for(self.taxonomy, None) - def test_validate(self): + def test_validate(self) -> None: action = DeleteTag( taxonomy=self.taxonomy, tag=TagItem( @@ -455,7 +459,7 @@ def test_validate(self): ) assert not action.validate(self.indexed_actions) - def test_execute(self): + def test_execute(self) -> None: tag_id = 'tag_3' tag_item = TagItem( id=tag_id, @@ -475,7 +479,7 @@ class TestWithoutChanges(TestImportActionMixin, TestCase): """ Test for 'without_changes' action """ - def test_applies_for(self): + def test_applies_for(self) -> None: result = WithoutChanges.applies_for( self.taxonomy, tag=TagItem( @@ -486,7 +490,7 @@ def test_applies_for(self): ) self.assertFalse(result) - def test_validate(self): + def test_validate(self) -> None: action = WithoutChanges( taxonomy=self.taxonomy, tag=TagItem( diff --git a/tests/openedx_tagging/core/tagging/import_export/test_api.py b/tests/openedx_tagging/core/tagging/import_export/test_api.py index 453979e01..3fa1d46cf 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_api.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_api.py @@ -6,14 +6,9 @@ from django.test.testcases import TestCase -from openedx_tagging.core.tagging.models import ( - TagImportTask, - TagImportTaskState, - Taxonomy, - LanguageTaxonomy, -) -from openedx_tagging.core.tagging.import_export import ParserFormat import openedx_tagging.core.tagging.import_export.api as import_export_api +from openedx_tagging.core.tagging.import_export import ParserFormat +from openedx_tagging.core.tagging.models import LanguageTaxonomy, TagImportTask, TagImportTaskState, Taxonomy from .mixins import TestImportExportMixin @@ -23,7 +18,7 @@ class TestImportExportApi(TestImportExportMixin, TestCase): Test import/export API functions """ - def setUp(self): + def setUp(self) -> None: self.tags = [ {"id": "tag_31", "value": "Tag 31"}, {"id": "tag_32", "value": "Tag 32"}, @@ -39,8 +34,8 @@ def setUp(self): ]} self.invalid_parser_file = BytesIO(json.dumps(json_data).encode()) json_data = {"tags": [ - {'id': 'tag_31', 'value': 'Tag 31',}, - {'id': 'tag_31', 'value': 'Tag 32',}, + {'id': 'tag_31', 'value': 'Tag 31'}, + {'id': 'tag_31', 'value': 'Tag 32'}, ]} self.invalid_plan_file = BytesIO(json.dumps(json_data).encode()) @@ -57,17 +52,17 @@ def setUp(self): self.system_taxonomy = self.system_taxonomy.cast() return super().setUp() - def test_check_status(self): + def test_check_status(self) -> None: TagImportTask.create(self.taxonomy) status = import_export_api.get_last_import_status(self.taxonomy) - assert status == TagImportTaskState.LOADING_DATA.value + assert status == TagImportTaskState.LOADING_DATA - def test_check_log(self): + def test_check_log(self) -> None: TagImportTask.create(self.taxonomy) log = import_export_api.get_last_import_log(self.taxonomy) assert "Import task created" in log - def test_invalid_import_tags(self): + def test_invalid_import_tags(self) -> None: TagImportTask.create(self.taxonomy) with self.assertRaises(ValueError): # Raise error if there is a current in progress task @@ -77,7 +72,7 @@ def test_invalid_import_tags(self): self.parser_format, ) - def test_import_export_validations(self): + def test_import_export_validations(self) -> None: # Check that import is invalid with open taxonomy with self.assertRaises(NotImplementedError): import_export_api.import_tags( @@ -94,7 +89,7 @@ def test_import_export_validations(self): self.parser_format, ) - def test_with_python_error(self): + def test_with_python_error(self) -> None: self.file.close() assert not import_export_api.import_tags( self.taxonomy, @@ -103,10 +98,10 @@ def test_with_python_error(self): ) status = import_export_api.get_last_import_status(self.taxonomy) log = import_export_api.get_last_import_log(self.taxonomy) - assert status == TagImportTaskState.ERROR.value + assert status == TagImportTaskState.ERROR assert "ValueError('I/O operation on closed file.')" in log - def test_with_parser_error(self): + def test_with_parser_error(self) -> None: assert not import_export_api.import_tags( self.taxonomy, self.invalid_parser_file, @@ -114,11 +109,11 @@ def test_with_parser_error(self): ) status = import_export_api.get_last_import_status(self.taxonomy) log = import_export_api.get_last_import_log(self.taxonomy) - assert status == TagImportTaskState.ERROR.value + assert status == TagImportTaskState.ERROR assert "Starting to load data from file" in log assert "Invalid '.json' format" in log - def test_with_plan_errors(self): + def test_with_plan_errors(self) -> None: assert not import_export_api.import_tags( self.taxonomy, self.invalid_plan_file, @@ -126,14 +121,14 @@ def test_with_plan_errors(self): ) status = import_export_api.get_last_import_status(self.taxonomy) log = import_export_api.get_last_import_log(self.taxonomy) - assert status == TagImportTaskState.ERROR.value + assert status == TagImportTaskState.ERROR assert "Starting to load data from file" in log assert "Load data finished" in log assert "Starting plan actions" in log assert "Plan finished" in log assert "Conflict with 'create'" in log - def test_valid(self): + def test_valid(self) -> None: assert import_export_api.import_tags( self.taxonomy, self.file, @@ -142,7 +137,7 @@ def test_valid(self): ) status = import_export_api.get_last_import_status(self.taxonomy) log = import_export_api.get_last_import_log(self.taxonomy) - assert status == TagImportTaskState.SUCCESS.value + assert status == TagImportTaskState.SUCCESS assert "Starting to load data from file" in log assert "Load data finished" in log assert "Starting plan actions" in log @@ -150,7 +145,7 @@ def test_valid(self): assert "Starting execute actions" in log assert "Execution finished" in log - def test_start_task_after_error(self): + def test_start_task_after_error(self) -> None: assert not import_export_api.import_tags( self.taxonomy, self.invalid_parser_file, @@ -162,7 +157,7 @@ def test_start_task_after_error(self): self.parser_format, ) - def test_start_task_after_success(self): + def test_start_task_after_success(self) -> None: assert import_export_api.import_tags( self.taxonomy, self.file, @@ -179,7 +174,7 @@ def test_start_task_after_success(self): self.parser_format, ) - def test_export_validations(self): + def test_export_validations(self) -> None: # Check that import is invalid with open taxonomy with self.assertRaises(NotImplementedError): import_export_api.export_tags( @@ -194,7 +189,7 @@ def test_export_validations(self): self.parser_format, ) - def test_import_with_export_output(self): + def test_import_with_export_output(self) -> None: for parser_format in ParserFormat: output = import_export_api.export_tags( self.taxonomy, @@ -215,5 +210,5 @@ def test_import_with_export_output(self): new_tag = new_taxonomy.tag_set.get(external_id=tag.external_id) assert new_tag.value == tag.value if tag.parent: + assert new_tag.parent assert tag.parent.external_id == new_tag.parent.external_id - \ No newline at end of file diff --git a/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py index af1e55f30..915ffb724 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py @@ -1,39 +1,39 @@ """ Test for import_plan functions """ -import ddt - +import ddt # type: ignore[import] from django.test.testcases import TestCase -from openedx_tagging.core.tagging.import_export.import_plan import TagItem, TagImportPlan from openedx_tagging.core.tagging.import_export.actions import CreateTag from openedx_tagging.core.tagging.import_export.exceptions import TagImportError +from openedx_tagging.core.tagging.import_export.import_plan import TagImportPlan, TagItem + from .test_actions import TestImportActionMixin + @ddt.ddt class TestTagImportPlan(TestImportActionMixin, TestCase): """ Test for import plan functions """ - def setUp(self): + def setUp(self) -> None: super().setUp() self.import_plan = TagImportPlan(self.taxonomy) - def test_tag_import_error(self): + def test_tag_import_error(self) -> None: message = "Error message" expected_repr = f"TagImportError({message})" error = TagImportError(message) assert str(error) == message assert repr(error) == expected_repr - @ddt.data( - ('tag_10', 1), # Test invalid - ('tag_30', 0), # Test valid + ('tag_10', 1), # Test invalid + ('tag_30', 0), # Test valid ) @ddt.unpack - def test_build_action(self, tag_id, errors_expected): + def test_build_action(self, tag_id: str, errors_expected: int): self.import_plan.indexed_actions = self.indexed_actions self.import_plan._build_action( # pylint: disable=protected-access CreateTag, @@ -48,7 +48,7 @@ def test_build_action(self, tag_id, errors_expected): assert self.import_plan.actions[0].name == 'create' assert self.import_plan.indexed_actions['create'][1].tag.id == tag_id - def test_build_delete_actions(self): + def test_build_delete_actions(self) -> None: tags = { tag.external_id: tag for tag in self.taxonomy.tag_set.exclude(pk=25) @@ -79,133 +79,142 @@ def test_build_delete_actions(self): assert self.import_plan.actions[4].tag.id == 'tag_3' @ddt.data( - ([ - { - 'id': 'tag_31', - 'value': 'Tag 31', - }, - { - 'id': 'tag_32', - 'value': 'Tag 32', - 'parent_id': 'tag_1', - }, - { - 'id': 'tag_2', - 'value': 'Tag 2 v2', - 'parent_id': 'tag_1' - }, - { - 'id': 'tag_4', - 'value': 'Tag 4 v2', - 'parent_id': 'tag_1', - }, - { - 'id': 'tag_1', - 'value': 'Tag 1', - }, - ], - False, - 0, - [ - { - 'name': 'create', - 'id': 'tag_31' - }, - { - 'name': 'create', - 'id': 'tag_32' - }, - { - 'name': 'rename', - 'id': 'tag_2' - }, - { - 'name': 'update_parent', - 'id': 'tag_4' - }, - { - 'name': 'rename', - 'id': 'tag_4' - }, - { - 'name': 'without_changes', - 'id': 'tag_1' - }, - ]), # Test valid actions - ([ - { - 'id': 'tag_31', - 'value': 'Tag 31', - }, - { - 'id': 'tag_31', - 'value': 'Tag 32', - }, - { - 'id': 'tag_1', - 'value': 'Tag 2', - }, - { - 'id': 'tag_4', - 'value': 'Tag 4', - 'parent_id': 'tag_100', - }, - ], - False, - 3, - [ - { - 'name': 'create', - 'id': 'tag_31', - }, - { - 'name': 'create', - 'id': 'tag_31', - }, - { - 'name': 'rename', - 'id': 'tag_1', - }, - { - 'name': 'update_parent', - 'id': 'tag_4', - } - ]), # Test with errors in actions - ([ - { - 'id': 'tag_4', - 'value': 'Tag 4', - 'parent_id': 'tag_3', - }, - ], - True, - 0, - [ - { - 'name': 'without_changes', - 'id': 'tag_4', - }, - { - 'name': 'update_parent', - 'id': 'tag_2', - }, - { - 'name': 'delete', - 'id': 'tag_1', - }, - { - 'name': 'delete', - 'id': 'tag_2', - }, - { - 'name': 'update_parent', - 'id': 'tag_4', - }, - { - 'name': 'delete', - 'id': 'tag_3', - }, - ]) # Test with deletes (replace=True) + # Test valid actions + ( + [ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], + False, + 0, + [ + { + 'name': 'create', + 'id': 'tag_31' + }, + { + 'name': 'create', + 'id': 'tag_32' + }, + { + 'name': 'rename', + 'id': 'tag_2' + }, + { + 'name': 'update_parent', + 'id': 'tag_4' + }, + { + 'name': 'rename', + 'id': 'tag_4' + }, + { + 'name': 'without_changes', + 'id': 'tag_1' + }, + ] + ), + # Test with errors in actions + ( + [ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_31', + 'value': 'Tag 32', + }, + { + 'id': 'tag_1', + 'value': 'Tag 2', + }, + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_100', + }, + ], + False, + 3, + [ + { + 'name': 'create', + 'id': 'tag_31', + }, + { + 'name': 'create', + 'id': 'tag_31', + }, + { + 'name': 'rename', + 'id': 'tag_1', + }, + { + 'name': 'update_parent', + 'id': 'tag_4', + } + ] + ), + # Test with deletes (replace=True) + ( + [ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], + True, + 0, + [ + { + 'name': 'without_changes', + 'id': 'tag_4', + }, + { + 'name': 'update_parent', + 'id': 'tag_2', + }, + { + 'name': 'delete', + 'id': 'tag_1', + }, + { + 'name': 'delete', + 'id': 'tag_2', + }, + { + 'name': 'update_parent', + 'id': 'tag_4', + }, + { + 'name': 'delete', + 'id': 'tag_3', + }, + ] + ) ) @ddt.unpack def test_generate_actions(self, tags, replace, expected_errors, expected_actions): @@ -220,109 +229,115 @@ def test_generate_actions(self, tags, replace, expected_errors, expected_actions assert self.import_plan.actions[index].index == index + 1 @ddt.data( - ([ - { - 'id': 'tag_31', - 'value': 'Tag 31', - }, - { - 'id': 'tag_31', - 'value': 'Tag 32', - }, - { - 'id': 'tag_1', - 'value': 'Tag 2', - }, - { - 'id': 'tag_4', - 'value': 'Tag 4', - 'parent_id': 'tag_100', - }, - { - 'id': 'tag_33', - 'value': 'Tag 32', - }, - { - 'id': 'tag_2', - 'value': 'Tag 31', - }, - ], - False, - "Import plan for Import Taxonomy Test\n" - "--------------------------------\n" - "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" - "#2: Create a new tag with values (external_id=tag_31, value=Tag 32, parent_id=None).\n" - "#3: Rename tag value of tag (external_id=tag_1) from 'Tag 1' to 'Tag 2'\n" - "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " - "to parent (external_id=tag_100).\n" - "#5: Create a new tag with values (external_id=tag_33, value=Tag 32, parent_id=None).\n" - "#6: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " - "to parent (external_id=None).\n" - "#7: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 31'\n" - "\nOutput errors\n" - "--------------------------------\n" - "Conflict with 'create' (#2) and action #1: Duplicated external_id tag.\n" - "Action error in 'rename' (#3): Duplicated tag value with tag in database (external_id=tag_2).\n" - "Action error in 'update_parent' (#4): Unknown parent tag (tag_100). " - "You need to add parent before the child in your file.\n" - "Conflict with 'create' (#5) and action #2: Duplicated tag value.\n" - "Conflict with 'rename' (#7) and action #1: Duplicated tag value.\n" - ), # Testing plan with errors - ([ - { - 'id': 'tag_31', - 'value': 'Tag 31', - }, - { - 'id': 'tag_32', - 'value': 'Tag 32', - 'parent_id': 'tag_1', - }, - { - 'id': 'tag_2', - 'value': 'Tag 2 v2', - 'parent_id': 'tag_1' - }, - { - 'id': 'tag_4', - 'value': 'Tag 4 v2', - 'parent_id': 'tag_1', - }, - { - 'id': 'tag_1', - 'value': 'Tag 1', - }, - ], - False, - "Import plan for Import Taxonomy Test\n" - "--------------------------------\n" - "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" - "#2: Create a new tag with values (external_id=tag_32, value=Tag 32, parent_id=tag_1).\n" - "#3: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 2 v2'\n" - "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " - "to parent (external_id=tag_1).\n" - "#5: Rename tag value of tag (external_id=tag_4) from 'Tag 4' to 'Tag 4 v2'\n" - "#6: No changes needed for tag (external_id=tag_1)\n" - ), # Testing valid plan - ([ - { - 'id': 'tag_4', - 'value': 'Tag 4', - 'parent_id': 'tag_3', - }, - ], - True, - "Import plan for Import Taxonomy Test\n" - "--------------------------------\n" - "#1: No changes needed for tag (external_id=tag_4)\n" - "#2: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " - "to parent (external_id=None).\n" - "#3: Delete tag (external_id=tag_1)\n" - "#4: Delete tag (external_id=tag_2)\n" - "#5: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " - "to parent (external_id=None).\n" - "#6: Delete tag (external_id=tag_3)\n" - ) # Testing deletes (replace=True) + # Testing plan with errors + ( + [ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_31', + 'value': 'Tag 32', + }, + { + 'id': 'tag_1', + 'value': 'Tag 2', + }, + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_100', + }, + { + 'id': 'tag_33', + 'value': 'Tag 32', + }, + { + 'id': 'tag_2', + 'value': 'Tag 31', + }, + ], + False, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" + "#2: Create a new tag with values (external_id=tag_31, value=Tag 32, parent_id=None).\n" + "#3: Rename tag value of tag (external_id=tag_1) from 'Tag 1' to 'Tag 2'\n" + "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=tag_100).\n" + "#5: Create a new tag with values (external_id=tag_33, value=Tag 32, parent_id=None).\n" + "#6: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " + "to parent (external_id=None).\n" + "#7: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 31'\n" + "\nOutput errors\n" + "--------------------------------\n" + "Conflict with 'create' (#2) and action #1: Duplicated external_id tag.\n" + "Action error in 'rename' (#3): Duplicated tag value with tag in database (external_id=tag_2).\n" + "Action error in 'update_parent' (#4): Unknown parent tag (tag_100). " + "You need to add parent before the child in your file.\n" + "Conflict with 'create' (#5) and action #2: Duplicated tag value.\n" + "Conflict with 'rename' (#7) and action #1: Duplicated tag value.\n" + ), + # Testing valid plan + ( + [ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], + False, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: Create a new tag with values (external_id=tag_31, value=Tag 31, parent_id=None).\n" + "#2: Create a new tag with values (external_id=tag_32, value=Tag 32, parent_id=tag_1).\n" + "#3: Rename tag value of tag (external_id=tag_2) from 'Tag 2' to 'Tag 2 v2'\n" + "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=tag_1).\n" + "#5: Rename tag value of tag (external_id=tag_4) from 'Tag 4' to 'Tag 4 v2'\n" + "#6: No changes needed for tag (external_id=tag_1)\n" + ), + # Testing deletes (replace=True) + ( + [ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], + True, + "Import plan for Import Taxonomy Test\n" + "--------------------------------\n" + "#1: No changes needed for tag (external_id=tag_4)\n" + "#2: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " + "to parent (external_id=None).\n" + "#3: Delete tag (external_id=tag_1)\n" + "#4: Delete tag (external_id=tag_2)\n" + "#5: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "to parent (external_id=None).\n" + "#6: Delete tag (external_id=tag_3)\n" + ), ) @ddt.unpack def test_plan(self, tags, replace, expected): @@ -339,38 +354,46 @@ def test_plan(self, tags, replace, expected): assert plan == expected @ddt.data( - ([ - { - 'id': 'tag_31', - 'value': 'Tag 31', - }, - { - 'id': 'tag_32', - 'value': 'Tag 32', - 'parent_id': 'tag_1', - }, - { - 'id': 'tag_2', - 'value': 'Tag 2 v2', - 'parent_id': 'tag_1' - }, - { - 'id': 'tag_4', - 'value': 'Tag 4 v2', - 'parent_id': 'tag_1', - }, - { - 'id': 'tag_1', - 'value': 'Tag 1', - }, - ], False), # Testing all actions - ([ - { - 'id': 'tag_4', - 'value': 'Tag 4', - 'parent_id': 'tag_3', - }, - ], True), # Testing deletes (replace=True) + # Testing all actions + ( + [ + { + 'id': 'tag_31', + 'value': 'Tag 31', + }, + { + 'id': 'tag_32', + 'value': 'Tag 32', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_2', + 'value': 'Tag 2 v2', + 'parent_id': 'tag_1' + }, + { + 'id': 'tag_4', + 'value': 'Tag 4 v2', + 'parent_id': 'tag_1', + }, + { + 'id': 'tag_1', + 'value': 'Tag 1', + }, + ], + False, + ), + # Testing deletes (replace=True) + ( + [ + { + 'id': 'tag_4', + 'value': 'Tag 4', + 'parent_id': 'tag_3', + }, + ], + True, + ), ) @ddt.unpack def test_execute(self, tags, replace): @@ -415,4 +438,3 @@ def test_error_in_execute(self): assert not self.taxonomy.tag_set.filter(external_id=created_tag).exists() assert not self.import_plan.execute() assert not self.taxonomy.tag_set.filter(external_id=created_tag).exists() - \ No newline at end of file diff --git a/tests/openedx_tagging/core/tagging/import_export/test_parsers.py b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py index 5b77e3a54..3bcd2667c 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_parsers.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py @@ -1,23 +1,18 @@ """ Test for import/export parsers """ -from io import BytesIO +from __future__ import annotations + import json -import ddt +from io import BytesIO +import ddt # type: ignore[import] from django.test.testcases import TestCase -from openedx_tagging.core.tagging.import_export.parsers import ( - Parser, - get_parser, - JSONParser, - CSVParser, - ParserFormat, -) -from openedx_tagging.core.tagging.import_export.exceptions import ( - TagParserError, -) +from openedx_tagging.core.tagging.import_export.exceptions import TagParserError +from openedx_tagging.core.tagging.import_export.parsers import CSVParser, JSONParser, Parser, ParserFormat, get_parser from openedx_tagging.core.tagging.models import Taxonomy + from .mixins import TestImportExportMixin @@ -26,16 +21,16 @@ class TestParser(TestCase): Test for general parser functions """ - def test_get_parser(self): + def test_get_parser(self) -> None: for parser_format in ParserFormat: parser = get_parser(parser_format) self.assertEqual(parser.format, parser_format) - def test_parser_not_found(self): + def test_parser_not_found(self) -> None: with self.assertRaises(ValueError): - get_parser(None) + get_parser(None) # type: ignore[arg-type] - def test_not_implemented(self): + def test_not_implemented(self) -> None: taxonomy = Taxonomy(name="Test taxonomy") taxonomy.save() with self.assertRaises(NotImplementedError): @@ -43,7 +38,7 @@ def test_not_implemented(self): with self.assertRaises(NotImplementedError): Parser.export(taxonomy) - def test_tag_parser_error(self): + def test_tag_parser_error(self) -> None: tag = {"id": 'tag_id', "value": "tag_value"} expected_str = f"Import parser error on {tag}" expected_repr = f"TagParserError(Import parser error on {tag})" @@ -58,7 +53,7 @@ class TestJSONParser(TestImportExportMixin, TestCase): Test for .json parser """ - def test_invalid_json(self): + def test_invalid_json(self) -> None: json_data = "{This is an invalid json}" json_file = BytesIO(json_data.encode()) tags, errors = JSONParser.parse_import(json_file) @@ -66,7 +61,7 @@ def test_invalid_json(self): assert len(errors) == 1 assert "Expecting property name enclosed in double quotes" in str(errors[0]) - def test_load_data_errors(self): + def test_load_data_errors(self) -> None: json_data = {"invalid": [ {"id": "tag_1", "name": "Tag 1"}, ]} @@ -84,7 +79,7 @@ def test_load_data_errors(self): @ddt.data( ( {"tags": [ - {"id": "tag_1", "value": "Tag 1"}, # Valid + {"id": "tag_1", "value": "Tag 1"}, # Valid ]}, [] ), @@ -105,7 +100,7 @@ def test_load_data_errors(self): {"tags": [ {"id": "", "value": "tag 1"}, {"id": "tag_2", "value": ""}, - {"id": "tag_3", "value": "tag 3", "parent_id": ""}, # Valid + {"id": "tag_3", "value": "tag 3", "parent_id": ""}, # Valid ]}, [ "Empty 'id' field on {'id': '', 'value': 'tag 1'}", @@ -114,7 +109,7 @@ def test_load_data_errors(self): ) ) @ddt.unpack - def test_parse_tags_errors(self, json_data, expected_errors): + def test_parse_tags_errors(self, json_data: dict, expected_errors: list[str]): json_file = BytesIO(json.dumps(json_data).encode()) _, errors = JSONParser.parse_import(json_file) @@ -123,7 +118,7 @@ def test_parse_tags_errors(self, json_data, expected_errors): for error in errors: self.assertIn(str(error), expected_errors) - def test_parse_tags(self): + def test_parse_tags(self) -> None: expected_tags = [ {"id": "tag_1", "value": "tag 1"}, {"id": "tag_2", "value": "tag 2"}, @@ -157,7 +152,7 @@ def test_parse_tags(self): index + JSONParser.inital_row ) - def test_export_data(self): + def test_export_data(self) -> None: result = JSONParser.export(self.taxonomy) tags = json.loads(result).get("tags") assert len(tags) == self.taxonomy.tag_set.count() @@ -167,7 +162,7 @@ def test_export_data(self): if tag.get("parent_id"): assert tag.get("parent_id") == taxonomy_tag.parent.external_id - def test_import_with_export_output(self): + def test_import_with_export_output(self) -> None: output = JSONParser.export(self.taxonomy) json_file = BytesIO(output.encode()) tags, errors = JSONParser.parse_import(json_file) @@ -176,7 +171,6 @@ def test_import_with_export_output(self): self.assertEqual(len(errors), 0) self.assertEqual(len(tags), len(output_tags)) - for tag in tags: output_tag = None for out_tag in output_tags: @@ -218,7 +212,7 @@ class TestCSVParser(TestImportExportMixin, TestCase): ) ) @ddt.unpack - def test_load_data_errors(self, csv_data, expected_errors): + def test_load_data_errors(self, csv_data: str, expected_errors: list[str]): csv_file = BytesIO(csv_data.encode()) tags, errors = CSVParser.parse_import(csv_file) @@ -237,7 +231,7 @@ def test_load_data_errors(self, csv_data, expected_errors): ] ), ( - "id,value\ntag_1,tag 1\n", # Valid + "id,value\ntag_1,tag 1\n", # Valid [] ) ) @@ -263,7 +257,7 @@ def _build_csv(self, tags): ) return csv - def test_parse_tags(self): + def test_parse_tags(self) -> None: expected_tags = [ {"id": "tag_1", "value": "tag 1"}, {"id": "tag_2", "value": "tag 2"}, @@ -296,7 +290,7 @@ def test_parse_tags(self): index + CSVParser.inital_row ) - def test_import_with_export_output(self): + def test_import_with_export_output(self) -> None: output = CSVParser.export(self.taxonomy) csv_file = BytesIO(output.encode()) tags, errors = CSVParser.parse_import(csv_file) diff --git a/tests/openedx_tagging/core/tagging/import_export/test_tasks.py b/tests/openedx_tagging/core/tagging/import_export/test_tasks.py index 4b88fc6d6..c3c3e3764 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_tasks.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_tasks.py @@ -6,8 +6,8 @@ from django.test.testcases import TestCase -from openedx_tagging.core.tagging.import_export import ParserFormat import openedx_tagging.core.tagging.import_export.tasks as import_export_tasks +from openedx_tagging.core.tagging.import_export import ParserFormat from .mixins import TestImportExportMixin diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index ae77d715b..0a19e276e 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -1,7 +1,12 @@ -""" Test the tagging APIs """ -import ddt +""" +Test the tagging APIs +""" +from __future__ import annotations -from django.test.testcases import TestCase, override_settings +from typing import Any + +import ddt # type: ignore[import] +from django.test import TestCase, override_settings import openedx_tagging.core.tagging.api as tagging_api from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy @@ -22,9 +27,8 @@ class TestApiTagging(TestTagTaxonomyMixin, TestCase): """ Test the Tagging API methods. """ - - def test_create_taxonomy(self): - params = { + def test_create_taxonomy(self) -> None: # Note: we must specify '-> None' to opt in to type checking + params: dict[str, Any] = { "name": "Difficulty", "description": "This taxonomy contains tags describing the difficulty of an activity", "enabled": False, @@ -46,13 +50,13 @@ def test_bad_taxonomy_class(self): ) assert " must be a subclass of Taxonomy" in str(exc.exception) - def test_get_taxonomy(self): + def test_get_taxonomy(self) -> None: tax1 = tagging_api.get_taxonomy(1) assert tax1 == self.taxonomy no_tax = tagging_api.get_taxonomy(10) assert no_tax is None - def test_get_taxonomies(self): + def test_get_taxonomies(self) -> None: tax1 = tagging_api.create_taxonomy("Enabled") tax2 = tagging_api.create_taxonomy("Disabled", enabled=False) tax3 = Taxonomy.objects.get(name="Import Taxonomy Test") @@ -90,7 +94,7 @@ def test_get_taxonomies(self): ] @override_settings(LANGUAGES=test_languages) - def test_get_tags(self): + def test_get_tags(self) -> None: self.setup_tag_depths() assert tagging_api.get_tags(self.taxonomy) == [ *self.domain_tags, @@ -103,7 +107,14 @@ def test_get_tags(self): expected_langs = [lang[0] for lang in test_languages] assert langs == expected_langs - def check_object_tag(self, object_tag, taxonomy, tag, name, value): + def check_object_tag( + self, + object_tag: ObjectTag, + taxonomy: Taxonomy | None, + tag: Tag | None, + name: str, + value: str, + ): """ Verifies that the properties of the given object_tag (once refreshed from the database) match those given. """ @@ -113,7 +124,7 @@ def check_object_tag(self, object_tag, taxonomy, tag, name, value): assert object_tag.name == name assert object_tag.value == value - def test_resync_object_tags(self): + def test_resync_object_tags(self) -> None: missing_links = ObjectTag.objects.create( object_id="abc", _name=self.taxonomy.name, @@ -228,7 +239,7 @@ def test_resync_object_tags(self): no_changes, first_taxonomy, None, "Life on Earth", "Anamelia" ) - def test_tag_object(self): + def test_tag_object(self) -> None: self.taxonomy.allow_multiple = True self.taxonomy.save() test_tags = [ @@ -289,7 +300,7 @@ def test_tag_object(self): == 0 ) - def test_tag_object_free_text(self): + def test_tag_object_free_text(self) -> None: self.taxonomy.allow_free_text = True self.taxonomy.save() object_tags = tagging_api.tag_object( @@ -306,7 +317,7 @@ def test_tag_object_free_text(self): assert object_tag.get_lineage() == ["Eukaryota Xenomorph"] assert object_tag.object_id == "biology101" - def test_tag_object_no_multiple(self): + def test_tag_object_no_multiple(self) -> None: with self.assertRaises(ValueError) as exc: tagging_api.tag_object( self.taxonomy, @@ -315,7 +326,7 @@ def test_tag_object_no_multiple(self): ) assert "only allows one tag per object" in str(exc.exception) - def test_tag_object_required(self): + def test_tag_object_required(self) -> None: self.taxonomy.required = True self.taxonomy.save() with self.assertRaises(ValueError) as exc: @@ -326,7 +337,7 @@ def test_tag_object_required(self): ) assert "requires at least one tag per object" in str(exc.exception) - def test_tag_object_invalid_tag(self): + def test_tag_object_invalid_tag(self) -> None: with self.assertRaises(ValueError) as exc: tagging_api.tag_object( self.taxonomy, @@ -338,7 +349,7 @@ def test_tag_object_invalid_tag(self): ) @override_settings(LANGUAGES=test_languages) - def test_tag_object_language_taxonomy(self): + def test_tag_object_language_taxonomy(self) -> None: tags_list = [ [get_tag("Azerbaijani").id], [get_tag("English").id], @@ -371,7 +382,7 @@ def test_tag_object_language_taxonomy(self): assert object_tag.object_id == "biology101" @override_settings(LANGUAGES=test_languages) - def test_tag_object_language_taxonomy_ivalid(self): + def test_tag_object_language_taxonomy_ivalid(self) -> None: tags = [get_tag("Spanish").id] with self.assertRaises(ValueError) as exc: tagging_api.tag_object( @@ -383,7 +394,7 @@ def test_tag_object_language_taxonomy_ivalid(self): exc.exception ) - def test_tag_object_model_system_taxonomy(self): + def test_tag_object_model_system_taxonomy(self) -> None: users = [ self.user_1, self.user_2, @@ -410,6 +421,7 @@ def test_tag_object_model_system_taxonomy(self): # And the expected number of tags were returned assert len(object_tags) == len(tags) for object_tag in object_tags: + assert object_tag.tag assert object_tag.tag.external_id == str(user.id) assert object_tag.tag.value == user.username assert object_tag.is_valid() @@ -417,7 +429,7 @@ def test_tag_object_model_system_taxonomy(self): assert object_tag.name == self.user_taxonomy.name assert object_tag.object_id == "biology101" - def test_tag_object_model_system_taxonomy_invalid(self): + def test_tag_object_model_system_taxonomy_invalid(self) -> None: tags = ["Invalid id"] with self.assertRaises(ValueError) as exc: tagging_api.tag_object( @@ -429,7 +441,7 @@ def test_tag_object_model_system_taxonomy_invalid(self): exc.exception ) - def test_get_object_tags(self): + def test_get_object_tags(self) -> None: # Alpha tag has no taxonomy alpha = ObjectTag(object_id="abc") alpha.name = self.taxonomy.name @@ -462,11 +474,11 @@ def test_get_object_tags(self): ] @ddt.data( - ("ChA", ["Archaea", "Archaebacteria"], [2,5]), - ("ar", ['Archaea', 'Archaebacteria', 'Arthropoda'], [2,5,14]), - ("aE", ['Archaea', 'Archaebacteria', 'Plantae'], [2,5,10]), + ("ChA", ["Archaea", "Archaebacteria"], [2, 5]), + ("ar", ['Archaea', 'Archaebacteria', 'Arthropoda'], [2, 5, 14]), + ("aE", ['Archaea', 'Archaebacteria', 'Plantae'], [2, 5, 10]), ( - "a", + "a", [ 'Animalia', 'Archaea', @@ -477,11 +489,11 @@ def test_get_object_tags(self): 'Placozoa', 'Plantae', ], - [9,2,5,14,16,13,19,10], + [9, 2, 5, 14, 16, 13, 19, 10], ), ) @ddt.unpack - def test_autocomplete_tags(self, search, expected_values, expected_ids): + def test_autocomplete_tags(self, search: str, expected_values: list[str], expected_ids: list[int | None]): tags = [ 'Archaea', 'Archaebacteria', @@ -531,17 +543,17 @@ def test_autocomplete_tags(self, search, expected_values, expected_ids): expected_ids, ) - def test_autocompleate_not_implemented(self): + def test_autocompleate_not_implemented(self) -> None: with self.assertRaises(NotImplementedError): tagging_api.autocomplete_tags(self.taxonomy, 'test', None, object_tags_only=False) - def _get_tag_values(self, tags): + def _get_tag_values(self, tags) -> list[str]: """ Get tag values from tagging_api.autocomplete_tags() result """ return [tag.get("value") for tag in tags] - def _get_tag_ids(self, tags): + def _get_tag_ids(self, tags) -> list[int]: """ Get tag ids from tagging_api.autocomplete_tags() result """ @@ -549,11 +561,11 @@ def _get_tag_ids(self, tags): def _validate_autocomplete_tags( self, - taxonomy, - search, - expected_values, - expected_ids, - ): + taxonomy: Taxonomy, + search: str, + expected_values: list[str], + expected_ids: list[int | None], + ) -> None: """ Validate autocomplete tags """ diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index a30093bd4..4b6f644aa 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -1,15 +1,12 @@ -""" Test the tagging base models """ -import ddt +""" +Test the tagging base models +""" +import ddt # type: ignore[import] from django.contrib.auth import get_user_model -from django.test.testcases import TestCase from django.db.utils import IntegrityError +from django.test.testcases import TestCase -from openedx_tagging.core.tagging.models import ( - ObjectTag, - Tag, - Taxonomy, - LanguageTaxonomy, -) +from openedx_tagging.core.tagging.models import LanguageTaxonomy, ObjectTag, Tag, Taxonomy def get_tag(value): diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py index cc4f50403..76300f851 100644 --- a/tests/openedx_tagging/core/tagging/test_rules.py +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -1,6 +1,7 @@ -"""Tests tagging rules-based permissions""" - -import ddt +""" +Tests tagging rules-based permissions +""" +import ddt # type: ignore[import] from django.contrib.auth import get_user_model from django.test.testcases import TestCase @@ -46,7 +47,9 @@ def setUp(self): "oel_tagging.change_taxonomy", ) def test_add_change_taxonomy(self, perm): - """Taxonomy administrators can create or modify any Taxonomy""" + """ + Taxonomy administrators can create or modify any Taxonomy + """ assert self.superuser.has_perm(perm) assert self.superuser.has_perm(perm, self.taxonomy) assert self.staff.has_perm(perm) @@ -60,7 +63,9 @@ def test_add_change_taxonomy(self, perm): "oel_tagging.delete_taxonomy", ) def test_system_taxonomy(self, perm): - """Taxonomy administrators cannot edit system taxonomies""" + """ + Taxonomy administrators cannot edit system taxonomies + """ assert self.superuser.has_perm(perm, self.system_taxonomy) assert not self.staff.has_perm(perm, self.system_taxonomy) assert not self.learner.has_perm(perm, self.system_taxonomy) @@ -70,7 +75,9 @@ def test_system_taxonomy(self, perm): False, ) def test_delete_taxonomy(self, enabled): - """Taxonomy administrators can delete any Taxonomy""" + """ + Taxonomy administrators can delete any Taxonomy + """ self.taxonomy.enabled = enabled assert self.superuser.has_perm("oel_tagging.delete_taxonomy") assert self.superuser.has_perm("oel_tagging.delete_taxonomy", self.taxonomy) @@ -84,7 +91,9 @@ def test_delete_taxonomy(self, enabled): False, ) def test_view_taxonomy_enabled(self, enabled): - """Anyone can see enabled taxonomies, but learners cannot see disabled taxonomies""" + """ + Anyone can see enabled taxonomies, but learners cannot see disabled taxonomies + """ self.taxonomy.enabled = enabled assert self.superuser.has_perm("oel_tagging.view_taxonomy") assert self.superuser.has_perm("oel_tagging.view_taxonomy", self.taxonomy) @@ -102,7 +111,9 @@ def test_view_taxonomy_enabled(self, enabled): "oel_tagging.change_tag", ) def test_add_change_tag(self, perm): - """Taxonomy administrators can modify tags on non-free-text taxonomies""" + """ + Taxonomy administrators can modify tags on non-free-text taxonomies + """ assert self.superuser.has_perm(perm) assert self.superuser.has_perm(perm, self.bacteria) assert self.staff.has_perm(perm) @@ -115,7 +126,9 @@ def test_add_change_tag(self, perm): "oel_tagging.change_tag", ) def test_tag_free_text_taxonomy(self, perm): - """Taxonomy administrators cannot modify tags on a free-text Taxonomy""" + """ + Taxonomy administrators cannot modify tags on a free-text Taxonomy + """ self.taxonomy.allow_free_text = True self.taxonomy.save() assert self.superuser.has_perm(perm, self.bacteria) @@ -128,7 +141,9 @@ def test_tag_free_text_taxonomy(self, perm): "oel_tagging.delete_tag", ) def test_tag_no_taxonomy(self, perm): - """Taxonomy administrators can modify any Tag, even those with no Taxonnmy.""" + """ + Taxonomy administrators can modify any Tag, even those with no Taxonnmy. + """ tag = Tag() assert self.superuser.has_perm(perm, tag) assert self.staff.has_perm(perm, tag) @@ -139,7 +154,9 @@ def test_tag_no_taxonomy(self, perm): False, ) def test_delete_tag(self, allow_free_text): - """Taxonomy administrators can delete any Tag, even those associated with a free-text Taxonomy.""" + """ + Taxonomy administrators can delete any Tag, even those associated with a free-text Taxonomy. + """ self.taxonomy.allow_free_text = allow_free_text self.taxonomy.save() assert self.superuser.has_perm("oel_tagging.delete_tag") @@ -150,7 +167,9 @@ def test_delete_tag(self, allow_free_text): assert not self.learner.has_perm("oel_tagging.delete_tag", self.bacteria) def test_view_tag(self): - """Anyone can view any Tag""" + """ + Anyone can view any Tag + """ assert self.superuser.has_perm("oel_tagging.view_tag") assert self.superuser.has_perm("oel_tagging.view_tag", self.bacteria) assert self.staff.has_perm("oel_tagging.view_tag") @@ -165,7 +184,9 @@ def test_view_tag(self): "oel_tagging.change_objecttag", ) def test_add_change_object_tag(self, perm): - """Taxonomy administrators can create/edit an ObjectTag with an enabled Taxonomy""" + """ + Taxonomy administrators can create/edit an ObjectTag with an enabled Taxonomy + """ assert self.superuser.has_perm(perm) assert self.superuser.has_perm(perm, self.object_tag) assert self.staff.has_perm(perm) @@ -178,7 +199,9 @@ def test_add_change_object_tag(self, perm): "oel_tagging.change_objecttag", ) def test_object_tag_disabled_taxonomy(self, perm): - """Taxonomy administrators cannot create/edit an ObjectTag with a disabled Taxonomy""" + """ + Taxonomy administrators cannot create/edit an ObjectTag with a disabled Taxonomy + """ self.taxonomy.enabled = False self.taxonomy.save() assert self.superuser.has_perm(perm, self.object_tag) @@ -190,7 +213,9 @@ def test_object_tag_disabled_taxonomy(self, perm): False, ) def test_delete_objecttag(self, enabled): - """Taxonomy administrators can delete any ObjectTag, even those associated with a disabled Taxonomy.""" + """ + Taxonomy administrators can delete any ObjectTag, even those associated with a disabled Taxonomy. + """ self.taxonomy.enabled = enabled self.taxonomy.save() assert self.superuser.has_perm("oel_tagging.delete_objecttag") @@ -208,14 +233,18 @@ def test_delete_objecttag(self, enabled): "oel_tagging.delete_objecttag", ) def test_object_tag_no_taxonomy(self, perm): - """Taxonomy administrators can modify an ObjectTag with no Taxonomy""" + """ + Taxonomy administrators can modify an ObjectTag with no Taxonomy + """ object_tag = ObjectTag() assert self.superuser.has_perm(perm, object_tag) assert self.staff.has_perm(perm, object_tag) assert not self.learner.has_perm(perm, object_tag) def test_view_object_tag(self): - """Anyone can view any ObjectTag""" + """ + Anyone can view any ObjectTag + """ assert self.superuser.has_perm("oel_tagging.view_objecttag") assert self.superuser.has_perm("oel_tagging.view_objecttag", self.object_tag) assert self.staff.has_perm("oel_tagging.view_objecttag") diff --git a/tests/openedx_tagging/core/tagging/test_system_defined_models.py b/tests/openedx_tagging/core/tagging/test_system_defined_models.py index 783a2604e..ff2d669be 100644 --- a/tests/openedx_tagging/core/tagging/test_system_defined_models.py +++ b/tests/openedx_tagging/core/tagging/test_system_defined_models.py @@ -1,14 +1,13 @@ -""" Test the tagging system-defined taxonomy models """ -import ddt +""" +Test the tagging system-defined taxonomy models +""" +from __future__ import annotations -from django.db.utils import IntegrityError -from django.test.testcases import TestCase, override_settings +import ddt # type: ignore[import] from django.contrib.auth import get_user_model +from django.db.utils import IntegrityError +from django.test import TestCase, override_settings -from openedx_tagging.core.tagging.models import ( - ObjectTag, - Tag, -) from openedx_tagging.core.tagging.models.system_defined import ( ModelObjectTag, ModelSystemDefinedTaxonomy, @@ -17,7 +16,6 @@ from .test_models import TestTagTaxonomyMixin - test_languages = [ ("en", "English"), ("az", "Azerbaijani"), @@ -98,31 +96,35 @@ def test_implementation_error(self, taxonomy_cls, expected_exception): with self.assertRaises(expected_exception): taxonomy_cls() - @ddt.data( - (1, "tag_id", True), # Valid - (0, "tag_id", False), # Invalid user - ("test_id", "tag_id", False), # Invalid user id - (1, None, False), # Testing parent validations - ) - @ddt.unpack - def test_validations(self, tag_external_id, tag_id, expected): - tag = Tag( - id=tag_id, - taxonomy=self.user_taxonomy, - value="_val", - external_id=tag_external_id, - ) - object_tag = ObjectTag( - object_id="id", - tag=tag, - ) - - assert self.user_taxonomy.validate_object_tag( - object_tag=object_tag, - check_object=False, - check_taxonomy=False, - check_tag=True, - ) == expected + # FIXME: something is wrong with this test case. It's setting the string + # "tag_id" as the primary key (integer) of the Tag instance, and it mentions + # "parent validation" but there is nothing to do with parents here. + # + # @ddt.data( + # ("1", "tag_id", True), # Valid + # ("0", "tag_id", False), # Invalid user + # ("test_id", "tag_id", False), # Invalid user id + # ("1", None, False), # Testing parent validations + # ) + # @ddt.unpack + # def test_validations(self, tag_external_id: str, tag_id: str | None, expected: bool) -> None: + # tag = Tag( + # id=tag_id, + # taxonomy=self.user_taxonomy, + # value="_val", + # external_id=tag_external_id, + # ) + # object_tag = ObjectTag( + # object_id="id", + # tag=tag, + # ) + # + # assert self.user_taxonomy.validate_object_tag( + # object_tag=object_tag, + # check_object=False, + # check_taxonomy=False, + # check_tag=True, + # ) == expected def test_tag_object_invalid_user(self): # Test user that doesn't exist @@ -205,29 +207,33 @@ class TestLanguageTaxonomy(TestTagTaxonomyMixin, TestCase): Test for Language taxonomy """ - @ddt.data( - ("en", "tag_id"), # Valid - ("es", "tag_id"), # Not available lang - ("en", None), # Test parent validations - ) - @ddt.unpack - def test_validations(self, lang, tag_id): - tag = Tag( - id=tag_id, - taxonomy=self.language_taxonomy, - value="_val", - external_id=lang, - ) - object_tag = ObjectTag( - object_id="id", - tag=tag, - ) - self.language_taxonomy.validate_object_tag( - object_tag=object_tag, - check_object=False, - check_taxonomy=False, - check_tag=True, - ) + # FIXME: something is wrong with this test case. It's setting the string + # "tag_id" as the primary key (integer) of the Tag instance, and it mentions + # "parent validation" but there is nothing to do with parents here. + # + # @ddt.data( + # ("en", "tag_id", True), # Valid + # ("es", "tag_id", False), # Not available lang + # ("en", None, False), # Test parent validations + # ) + # @ddt.unpack + # def test_validations(self, lang: str, tag_id: str | None, expected: bool): + # tag = Tag( + # id=tag_id, + # taxonomy=self.language_taxonomy, + # value="_val", + # external_id=lang, + # ) + # object_tag = ObjectTag( + # object_id="id", + # tag=tag, + # ) + # assert self.language_taxonomy.validate_object_tag( + # object_tag=object_tag, + # check_object=False, + # check_taxonomy=False, + # check_tag=True, + # ) == expected def test_get_tags(self): tags = self.language_taxonomy.get_tags() diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index a0b946cbf..527625194 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -1,14 +1,16 @@ """ Tests tagging rest api views """ +from __future__ import annotations -import ddt +from urllib.parse import parse_qs, urlparse + +import ddt # type: ignore[import] from django.contrib.auth import get_user_model from rest_framework import status from rest_framework.test import APITestCase -from urllib.parse import urlparse, parse_qs -from openedx_tagging.core.tagging.models import Taxonomy, ObjectTag, Tag +from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy User = get_user_model() @@ -21,10 +23,10 @@ def check_taxonomy( - data, - id, + data: dict, + id, # pylint: disable=redefined-builtin name, - description=None, + description="", enabled=True, required=False, allow_multiple=False, @@ -32,6 +34,9 @@ def check_taxonomy( system_defined=False, visible_to_authors=True, ): + """ + Helper method to check expected fields of a Taxonomy + """ assert data["id"] == id assert data["name"] == name assert data["description"] == description @@ -45,7 +50,10 @@ def check_taxonomy( @ddt.ddt class TestTaxonomyViewSet(APITestCase): - def setUp(self): + """ + Test of the Taxonomy REST API + """ + def setUp(self) -> None: super().setUp() self.user = User.objects.create( @@ -73,7 +81,7 @@ def setUp(self): ("invalid", status.HTTP_400_BAD_REQUEST, None), ) @ddt.unpack - def test_list_taxonomy_queryparams(self, enabled, expected_status, expected_count): + def test_list_taxonomy_queryparams(self, enabled, expected_status: int, expected_count: int | None): Taxonomy.objects.create(name="Taxonomy enabled 1", enabled=True).save() Taxonomy.objects.create(name="Taxonomy enabled 2", enabled=True).save() Taxonomy.objects.create(name="Taxonomy disabled", enabled=False).save() @@ -98,7 +106,7 @@ def test_list_taxonomy_queryparams(self, enabled, expected_status, expected_coun ("staff", status.HTTP_200_OK), ) @ddt.unpack - def test_list_taxonomy(self, user_attr, expected_status): + def test_list_taxonomy(self, user_attr: str | None, expected_status: int): url = TAXONOMY_LIST_URL if user_attr: @@ -108,7 +116,7 @@ def test_list_taxonomy(self, user_attr, expected_status): response = self.client.get(url) assert response.status_code == expected_status - def test_list_taxonomy_pagination(self): + def test_list_taxonomy_pagination(self) -> None: url = TAXONOMY_LIST_URL Taxonomy.objects.create(name="T1", enabled=True).save() Taxonomy.objects.create(name="T2", enabled=True).save() @@ -126,10 +134,10 @@ def test_list_taxonomy_pagination(self): self.assertEqual(set(t["name"] for t in response.data["results"]), set(("T2", "T3"))) parsed_url = urlparse(response.data["next"]) - next_page = parse_qs(parsed_url.query).get("page", [None])[0] + next_page = parse_qs(parsed_url.query).get("page", [""])[0] assert next_page == "3" - def test_list_invalid_page(self): + def test_list_invalid_page(self) -> None: url = TAXONOMY_LIST_URL self.client.force_authenticate(user=self.user) @@ -149,8 +157,8 @@ def test_list_invalid_page(self): ("staff", {"enabled": False}, status.HTTP_200_OK), ) @ddt.unpack - def test_detail_taxonomy(self, user_attr, taxonomy_data, expected_status): - create_data = {**{"name": "taxonomy detail test"}, **taxonomy_data} + def test_detail_taxonomy(self, user_attr: str | None, taxonomy_data: dict[str, bool], expected_status: int): + create_data = {"name": "taxonomy detail test", **taxonomy_data} taxonomy = Taxonomy.objects.create(**create_data) url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) @@ -164,7 +172,7 @@ def test_detail_taxonomy(self, user_attr, taxonomy_data, expected_status): if status.is_success(expected_status): check_taxonomy(response.data, taxonomy.pk, **create_data) - def test_detail_taxonomy_404(self): + def test_detail_taxonomy_404(self) -> None: url = TAXONOMY_DETAIL_URL.format(pk=123123) self.client.force_authenticate(user=self.staff) @@ -177,7 +185,7 @@ def test_detail_taxonomy_404(self): ("staff", status.HTTP_201_CREATED), ) @ddt.unpack - def test_create_taxonomy(self, user_attr, expected_status): + def test_create_taxonomy(self, user_attr: str | None, expected_status: int): url = TAXONOMY_LIST_URL create_data = { @@ -208,7 +216,7 @@ def test_create_taxonomy(self, user_attr, expected_status): {"name": "Error taxonomy 2", "required": "Invalid value"}, {"name": "Error taxonomy 3", "enabled": "Invalid value"}, ) - def test_create_taxonomy_error(self, create_data): + def test_create_taxonomy_error(self, create_data: dict[str, str]): url = TAXONOMY_LIST_URL self.client.force_authenticate(user=self.staff) @@ -225,7 +233,7 @@ def test_create_taxonomy_system_defined(self, create_data): self.client.force_authenticate(user=self.staff) response = self.client.post(url, create_data, format="json") assert response.status_code == status.HTTP_201_CREATED - assert response.data["system_defined"] == False + assert response.data["system_defined"] is False @ddt.data( (None, status.HTTP_403_FORBIDDEN), diff --git a/tox.ini b/tox.ini index 15f4bb8bb..c42101f58 100644 --- a/tox.ini +++ b/tox.ini @@ -73,9 +73,8 @@ whitelist_externals = deps = -r{toxinidir}/requirements/quality.txt commands = - touch tests/__init__.py pylint openedx_learning openedx_tagging tests test_utils manage.py setup.py - rm tests/__init__.py + mypy pycodestyle openedx_learning openedx_tagging tests manage.py setup.py pydocstyle openedx_learning openedx_tagging tests manage.py setup.py isort --check-only --diff --recursive tests test_utils openedx_learning openedx_tagging manage.py setup.py test_settings.py From f73adefbcf9748dafe8ebfb7e8e6590df6780bb2 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 24 Aug 2023 10:13:33 -0700 Subject: [PATCH 039/282] docs: remove a comment copy-pasta --- mypy.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/mypy.ini b/mypy.ini index d3bb5b9f8..b97332862 100644 --- a/mypy.ini +++ b/mypy.ini @@ -11,5 +11,4 @@ files = tests [mypy.plugins.django-stubs] -# content_staging only works with CMS; others work with either, so we run mypy with CMS settings. django_settings_module = "projects.dev" From 4fca1ee1078713174633dba8c928b0004958abda Mon Sep 17 00:00:00 2001 From: Zubair Shakoor <57657330+zubairshakoorarbisoft@users.noreply.github.com> Date: Fri, 1 Sep 2023 16:15:22 +0500 Subject: [PATCH 040/282] fix: readthedocs file renamed --- .readthedocs.yml => .readthedocs.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .readthedocs.yml => .readthedocs.yaml (100%) diff --git a/.readthedocs.yml b/.readthedocs.yaml similarity index 100% rename from .readthedocs.yml rename to .readthedocs.yaml From 8f39d07c073bbf84163939518b4a6a4018f7adbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Fri, 1 Sep 2023 20:36:24 -0300 Subject: [PATCH 041/282] feat: add tag_object rest api (#74) --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/models/base.py | 35 +- .../core/tagging/rest_api/v1/serializers.py | 16 + .../core/tagging/rest_api/v1/views.py | 83 ++++- openedx_tagging/core/tagging/rules.py | 56 +++- .../openedx_tagging/core/tagging/test_api.py | 110 ++++++- .../core/tagging/test_rules.py | 92 +++--- .../core/tagging/test_views.py | 309 ++++++++++++------ 8 files changed, 529 insertions(+), 174 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 5f1cea0b2..75d55cb5e 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.1.5" +__version__ = "0.1.6" diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index a8e0ea718..20f610e49 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -385,11 +385,30 @@ def tag_object( """ Replaces the existing ObjectTag entries for the current taxonomy + object_id with the given list of tags. If self.allows_free_text, then the list should be a list of tag values. - Otherwise, it should be a list of existing Tag IDs. + Otherwise, it should be either a list of existing Tag Values or IDs. Raised ValueError if the proposed tags are invalid for this taxonomy. Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. """ + def _find_object_tag_index(tag_ref, object_tags) -> int: + """ + Search for Tag in the given list of ObjectTags by tag_ref or value, + returning its index or -1 if not found. + """ + return next( + ( + i + for i, object_tag in enumerate(object_tags) + if object_tag.tag_ref == tag_ref or object_tag.value == tag_ref + ), + -1, + ) + + if not isinstance(tags, list): + raise ValueError(_(f"Tags must be a list, not {type(tags).__name__}.")) + + tags = list(dict.fromkeys(tags)) # Remove duplicates preserving order + if not self.allow_multiple and len(tags) > 1: raise ValueError(_(f"Taxonomy ({self.id}) only allows one tag per object.")) @@ -399,17 +418,17 @@ def tag_object( ) ObjectTagClass = self.object_tag_class - current_tags = { - tag.tag_ref: tag - for tag in ObjectTagClass.objects.filter( + current_tags = list( + ObjectTagClass.objects.filter( taxonomy=self, object_id=object_id, ) - } + ) updated_tags = [] for tag_ref in tags: - if tag_ref in current_tags: - object_tag = current_tags.pop(tag_ref) + object_tag_index = _find_object_tag_index(tag_ref, current_tags) + if object_tag_index >= 0: + object_tag = current_tags.pop(object_tag_index) else: object_tag = ObjectTagClass( taxonomy=self, @@ -429,7 +448,7 @@ def tag_object( object_tag.save() # ...and delete any omitted existing tags - for old_tag in current_tags.values(): + for old_tag in current_tags: old_tag.delete() return updated_tags diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index e278b84fe..787d3427b 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -55,3 +55,19 @@ class Meta: "tag_ref", "is_valid", ] + + +class ObjectTagUpdateBodySerializer(serializers.Serializer): + """ + Serializer of the body for the ObjectTag UPDATE view + """ + + tags = serializers.ListField(child=serializers.CharField(), required=True) + + +class ObjectTagUpdateQueryParamsSerializer(serializers.Serializer): + """ + Serializer of the query params for the ObjectTag UPDATE view + """ + + taxonomy = serializers.PrimaryKeyRelatedField(queryset=Taxonomy.objects.all(), required=True) diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 3e8ae18c2..008859f5a 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -3,17 +3,19 @@ """ from django.db import models from django.http import Http404 -from django.shortcuts import get_object_or_404 -from rest_framework import status -from rest_framework.response import Response -from rest_framework.viewsets import ModelViewSet, ReadOnlyModelViewSet +from rest_framework import mixins +from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError +from rest_framework.viewsets import GenericViewSet, ModelViewSet -from ...api import create_taxonomy, get_object_tags, get_taxonomies, get_taxonomy +from ...api import create_taxonomy, get_object_tags, get_taxonomies, get_taxonomy, tag_object from ...models import Taxonomy +from ...rules import ChangeObjectTagPermissionItem from .permissions import ObjectTagObjectPermissions, TaxonomyObjectPermissions from .serializers import ( ObjectTagListQueryParamsSerializer, ObjectTagSerializer, + ObjectTagUpdateBodySerializer, + ObjectTagUpdateQueryParamsSerializer, TaxonomyListQueryParamsSerializer, TaxonomySerializer, ) @@ -167,10 +169,9 @@ def perform_create(self, serializer) -> None: serializer.instance = create_taxonomy(**serializer.validated_data) -class ObjectTagView(ReadOnlyModelViewSet): +class ObjectTagView(mixins.RetrieveModelMixin, mixins.UpdateModelMixin, mixins.ListModelMixin, GenericViewSet): """ - View to retrieve paginated ObjectTags for an Object, given its Object ID. - (What tags does this object have?) + View to retrieve paginated ObjectTags for a provided Object ID (object_id). **Retrieve Parameters** * object_id (required): - The Object ID to retrieve ObjectTags for. @@ -195,7 +196,14 @@ class ObjectTagView(ReadOnlyModelViewSet): * 403 - Permission denied * 405 - Method not allowed + **Update Parameters** + * object_id (required): - The Object ID to add ObjectTags for. + + **Update Request Body** + * tags: List of tags to be applied to a object id. Must be a list of Tag ids or Tag values. + **Update Query Returns** + * 200 - Success * 403 - Permission denied * 405 - Method not allowed @@ -224,8 +232,8 @@ def get_queryset(self) -> models.QuerySet: def retrieve(self, request, object_id=None): """ - Retrieve ObjectTags that belong to a given Object given its - object_id and return paginated results. + Retrieve ObjectTags that belong to a given object_id and + return paginated results. Note: We override `retrieve` here instead of `list` because we are passing in the Object ID (object_id) in the path (as opposed to passing @@ -238,3 +246,58 @@ def retrieve(self, request, object_id=None): paginated_object_tags = self.paginate_queryset(object_tags) serializer = ObjectTagSerializer(paginated_object_tags, many=True) return self.get_paginated_response(serializer.data) + + def update(self, request, object_id, partial=False): + """ + Update ObjectTags that belong to a given object_id and + return the list of these ObjecTags paginated. + + Pass a list of Tag ids or Tag values to be applied to an object id in the + body `tag` parameter. Passing an empty list will remove all tags from + the object id. + + **Example Body Requests** + + PUT api/tagging/v1/object_tags/:object_id + + **Example Body Requests** + ```json + { + "tags": [1, 2, 3] + }, + { + "tags": ["Tag 1", "Tag 2"] + }, + { + "tags": [] + } + """ + + if partial: + raise MethodNotAllowed("PATCH", detail="PATCH not allowed") + + query_params = ObjectTagUpdateQueryParamsSerializer(data=request.query_params.dict()) + query_params.is_valid(raise_exception=True) + taxonomy = query_params.validated_data.get("taxonomy", None) + taxonomy = taxonomy.cast() + + perm = f"{taxonomy._meta.app_label}.change_objecttag" + + perm_obj = ChangeObjectTagPermissionItem( + taxonomy=taxonomy, + object_id=object_id, + ) + + if not request.user.has_perm(perm, perm_obj): + raise PermissionDenied("You do not have permission to change object tags for this taxonomy or object_id.") + + body = ObjectTagUpdateBodySerializer(data=request.data) + body.is_valid(raise_exception=True) + + tags = body.data.get("tags", []) + try: + tag_object(taxonomy, tags, object_id) + except ValueError as e: + raise ValidationError(e) + + return self.retrieve(request, object_id) diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 8f0852a10..880fe19c8 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -8,8 +8,9 @@ import django.contrib.auth.models # typing support in rules depends on https://github.com/dfunckt/django-rules/pull/177 import rules # type: ignore[import] +from attrs import define -from .models import ObjectTag, Tag, Taxonomy +from .models import Tag, Taxonomy UserType = Union[django.contrib.auth.models.User, django.contrib.auth.models.AnonymousUser] @@ -19,6 +20,16 @@ is_taxonomy_admin: Callable[[UserType], bool] = rules.is_staff +@define +class ChangeObjectTagPermissionItem: + """ + Pair of taxonomy and object_id used for permission checking. + """ + + taxonomy: Taxonomy + object_id: str + + @rules.predicate def can_view_taxonomy(user: UserType, taxonomy: Taxonomy | None = None) -> bool: """ @@ -53,18 +64,39 @@ def can_change_tag(user: UserType, tag: Tag | None = None) -> bool: @rules.predicate -def can_change_object_tag(user: UserType, object_tag: ObjectTag | None = None) -> bool: +def can_change_object_tag_objectid(_user: UserType, _object_id: str) -> bool: """ - Taxonomy admins can create or modify object tags on enabled taxonomies. + Nobody can create or modify object tags without checking the permission for the tagged object. + + This rule should be defined in other apps for proper permission checking. """ - taxonomy = ( - object_tag.taxonomy.cast() if (object_tag and object_tag.taxonomy) else None - ) - object_tag = taxonomy.object_tag_class.cast(object_tag) if taxonomy else object_tag - return is_taxonomy_admin(user) and ( - not object_tag or not taxonomy or (taxonomy and taxonomy.enabled) + return False + + +@rules.predicate +def can_change_object_tag(user: UserType, perm_obj: ChangeObjectTagPermissionItem | None = None) -> bool: + """ + Checks if the user has permissions to create or modify tags on the given taxonomy and object_id. + """ + + # The following code allows METHOD permission (PUT) in the viewset for everyone + if perm_obj is None: + return True + + # Checks the permission for the taxonomy + taxonomy_perm = user.has_perm("oel_tagging.change_objecttag_taxonomy", perm_obj.taxonomy) + if not taxonomy_perm: + return False + + # Checks the permission for the object_id + objectid_perm = user.has_perm( + "oel_tagging.change_objecttag_objectid", + # The obj arg expects an object, but we are passing a string + perm_obj.object_id, # type: ignore[arg-type] ) + return objectid_perm + # Taxonomy rules.add_perm("oel_tagging.add_taxonomy", can_change_taxonomy) @@ -81,5 +113,9 @@ def can_change_object_tag(user: UserType, object_tag: ObjectTag | None = None) - # ObjectTag rules.add_perm("oel_tagging.add_objecttag", can_change_object_tag) rules.add_perm("oel_tagging.change_objecttag", can_change_object_tag) -rules.add_perm("oel_tagging.delete_objecttag", is_taxonomy_admin) +rules.add_perm("oel_tagging.delete_objecttag", can_change_object_tag) rules.add_perm("oel_tagging.view_objecttag", rules.always_allow) + +# Users can tag objects using tags from any taxonomy that they have permission to view +rules.add_perm("oel_tagging.change_objecttag_taxonomy", can_view_taxonomy) +rules.add_perm("oel_tagging.change_objecttag_objectid", can_change_object_tag_objectid) diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 0a19e276e..46e8c190e 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -27,6 +27,7 @@ class TestApiTagging(TestTagTaxonomyMixin, TestCase): """ Test the Tagging API methods. """ + def test_create_taxonomy(self) -> None: # Note: we must specify '-> None' to opt in to type checking params: dict[str, Any] = { "name": "Difficulty", @@ -42,11 +43,11 @@ def test_create_taxonomy(self) -> None: # Note: we must specify '-> None' to op assert not taxonomy.system_defined assert taxonomy.visible_to_authors - def test_bad_taxonomy_class(self): + def test_bad_taxonomy_class(self) -> None: with self.assertRaises(ValueError) as exc: tagging_api.create_taxonomy( name="Bad class", - taxonomy_class=str, + taxonomy_class=str, # type: ignore[arg-type] ) assert " must be a subclass of Taxonomy" in str(exc.exception) @@ -114,7 +115,7 @@ def check_object_tag( tag: Tag | None, name: str, value: str, - ): + ) -> None: """ Verifies that the properties of the given object_tag (once refreshed from the database) match those given. """ @@ -344,9 +345,106 @@ def test_tag_object_invalid_tag(self) -> None: ["Eukaryota Xenomorph"], "biology101", ) - assert "Invalid object tag for taxonomy (1): Eukaryota Xenomorph" in str( - exc.exception + assert "Invalid object tag for taxonomy (1): Eukaryota Xenomorph" in str(exc.exception) + + def test_tag_object_string(self) -> None: + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.taxonomy, + 'string', # type: ignore[arg-type] + "biology101", + ) + assert "Tags must be a list, not str." in str(exc.exception) + + def test_tag_object_integer(self) -> None: + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.taxonomy, + 1, # type: ignore[arg-type] + "biology101", + ) + assert "Tags must be a list, not int." in str(exc.exception) + + def test_tag_object_same_id(self) -> None: + # Tag the object with the same id twice + tagging_api.tag_object( + self.taxonomy, + [self.eubacteria.id], + "biology101", + ) + object_tags = tagging_api.tag_object( + self.taxonomy, + [self.eubacteria.id], + "biology101", + ) + assert len(object_tags) == 1 + assert str(object_tags[0]) == " biology101: Life on Earth=Eubacteria" + + def test_tag_object_same_value(self) -> None: + # Tag the object with the same value twice + tagging_api.tag_object( + self.taxonomy, + ["Eubacteria"], + "biology101", + ) + object_tags = tagging_api.tag_object( + self.taxonomy, + ["Eubacteria"], + "biology101", + ) + + assert len(object_tags) == 1 + assert str(object_tags[0]) == " biology101: Life on Earth=Eubacteria" + + def test_tag_object_same_mixed(self) -> None: + # Tag the object with the same id/value twice + tagging_api.tag_object( + self.taxonomy, + [self.eubacteria.id], + "biology101", + ) + object_tags = tagging_api.tag_object( + self.taxonomy, + ["Eubacteria"], + "biology101", + ) + + assert len(object_tags) == 1 + assert str(object_tags[0]) == " biology101: Life on Earth=Eubacteria" + + def test_tag_object_same_id_multiple(self) -> None: + self.taxonomy.allow_multiple = True + self.taxonomy.save() + # Tag the object with the same value twice + object_tags = tagging_api.tag_object( + self.taxonomy, + [self.eubacteria.id, self.eubacteria.id], + "biology101", + ) + assert len(object_tags) == 1 + + def test_tag_object_same_value_multiple(self) -> None: + self.taxonomy.allow_multiple = True + self.taxonomy.save() + # Tag the object with the same value twice + object_tags = tagging_api.tag_object( + self.taxonomy, + ["Eubacteria", "Eubacteria"], + "biology101", ) + assert len(object_tags) == 1 + + def test_tag_object_same_value_multiple_free(self) -> None: + self.taxonomy.allow_multiple = True + self.taxonomy.allow_free_text = True + self.taxonomy.save() + # Tag the object with the same value twice + object_tags = tagging_api.tag_object( + self.taxonomy, + ["tag1", "tag1"], + "biology101", + ) + assert len(object_tags) == 1 @override_settings(LANGUAGES=test_languages) def test_tag_object_language_taxonomy(self) -> None: @@ -493,7 +591,7 @@ def test_get_object_tags(self) -> None: ), ) @ddt.unpack - def test_autocomplete_tags(self, search: str, expected_values: list[str], expected_ids: list[int | None]): + def test_autocomplete_tags(self, search: str, expected_values: list[str], expected_ids: list[int | None]) -> None: tags = [ 'Archaea', 'Archaebacteria', diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py index 76300f851..d49f8b997 100644 --- a/tests/openedx_tagging/core/tagging/test_rules.py +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -2,10 +2,12 @@ Tests tagging rules-based permissions """ import ddt # type: ignore[import] +import rules # type: ignore[import] from django.contrib.auth import get_user_model from django.test.testcases import TestCase -from openedx_tagging.core.tagging.models import ObjectTag, Tag +from openedx_tagging.core.tagging.models import ObjectTag +from openedx_tagging.core.tagging.rules import ChangeObjectTagPermissionItem from .test_models import TestTagTaxonomyMixin @@ -19,7 +21,15 @@ class TestRulesTagging(TestTagTaxonomyMixin, TestCase): """ def setUp(self): + + def _object_permission(_user, object_id: str) -> bool: + """ + Everyone have object permission on object_id "abc" + """ + return object_id == "abc" + super().setUp() + self.superuser = User.objects.create( username="superuser", email="superuser@example.com", @@ -37,9 +47,13 @@ def setUp(self): self.object_tag = ObjectTag.objects.create( taxonomy=self.taxonomy, tag=self.bacteria, + object_id="abc", ) self.object_tag.save() + # Override the object permission for the test + rules.set_perm("oel_tagging.change_objecttag_objectid", _object_permission) + # Taxonomy @ddt.data( @@ -135,20 +149,6 @@ def test_tag_free_text_taxonomy(self, perm): assert not self.staff.has_perm(perm, self.bacteria) assert not self.learner.has_perm(perm, self.bacteria) - @ddt.data( - "oel_tagging.add_tag", - "oel_tagging.change_tag", - "oel_tagging.delete_tag", - ) - def test_tag_no_taxonomy(self, perm): - """ - Taxonomy administrators can modify any Tag, even those with no Taxonnmy. - """ - tag = Tag() - assert self.superuser.has_perm(perm, tag) - assert self.staff.has_perm(perm, tag) - assert not self.learner.has_perm(perm, tag) - @ddt.data( True, False, @@ -182,64 +182,60 @@ def test_view_tag(self): @ddt.data( "oel_tagging.add_objecttag", "oel_tagging.change_objecttag", + "oel_tagging.delete_objecttag", ) def test_add_change_object_tag(self, perm): """ - Taxonomy administrators can create/edit an ObjectTag with an enabled Taxonomy + Everyone can create/edit an ObjectTag with an enabled Taxonomy """ + obj_perm = ChangeObjectTagPermissionItem( + taxonomy=self.object_tag.taxonomy, + object_id=self.object_tag.object_id, + ) assert self.superuser.has_perm(perm) - assert self.superuser.has_perm(perm, self.object_tag) + assert self.superuser.has_perm(perm, obj_perm) assert self.staff.has_perm(perm) - assert self.staff.has_perm(perm, self.object_tag) - assert not self.learner.has_perm(perm) - assert not self.learner.has_perm(perm, self.object_tag) + assert self.staff.has_perm(perm, obj_perm) + assert self.learner.has_perm(perm) + assert self.learner.has_perm(perm, obj_perm) @ddt.data( "oel_tagging.add_objecttag", "oel_tagging.change_objecttag", + "oel_tagging.delete_objecttag", ) def test_object_tag_disabled_taxonomy(self, perm): """ - Taxonomy administrators cannot create/edit an ObjectTag with a disabled Taxonomy + Only Taxonomy administrators can create/edit an ObjectTag with a disabled Taxonomy """ self.taxonomy.enabled = False self.taxonomy.save() - assert self.superuser.has_perm(perm, self.object_tag) - assert not self.staff.has_perm(perm, self.object_tag) - assert not self.learner.has_perm(perm, self.object_tag) - - @ddt.data( - True, - False, - ) - def test_delete_objecttag(self, enabled): - """ - Taxonomy administrators can delete any ObjectTag, even those associated with a disabled Taxonomy. - """ - self.taxonomy.enabled = enabled - self.taxonomy.save() - assert self.superuser.has_perm("oel_tagging.delete_objecttag") - assert self.superuser.has_perm("oel_tagging.delete_objecttag", self.object_tag) - assert self.staff.has_perm("oel_tagging.delete_objecttag") - assert self.staff.has_perm("oel_tagging.delete_objecttag", self.object_tag) - assert not self.learner.has_perm("oel_tagging.delete_objecttag") - assert not self.learner.has_perm( - "oel_tagging.delete_objecttag", self.object_tag + obj_perm = ChangeObjectTagPermissionItem( + taxonomy=self.object_tag.taxonomy, + object_id=self.object_tag.object_id, ) + assert self.superuser.has_perm(perm, obj_perm) + assert self.staff.has_perm(perm, obj_perm) + assert not self.learner.has_perm(perm, obj_perm) @ddt.data( "oel_tagging.add_objecttag", "oel_tagging.change_objecttag", "oel_tagging.delete_objecttag", ) - def test_object_tag_no_taxonomy(self, perm): + def test_object_tag_without_object_permission(self, perm): """ - Taxonomy administrators can modify an ObjectTag with no Taxonomy + Only superusers can create/edit an ObjectTag without object permission """ - object_tag = ObjectTag() - assert self.superuser.has_perm(perm, object_tag) - assert self.staff.has_perm(perm, object_tag) - assert not self.learner.has_perm(perm, object_tag) + self.taxonomy.enabled = False + self.taxonomy.save() + obj_perm = ChangeObjectTagPermissionItem( + taxonomy=self.object_tag.taxonomy, + object_id="not abc", + ) + assert self.superuser.has_perm(perm, obj_perm) + assert not self.staff.has_perm(perm, obj_perm) + assert not self.learner.has_perm(perm, obj_perm) def test_view_object_tag(self): """ diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 527625194..a50ac3e65 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -6,6 +6,8 @@ from urllib.parse import parse_qs, urlparse import ddt # type: ignore[import] +# typing support in rules depends on https://github.com/dfunckt/django-rules/pull/177 +import rules # type: ignore[import] from django.contrib.auth import get_user_model from rest_framework import status from rest_framework.test import APITestCase @@ -19,7 +21,10 @@ TAXONOMY_DETAIL_URL = "/tagging/rest_api/v1/taxonomies/{pk}/" -OBJECT_TAGS_RETRIEVE_URL = '/tagging/rest_api/v1/object_tags/{object_id}/' +OBJECT_TAGS_RETRIEVE_URL = "/tagging/rest_api/v1/object_tags/{object_id}/" +OBJECT_TAGS_UPDATE_URL = "/tagging/rest_api/v1/object_tags/{object_id}/?taxonomy={taxonomy_id}" + +LANGUAGE_TAXONOMY_ID = -1 def check_taxonomy( @@ -396,6 +401,13 @@ class TestObjectTagViewSet(APITestCase): """ def setUp(self): + + def _object_permission(_user, object_id: str) -> bool: + """ + Everyone have object permission on object_id "abc" + """ + return object_id == "abc" + super().setUp() self.user = User.objects.create( @@ -410,70 +422,55 @@ def setUp(self): ) # System-defined language taxonomy with valid ObjectTag - self.system_taxonomy = SystemDefinedTaxonomy.objects.create( - name="System Taxonomy" - ) - self.tag1 = Tag.objects.create( - taxonomy=self.system_taxonomy, value="Tag 1" - ) - ObjectTag.objects.create( - object_id="abc", - taxonomy=self.system_taxonomy, - tag=self.tag1 - ) + self.system_taxonomy = SystemDefinedTaxonomy.objects.create(name="System Taxonomy") + self.tag1 = Tag.objects.create(taxonomy=self.system_taxonomy, value="Tag 1") + ObjectTag.objects.create(object_id="abc", taxonomy=self.system_taxonomy, tag=self.tag1) + + # Language system-defined language taxonomy + self.language_taxonomy = Taxonomy.objects.get(pk=LANGUAGE_TAXONOMY_ID) # Closed Taxonomies created by taxonomy admins, each with 20 ObjectTags self.enabled_taxonomy = Taxonomy.objects.create(name="Enabled Taxonomy") + self.disabled_taxonomy = Taxonomy.objects.create(name="Disabled Taxonomy", enabled=False) + self.multiple_taxonomy = Taxonomy.objects.create(name="Multiple Taxonomy", allow_multiple=True) for i in range(20): # Valid ObjectTags - tag = Tag.objects.create( - taxonomy=self.enabled_taxonomy, value=f"Tag {i}" + tag_enabled = Tag.objects.create(taxonomy=self.enabled_taxonomy, value=f"Tag {i}") + tag_disabled = Tag.objects.create(taxonomy=self.disabled_taxonomy, value=f"Tag {i}") + tag_multiple = Tag.objects.create(taxonomy=self.multiple_taxonomy, value=f"Tag {i}") + ObjectTag.objects.create( + object_id="abc", taxonomy=self.enabled_taxonomy, tag=tag_enabled, _value=tag_enabled.value ) ObjectTag.objects.create( - object_id="abc", taxonomy=self.enabled_taxonomy, - tag=tag, _value=tag.value + object_id="abc", taxonomy=self.disabled_taxonomy, tag=tag_disabled, _value=tag_disabled.value + ) + ObjectTag.objects.create( + object_id="abc", taxonomy=self.multiple_taxonomy, tag=tag_multiple, _value=tag_multiple.value ) - - # Taxonomy with invalid ObjectTag - self.taxonomy_with_invalid_object_tag = Taxonomy.objects.create() - self.to_be_deleted_tag = Tag.objects.create( - taxonomy=self.enabled_taxonomy, value="Deleted Tag" - ) - ObjectTag.objects.create( - object_id="abc", taxonomy=self.taxonomy_with_invalid_object_tag, - tag=self.to_be_deleted_tag, _value=self.to_be_deleted_tag.value - ) - self.to_be_deleted_tag.delete() # Delete tag so ObjectTag is invalid # Free-Text Taxonomies created by taxonomy admins, each linked # to 200 ObjectTags - self.open_taxonomy_enabled = Taxonomy.objects.create( - name="Enabled Free-Text Taxonomy", allow_free_text=True - ) + self.open_taxonomy_enabled = Taxonomy.objects.create(name="Enabled Free-Text Taxonomy", allow_free_text=True) self.open_taxonomy_disabled = Taxonomy.objects.create( - name="Disabled Free-Text Taxonomy", enabled=False, allow_free_text=True + name="Disabled Free-Text Taxonomy", allow_free_text=True, enabled=False ) for i in range(200): - ObjectTag.objects.create( - object_id="abc", taxonomy=self.open_taxonomy_enabled, _value=f"Free Text {i}" - ) - ObjectTag.objects.create( - object_id="abc", taxonomy=self.open_taxonomy_disabled, _value=f"Free Text {i}" - ) + ObjectTag.objects.create(object_id="abc", taxonomy=self.open_taxonomy_enabled, _value=f"Free Text {i}") + ObjectTag.objects.create(object_id="abc", taxonomy=self.open_taxonomy_disabled, _value=f"Free Text {i}") + + # Override the object permission for the test + rules.set_perm("oel_tagging.change_objecttag_objectid", _object_permission) @ddt.data( (None, "abc", status.HTTP_403_FORBIDDEN, None, None), - ("user", "abc", status.HTTP_200_OK, 422, 10), - ("staff", "abc", status.HTTP_200_OK, 422, 10), + ("user", "abc", status.HTTP_200_OK, 461, 10), + ("staff", "abc", status.HTTP_200_OK, 461, 10), (None, "non-existing-id", status.HTTP_403_FORBIDDEN, None, None), ("user", "non-existing-id", status.HTTP_200_OK, 0, 0), ("staff", "non-existing-id", status.HTTP_200_OK, 0, 0), ) @ddt.unpack - def test_retrieve_object_tags( - self, user_attr, object_id, expected_status, expected_count, expected_results - - ): + def test_retrieve_object_tags(self, user_attr, object_id, expected_status, expected_count, expected_results): """ Test retrieving object tags """ @@ -492,15 +489,13 @@ def test_retrieve_object_tags( assert len(response.data.get("results")) == expected_results @ddt.data( - (None, "abc", status.HTTP_403_FORBIDDEN, None, None, None, None), - ("user", "abc", status.HTTP_200_OK, 20, 10, 1, 1), - ("staff", "abc", status.HTTP_200_OK, 20, 10, 1, 1), + (None, "abc", status.HTTP_403_FORBIDDEN, None, None), + ("user", "abc", status.HTTP_200_OK, 20, 10), + ("staff", "abc", status.HTTP_200_OK, 20, 10), ) @ddt.unpack def test_retrieve_object_tags_taxonomy_queryparam( - self, user_attr, object_id, expected_status, - expected_count, expected_results, - expected_invalid_count, expected_invalid_results + self, user_attr, object_id, expected_status, expected_count, expected_results ): """ Test retrieving object tags for specific taxonomies provided @@ -511,7 +506,6 @@ def test_retrieve_object_tags_taxonomy_queryparam( user = getattr(self, user_attr) self.client.force_authenticate(user=user) - # Check valid object tags response = self.client.get(url, {"taxonomy": self.enabled_taxonomy.pk}) assert response.status_code == expected_status if status.is_success(expected_status): @@ -523,30 +517,13 @@ def test_retrieve_object_tags_taxonomy_queryparam( assert object_tag.get("is_valid") is True assert object_tag.get("taxonomy_id") == self.enabled_taxonomy.pk - # Check invalid object tags - response = self.client.get( - url, {"taxonomy": self.taxonomy_with_invalid_object_tag.pk} - ) - assert response.status_code == expected_status - if status.is_success(expected_status): - assert response.data.get("count") == expected_invalid_count - assert response.data.get("results") is not None - assert len(response.data.get("results")) == expected_invalid_results - object_tags = response.data.get("results") - for object_tag in object_tags: - assert object_tag.get("is_valid") is False - assert object_tag.get("taxonomy_id") == \ - self.taxonomy_with_invalid_object_tag.pk - @ddt.data( (None, "abc", status.HTTP_403_FORBIDDEN), ("user", "abc", status.HTTP_400_BAD_REQUEST), ("staff", "abc", status.HTTP_400_BAD_REQUEST), ) @ddt.unpack - def test_retrieve_object_tags_invalid_taxonomy_queryparam( - self, user_attr, object_id, expected_status - ): + def test_retrieve_object_tags_invalid_taxonomy_queryparam(self, user_attr, object_id, expected_status): """ Test retrieving object tags for invalid taxonomy """ @@ -591,10 +568,7 @@ def test_retrieve_object_tags_pagination( user = getattr(self, user_attr) self.client.force_authenticate(user=user) - query_params = { - "taxonomy": self.open_taxonomy_enabled.pk, - "page": page - } + query_params = {"taxonomy": self.open_taxonomy_enabled.pk, "page": page} if page_size: query_params["page_size"] = page_size @@ -610,25 +584,24 @@ def test_retrieve_object_tags_pagination( @ddt.data( (None, "POST", status.HTTP_403_FORBIDDEN), - (None, "PUT", status.HTTP_403_FORBIDDEN), (None, "PATCH", status.HTTP_403_FORBIDDEN), (None, "DELETE", status.HTTP_403_FORBIDDEN), - ("user", "POST", status.HTTP_403_FORBIDDEN), - ("user", "PUT", status.HTTP_403_FORBIDDEN), - ("user", "PATCH", status.HTTP_403_FORBIDDEN), - ("user", "DELETE", status.HTTP_403_FORBIDDEN), + ("user", "POST", status.HTTP_405_METHOD_NOT_ALLOWED), + ("user", "PATCH", status.HTTP_405_METHOD_NOT_ALLOWED), + ("user", "DELETE", status.HTTP_405_METHOD_NOT_ALLOWED), ("staff", "POST", status.HTTP_405_METHOD_NOT_ALLOWED), - ("staff", "PUT", status.HTTP_405_METHOD_NOT_ALLOWED), ("staff", "PATCH", status.HTTP_405_METHOD_NOT_ALLOWED), ("staff", "DELETE", status.HTTP_405_METHOD_NOT_ALLOWED), ) @ddt.unpack def test_object_tags_remaining_http_methods( - self, user_attr, http_method, expected_status, - + self, + user_attr, + http_method, + expected_status, ): """ - Test POST/PUT/PATCH/DELETE method for ObjectTagView + Test POST/PATCH/DELETE method for ObjectTagView Only staff users should have permissions to perform the actions, however the methods are currently not allowed. @@ -640,18 +613,172 @@ def test_object_tags_remaining_http_methods( self.client.force_authenticate(user=user) if http_method == "POST": - response = self.client.post( - url, {"test": "payload"}, format="json" - ) - elif http_method == "PUT": - response = self.client.put( - url, {"test": "payload"}, format="json" - ) + response = self.client.post(url, {"test": "payload"}, format="json") elif http_method == "PATCH": - response = self.client.patch( - url, {"test": "payload"}, format="json" - ) + response = self.client.patch(url, {"test": "payload"}, format="json") elif http_method == "DELETE": response = self.client.delete(url) assert response.status_code == expected_status + + @ddt.data( + # Users and staff can add tags to a taxonomy + (None, "language_taxonomy", ["Portuguese"], status.HTTP_403_FORBIDDEN), + ("user", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), + ("staff", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), + # Users and staff can clear add tags to a taxonomy + (None, "enabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + ("user", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), + ("staff", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), + # Only staff can add tag to a disabled taxonomy + (None, "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + ("user", "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + ("staff", "disabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), + # Users and staff can add a single tag to a allow_multiple=True taxonomy + (None, "multiple_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + ("user", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), + ("staff", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), + # Users and staff can add tags to an open taxonomy + (None, "open_taxonomy_enabled", ["tag1"], status.HTTP_403_FORBIDDEN), + ("user", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), + ("staff", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), + # Only staff can add tags to a disabled open taxonomy + (None, "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), + ("user", "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), + ("staff", "open_taxonomy_disabled", ["tag1"], status.HTTP_200_OK), + ) + @ddt.unpack + def test_tag_object(self, user_attr, taxonomy_attr, tag_values, expected_status): + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + taxonomy = getattr(self, taxonomy_attr) + + url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + + response = self.client.put(url, {"tags": tag_values}, format="json") + assert response.status_code == expected_status + if status.is_success(expected_status): + assert len(response.data.get("results")) == len(tag_values) + assert set(t["value"] for t in response.data["results"]) == set(tag_values) + + @ddt.data( + # Can't add invalid tags to a closed taxonomy + (None, "language_taxonomy", ["Invalid"], status.HTTP_403_FORBIDDEN), + ("user", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), + ("staff", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), + (None, "enabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), + ("user", "enabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), + ("staff", "enabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), + (None, "multiple_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), + ("user", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), + ("staff", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), + # Users can't edit tags from a disabled taxonomy. Staff can't add invalid tags to a closed taxonomy + (None, "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), + ("user", "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), + ("staff", "disabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), + ) + @ddt.unpack + def test_tag_object_invalid(self, user_attr, taxonomy_attr, tag_values, expected_status): + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + taxonomy = getattr(self, taxonomy_attr) + + url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + + response = self.client.put(url, {"tags": tag_values}, format="json") + assert response.status_code == expected_status + assert not status.is_success(expected_status) # No success cases here + + @ddt.data( + # Users and staff can clear tags from a taxonomy + (None, "enabled_taxonomy", [], status.HTTP_403_FORBIDDEN), + ("user", "enabled_taxonomy", [], status.HTTP_200_OK), + ("staff", "enabled_taxonomy", [], status.HTTP_200_OK), + # Users and staff can clear tags from a allow_multiple=True taxonomy + (None, "multiple_taxonomy", [], status.HTTP_403_FORBIDDEN), + ("user", "multiple_taxonomy", [], status.HTTP_200_OK), + ("staff", "multiple_taxonomy", [], status.HTTP_200_OK), + # Only staff can clear tags from a disabled taxonomy + (None, "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), + ("user", "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), + ("staff", "disabled_taxonomy", [], status.HTTP_200_OK), + (None, "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), + ("user", "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), + ("staff", "open_taxonomy_disabled", [], status.HTTP_200_OK), + # Users and staff can't clear a taxonomy with required=True + (None, "language_taxonomy", [], status.HTTP_403_FORBIDDEN), + ("user", "language_taxonomy", [], status.HTTP_400_BAD_REQUEST), + ("staff", "language_taxonomy", [], status.HTTP_400_BAD_REQUEST), + ) + @ddt.unpack + def test_tag_object_clear(self, user_attr, taxonomy_attr, tag_values, expected_status): + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + taxonomy = getattr(self, taxonomy_attr) + + url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + + response = self.client.put(url, {"tags": tag_values}, format="json") + assert response.status_code == expected_status + if status.is_success(expected_status): + assert len(response.data.get("results")) == len(tag_values) + assert set(t["value"] for t in response.data["results"]) == set(tag_values) + + @ddt.data( + # Users and staff can add multiple tags to a allow_multiple=True taxonomy + (None, "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), + ("user", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), + ("staff", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), + (None, "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_403_FORBIDDEN), + ("user", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), + ("staff", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), + # Users and staff can't add multple tags to a allow_multiple=False taxonomy + (None, "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), + ("user", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), + ("staff", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), + (None, "language_taxonomy", ["Portuguese", "English"], status.HTTP_403_FORBIDDEN), + ("user", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), + ("staff", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), + # Users can't edit tags from a disabled taxonomy. Staff can't add multiple tags to + # a taxonomy with allow_multiple=False + (None, "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), + ("user", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), + ("staff", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), + ) + @ddt.unpack + def test_tag_object_multiple(self, user_attr, taxonomy_attr, tag_values, expected_status): + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + taxonomy = getattr(self, taxonomy_attr) + + url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + + response = self.client.put(url, {"tags": tag_values}, format="json") + assert response.status_code == expected_status + if status.is_success(expected_status): + assert len(response.data.get("results")) == len(tag_values) + assert set(t["value"] for t in response.data["results"]) == set(tag_values) + + @ddt.data( + (None, status.HTTP_403_FORBIDDEN), + ("user", status.HTTP_403_FORBIDDEN), + ("staff", status.HTTP_403_FORBIDDEN), + ) + @ddt.unpack + def test_tag_object_without_permission(self, user_attr, expected_status): + if user_attr: + user = getattr(self, user_attr) + self.client.force_authenticate(user=user) + + url = OBJECT_TAGS_UPDATE_URL.format(object_id="not abc", taxonomy_id=self.enabled_taxonomy.pk) + + response = self.client.put(url, {"tags": ["Tag 1"]}, format="json") + assert response.status_code == expected_status From 6335f6594c9c3fb927a47c01c1513938e546071b Mon Sep 17 00:00:00 2001 From: Zaeema Anwar <55206089+thezaeemaanwar@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:49:34 +0500 Subject: [PATCH 042/282] chore: created python-upgrade-requirements workflow and tag aximprovements team in weekly maintenance PRs --- .../workflows/upgrade-python-requirements.yml | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/upgrade-python-requirements.yml diff --git a/.github/workflows/upgrade-python-requirements.yml b/.github/workflows/upgrade-python-requirements.yml new file mode 100644 index 000000000..14b5bc745 --- /dev/null +++ b/.github/workflows/upgrade-python-requirements.yml @@ -0,0 +1,31 @@ +name: Upgrade Python Requirements + +on: + schedule: + - cron: "0 0 * * 1" + workflow_dispatch: + inputs: + branch: + description: "Target branch against which to create requirements PR" + required: true + # If copying this template manually, you must provide your default branch name + # in quotes, such as 'master' + default: "main" + +jobs: + call-upgrade-python-requirements-workflow: + uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master + with: + # If copying manually, also provide your default branch name in quotes here + branch: ${{ github.event.inputs.branch || 'main' }} + # optional parameters below; fill in if you'd like github or email notifications + # user_reviewers: "" + team_reviewers: "axim-aximprovements" + email_address: "aximimprovements@axim.org" + send_success_notification: false + # python_version: "" + secrets: + requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }} + requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }} + edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }} + edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }} From ea311e143d7eb86cddf954d20889857f19425aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Tue, 12 Sep 2023 15:39:44 -0500 Subject: [PATCH 043/282] docs: ADR for pagination and repr of single taxonomy view api (#72) --- .../0014-single-taxonomy-view-api.rst | 193 ++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 docs/decisions/0014-single-taxonomy-view-api.rst diff --git a/docs/decisions/0014-single-taxonomy-view-api.rst b/docs/decisions/0014-single-taxonomy-view-api.rst new file mode 100644 index 000000000..a71a1ce7f --- /dev/null +++ b/docs/decisions/0014-single-taxonomy-view-api.rst @@ -0,0 +1,193 @@ +14. Single taxonomy view API +===================================== + +Context +-------- + +This view returns tags of a closed taxonomy (for MVP has not been implemented yet +for open taxonomies). It is necessary to make a decision about what structure the tags are going +to have, how the pagination is going to work and how will the search for tags be implemented. +It was taken into account that taxonomies commonly have the following characteristics: + +- It has few root tags. +- It may a very large number of children for each tag. +- It is mostly represented as trees on frontend, with a depth of up to 3 levels. + +For the decisions, the following use cases were taken into account: + +- As a taxonomy administrator, I want to see all the tags available for use with a closed taxonomy, + so that I can see the taxonomy's structure in the management interface. + - As a taxonomy administrator, I want to see the available tags as a lits of root tags + that can be expanded to show children tags. + - As a taxonomy administrator, I want to sort the list of root tags alphabetically: A-Z (default) and Z-A. + - As a taxonomy administrator, I want to expand all root tags to see all children tags. + - As a taxonomy administrator, I want to search for tags, so I can find root and children tags more easily. +- As a course author, when I am editing the tags of a component, I want to see all the tags available + from a particular taxonomy that I can use. + - As a course author, I want to see the available tags as a lits of root tags + that can be expanded to show children tags. + - As a course author, I want to search for tags, so I can find root and children tags more easily. + +Excluded use cases: + +- As a content author, when searching/filtering a course/library, I want to see which tags are applied to the content + and use them to refine my search. - This is excluded from this API's use case because this is automatically handled + by elasticsearch/opensearch. + + +Decision +--------- + +Views & Pagination +~~~~~~~~~~~~~~~~~~~ + +Make one view: + +**get_matching_tags(parent_tag_id: str = None, search_term: str = None)** + +that can handle this cases: + +- Get the root tags of the taxonomy. If ``parent_tag_id`` is ``None``. +- Get the children of a tag. Called each time the user expands a parent tag to see its children. + If ``parent_tag_id`` is not ``None``. + +In both cases the results are paginated. In addition to the common pagination metadata, it is necessary to return: + +- Total number of pages. +- Total number of root/children tags. +- Range index of current page, Ex. Page 1: 1-12, Page 2: 13-24. +- Total number of children of each tag. + +The pagination of root tags and child tags are independent. +In order to be able to fulfill the functionality of "Expand-all" in a scalable way, +the following has been agreed: + +- Create a ``TAGS_THRESHOLD`` (default: 1000). +- If ``taxonomy.tags.count < TAGS_THRESHOLD``, then ``get_matching_tags()`` will return all tags on the taxonomy, + roots and children. +- Otherwise, ``get_matching_tags()`` will only return paginated root tags, and it will be necessary + to use ``get_matching_tags()`` to return paginated children. Also the "Expand-all" functionality will be disabled. + +For search you can see the next section (Search tags) + +**Pros** + +- It is the simplest way. +- Paging both root tags and children mitigates the huge number of tags that can be found in large taxonomies. + +Search tags +~~~~~~~~~~~~ + +Support tag search on the backend. Return a subset of matching tags. +We will use the same view to perform a search with the same logic: + +**get_matching_tags(parent_tag_id: str = None, search_term: str = None)** + +We can use ``search_term`` to perferom a search on root tags or children tags depending of ``parent_tag_id``. + +For the search, ``SEARCH_TAGS_THRESHOLD`` will be used. (It is recommended that it be 20% of ``TAGS_THRESHOLD``). +It will work in the same way of ``TAGS_THRESHOLD`` (see Views & Pagination) + +**Pros** + +- It is the most scalable way. + +Tag representation +~~~~~~~~~~~~~~~~~~~ + +Return a list of root tags and within a link to obtain the children tags +or the complete list of children tags depending of ``TAGS_THRESHOLD`` or ``SEARCH_TAGS_THRESHOLD``. +The list of root tags will be ordered alphabetically. If it has child tags, they must also +be ordered alphabetically. + +**(taxonomy.tags.count < *_THRESHOLD)**:: + + { + "count": 100, + "tags": [ + { + "id": "tag_1", + "value": "Tag 1", + "taxonomy_id": "1", + "sub_tags": [ + { + "id": "tag_2", + "value": "Tag 2", + "taxonomy_id": "1", + "sub_tags": [ + (....) + ] + }, + (....) + ] + } + + +**Otherwise**:: + + { + "count": 100, + "tags": [ + { + "id": "tag_1", + "value": "Tag 1", + "taxonomy_id": "1", + "sub_tags_link": "http//api-call-to-get-children.com" + }, + (....) + ] + } + + +**Pros:** + +- The edX's interfaces show the tags in the form of a tree. +- The frontend needs no further processing as it is in a displayable format. +- It is kept as a simple implementation. + + +Rejected Options +----------------- + + +Render as a simple list of tags +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Return a simple list of tags, regardless of whether it is root or leaf. + +**Pros:** + +- It is simple and does not need further implementation and processing in the API. + +**Cons:** + +- It is more work to re-process all that list in the frontend to know who it is whose father. +- In no edX's interface is it used this way and it would be a very specific use case. +- Pagination would be more complicated to perform. + + +Add the children to the root pagination +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Ex. If the ``page_size`` is 100, when fetching the first root tag, which has 10 children tags, +11 tags are counted for the total and there would be reamin 89 tags to be obtained. + +**Cons:** + +- If there is a branch with a number of tags that exceeds ``page_size``, + it would only return that branch. +- All branches are variable in size, therefore a variable number of root tags + would be returned. This would cause interfaces between taxonomies to be inconsistent + in the number of root tags shown. + + +Search on frontend +~~~~~~~~~~~~~~~~~~ + +We constrain the number of tags allowed in a taxonomy for MVP, so that the API +can return all the tags in one page. So we can perform the tag search on the frontend. + +**Cons:** + +- It is not scalable. +- Sets limits of tags that can be created in the taxonomy. From 69519f4be37e1b49a8f064f4ecc8ae2bdd5390dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Wed, 20 Sep 2023 18:11:37 -0500 Subject: [PATCH 044/282] feat: Single taxonomy view API for tags (#78) --- .../0014-single-taxonomy-view-api.rst | 13 +- openedx_tagging/core/tagging/api.py | 50 ++- .../core/tagging/import_export/import_plan.py | 1 + .../core/tagging/import_export/parsers.py | 4 +- .../0007_tag_import_task_log_null_fix.py | 11 +- .../0008_taxonomy_description_not_null.py | 13 +- openedx_tagging/core/tagging/models/base.py | 54 ++- .../core/tagging/models/system_defined.py | 33 +- .../core/tagging/rest_api/paginators.py | 29 ++ .../core/tagging/rest_api/v1/permissions.py | 14 +- .../core/tagging/rest_api/v1/serializers.py | 89 ++++- .../core/tagging/rest_api/v1/urls.py | 9 +- .../core/tagging/rest_api/v1/views.py | 215 ++++++++++- openedx_tagging/core/tagging/rules.py | 13 +- .../openedx_tagging/core/tagging/test_api.py | 48 +++ .../core/tagging/test_models.py | 60 +++ .../core/tagging/test_views.py | 345 +++++++++++++++++- 17 files changed, 953 insertions(+), 48 deletions(-) create mode 100644 openedx_tagging/core/tagging/rest_api/paginators.py diff --git a/docs/decisions/0014-single-taxonomy-view-api.rst b/docs/decisions/0014-single-taxonomy-view-api.rst index a71a1ce7f..ddd0de26b 100644 --- a/docs/decisions/0014-single-taxonomy-view-api.rst +++ b/docs/decisions/0014-single-taxonomy-view-api.rst @@ -83,9 +83,18 @@ We will use the same view to perform a search with the same logic: **get_matching_tags(parent_tag_id: str = None, search_term: str = None)** -We can use ``search_term`` to perferom a search on root tags or children tags depending of ``parent_tag_id``. +We can use ``search_term`` to perform a search on all taxonomy tags or children tags depending of ``parent_tag_id``. +The result will be a pruned tree with the necessary tags to be able to reach the results from a root tag. +Ex. if in the result there may be a child tag of ``depth=2``, but the parents are not found in the result. +In this case, it is necessary to add the parent and the parent of the parent (root tag) to be able to show +the child tag that is in the result. For the search, ``SEARCH_TAGS_THRESHOLD`` will be used. (It is recommended that it be 20% of ``TAGS_THRESHOLD``). +It will work in the following way: + +- If ``search_result.count() < SEARCH_TAGS_THRESHOLD``, then it will return all tags on the result tree without pagination. +- Otherwise, it will return the roots of the result tree with pagination. Each root will have the entire pruned branch. + It will work in the same way of ``TAGS_THRESHOLD`` (see Views & Pagination) **Pros** @@ -190,4 +199,4 @@ can return all the tags in one page. So we can perform the tag search on the fro **Cons:** - It is not scalable. -- Sets limits of tags that can be created in the taxonomy. +- Sets limits of tags that can be created in the taxonomy. \ No newline at end of file diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index e0295df59..ca2cb1f11 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -79,6 +79,47 @@ def get_tags(taxonomy: Taxonomy) -> list[Tag]: return taxonomy.cast().get_tags() +def get_root_tags(taxonomy: Taxonomy) -> list[Tag]: + """ + Returns a list of the root tags for the given taxonomy. + + Note that if the taxonomy allows free-text tags, then the returned list will be empty. + """ + return list(taxonomy.cast().get_filtered_tags()) + + +def search_tags(taxonomy: Taxonomy, search_term: str) -> list[Tag]: + """ + Returns a list of all tags that contains `search_term` of the given taxonomy. + + Note that if the taxonomy allows free-text tags, then the returned list will be empty. + """ + return list( + taxonomy.cast().get_filtered_tags( + search_term=search_term, + search_in_all=True, + ) + ) + + +def get_children_tags( + taxonomy: Taxonomy, + parent_tag_id: int, + search_term: str | None = None, +) -> list[Tag]: + """ + Returns a list of children tags for the given parent tag. + + Note that if the taxonomy allows free-text tags, then the returned list will be empty. + """ + return list( + taxonomy.cast().get_filtered_tags( + parent_tag_id=parent_tag_id, + search_term=search_term, + ) + ) + + def resync_object_tags(object_tags: QuerySet | None = None) -> int: """ Reconciles ObjectTag entries with any changes made to their associated taxonomies and tags. @@ -98,8 +139,7 @@ def resync_object_tags(object_tags: QuerySet | None = None) -> int: def get_object_tags( - object_id: str, - taxonomy_id: str | None = None + object_id: str, taxonomy_id: str | None = None ) -> QuerySet[ObjectTag]: """ Returns a Queryset of object tags for a given object. @@ -124,10 +164,8 @@ def delete_object_tags(object_id: str): """ Delete all ObjectTag entries for a given object. """ - tags = ( - ObjectTag.objects.filter( - object_id=object_id, - ) + tags = ObjectTag.objects.filter( + object_id=object_id, ) tags.delete() diff --git a/openedx_tagging/core/tagging/import_export/import_plan.py b/openedx_tagging/core/tagging/import_export/import_plan.py index 774afcadb..f58390df1 100644 --- a/openedx_tagging/core/tagging/import_export/import_plan.py +++ b/openedx_tagging/core/tagging/import_export/import_plan.py @@ -16,6 +16,7 @@ class TagItem: """ Tag representation on the tag import plan """ + id: str value: str index: int | None = 0 diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py index 1fb714735..b0132f3d1 100644 --- a/openedx_tagging/core/tagging/import_export/parsers.py +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -103,7 +103,9 @@ def _export_data(cls, tags: list[dict], taxonomy: Taxonomy) -> str: raise NotImplementedError @classmethod - def _parse_tags(cls, tags_data: list[dict]) -> tuple[list[TagItem], list[TagParserError]]: + def _parse_tags( + cls, tags_data: list[dict] + ) -> tuple[list[TagItem], list[TagParserError]]: """ Validate the required fields of each tag. diff --git a/openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py b/openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py index c4a4067a2..48e278110 100644 --- a/openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py +++ b/openedx_tagging/core/tagging/migrations/0007_tag_import_task_log_null_fix.py @@ -4,15 +4,16 @@ class Migration(migrations.Migration): - dependencies = [ - ('oel_tagging', '0006_auto_20230802_1631'), + ("oel_tagging", "0006_auto_20230802_1631"), ] operations = [ migrations.AlterField( - model_name='tagimporttask', - name='log', - field=models.TextField(blank=True, default=None, help_text='Action execution logs'), + model_name="tagimporttask", + name="log", + field=models.TextField( + blank=True, default=None, help_text="Action execution logs" + ), ), ] diff --git a/openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py b/openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py index 37b352823..73da9320b 100644 --- a/openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py +++ b/openedx_tagging/core/tagging/migrations/0008_taxonomy_description_not_null.py @@ -6,16 +6,19 @@ class Migration(migrations.Migration): - dependencies = [ - ('oel_tagging', '0007_tag_import_task_log_null_fix'), + ("oel_tagging", "0007_tag_import_task_log_null_fix"), ] operations = [ migrations.AlterField( - model_name='taxonomy', - name='description', - field=openedx_learning.lib.fields.MultiCollationTextField(blank=True, default='', help_text='Provides extra information for the user when applying tags from this taxonomy to an object.'), + model_name="taxonomy", + name="description", + field=openedx_learning.lib.fields.MultiCollationTextField( + blank=True, + default="", + help_text="Provides extra information for the user when applying tags from this taxonomy to an object.", + ), preserve_default=False, ), ] diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index 20f610e49..829909b6a 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -268,7 +268,10 @@ def copy(self, taxonomy: Taxonomy) -> Taxonomy: self._taxonomy_class = taxonomy._taxonomy_class return self - def get_tags(self, tag_set: models.QuerySet | None = None) -> list[Tag]: + def get_tags( + self, + tag_set: models.QuerySet[Tag] | None = None, + ) -> list[Tag]: """ Returns a list of all Tags in the current taxonomy, from the root(s) down to TAXONOMY_MAX_DEPTH tags, in tree order. @@ -289,6 +292,7 @@ def get_tags(self, tag_set: models.QuerySet | None = None) -> list[Tag]: tag_set = self.tag_set.all() parents = None + for depth in range(TAXONOMY_MAX_DEPTH): filtered_tags = tag_set.prefetch_related("parent") if parents is None: @@ -310,6 +314,42 @@ def get_tags(self, tag_set: models.QuerySet | None = None) -> list[Tag]: break return tags + def get_filtered_tags( + self, + tag_set: models.QuerySet[Tag] | None = None, + parent_tag_id: int | None = None, + search_term: str | None = None, + search_in_all: bool = False, + ) -> models.QuerySet[Tag]: + """ + Returns a filtered QuerySet of tags. + By default returns the root tags of the given taxonomy + + Use `parent_tag_id` to return the children of a tag. + + Use `search_term` to filter the results by values that contains `search_term`. + + Set `search_in_all` to True to make the search in all tags on the given taxonomy. + + Note: This is mostly an 'internal' API and generally code outside of openedx_tagging + should use the APIs in openedx_tagging.api which in turn use this. + """ + if tag_set is None: + tag_set = self.tag_set.all() + + if self.allow_free_text: + return tag_set.none() + + if not search_in_all: + # If not search in all taxonomy, then apply parent filter. + tag_set = tag_set.filter(parent=parent_tag_id) + + if search_term: + # Apply search filter + tag_set = tag_set.filter(value__icontains=search_term) + + return tag_set.order_by("value", "id") + def validate_object_tag( self, object_tag: "ObjectTag", @@ -348,7 +388,9 @@ def _check_taxonomy( Subclasses can override this method to perform their own taxonomy validation checks. """ # Must be linked to this taxonomy - return (object_tag.taxonomy_id is not None) and object_tag.taxonomy_id == self.id + return ( + object_tag.taxonomy_id is not None + ) and object_tag.taxonomy_id == self.id def _check_tag( self, @@ -481,9 +523,11 @@ def autocomplete_tags( # Fetch tags that the object already has to exclude them from the result excluded_tags: list[str] = [] if object_id: - excluded_tags = list(self.objecttag_set.filter(object_id=object_id).values_list( - "_value", flat=True - )) + excluded_tags = list( + self.objecttag_set.filter(object_id=object_id).values_list( + "_value", flat=True + ) + ) return ( # Fetch object tags from this taxonomy whose value contains the search self.objecttag_set.filter(_value__icontains=search) diff --git a/openedx_tagging/core/tagging/models/system_defined.py b/openedx_tagging/core/tagging/models/system_defined.py index 275d6e97d..c78e9aa83 100644 --- a/openedx_tagging/core/tagging/models/system_defined.py +++ b/openedx_tagging/core/tagging/models/system_defined.py @@ -10,7 +10,7 @@ from django.contrib.auth import get_user_model from django.db import models -from openedx_tagging.core.tagging.models.base import ObjectTag +from openedx_tagging.core.tagging.models.base import ObjectTag, Tag from .base import ObjectTag, Tag, Taxonomy @@ -241,7 +241,10 @@ class LanguageTaxonomy(SystemDefinedTaxonomy): class Meta: proxy = True - def get_tags(self, tag_set: models.QuerySet | None = None) -> list[Tag]: + def get_tags( + self, + tag_set: models.QuerySet[Tag] | None = None, + ) -> list[Tag]: """ Returns a list of all the available Language Tags, annotated with ``depth`` = 0. """ @@ -249,6 +252,32 @@ def get_tags(self, tag_set: models.QuerySet | None = None) -> list[Tag]: tag_set = self.tag_set.filter(external_id__in=available_langs) return super().get_tags(tag_set=tag_set) + def get_filtered_tags( + self, + tag_set: models.QuerySet[Tag] | None = None, + parent_tag_id: int | None = None, + search_term: str | None = None, + search_in_all: bool = False, + ) -> models.QuerySet[Tag]: + """ + Returns a filtered QuerySet of available Language Tags. + By default returns all the available Language Tags. + + `parent_tag_id` returns an empty result because all Language tags are root tags. + + Use `search_term` to filter the results by values that contains `search_term`. + """ + if parent_tag_id: + return self.tag_set.none() + + available_langs = self._get_available_languages() + tag_set = self.tag_set.filter(external_id__in=available_langs) + return super().get_filtered_tags( + tag_set=tag_set, + search_term=search_term, + search_in_all=search_in_all, + ) + def _get_available_languages(cls) -> set[str]: """ Get available languages from Django LANGUAGE. diff --git a/openedx_tagging/core/tagging/rest_api/paginators.py b/openedx_tagging/core/tagging/rest_api/paginators.py new file mode 100644 index 000000000..6e2fd99f5 --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/paginators.py @@ -0,0 +1,29 @@ +from edx_rest_framework_extensions.paginators import DefaultPagination # type: ignore[import] + +# From this point, the tags begin to be paginated +TAGS_THRESHOLD = 1000 + +# From this point, search tags begin to be paginated +SEARCH_TAGS_THRESHOLD = 200 + + +class TagsPagination(DefaultPagination): + """ + Custom pagination configuration for taxonomies + with a large number of tags. Used on the get tags API view. + """ + page_size = 10 + max_page_size = 300 + + +class DisabledTagsPagination(DefaultPagination): + """ + Custom pagination configuration for taxonomies + with a small number of tags. Used on the get tags API view + + This class allows to bring all the tags of the taxonomy. + It should be used if the number of tags within + the taxonomy does not exceed `TAGS_THRESHOLD`. + """ + page_size = TAGS_THRESHOLD + max_page_size = TAGS_THRESHOLD + 1 diff --git a/openedx_tagging/core/tagging/rest_api/v1/permissions.py b/openedx_tagging/core/tagging/rest_api/v1/permissions.py index 245fc3cb2..2e7f921c9 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/permissions.py +++ b/openedx_tagging/core/tagging/rest_api/v1/permissions.py @@ -1,7 +1,7 @@ """ Tagging permissions """ - +import rules # type: ignore[import] from rest_framework.permissions import DjangoObjectPermissions @@ -27,3 +27,15 @@ class ObjectTagObjectPermissions(DjangoObjectPermissions): "PATCH": ["%(app_label)s.change_%(model_name)s"], "DELETE": ["%(app_label)s.delete_%(model_name)s"], } + + +class TagListPermissions(DjangoObjectPermissions): + def has_permission(self, request, view): + if not request.user or ( + not request.user.is_authenticated and self.authenticated_users_only + ): + return False + return True + + def has_object_permission(self, request, view, obj): + return rules.has_perm("oel_tagging.list_tag", request.user, obj) diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 787d3427b..756d4a793 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -3,8 +3,9 @@ """ from rest_framework import serializers +from rest_framework.reverse import reverse -from openedx_tagging.core.tagging.models import ObjectTag, Taxonomy +from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy class TaxonomyListQueryParamsSerializer(serializers.Serializer): @@ -70,4 +71,88 @@ class ObjectTagUpdateQueryParamsSerializer(serializers.Serializer): Serializer of the query params for the ObjectTag UPDATE view """ - taxonomy = serializers.PrimaryKeyRelatedField(queryset=Taxonomy.objects.all(), required=True) + taxonomy = serializers.PrimaryKeyRelatedField( + queryset=Taxonomy.objects.all(), required=True + ) + + +class TagsSerializer(serializers.ModelSerializer): + """ + Serializer for Tags + + Adds a link to get the sub tags + """ + + sub_tags_link = serializers.SerializerMethodField() + children_count = serializers.SerializerMethodField() + + class Meta: + model = Tag + fields = ( + "id", + "value", + "taxonomy_id", + "parent_id", + "sub_tags_link", + "children_count", + ) + + def get_sub_tags_link(self, obj): + if obj.children.count(): + query_params = f"?parent_tag_id={obj.id}" + url = ( + reverse("oel_tagging:taxonomy-tags", args=[str(obj.taxonomy_id)]) + + query_params + ) + request = self.context.get("request") + return request.build_absolute_uri(url) + + def get_children_count(self, obj): + return obj.children.count() + + +class TagsWithSubTagsSerializer(serializers.ModelSerializer): + """ + Serializer for Tags. + + Represents a tree with a list of sub tags + """ + + sub_tags = serializers.SerializerMethodField() + children_count = serializers.SerializerMethodField() + + class Meta: + model = Tag + fields = ( + "id", + "value", + "taxonomy_id", + "sub_tags", + "children_count", + ) + + def get_sub_tags(self, obj): + serializer = TagsWithSubTagsSerializer( + obj.children.all().order_by("value", "id"), + many=True, + read_only=True, + ) + return serializer.data + + def get_children_count(self, obj): + return obj.children.count() + + +class TagsForSearchSerializer(TagsWithSubTagsSerializer): + """ + Serializer for Tags + + Used to filter sub tags of a given tag + """ + + def get_sub_tags(self, obj): + serializer = TagsWithSubTagsSerializer(obj.sub_tags, many=True, read_only=True) + return serializer.data + + def get_children_count(self, obj): + return len(obj.sub_tags) diff --git a/openedx_tagging/core/tagging/rest_api/v1/urls.py b/openedx_tagging/core/tagging/rest_api/v1/urls.py index 02cb48e40..7b96cf98c 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/urls.py +++ b/openedx_tagging/core/tagging/rest_api/v1/urls.py @@ -11,4 +11,11 @@ router.register("taxonomies", views.TaxonomyView, basename="taxonomy") router.register("object_tags", views.ObjectTagView, basename="object_tag") -urlpatterns = [path("", include(router.urls))] +urlpatterns = [ + path("", include(router.urls)), + path( + "taxonomies//tags/", + views.TaxonomyTagsView.as_view(), + name="taxonomy-tags", + ), +] diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 008859f5a..52029e046 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -1,21 +1,39 @@ """ Tagging API Views """ +from __future__ import annotations + from django.db import models from django.http import Http404 from rest_framework import mixins from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError +from rest_framework.generics import ListAPIView from rest_framework.viewsets import GenericViewSet, ModelViewSet -from ...api import create_taxonomy, get_object_tags, get_taxonomies, get_taxonomy, tag_object +from openedx_tagging.core.tagging.models.base import Tag + +from ...api import ( + create_taxonomy, + get_children_tags, + get_object_tags, + get_root_tags, + get_taxonomies, + get_taxonomy, + search_tags, + tag_object, +) from ...models import Taxonomy from ...rules import ChangeObjectTagPermissionItem -from .permissions import ObjectTagObjectPermissions, TaxonomyObjectPermissions +from ..paginators import SEARCH_TAGS_THRESHOLD, TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination +from .permissions import ObjectTagObjectPermissions, TagListPermissions, TaxonomyObjectPermissions from .serializers import ( ObjectTagListQueryParamsSerializer, ObjectTagSerializer, ObjectTagUpdateBodySerializer, ObjectTagUpdateQueryParamsSerializer, + TagsForSearchSerializer, + TagsSerializer, + TagsWithSubTagsSerializer, TaxonomyListQueryParamsSerializer, TaxonomySerializer, ) @@ -169,7 +187,12 @@ def perform_create(self, serializer) -> None: serializer.instance = create_taxonomy(**serializer.validated_data) -class ObjectTagView(mixins.RetrieveModelMixin, mixins.UpdateModelMixin, mixins.ListModelMixin, GenericViewSet): +class ObjectTagView( + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.ListModelMixin, + GenericViewSet, +): """ View to retrieve paginated ObjectTags for a provided Object ID (object_id). @@ -276,7 +299,9 @@ def update(self, request, object_id, partial=False): if partial: raise MethodNotAllowed("PATCH", detail="PATCH not allowed") - query_params = ObjectTagUpdateQueryParamsSerializer(data=request.query_params.dict()) + query_params = ObjectTagUpdateQueryParamsSerializer( + data=request.query_params.dict() + ) query_params.is_valid(raise_exception=True) taxonomy = query_params.validated_data.get("taxonomy", None) taxonomy = taxonomy.cast() @@ -289,7 +314,9 @@ def update(self, request, object_id, partial=False): ) if not request.user.has_perm(perm, perm_obj): - raise PermissionDenied("You do not have permission to change object tags for this taxonomy or object_id.") + raise PermissionDenied( + "You do not have permission to change object tags for this taxonomy or object_id." + ) body = ObjectTagUpdateBodySerializer(data=request.data) body.is_valid(raise_exception=True) @@ -301,3 +328,181 @@ def update(self, request, object_id, partial=False): raise ValidationError(e) return self.retrieve(request, object_id) + + +class TaxonomyTagsView(ListAPIView): + """ + View to list tags of a taxonomy. + + **List Query Parameters** + * pk (required) - The pk of the taxonomy to retrieve tags. + * parent_tag_id (optional) - Id of the tag to retrieve children tags. + * page (optional) - Page number (default: 1) + * page_size (optional) - Number of items per page (default: 10) + + **List Example Requests** + GET api/tagging/v1/taxonomy/:pk/tags - Get tags of taxonomy + GET api/tagging/v1/taxonomy/:pk/tags?parent_tag_id=30 - Get children tags of tag + + **List Query Returns** + * 200 - Success + * 400 - Invalid query parameter + * 403 - Permission denied + * 404 - Taxonomy not found + """ + + permission_classes = [TagListPermissions] + pagination_enabled = True + + def __init__(self): + # Initialized here to avoid errors on type hints + self.serializer_class = TagsSerializer + + def get_pagination_class(self): + """ + Get the corresponding class depending if the pagination is enabled. + + It is necessary to call this function before returning the data. + """ + if self.pagination_enabled: + return TagsPagination + else: + return DisabledTagsPagination + + def get_taxonomy(self, pk: int) -> Taxonomy: + """ + Get the taxonomy from `pk` or raise 404. + """ + taxonomy = get_taxonomy(pk) + if not taxonomy: + raise Http404("Taxonomy not found") + self.check_object_permissions(self.request, taxonomy) + return taxonomy + + def _build_search_tree(self, tags: list[Tag]) -> list[Tag]: + """ + Builds a tree with the result tags for a search. + + The retult is a pruned tree that contains + the path from root tags to tags that match the search. + """ + tag_ids = [tag.id for tag in tags] + + # Get missing parents. + # Not all parents are in the search result. + # This occurs when a child tag is on the search result, but its parent not, + # we need to add the parent to show the tree from the root to the child. + for tag in tags: + if tag.parent and tag.parent_id and tag.parent_id not in tag_ids: + tag_ids.append(tag.parent_id) + tags.append(tag.parent) # Our loop will iterate over this new parent tag too. + + groups: dict[int, list[Tag]] = {} + roots: list[Tag] = [] + + # Group tags by parent + for tag in tags: + if tag.parent_id is not None: + if tag.parent_id not in groups: + groups[tag.parent_id] = [] + groups[tag.parent_id].append(tag) + else: + roots.append(tag) + + for tag in tags: + # Used to serialize searched childrens + tag.sub_tags = groups.get(tag.id, []) # type: ignore[attr-defined] + + return roots + + def get_matching_tags( + self, + taxonomy_id: int, + parent_tag_id: str | None = None, + search_term: str | None = None, + ) -> list[Tag]: + """ + Returns a list of tags for the given taxonomy. + + The pagination can be enabled or disabled depending of the taxonomy size. + You can read the desicion '0014_*' to more info about this logic. + Also, determines the serializer to be used. + + Use `parent_tag_id` to get the children of the given tag. + + Use `search_term` to filter tags values that contains the given term. + """ + taxonomy = self.get_taxonomy(taxonomy_id) + if parent_tag_id: + # Get children of a tag. + + # If you need to get the children, then the roots are + # paginated, so we need to paginate the childrens too. + self.pagination_enabled = True + + # Normal serializer, with children link. + self.serializer_class = TagsSerializer + return get_children_tags( + taxonomy, + int(parent_tag_id), + search_term=search_term, + ) + else: + if search_term: + # Search tags + result = search_tags( + taxonomy, + search_term, + ) + # Checks the result size to determine whether + # to turn pagination on or off. + self.pagination_enabled = len(result) > SEARCH_TAGS_THRESHOLD + + # Use the special serializer to only show the tree + # of the search result. + self.serializer_class = TagsForSearchSerializer + + result = self._build_search_tree(result) + else: + # Get root tags of taxonomy + + # Checks the taxonomy size to determine whether + # to turn pagination on or off. + self.pagination_enabled = taxonomy.tag_set.count() > TAGS_THRESHOLD + + if self.pagination_enabled: + # If pagination is enabled, use the normal serializer + # with children link. + self.serializer_class = TagsSerializer + else: + # If pagination is disabled, use the special serializer + # to show children. In this case, we return all taxonomy tags + # in a tree structure. + self.serializer_class = TagsWithSubTagsSerializer + + result = get_root_tags(taxonomy) + + return result + + def get_queryset(self) -> list[Tag]: # type: ignore[override] + """ + Builds and returns the queryset to be paginated. + + The return type is not a QuerySet because the tagging python api functions + return lists, and on this point convert the list to a query set + is an unnecesary operation. + """ + pk = self.kwargs.get("pk") + parent_tag_id = self.request.query_params.get("parent_tag_id", None) + search_term = self.request.query_params.get("search_term", None) + + result = self.get_matching_tags( + pk, + parent_tag_id=parent_tag_id, + search_term=search_term, + ) + + # This function is not called automatically + self.pagination_class = self.get_pagination_class() + + return result diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 880fe19c8..00ec88116 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -12,7 +12,9 @@ from .models import Tag, Taxonomy -UserType = Union[django.contrib.auth.models.User, django.contrib.auth.models.AnonymousUser] +UserType = Union[ + django.contrib.auth.models.User, django.contrib.auth.models.AnonymousUser +] # Global staff are taxonomy admins. @@ -74,7 +76,9 @@ def can_change_object_tag_objectid(_user: UserType, _object_id: str) -> bool: @rules.predicate -def can_change_object_tag(user: UserType, perm_obj: ChangeObjectTagPermissionItem | None = None) -> bool: +def can_change_object_tag( + user: UserType, perm_obj: ChangeObjectTagPermissionItem | None = None +) -> bool: """ Checks if the user has permissions to create or modify tags on the given taxonomy and object_id. """ @@ -84,7 +88,9 @@ def can_change_object_tag(user: UserType, perm_obj: ChangeObjectTagPermissionIte return True # Checks the permission for the taxonomy - taxonomy_perm = user.has_perm("oel_tagging.change_objecttag_taxonomy", perm_obj.taxonomy) + taxonomy_perm = user.has_perm( + "oel_tagging.change_objecttag_taxonomy", perm_obj.taxonomy + ) if not taxonomy_perm: return False @@ -109,6 +115,7 @@ def can_change_object_tag(user: UserType, perm_obj: ChangeObjectTagPermissionIte rules.add_perm("oel_tagging.change_tag", can_change_tag) rules.add_perm("oel_tagging.delete_tag", is_taxonomy_admin) rules.add_perm("oel_tagging.view_tag", rules.always_allow) +rules.add_perm("oel_tagging.list_tag", can_view_taxonomy) # ObjectTag rules.add_perm("oel_tagging.add_objecttag", can_change_object_tag) diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 46e8c190e..8f185e751 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -17,9 +17,17 @@ ("az", "Azerbaijani"), ("en", "English"), ("id", "Indonesian"), + ("ga", "Irish"), + ("pl", "Polish"), ("qu", "Quechua"), ("zu", "Zulu"), ] +# Languages that contains 'ish' +filtered_test_languages = [ + ("en", "English"), + ("ga", "Irish"), + ("pl", "Polish"), +] @ddt.ddt @@ -108,6 +116,46 @@ def test_get_tags(self) -> None: expected_langs = [lang[0] for lang in test_languages] assert langs == expected_langs + @override_settings(LANGUAGES=test_languages) + def test_get_root_tags(self): + assert tagging_api.get_root_tags(self.taxonomy) == self.domain_tags + assert tagging_api.get_root_tags(self.system_taxonomy) == self.system_tags + tags = tagging_api.get_root_tags(self.language_taxonomy) + langs = [tag.external_id for tag in tags] + expected_langs = [lang[0] for lang in test_languages] + assert langs == expected_langs + + @override_settings(LANGUAGES=test_languages) + def test_search_tags(self): + assert tagging_api.search_tags( + self.taxonomy, + search_term='eU' + ) == self.filtered_tags + + tags = tagging_api.search_tags(self.language_taxonomy, search_term='IsH') + langs = [tag.external_id for tag in tags] + expected_langs = [lang[0] for lang in filtered_test_languages] + assert langs == expected_langs + + def test_get_children_tags(self): + assert tagging_api.get_children_tags( + self.taxonomy, + self.animalia.id, + ) == self.phylum_tags + assert tagging_api.get_children_tags( + self.taxonomy, + self.animalia.id, + search_term='dA', + ) == self.filtered_phylum_tags + assert not tagging_api.get_children_tags( + self.system_taxonomy, + self.system_taxonomy_tag.id, + ) + assert not tagging_api.get_children_tags( + self.language_taxonomy, + self.english_tag, + ) + def check_object_tag( self, object_tag: ObjectTag, diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 4b6f644aa..6cef2bcf4 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -39,7 +39,9 @@ def setUp(self): self.eubacteria = get_tag("Eubacteria") self.chordata = get_tag("Chordata") self.mammalia = get_tag("Mammalia") + self.animalia = get_tag("Animalia") self.system_taxonomy_tag = get_tag("System Tag 1") + self.english_tag = get_tag("English") self.user_1 = get_user_model()( id=1, username="test_user_1", @@ -58,6 +60,12 @@ def setUp(self): get_tag("Bacteria"), get_tag("Eukaryota"), ] + # Domain tags that contains 'ar' + self.filtered_domain_tags = [ + get_tag("Archaea"), + get_tag("Eukaryota"), + ] + # Kingdom tags (depth=1) self.kingdom_tags = [ # Kingdoms of https://en.wikipedia.org/wiki/Archaea @@ -74,6 +82,7 @@ def setUp(self): get_tag("Plantae"), get_tag("Protista"), ] + # Phylum tags (depth=2) self.phylum_tags = [ # Some phyla of https://en.wikipedia.org/wiki/Animalia @@ -85,6 +94,19 @@ def setUp(self): get_tag("Placozoa"), get_tag("Porifera"), ] + # Phylum tags that contains 'da' + self.filtered_phylum_tags = [ + get_tag("Arthropoda"), + get_tag("Chordata"), + get_tag("Cnidaria"), + ] + + # Biology tags that contains 'eu' + self.filtered_tags = [ + get_tag("Eubacteria"), + get_tag("Eukaryota"), + get_tag("Euryarchaeida"), + ] self.system_tags = [ get_tag("System Tag 1"), @@ -220,11 +242,49 @@ def test_get_tags(self): *self.phylum_tags, ] + def test_get_root_tags(self): + assert list(self.taxonomy.get_filtered_tags()) == self.domain_tags + assert list( + self.taxonomy.get_filtered_tags(search_term='aR') + ) == self.filtered_domain_tags + def test_get_tags_free_text(self): self.taxonomy.allow_free_text = True with self.assertNumQueries(0): assert self.taxonomy.get_tags() == [] + def test_get_children_tags(self): + assert list( + self.taxonomy.get_filtered_tags(parent_tag_id=self.animalia.id) + ) == self.phylum_tags + assert list( + self.taxonomy.get_filtered_tags( + parent_tag_id=self.animalia.id, + search_term='dA', + ) + ) == self.filtered_phylum_tags + assert not list( + self.system_taxonomy.get_filtered_tags( + parent_tag_id=self.system_taxonomy_tag.id + ) + ) + + def test_get_children_tags_free_text(self): + self.taxonomy.allow_free_text = True + assert not list(self.taxonomy.get_filtered_tags( + parent_tag_id=self.animalia.id + )) + assert not list(self.taxonomy.get_filtered_tags( + parent_tag_id=self.animalia.id, + search_term='dA', + )) + + def test_search_tags(self): + assert list(self.taxonomy.get_filtered_tags( + search_term='eU', + search_in_all=True + )) == self.filtered_tags + def test_get_tags_shallow_taxonomy(self): taxonomy = Taxonomy.objects.create(name="Difficulty") tags = [ diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index a50ac3e65..99c552564 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -14,11 +14,13 @@ from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy +from openedx_tagging.core.tagging.rest_api.paginators import TagsPagination User = get_user_model() TAXONOMY_LIST_URL = "/tagging/rest_api/v1/taxonomies/" TAXONOMY_DETAIL_URL = "/tagging/rest_api/v1/taxonomies/{pk}/" +TAXONOMY_TAGS_URL = "/tagging/rest_api/v1/taxonomies/{pk}/tags/" OBJECT_TAGS_RETRIEVE_URL = "/tagging/rest_api/v1/object_tags/{object_id}/" @@ -28,8 +30,8 @@ def check_taxonomy( - data: dict, - id, # pylint: disable=redefined-builtin + data, + taxonomy_id, name, description="", enabled=True, @@ -40,9 +42,9 @@ def check_taxonomy( visible_to_authors=True, ): """ - Helper method to check expected fields of a Taxonomy + Check taxonomy data """ - assert data["id"] == id + assert data["id"] == taxonomy_id assert data["name"] == name assert data["description"] == description assert data["enabled"] == enabled @@ -53,12 +55,12 @@ def check_taxonomy( assert data["visible_to_authors"] == visible_to_authors -@ddt.ddt -class TestTaxonomyViewSet(APITestCase): +class TestTaxonomyViewMixin(APITestCase): """ - Test of the Taxonomy REST API + Mixin for taxonomy views. Adds users. """ - def setUp(self) -> None: + + def setUp(self): super().setUp() self.user = User.objects.create( @@ -72,6 +74,13 @@ def setUp(self) -> None: is_staff=True, ) + +@ddt.ddt +class TestTaxonomyViewSet(TestTaxonomyViewMixin): + """ + Test taxonomy view set + """ + @ddt.data( (None, status.HTTP_200_OK, 4), (1, status.HTTP_200_OK, 3), @@ -238,7 +247,7 @@ def test_create_taxonomy_system_defined(self, create_data): self.client.force_authenticate(user=self.staff) response = self.client.post(url, create_data, format="json") assert response.status_code == status.HTTP_201_CREATED - assert response.data["system_defined"] is False + assert not response.data["system_defined"] @ddt.data( (None, status.HTTP_403_FORBIDDEN), @@ -391,7 +400,7 @@ def test_delete_taxonomy_404(self): self.client.force_authenticate(user=self.staff) response = self.client.delete(url) - assert response.status_code, status.HTTP_404_NOT_FOUND + assert response.status_code == status.HTTP_404_NOT_FOUND @ddt.ddt @@ -782,3 +791,319 @@ def test_tag_object_without_permission(self, user_attr, expected_status): response = self.client.put(url, {"tags": ["Tag 1"]}, format="json") assert response.status_code == expected_status + + +class TestTaxonomyTagsView(TestTaxonomyViewMixin): + """ + Tests the list tags of taxonomy view + """ + + fixtures = ["tests/openedx_tagging/core/fixtures/tagging.yaml"] + + def setUp(self): + self.small_taxonomy = Taxonomy.objects.get(name="Life on Earth") + self.large_taxonomy = Taxonomy(name="Large Taxonomy") + self.large_taxonomy.save() + + self.small_taxonomy_url = TAXONOMY_TAGS_URL.format(pk=self.small_taxonomy.pk) + self.large_taxonomy_url = TAXONOMY_TAGS_URL.format(pk=self.large_taxonomy.pk) + + self.root_tags_count = 51 + self.children_tags_count = [12, 12] # 51 * 12 * 12 = 7344 tags + + self.page_size = TagsPagination().page_size + + return super().setUp() + + def _create_tag(self, depth: int, parent: Tag | None = None): + """ + Creates tags and children in a recursive way. + """ + tag_count = self.large_taxonomy.tag_set.count() + tag = Tag( + taxonomy=self.large_taxonomy, + parent=parent, + value=f"Tag {tag_count}", + ) + tag.save() + if depth < len(self.children_tags_count): + for _ in range(self.children_tags_count[depth]): + self._create_tag(depth + 1, parent=tag) + return tag + + def _build_large_taxonomy(self): + # Pupulates the large taxonomy with tags + for _ in range(self.root_tags_count): + self._create_tag(0) + + def test_invalid_taxonomy(self): + url = TAXONOMY_TAGS_URL.format(pk=212121) + + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_not_authorized_user(self): + # Not authenticated user + response = self.client.get(self.small_taxonomy_url) + assert response.status_code == status.HTTP_403_FORBIDDEN + + self.small_taxonomy.enabled = False + self.small_taxonomy.save() + self.client.force_authenticate(user=self.user) + response = self.client.get(self.small_taxonomy_url) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_small_taxonomy(self): + self.client.force_authenticate(user=self.staff) + response = self.client.get(self.small_taxonomy_url) + assert response.status_code == status.HTTP_200_OK + + data = response.data + results = data.get("results", []) + + # Count of root tags + root_count = self.small_taxonomy.tag_set.filter(parent=None).count() + assert len(results) == root_count + + # Checking tag fields + root_tag = self.small_taxonomy.tag_set.get(id=results[0].get("id")) + root_children_count = root_tag.children.count() + assert results[0].get("value") == root_tag.value + assert results[0].get("taxonomy_id") == self.small_taxonomy.id + assert results[0].get("children_count") == root_children_count + assert len(results[0].get("sub_tags")) == root_children_count + + # Checking pagination values + assert data.get("next") is None + assert data.get("previous") is None + assert data.get("count") == root_count + assert data.get("num_pages") == 1 + assert data.get("current_page") == 1 + + def test_small_search(self): + search_term = 'eU' + url = f"{self.small_taxonomy_url}?search_term={search_term}" + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + assert response.status_code == status.HTTP_200_OK + + data = response.data + results = data.get("results", []) + + assert len(results) == 3 + + # Checking pagination values + assert data.get("next") is None + assert data.get("previous") is None + assert data.get("count") == 3 + assert data.get("num_pages") == 1 + assert data.get("current_page") == 1 + + def test_large_taxonomy(self): + self._build_large_taxonomy() + self.client.force_authenticate(user=self.staff) + response = self.client.get(self.large_taxonomy_url) + assert response.status_code == status.HTTP_200_OK + + data = response.data + results = data.get("results", []) + + # Count of paginated root tags + assert len(results) == self.page_size + + # Checking tag fields + root_tag = self.large_taxonomy.tag_set.get(id=results[0].get("id")) + assert results[0].get("value") == root_tag.value + assert results[0].get("taxonomy_id") == self.large_taxonomy.id + assert results[0].get("parent_id") == root_tag.parent_id + assert results[0].get("children_count") == root_tag.children.count() + assert results[0].get("sub_tags_link") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" + f"/tags/?parent_tag_id={root_tag.id}" + ) + + # Checking pagination values + assert data.get("next") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/?page=2" + ) + assert data.get("previous") is None + assert data.get("count") == self.root_tags_count + assert data.get("num_pages") == 6 + assert data.get("current_page") == 1 + + def test_next_page_large_taxonomy(self): + self._build_large_taxonomy() + self.client.force_authenticate(user=self.staff) + + # Gets the root tags to obtain the next links + response = self.client.get(self.large_taxonomy_url) + + # Gets the next root tags + response = self.client.get(response.data.get("next")) + assert response.status_code == status.HTTP_200_OK + + data = response.data + + # Checking pagination values + assert data.get("next") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/?page=3" + ) + assert data.get("previous") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/" + ) + assert data.get("count") == self.root_tags_count + assert data.get("num_pages") == 6 + assert data.get("current_page") == 2 + + def test_large_search(self): + self._build_large_taxonomy() + search_term = '1' + url = f"{self.large_taxonomy_url}?search_term={search_term}" + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + assert response.status_code == status.HTTP_200_OK + + data = response.data + results = data.get("results", []) + + # Count of paginated root tags + assert len(results) == self.page_size + + # Checking pagination values + assert data.get("next") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" + f"/tags/?page=2&search_term={search_term}" + ) + assert data.get("previous") is None + assert data.get("count") == 51 + assert data.get("num_pages") == 6 + assert data.get("current_page") == 1 + + def test_next_large_search(self): + self._build_large_taxonomy() + search_term = '1' + url = f"{self.large_taxonomy_url}?search_term={search_term}" + + # Get first page of the search + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + + # Get next page + response = self.client.get(response.data.get("next")) + + data = response.data + results = data.get("results", []) + + # Count of paginated root tags + assert len(results) == self.page_size + + # Checking pagination values + assert data.get("next") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" + f"/tags/?page=3&search_term={search_term}" + ) + assert data.get("previous") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" + f"/tags/?search_term={search_term}" + ) + assert data.get("count") == 51 + assert data.get("num_pages") == 6 + assert data.get("current_page") == 2 + + def test_get_children(self): + self._build_large_taxonomy() + self.client.force_authenticate(user=self.staff) + + # Get root tags to obtain the children link of a tag. + response = self.client.get(self.large_taxonomy_url) + results = response.data.get("results", []) + + # Get children tags + response = self.client.get(results[0].get("sub_tags_link")) + assert response.status_code == status.HTTP_200_OK + + data = response.data + results = data.get("results", []) + + # Count of paginated children tags + assert len(results) == self.page_size + + # Checking tag fields + tag = self.large_taxonomy.tag_set.get(id=results[0].get("id")) + assert results[0].get("value") == tag.value + assert results[0].get("taxonomy_id") == self.large_taxonomy.id + assert results[0].get("parent_id") == tag.parent_id + assert results[0].get("children_count") == tag.children.count() + assert results[0].get("sub_tags_link") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" + f"/tags/?parent_tag_id={tag.id}" + ) + + # Checking pagination values + assert data.get("next") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" + f"/tags/?page=2&parent_tag_id={tag.parent_id}" + ) + assert data.get("previous") is None + assert data.get("count") == self.children_tags_count[0] + assert data.get("num_pages") == 2 + assert data.get("current_page") == 1 + + def test_get_leaves(self): + # Get tags depth=2 + self.client.force_authenticate(user=self.staff) + parent_tag = Tag.objects.get(value="Animalia") + + # Build url to get tags depth=2 + url = f"{self.small_taxonomy_url}?parent_tag_id={parent_tag.id}" + response = self.client.get(url) + results = response.data.get("results", []) + + # Checking tag fields + tag = self.small_taxonomy.tag_set.get(id=results[0].get("id")) + assert results[0].get("value") == tag.value + assert results[0].get("taxonomy_id") == self.small_taxonomy.id + assert results[0].get("parent_id") == tag.parent_id + assert results[0].get("children_count") == tag.children.count() + assert results[0].get("sub_tags_link") is None + + def test_next_children(self): + self._build_large_taxonomy() + self.client.force_authenticate(user=self.staff) + + # Get roots to obtain children link of a tag + response = self.client.get(self.large_taxonomy_url) + results = response.data.get("results", []) + + # Get children to obtain next link + response = self.client.get(results[0].get("sub_tags_link")) + + # Get next children + response = self.client.get(response.data.get("next")) + assert response.status_code == status.HTTP_200_OK + + data = response.data + results = data.get("results", []) + tag = self.large_taxonomy.tag_set.get(id=results[0].get("id")) + + # Checking pagination values + assert data.get("next") is None + assert data.get("previous") == ( + "http://testserver/tagging/" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/?parent_tag_id={tag.parent_id}" + ) + assert data.get("count") == self.children_tags_count[0] + assert data.get("num_pages") == 2 + assert data.get("current_page") == 2 From 681c1a00180832124cd58e57e0d356638915c0d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 25 Sep 2023 16:25:01 -0300 Subject: [PATCH 045/282] feat: Enforce limit on number of tags per object (#81) --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/models/base.py | 14 ++ .../core/tagging/rest_api/v1/views.py | 18 +-- .../openedx_tagging/core/tagging/test_api.py | 47 +++++- .../core/tagging/test_models.py | 15 ++ .../core/tagging/test_views.py | 135 ++++++++---------- 6 files changed, 142 insertions(+), 89 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 75d55cb5e..8b70cf96d 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.1.6" +__version__ = "0.1.7" diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index 829909b6a..abb4eae1d 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -446,11 +446,25 @@ def _find_object_tag_index(tag_ref, object_tags) -> int: -1, ) + def _check_new_tag_count(new_tag_count: int) -> None: + """ + Checks if the new count of tags for the object is equal or less than 100 + """ + # Exclude self.id to avoid counting the tags that are going to be updated + current_count = ObjectTag.objects.filter(object_id=object_id).exclude(taxonomy_id=self.id).count() + + if current_count + new_tag_count > 100: + raise ValueError( + _(f"Cannot add more than 100 tags to ({object_id}).") + ) + if not isinstance(tags, list): raise ValueError(_(f"Tags must be a list, not {type(tags).__name__}.")) tags = list(dict.fromkeys(tags)) # Remove duplicates preserving order + _check_new_tag_count(len(tags)) + if not self.allow_multiple and len(tags) > 1: raise ValueError(_(f"Taxonomy ({self.id}) only allows one tag per object.")) diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 52029e046..91208a0f3 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -8,6 +8,7 @@ from rest_framework import mixins from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError from rest_framework.generics import ListAPIView +from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet, ModelViewSet from openedx_tagging.core.tagging.models.base import Tag @@ -194,21 +195,17 @@ class ObjectTagView( GenericViewSet, ): """ - View to retrieve paginated ObjectTags for a provided Object ID (object_id). + View to retrieve ObjectTags for a provided Object ID (object_id). **Retrieve Parameters** * object_id (required): - The Object ID to retrieve ObjectTags for. **Retrieve Query Parameters** * taxonomy (optional) - PK of taxonomy to filter ObjectTags for. - * page (optional) - Page number of paginated results. - * page_size (optional) - Number of results included in each page. **Retrieve Example Requests** GET api/tagging/v1/object_tags/:object_id GET api/tagging/v1/object_tags/:object_id?taxonomy=1 - GET api/tagging/v1/object_tags/:object_id?taxonomy=1&page=2 - GET api/tagging/v1/object_tags/:object_id?taxonomy=1&page=2&page_size=10 **Retrieve Query Returns** * 200 - Success @@ -255,8 +252,7 @@ def get_queryset(self) -> models.QuerySet: def retrieve(self, request, object_id=None): """ - Retrieve ObjectTags that belong to a given object_id and - return paginated results. + Retrieve ObjectTags that belong to a given object_id Note: We override `retrieve` here instead of `list` because we are passing in the Object ID (object_id) in the path (as opposed to passing @@ -266,14 +262,12 @@ def retrieve(self, request, object_id=None): behavior we want. """ object_tags = self.get_queryset() - paginated_object_tags = self.paginate_queryset(object_tags) - serializer = ObjectTagSerializer(paginated_object_tags, many=True) - return self.get_paginated_response(serializer.data) + serializer = ObjectTagSerializer(object_tags, many=True) + return Response(serializer.data) def update(self, request, object_id, partial=False): """ - Update ObjectTags that belong to a given object_id and - return the list of these ObjecTags paginated. + Update ObjectTags that belong to a given object_id Pass a list of Tag ids or Tag values to be applied to an object id in the body `tag` parameter. Passing an empty list will remove all tags from diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 8f185e751..3525bc5c9 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -62,7 +62,7 @@ def test_bad_taxonomy_class(self) -> None: def test_get_taxonomy(self) -> None: tax1 = tagging_api.get_taxonomy(1) assert tax1 == self.taxonomy - no_tax = tagging_api.get_taxonomy(10) + no_tax = tagging_api.get_taxonomy(200) assert no_tax is None def test_get_taxonomies(self) -> None: @@ -78,7 +78,7 @@ def test_get_taxonomies(self) -> None: self.taxonomy, self.system_taxonomy, self.user_taxonomy, - ] + ] + self.dummy_taxonomies assert str(enabled[0]) == f" ({tax1.id}) Enabled" assert str(enabled[1]) == " (5) Import Taxonomy Test" assert str(enabled[2]) == " (-1) Languages" @@ -100,7 +100,7 @@ def test_get_taxonomies(self) -> None: self.taxonomy, self.system_taxonomy, self.user_taxonomy, - ] + ] + self.dummy_taxonomies @override_settings(LANGUAGES=test_languages) def test_get_tags(self) -> None: @@ -587,6 +587,47 @@ def test_tag_object_model_system_taxonomy_invalid(self) -> None: exc.exception ) + def test_tag_object_limit(self) -> None: + """ + Test that the tagging limit is enforced. + """ + # The user can add up to 100 tags to a object + for taxonomy in self.dummy_taxonomies: + tagging_api.tag_object( + taxonomy, + ["Dummy Tag"], + "object_1", + ) + + # Adding a new tag should fail + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + self.taxonomy, + ["Eubacteria"], + "object_1", + ) + assert exc.exception + assert "Cannot add more than 100 tags to" in str(exc.exception) + + # Updating existing tags should work + for taxonomy in self.dummy_taxonomies: + tagging_api.tag_object( + taxonomy, + ["New Dummy Tag"], + "object_1", + ) + + # Updating existing tags adding a new one should fail + for taxonomy in self.dummy_taxonomies: + with self.assertRaises(ValueError) as exc: + tagging_api.tag_object( + taxonomy, + ["New Dummy Tag 1", "New Dummy Tag 2"], + "object_1", + ) + assert exc.exception + assert "Cannot add more than 100 tags to" in str(exc.exception) + def test_get_object_tags(self) -> None: # Alpha tag has no taxonomy alpha = ObjectTag(object_id="abc") diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 6cef2bcf4..44905c3ad 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -115,6 +115,21 @@ def setUp(self): get_tag("System Tag 4"), ] + self.dummy_taxonomies = [] + for i in range(100): + taxonomy = Taxonomy.objects.create( + name=f"ZZ Dummy Taxonomy {i:03}", + allow_free_text=True, + allow_multiple=True + ) + ObjectTag.objects.create( + object_id="limit_tag_count", + taxonomy=taxonomy, + _name=taxonomy.name, + _value="Dummy Tag", + ) + self.dummy_taxonomies.append(taxonomy) + def setup_tag_depths(self): """ Annotate our tags with depth so we can compare them. diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 99c552564..b7b12cd31 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -415,7 +415,7 @@ def _object_permission(_user, object_id: str) -> bool: """ Everyone have object permission on object_id "abc" """ - return object_id == "abc" + return object_id in ("abc", "limit_tag_count") super().setUp() @@ -458,28 +458,43 @@ def _object_permission(_user, object_id: str) -> bool: ) # Free-Text Taxonomies created by taxonomy admins, each linked - # to 200 ObjectTags + # to 10 ObjectTags self.open_taxonomy_enabled = Taxonomy.objects.create(name="Enabled Free-Text Taxonomy", allow_free_text=True) self.open_taxonomy_disabled = Taxonomy.objects.create( name="Disabled Free-Text Taxonomy", allow_free_text=True, enabled=False ) - for i in range(200): + for i in range(10): ObjectTag.objects.create(object_id="abc", taxonomy=self.open_taxonomy_enabled, _value=f"Free Text {i}") ObjectTag.objects.create(object_id="abc", taxonomy=self.open_taxonomy_disabled, _value=f"Free Text {i}") + self.dummy_taxonomies = [] + for i in range(100): + taxonomy = Taxonomy.objects.create( + name=f"Dummy Taxonomy {i}", + allow_free_text=True, + allow_multiple=True + ) + ObjectTag.objects.create( + object_id="limit_tag_count", + taxonomy=taxonomy, + _name=taxonomy.name, + _value="Dummy Tag" + ) + self.dummy_taxonomies.append(taxonomy) + # Override the object permission for the test rules.set_perm("oel_tagging.change_objecttag_objectid", _object_permission) @ddt.data( - (None, "abc", status.HTTP_403_FORBIDDEN, None, None), - ("user", "abc", status.HTTP_200_OK, 461, 10), - ("staff", "abc", status.HTTP_200_OK, 461, 10), - (None, "non-existing-id", status.HTTP_403_FORBIDDEN, None, None), - ("user", "non-existing-id", status.HTTP_200_OK, 0, 0), - ("staff", "non-existing-id", status.HTTP_200_OK, 0, 0), + (None, "abc", status.HTTP_403_FORBIDDEN, None), + ("user", "abc", status.HTTP_200_OK, 81), + ("staff", "abc", status.HTTP_200_OK, 81), + (None, "non-existing-id", status.HTTP_403_FORBIDDEN, None), + ("user", "non-existing-id", status.HTTP_200_OK, 0), + ("staff", "non-existing-id", status.HTTP_200_OK, 0), ) @ddt.unpack - def test_retrieve_object_tags(self, user_attr, object_id, expected_status, expected_count, expected_results): + def test_retrieve_object_tags(self, user_attr, object_id, expected_status, expected_count): """ Test retrieving object tags """ @@ -493,18 +508,16 @@ def test_retrieve_object_tags(self, user_attr, object_id, expected_status, expec assert response.status_code == expected_status if status.is_success(expected_status): - assert response.data.get("count") == expected_count - assert response.data.get("results") is not None - assert len(response.data.get("results")) == expected_results + assert len(response.data) == expected_count @ddt.data( - (None, "abc", status.HTTP_403_FORBIDDEN, None, None), - ("user", "abc", status.HTTP_200_OK, 20, 10), - ("staff", "abc", status.HTTP_200_OK, 20, 10), + (None, "abc", status.HTTP_403_FORBIDDEN, None), + ("user", "abc", status.HTTP_200_OK, 20), + ("staff", "abc", status.HTTP_200_OK, 20), ) @ddt.unpack def test_retrieve_object_tags_taxonomy_queryparam( - self, user_attr, object_id, expected_status, expected_count, expected_results + self, user_attr, object_id, expected_status, expected_count ): """ Test retrieving object tags for specific taxonomies provided @@ -518,11 +531,8 @@ def test_retrieve_object_tags_taxonomy_queryparam( response = self.client.get(url, {"taxonomy": self.enabled_taxonomy.pk}) assert response.status_code == expected_status if status.is_success(expected_status): - assert response.data.get("count") == expected_count - assert response.data.get("results") is not None - assert len(response.data.get("results")) == expected_results - object_tags = response.data.get("results") - for object_tag in object_tags: + assert len(response.data) == expected_count + for object_tag in response.data: assert object_tag.get("is_valid") is True assert object_tag.get("taxonomy_id") == self.enabled_taxonomy.pk @@ -546,51 +556,6 @@ def test_retrieve_object_tags_invalid_taxonomy_queryparam(self, user_attr, objec response = self.client.get(url, {"taxonomy": 123123}) assert response.status_code == expected_status - @ddt.data( - # Page 1, default page size 10, total count 200, returns 10 results - (None, 1, None, status.HTTP_403_FORBIDDEN, None, None), - ("user", 1, None, status.HTTP_200_OK, 200, 10), - ("staff", 1, None, status.HTTP_200_OK, 200, 10), - # Page 2, default page size 10, total count 200, returns 10 results - (None, 2, None, status.HTTP_403_FORBIDDEN, None, None), - ("user", 2, None, status.HTTP_200_OK, 200, 10), - ("staff", 2, None, status.HTTP_200_OK, 200, 10), - # Page 21, default page size 10, total count 200, no more results - (None, 21, None, status.HTTP_403_FORBIDDEN, None, None), - ("user", 21, None, status.HTTP_404_NOT_FOUND, None, None), - ("staff", 21, None, status.HTTP_404_NOT_FOUND, None, None), - # Page 3, page size 2, total count 200, returns 2 results - (None, 3, 2, status.HTTP_403_FORBIDDEN, 200, 2), - ("user", 3, 2, status.HTTP_200_OK, 200, 2), - ("staff", 3, 2, status.HTTP_200_OK, 200, 2), - ) - @ddt.unpack - def test_retrieve_object_tags_pagination( - self, user_attr, page, page_size, expected_status, expected_count, expected_results - ): - """ - Test pagination for retrieve object tags - """ - url = OBJECT_TAGS_RETRIEVE_URL.format(object_id="abc") - - if user_attr: - user = getattr(self, user_attr) - self.client.force_authenticate(user=user) - - query_params = {"taxonomy": self.open_taxonomy_enabled.pk, "page": page} - if page_size: - query_params["page_size"] = page_size - - response = self.client.get(url, query_params) - assert response.status_code == expected_status - if status.is_success(expected_status): - assert response.data.get("count") == expected_count - assert response.data.get("results") is not None - assert len(response.data.get("results")) == expected_results - object_tags = response.data.get("results") - for object_tag in object_tags: - assert object_tag.get("taxonomy_id") == self.open_taxonomy_enabled.pk - @ddt.data( (None, "POST", status.HTTP_403_FORBIDDEN), (None, "PATCH", status.HTTP_403_FORBIDDEN), @@ -669,8 +634,8 @@ def test_tag_object(self, user_attr, taxonomy_attr, tag_values, expected_status) response = self.client.put(url, {"tags": tag_values}, format="json") assert response.status_code == expected_status if status.is_success(expected_status): - assert len(response.data.get("results")) == len(tag_values) - assert set(t["value"] for t in response.data["results"]) == set(tag_values) + assert len(response.data) == len(tag_values) + assert set(t["value"] for t in response.data) == set(tag_values) @ddt.data( # Can't add invalid tags to a closed taxonomy @@ -736,8 +701,8 @@ def test_tag_object_clear(self, user_attr, taxonomy_attr, tag_values, expected_s response = self.client.put(url, {"tags": tag_values}, format="json") assert response.status_code == expected_status if status.is_success(expected_status): - assert len(response.data.get("results")) == len(tag_values) - assert set(t["value"] for t in response.data["results"]) == set(tag_values) + assert len(response.data) == len(tag_values) + assert set(t["value"] for t in response.data) == set(tag_values) @ddt.data( # Users and staff can add multiple tags to a allow_multiple=True taxonomy @@ -773,8 +738,8 @@ def test_tag_object_multiple(self, user_attr, taxonomy_attr, tag_values, expecte response = self.client.put(url, {"tags": tag_values}, format="json") assert response.status_code == expected_status if status.is_success(expected_status): - assert len(response.data.get("results")) == len(tag_values) - assert set(t["value"] for t in response.data["results"]) == set(tag_values) + assert len(response.data) == len(tag_values) + assert set(t["value"] for t in response.data) == set(tag_values) @ddt.data( (None, status.HTTP_403_FORBIDDEN), @@ -791,6 +756,30 @@ def test_tag_object_without_permission(self, user_attr, expected_status): response = self.client.put(url, {"tags": ["Tag 1"]}, format="json") assert response.status_code == expected_status + assert not status.is_success(expected_status) # No success cases here + + def test_tag_object_count_limit(self): + """ + Checks if the limit of 100 tags per object is enforced + """ + object_id = "limit_tag_count" + url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=self.enabled_taxonomy.pk) + self.client.force_authenticate(user=self.staff) + response = self.client.put(url, {"tags": ["Tag 1"]}, format="json") + # Can't add another tag because the object already has 100 tags + assert response.status_code == status.HTTP_400_BAD_REQUEST + + # The user can edit the tags that are already on the object + for taxonomy in self.dummy_taxonomies: + url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=taxonomy.pk) + response = self.client.put(url, {"tags": ["New Tag"]}, format="json") + assert response.status_code == status.HTTP_200_OK + + # Editing tags adding another one will fail + for taxonomy in self.dummy_taxonomies: + url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=taxonomy.pk) + response = self.client.put(url, {"tags": ["New Tag 1", "New Tag 2"]}, format="json") + assert response.status_code == status.HTTP_400_BAD_REQUEST class TestTaxonomyTagsView(TestTaxonomyViewMixin): From 0c4ca76715bc76d4caf6718ca36a29d061b060ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Thu, 28 Sep 2023 15:47:30 -0300 Subject: [PATCH 046/282] feat: make object_id field case insensitive (#86) --- openedx_learning/__init__.py | 2 +- .../0009_alter_objecttag_object_id.py | 20 ++++++++++ openedx_tagging/core/tagging/models/base.py | 4 +- .../openedx_tagging/core/tagging/test_api.py | 32 ++++++++++++++++ .../core/tagging/test_models.py | 37 +++++++++++++++++++ 5 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 openedx_tagging/core/tagging/migrations/0009_alter_objecttag_object_id.py diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 8b70cf96d..576d6b42d 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.1.7" +__version__ = "0.1.8" diff --git a/openedx_tagging/core/tagging/migrations/0009_alter_objecttag_object_id.py b/openedx_tagging/core/tagging/migrations/0009_alter_objecttag_object_id.py new file mode 100644 index 000000000..9e1150b27 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0009_alter_objecttag_object_id.py @@ -0,0 +1,20 @@ +# Generated by Django 3.2.19 on 2023-09-26 13:36 + +from django.db import migrations + +import openedx_learning.lib.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_tagging', '0008_taxonomy_description_not_null'), + ] + + operations = [ + migrations.AlterField( + model_name='objecttag', + name='object_id', + field=openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, db_index=True, editable=False, help_text='Identifier for the object being tagged', max_length=255), + ), + ] diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index abb4eae1d..58a7c950d 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -11,7 +11,7 @@ from django.utils.translation import gettext_lazy as _ from typing_extensions import Self # Until we upgrade to python 3.11 -from openedx_learning.lib.fields import MultiCollationTextField, case_insensitive_char_field +from openedx_learning.lib.fields import MultiCollationTextField, case_insensitive_char_field, case_sensitive_char_field log = logging.getLogger(__name__) @@ -577,7 +577,7 @@ class ObjectTag(models.Model): """ id = models.BigAutoField(primary_key=True) - object_id = case_insensitive_char_field( + object_id = case_sensitive_char_field( max_length=255, db_index=True, editable=False, diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 3525bc5c9..422b15c1b 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -494,6 +494,38 @@ def test_tag_object_same_value_multiple_free(self) -> None: ) assert len(object_tags) == 1 + def test_tag_object_case_id(self) -> None: + """ + Test that the case of the object_id is preserved. + """ + tagging_api.tag_object( + self.taxonomy, + [self.eubacteria.id], + "biology101", + ) + + tagging_api.tag_object( + self.taxonomy, + [self.archaea.id], + "BIOLOGY101", + ) + + object_tags_lower = tagging_api.get_object_tags( + taxonomy_id=self.taxonomy.pk, + object_id="biology101", + ) + + assert len(object_tags_lower) == 1 + assert object_tags_lower[0].tag_id == self.eubacteria.id + + object_tags_upper = tagging_api.get_object_tags( + taxonomy_id=self.taxonomy.pk, + object_id="BIOLOGY101", + ) + + assert len(object_tags_upper) == 1 + assert object_tags_upper[0].tag_id == self.archaea.id + @override_settings(LANGUAGES=test_languages) def test_tag_object_language_taxonomy(self) -> None: tags_list = [ diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 44905c3ad..aa7bca0a0 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -3,6 +3,7 @@ """ import ddt # type: ignore[import] from django.contrib.auth import get_user_model +from django.db import transaction from django.db.utils import IntegrityError from django.test.testcases import TestCase @@ -525,3 +526,39 @@ def test_tag_object_invalid_tag(self): "biology101", ) assert "Invalid object tag for taxonomy" in str(exc.exception) + + def test_tag_case(self) -> None: + """ + Test that the object_id is case sensitive. + """ + # Tag with object_id with lower case + ObjectTag( + object_id="case:id:2", + taxonomy=self.taxonomy, + tag=self.domain_tags[0], + ).save() + + # Tag with object_id with upper case should not trigger IntegrityError + ObjectTag( + object_id="CASE:id:2", + taxonomy=self.taxonomy, + tag=self.domain_tags[0], + ).save() + + # Create another ObjectTag with lower case object_id should trigger IntegrityError + with transaction.atomic(): + with self.assertRaises(IntegrityError): + ObjectTag( + object_id="case:id:2", + taxonomy=self.taxonomy, + tag=self.domain_tags[0], + ).save() + + # Create another ObjectTag with upper case object_id should trigger IntegrityError + with transaction.atomic(): + with self.assertRaises(IntegrityError): + ObjectTag( + object_id="CASE:id:2", + taxonomy=self.taxonomy, + tag=self.domain_tags[0], + ).save() From 5de0969f73b76d69a5f4957342e1f4c73bd3735b Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Tue, 3 Oct 2023 14:36:56 -0700 Subject: [PATCH 047/282] feat: Simplify Tag Models [FC-0030] (#87) * feat: Remove object_tag_class from Taxonomy * feat: set tags by value, not tag_ref. Remove ObjectTag from Taxonomy. * chore: updated tests for system defined taxonomies * feat: more cleanups and validation for ObjectTag * feat: Remove "re-sync" of deleted taxonomies, update test_api.py * fix: case-insensitive values on MySQL * feat: minor cleanups * fix: fix flaky test * docs: Update system defined taxonomy creation ADR * chore: version bump: 0.2.0 --- .../0012-system-taxonomy-creation.rst | 95 +++-- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 143 +++++-- .../tagging/fixtures/language_taxonomy.yaml | 1 + .../core/tagging/import_export/api.py | 2 +- .../core/tagging/migrations/0010_cleanups.py | 32 ++ .../core/tagging/models/__init__.py | 5 +- openedx_tagging/core/tagging/models/base.py | 351 ++++------------ .../core/tagging/models/system_defined.py | 291 +++++++------- .../core/tagging/rest_api/v1/serializers.py | 4 +- .../core/tagging/rest_api/v1/views.py | 2 + .../openedx_tagging/core/tagging/test_api.py | 376 +++++------------- .../core/tagging/test_models.py | 201 ++++------ .../tagging/test_system_defined_models.py | 369 +++++++++-------- .../core/tagging/test_views.py | 2 +- 15 files changed, 811 insertions(+), 1065 deletions(-) create mode 100644 openedx_tagging/core/tagging/migrations/0010_cleanups.py diff --git a/docs/decisions/0012-system-taxonomy-creation.rst b/docs/decisions/0012-system-taxonomy-creation.rst index 0036432bf..c7bb44ff8 100644 --- a/docs/decisions/0012-system-taxonomy-creation.rst +++ b/docs/decisions/0012-system-taxonomy-creation.rst @@ -4,9 +4,10 @@ Context -------- -System-defined taxonomies are taxonomies created by the system. Some of these are totally static (e.g Language) -and some depends on a core data model (e.g. Organizations). It is necessary to define how to create and validate -the System-defined taxonomies and their tags. +System-defined taxonomies are taxonomies created by the system. Some of these +depend on Django settings (e.g. Languages) and others depends on a core data +model (e.g. Organizations or Users). It is necessary to define how to create and +validate the System-defined taxonomies and their tags. Decision @@ -15,44 +16,50 @@ Decision System Tag lists and validation ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Each System-defined Taxonomy will have its own ``ObjectTag`` subclass which is used for tag validation (e.g. ``LanguageObjectTag``, ``OrganizationObjectTag``). -Each subclass can overwrite ``get_tags``; to configure the valid tags, and ``is_valid``; to check if a list of tags are valid. Both functions are implemented on the ``ObjectTag`` base class, but can be overwritten to handle special cases. - -We need to create an instance of each System-defined Taxonomy in a fixture. With their respective characteristics and subclasses. -The ``pk`` of these instances must be negative so as not to affect the auto-incremented ``pk`` of Taxonomies. - -Later, we need to create content-side ObjectTags that live on ``openedx.features.content_tagging`` for each content and taxonomy to be used (eg. ``CourseLanguageObjectTag``, ``CourseOrganizationObjectTag``). -This new class is used to configure the automatic content tagging. You can read the `document number 0013`_ to see this configuration. - -Tags creation -~~~~~~~~~~~~~~ - -We have two ways to handle Tags creation and validation for System-defined Taxonomies: - -**Hardcoded by fixtures/migrations** - -#. If the tags don't change over the time, you can create all on a fixture (e.g Languages). - The ``pk`` of these instances must be negative. -#. If the tags change over the time, you can create all on a migration. If you edit, delete, or add new tags, you should also do it in a migration. - -**Dynamic tags** - -Closed Taxonomies that depends on a core data model. Ex. AuthorTaxonomy with Users as Tags - -#. Tags are created on the fly when new ObjectTags are added. -#. Tag.external_id we store an identifier from the instance (eg. User.pk). -#. Tag.value we store a human readable representation of the instance (eg. User.username). -#. Resync the tags to re-fetch the value. - - -Rejected Options ------------------ - -Free-form tags -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Open Taxonomy that depends on a core data model, but simplifies the creation of Tags by allowing free-form tags, - -Rejected because it has been seen that using dynamic tags provides more functionality and more advantages. - -.. _document number 0013: https://github.com/openedx/openedx-learning/blob/main/docs/decisions/0013-system-taxonomy-auto-tagging.rst +Each Taxonomy has two methods for validating tags: +#. ``validate_value`` +#. ``validate_external_id`` + +These functions will return ``True`` if a given tag is valid, based on its +external ID or value. Subclasses should override these as needed, to implement +different types of taxonomy behavior (e.g. based on a model or baed on Django +settings). + +For example ``validate_value("English")`` will return ``True`` for the language +taxonomy if the English language is enabled in the Django settings. Likewise, +``validate_external_id("en")`` would return true, but +``validate_external_id("zz")`` would be ``False`` because there is no such +language. Or, for a User taxonomy, ``validate_value("username")`` would return +``True`` if a user with that username exists, or ``validate_external_id(...)`` +could validate if a user with that ID exists (note that the ID must be converted +to a string). + +In all of these cases, a ``Tag`` instance may or may not exist in the database. +Before saving an ``ObjectTag`` which references a tag in these taxonomies, the +tagging API will use either ``Taxonomy.tag_for_value`` or +``Taxonomy.tag_for_external_id``. These methods are responsible for both +validating the tag (like ``validate_...``) but also auto-creating the ``Tag`` +instance in case it doesn't already exist. Subclasses should override these as +needed. + +In this way, the system-defined taxonomies are fully dynamic and can represent +tags based on Languages, Users, or Organizations that may exist in large numbers +or be constantly created. + +At present, there isn't a good way to *list* all of the [potential] tags that +exist in a system-defined Taxonomy. We may add an API for that in the future, +for example to list all of the available languages. However for other cases like +users it doesn't make sense to even try to list all of the available tags. So +for now, the assumption is that the UI will not even try to display a list of +available tags for system-defined taxonomies. After all, system-defined tags are +usually applied automatically, rather than a user manually selecting from a +list. If there is a need to show a list of tags to the user, use the API that +lists the actually applied tags - i.e. the values of the ``ObjectTag``s +currently applied to objects using the taxonomy. + +Tags hard-coded by fixtures/migrations +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In the future there may be system-defined taxonomies that are not dynamics at +all, where the list of tags are defined by ``Tag`` instances created by a +fixture or migration. However, as of now we don't have a use case for that. diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 576d6b42d..574c53b52 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.1.8" +__version__ = "0.2.0" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index ca2cb1f11..14595a1ae 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -12,13 +12,15 @@ """ from __future__ import annotations -from typing import Iterator - -from django.db.models import QuerySet +from django.db import transaction +from django.db.models import F, QuerySet from django.utils.translation import gettext_lazy as _ from .models import ObjectTag, Tag, Taxonomy +# Export this as part of the API +TagDoesNotExist = Tag.DoesNotExist + def create_taxonomy( name: str, @@ -46,11 +48,11 @@ def create_taxonomy( return taxonomy.cast() -def get_taxonomy(id: int) -> Taxonomy | None: +def get_taxonomy(taxonomy_id: int) -> Taxonomy | None: """ Returns a Taxonomy cast to the appropriate subclass which has the given ID. """ - taxonomy = Taxonomy.objects.filter(id=id).first() + taxonomy = Taxonomy.objects.filter(pk=taxonomy_id).first() return taxonomy.cast() if taxonomy else None @@ -139,21 +141,18 @@ def resync_object_tags(object_tags: QuerySet | None = None) -> int: def get_object_tags( - object_id: str, taxonomy_id: str | None = None + object_id: str, + taxonomy_id: int | None = None, + object_tag_class: type[ObjectTag] = ObjectTag ) -> QuerySet[ObjectTag]: """ Returns a Queryset of object tags for a given object. - Pass taxonomy to limit the returned object_tags to a specific taxonomy. + Pass taxonomy_id to limit the returned object_tags to a specific taxonomy. """ - ObjectTagClass = ObjectTag - extra_filters = {} - if taxonomy_id is not None: - taxonomy = Taxonomy.objects.get(pk=taxonomy_id) - ObjectTagClass = taxonomy.object_tag_class - extra_filters["taxonomy_id"] = taxonomy_id + filters = {"taxonomy_id": taxonomy_id} if taxonomy_id else {} tags = ( - ObjectTagClass.objects.filter(object_id=object_id, **extra_filters) + object_tag_class.objects.filter(object_id=object_id, **filters) .select_related("tag", "taxonomy") .order_by("id") ) @@ -164,30 +163,103 @@ def delete_object_tags(object_id: str): """ Delete all ObjectTag entries for a given object. """ - tags = ObjectTag.objects.filter( - object_id=object_id, - ) + tags = ObjectTag.objects.filter(object_id=object_id) tags.delete() +# TODO: a function called "tag_object" should take "object_id" as its first parameter, not taxonomy def tag_object( taxonomy: Taxonomy, tags: list[str], object_id: str, -) -> list[ObjectTag]: + object_tag_class: type[ObjectTag] = ObjectTag, +) -> None: """ - Replaces the existing ObjectTag entries for the given taxonomy + object_id with the given list of tags. + Replaces the existing ObjectTag entries for the given taxonomy + object_id + with the given list of tags. + + tags: A list of the values of the tags from this taxonomy to apply. - If taxonomy.allows_free_text, then the list should be a list of tag values. - Otherwise, it should be a list of existing Tag IDs. + object_tag_class: Optional. Use a proxy subclass of ObjectTag for additional + validation. (e.g. only allow tagging certain types of objects.) - Raised ValueError if the proposed tags are invalid for this taxonomy. - Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. + Raised Tag.DoesNotExist if the proposed tags are invalid for this taxonomy. + Preserves existing (valid) tags, adds new (valid) tags, and removes omitted + (or invalid) tags. """ - return taxonomy.cast().tag_object(tags, object_id) + def _check_new_tag_count(new_tag_count: int) -> None: + """ + Checks if the new count of tags for the object is equal or less than 100 + """ + # Exclude self.id to avoid counting the tags that are going to be updated + current_count = ObjectTag.objects.filter(object_id=object_id).exclude(taxonomy_id=taxonomy.id).count() + + if current_count + new_tag_count > 100: + raise ValueError( + _(f"Cannot add more than 100 tags to ({object_id}).") + ) + + if not isinstance(tags, list): + raise ValueError(_(f"Tags must be a list, not {type(tags).__name__}.")) + + ObjectTagClass = object_tag_class + taxonomy = taxonomy.cast() # Make sure we're using the right subclass. This is a no-op if we are already. + tags = list(dict.fromkeys(tags)) # Remove duplicates preserving order + + _check_new_tag_count(len(tags)) + if not taxonomy.allow_multiple and len(tags) > 1: + raise ValueError(_(f"Taxonomy ({taxonomy.name}) only allows one tag per object.")) + + if taxonomy.required and len(tags) == 0: + raise ValueError( + _(f"Taxonomy ({taxonomy.id}) requires at least one tag per object.") + ) + + current_tags = list( + ObjectTagClass.objects.filter(taxonomy=taxonomy, object_id=object_id) + ) + updated_tags = [] + if taxonomy.allow_free_text: + for tag_value in tags: + object_tag_index = next((i for (i, t) in enumerate(current_tags) if t.value == tag_value), -1) + if object_tag_index >= 0: + # This tag is already applied. + object_tag = current_tags.pop(object_tag_index) + else: + object_tag = ObjectTagClass(taxonomy=taxonomy, object_id=object_id, _value=tag_value) + updated_tags.append(object_tag) + else: + # Handle closed taxonomies: + for tag_value in tags: + tag = taxonomy.tag_for_value(tag_value) # Will raise Tag.DoesNotExist if the value is invalid. + object_tag_index = next((i for (i, t) in enumerate(current_tags) if t.tag_id == tag.id), -1) + if object_tag_index >= 0: + # This tag is already applied. + object_tag = current_tags.pop(object_tag_index) + if object_tag._value != tag.value: # pylint: disable=protected-access + # The ObjectTag's cached '_value' is out of sync with the Tag, so update it: + object_tag._value = tag.value # pylint: disable=protected-access + updated_tags.append(object_tag) + else: + # We are newly applying this tag: + object_tag = ObjectTagClass(taxonomy=taxonomy, object_id=object_id, tag=tag) + updated_tags.append(object_tag) + + # Save all updated tags at once to avoid partial updates + with transaction.atomic(): + # delete any omitted existing tags. We do this first to reduce chances of UNIQUE constraint edge cases + for old_tag in current_tags: + old_tag.delete() + # add the new tags: + for object_tag in updated_tags: + object_tag.full_clean() # Run validation + object_tag.save() + + +# TODO: return tags from closed taxonomies as well as the count of how many times each is used. def autocomplete_tags( taxonomy: Taxonomy, search: str, @@ -228,4 +300,25 @@ def autocomplete_tags( "using get_tags() and filtering them on the frontend." ) ) - return taxonomy.cast().autocomplete_tags(search, object_id) + # Fetch tags that the object already has to exclude them from the result + excluded_tags: list[str] = [] + if object_id: + excluded_tags = list( + taxonomy.objecttag_set.filter(object_id=object_id).values_list( + "_value", flat=True + ) + ) + return ( + # Fetch object tags from this taxonomy whose value contains the search + taxonomy.objecttag_set.filter(_value__icontains=search) + # omit any tags whose values match the tags on the given object + .exclude(_value__in=excluded_tags) + # alphabetical ordering + .order_by("_value") + # Alias the `_value` field to `value` to make it nicer for users + .annotate(value=F("_value")) + # obtain tag values + .values("value", "tag_id") + # remove repeats + .distinct() + ) diff --git a/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml b/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml index 38355b99b..b300d8ac6 100644 --- a/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml +++ b/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml @@ -1296,3 +1296,4 @@ allow_multiple: false allow_free_text: false visible_to_authors: true + _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.LanguageTaxonomy diff --git a/openedx_tagging/core/tagging/import_export/api.py b/openedx_tagging/core/tagging/import_export/api.py index 80f01ad64..77203bf5e 100644 --- a/openedx_tagging/core/tagging/import_export/api.py +++ b/openedx_tagging/core/tagging/import_export/api.py @@ -183,7 +183,7 @@ def _import_export_validations(taxonomy: Taxonomy): if taxonomy.allow_free_text: raise NotImplementedError( _( - f"Import/export for free-form taxonomies will be implemented in the future." + "Import/export for free-form taxonomies will be implemented in the future." ) ) if taxonomy.system_defined: diff --git a/openedx_tagging/core/tagging/migrations/0010_cleanups.py b/openedx_tagging/core/tagging/migrations/0010_cleanups.py new file mode 100644 index 000000000..35b74a2d1 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0010_cleanups.py @@ -0,0 +1,32 @@ +# Generated by Django 3.2.19 on 2023-09-29 16:59 + +import django.db.models.expressions +from django.db import migrations, models + +import openedx_learning.lib.fields + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_tagging', '0009_alter_objecttag_object_id'), + ] + + operations = [ + migrations.DeleteModel( + name='ModelObjectTag', + ), + migrations.DeleteModel( + name='UserModelObjectTag', + ), + migrations.AlterUniqueTogether( + name='objecttag', + unique_together={('object_id', 'taxonomy', 'tag_id'), ('object_id', 'taxonomy', '_value')}, + ), + # ObjectTag.Tag can be blank + migrations.AlterField( + model_name='objecttag', + name='tag', + field=models.ForeignKey(blank=True, default=None, help_text="Tag associated with this object tag. Provides the tag's 'value' if set.", null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_tagging.tag'), + ), + ] diff --git a/openedx_tagging/core/tagging/models/__init__.py b/openedx_tagging/core/tagging/models/__init__.py index 226d9991b..c6b31e052 100644 --- a/openedx_tagging/core/tagging/models/__init__.py +++ b/openedx_tagging/core/tagging/models/__init__.py @@ -1,3 +1,6 @@ +""" +Core models for Tagging +""" from .base import ObjectTag, Tag, Taxonomy from .import_export import TagImportTask, TagImportTaskState -from .system_defined import LanguageTaxonomy, ModelObjectTag, ModelSystemDefinedTaxonomy, UserSystemDefinedTaxonomy +from .system_defined import LanguageTaxonomy, ModelSystemDefinedTaxonomy, UserSystemDefinedTaxonomy diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index 58a7c950d..eb4678bdb 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -6,6 +6,7 @@ import logging from typing import List +from django.core.exceptions import ValidationError from django.db import models from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ @@ -59,7 +60,7 @@ class Tag(models.Model): ) external_id = case_insensitive_char_field( max_length=255, - null=True, + null=True, # To allow multiple values with our UNIQUE constraint, we need to use NULL values here instead of "" blank=True, help_text=_( "Used to link an Open edX Tag with a tag in an externally-defined taxonomy." @@ -137,7 +138,7 @@ class Taxonomy(models.Model): ), ) allow_multiple = models.BooleanField( - default=False, + default=False, # TODO: This should be true, or perhaps remove this property altogether help_text=_( "Indicates that multiple tags from this taxonomy may be added to an object." ), @@ -187,15 +188,6 @@ def __str__(self): ) return f"<{self.__class__.__name__}> ({self.id}) {self.name}" - @property - def object_tag_class(self) -> type[ObjectTag]: - """ - Returns the ObjectTag subclass associated with this taxonomy, which is ObjectTag by default. - - Taxonomy subclasses may override this method to use different subclasses of ObjectTag. - """ - return ObjectTag - @property def taxonomy_class(self) -> type[Taxonomy] | None: """ @@ -253,6 +245,13 @@ def cast(self): return self + def check_casted(self): + """ + Double-check that this taxonomy has been cast() to a subclass if needed. + """ + if self.cast() is not self: + raise TypeError("Taxonomy was used incorrectly - without .cast()") + def copy(self, taxonomy: Taxonomy) -> Taxonomy: """ Copy the fields from the given Taxonomy into the current instance. @@ -265,7 +264,7 @@ def copy(self, taxonomy: Taxonomy) -> Taxonomy: self.allow_multiple = taxonomy.allow_multiple self.allow_free_text = taxonomy.allow_free_text self.visible_to_authors = taxonomy.visible_to_authors - self._taxonomy_class = taxonomy._taxonomy_class + self._taxonomy_class = taxonomy._taxonomy_class # pylint: disable=protected-access return self def get_tags( @@ -350,212 +349,54 @@ def get_filtered_tags( return tag_set.order_by("value", "id") - def validate_object_tag( - self, - object_tag: "ObjectTag", - check_taxonomy=True, - check_tag=True, - check_object=True, - ) -> bool: + def validate_value(self, value: str) -> bool: """ - Returns True if the given object tag is valid for the current Taxonomy. - - Subclasses should override the internal _validate* methods to perform their own validation checks, e.g. against - dynamically generated tag lists. - - If `check_taxonomy` is False, then we skip validating the object tag's taxonomy reference. - If `check_tag` is False, then we skip validating the object tag's tag reference. - If `check_object` is False, then we skip validating the object ID/type. + Check if 'value' is part of this Taxonomy. + A 'Tag' object may not exist for the value (e.g. if this is a free text + taxonomy, then any value is allowed but no Tags are created; if this is + a user taxonomy, Tag entries may only get created as needed.), but if + this returns True then the value conceptually exists in this taxonomy + and can be used to tag objects. """ - if check_taxonomy and not self._check_taxonomy(object_tag): - return False - - if check_tag and not self._check_tag(object_tag): - return False - - if check_object and not self._check_object(object_tag): - return False - - return True + self.check_casted() + if self.allow_free_text: + return value != "" and isinstance(value, str) + return self.tag_set.filter(value__iexact=value).exists() - def _check_taxonomy( - self, - object_tag: ObjectTag, - ) -> bool: + def tag_for_value(self, value: str) -> Tag: """ - Returns True if the given object tag is valid for the current Taxonomy. + Get the Tag object for the given value. + Some Taxonomies may auto-create the Tag at this point, e.g. a User + Taxonomy will create User Tags "just in time". - Subclasses can override this method to perform their own taxonomy validation checks. + Will raise Tag.DoesNotExist if the value is not valid for this taxonomy. """ - # Must be linked to this taxonomy - return ( - object_tag.taxonomy_id is not None - ) and object_tag.taxonomy_id == self.id + self.check_casted() + if self.allow_free_text: + raise ValueError("tag_for_value() doesn't work for free text taxonomies. They don't use Tag instances.") + return self.tag_set.get(value__iexact=value) - def _check_tag( - self, - object_tag: ObjectTag, - ) -> bool: + def validate_external_id(self, external_id: str) -> bool: """ - Returns True if the given object tag's value is valid for the current Taxonomy. - - Subclasses can override this method to perform their own taxonomy validation checks. + Check if 'external_id' is part of this Taxonomy. """ - # Open taxonomies only need a value. + self.check_casted() if self.allow_free_text: - return bool(object_tag.value) - - # Closed taxonomies need an associated tag in this taxonomy - return (object_tag.tag is not None) and object_tag.tag.taxonomy_id == self.id + return False # Free text taxonomies don't use 'external_id' on their tags + return self.tag_set.filter(external_id__iexact=external_id).exists() - def _check_object( - self, - object_tag: ObjectTag, - ) -> bool: + def tag_for_external_id(self, external_id: str) -> Tag: """ - Returns True if the given object tag's object is valid for the current Taxonomy. + Get the Tag object for the given external_id. + Some Taxonomies may auto-create the Tag at this point, e.g. a User + Taxonomy will create User Tags "just in time". - Subclasses can override this method to perform their own taxonomy validation checks. + Will raise Tag.DoesNotExist if the tag is not valid for this taxonomy. """ - return bool(object_tag.object_id) - - def tag_object( - self, - tags: list[str], - object_id: str, - ) -> list[ObjectTag]: - """ - Replaces the existing ObjectTag entries for the current taxonomy + object_id with the given list of tags. - If self.allows_free_text, then the list should be a list of tag values. - Otherwise, it should be either a list of existing Tag Values or IDs. - Raised ValueError if the proposed tags are invalid for this taxonomy. - Preserves existing (valid) tags, adds new (valid) tags, and removes omitted (or invalid) tags. - """ - - def _find_object_tag_index(tag_ref, object_tags) -> int: - """ - Search for Tag in the given list of ObjectTags by tag_ref or value, - returning its index or -1 if not found. - """ - return next( - ( - i - for i, object_tag in enumerate(object_tags) - if object_tag.tag_ref == tag_ref or object_tag.value == tag_ref - ), - -1, - ) - - def _check_new_tag_count(new_tag_count: int) -> None: - """ - Checks if the new count of tags for the object is equal or less than 100 - """ - # Exclude self.id to avoid counting the tags that are going to be updated - current_count = ObjectTag.objects.filter(object_id=object_id).exclude(taxonomy_id=self.id).count() - - if current_count + new_tag_count > 100: - raise ValueError( - _(f"Cannot add more than 100 tags to ({object_id}).") - ) - - if not isinstance(tags, list): - raise ValueError(_(f"Tags must be a list, not {type(tags).__name__}.")) - - tags = list(dict.fromkeys(tags)) # Remove duplicates preserving order - - _check_new_tag_count(len(tags)) - - if not self.allow_multiple and len(tags) > 1: - raise ValueError(_(f"Taxonomy ({self.id}) only allows one tag per object.")) - - if self.required and len(tags) == 0: - raise ValueError( - _(f"Taxonomy ({self.id}) requires at least one tag per object.") - ) - - ObjectTagClass = self.object_tag_class - current_tags = list( - ObjectTagClass.objects.filter( - taxonomy=self, - object_id=object_id, - ) - ) - updated_tags = [] - for tag_ref in tags: - object_tag_index = _find_object_tag_index(tag_ref, current_tags) - if object_tag_index >= 0: - object_tag = current_tags.pop(object_tag_index) - else: - object_tag = ObjectTagClass( - taxonomy=self, - object_id=object_id, - ) - - object_tag.tag_ref = tag_ref - object_tag.resync() - if not self.validate_object_tag(object_tag): - raise ValueError( - _(f"Invalid object tag for taxonomy ({self.id}): {tag_ref}") - ) - updated_tags.append(object_tag) - - # Save all updated tags at once to avoid partial updates - for object_tag in updated_tags: - object_tag.save() - - # ...and delete any omitted existing tags - for old_tag in current_tags: - old_tag.delete() - - return updated_tags - - def autocomplete_tags( - self, - search: str, - object_id: str | None = None, - ) -> models.QuerySet: - """ - Provides auto-complete suggestions by matching the `search` string against existing - ObjectTags linked to the given taxonomy. A case-insensitive search is used in order - to return the highest number of relevant tags. - - If `object_id` is provided, then object tag values already linked to this object - are omitted from the returned suggestions. (ObjectTag values must be unique for a - given object + taxonomy, and so omitting these suggestions helps users avoid - duplication errors.). - - Returns a QuerySet of dictionaries containing distinct `value` (string) and `tag` - (numeric ID) values, sorted alphabetically by `value`. - - Subclasses can override this method to perform their own autocomplete process. - Subclass use cases: - * Large taxonomy associated with a model. It can be overridden to get - the suggestions directly from the model by doing own filtering. - * Taxonomy with a list of available tags: It can be overridden to only - search the suggestions on a list of available tags. - """ - # Fetch tags that the object already has to exclude them from the result - excluded_tags: list[str] = [] - if object_id: - excluded_tags = list( - self.objecttag_set.filter(object_id=object_id).values_list( - "_value", flat=True - ) - ) - return ( - # Fetch object tags from this taxonomy whose value contains the search - self.objecttag_set.filter(_value__icontains=search) - # omit any tags whose values match the tags on the given object - .exclude(_value__in=excluded_tags) - # alphabetical ordering - .order_by("_value") - # Alias the `_value` field to `value` to make it nicer for users - .annotate(value=models.F("_value")) - # obtain tag values - .values("value", "tag_id") - # remove repeats - .distinct() - ) + self.check_casted() + if self.allow_free_text: + raise ValueError("tag_for_external_id() doesn't work for free text taxonomies.") + return self.tag_set.get(external_id__iexact=external_id) class ObjectTag(models.Model): @@ -595,7 +436,8 @@ class ObjectTag(models.Model): ) tag = models.ForeignKey( Tag, - null=True, + null=True, # NULL in the case of free text taxonomies or when the Tag gets deleted. + blank=True, default=None, on_delete=models.SET_NULL, help_text=_( @@ -603,7 +445,6 @@ class ObjectTag(models.Model): ), ) _name = case_insensitive_char_field( - null=False, max_length=255, help_text=_( "User-facing label used for this tag, stored in case taxonomy is (or becomes) null." @@ -611,7 +452,6 @@ class ObjectTag(models.Model): ), ) _value = case_insensitive_char_field( - null=False, max_length=500, help_text=_( "User-facing value used for this tag, stored in case tag is null, e.g if taxonomy is free text, or if it" @@ -625,7 +465,19 @@ class Meta: models.Index(fields=["taxonomy", "object_id"]), models.Index(fields=["taxonomy", "_value"]), ] - unique_together = ("taxonomy", "_value", "object_id") + unique_together = [ + ("object_id", "taxonomy", "tag_id"), + ("object_id", "taxonomy", "_value"), + ] + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + if not self.pk: # This is a new instance: + # Set _name and _value automatically on creation, if they weren't set: + if not self._name and self.taxonomy: + self._name = self.taxonomy.name + if not self._value and self.tag: + self._value = self.tag.value def __repr__(self): """ @@ -674,40 +526,33 @@ def value(self, value: str): self._value = value @property - def tag_ref(self) -> str: + def is_deleted(self) -> bool: """ - Returns this tag's reference string. - - If tag is set, then returns its id. - Otherwise, returns the cached _value field. + Has this Tag been deleted from the Taxonomy? If so, we preserve this + ObjecTag in the DB but it shouldn't be shown to the user. """ - return self.tag.id if self.tag else self._value + return self.taxonomy is None or (self.tag is None and not self.taxonomy.allow_free_text) - @tag_ref.setter - def tag_ref(self, tag_ref: str): + def clean(self): """ - Sets the ObjectTag's Tag and/or value, depending on whether a valid Tag is found. + Validate this ObjectTag. - Subclasses may override this method to dynamically create Tags. + Note: this doesn't happen automatically on save(); only when edited in + the django admin. So it's best practice to call obj_tag.full_clean() + before saving. """ - self.value = tag_ref - - if self.taxonomy: - try: - self.tag = self.taxonomy.tag_set.get(pk=tag_ref) - self.value = self.tag.value - except (ValueError, Tag.DoesNotExist): - # This might be ok, e.g. if our taxonomy.allow_free_text, so we just pass through here. - # We rely on the caller to validate before saving. - pass - - def is_valid(self) -> bool: - """ - Returns True if this ObjectTag represents a valid taxonomy tag. - - A valid ObjectTag must be linked to a Taxonomy, and be a valid tag in that taxonomy. - """ - return self.taxonomy.validate_object_tag(self) if self.taxonomy else False + if self.tag: + if self.tag.taxonomy_id != self.taxonomy_id: + raise ValidationError("ObjectTag's Taxonomy does not match Tag taxonomy") + if self.tag.value != self._value: + raise ValidationError("ObjectTag's _value is out of sync with Tag.value") + else: + # Note: self.taxonomy and/or self.tag may be NULL which is OK, because it means the Tag/Taxonomy + # was deleted, but we still preserve this _value here in case the Taxonomy or Tag get re-created in future. + if self._value == "": + raise ValidationError("Invalid _value - empty string") + if self.taxonomy and self.taxonomy.name != self._name: + raise ValidationError("ObjectTag's _name is out of sync with Taxonomy.name") def get_lineage(self) -> Lineage: """ @@ -731,34 +576,8 @@ def resync(self) -> bool: """ changed = False - # Locate an enabled taxonomy matching _name, and maybe a tag matching _value - if not self.taxonomy_id: - # Use the linked tag's taxonomy if there is one. - if self.tag: - self.taxonomy_id = self.tag.taxonomy_id - changed = True - else: - for taxonomy in Taxonomy.objects.filter( - name=self.name, enabled=True - ).order_by("allow_free_text", "id"): - # Cast to the subclass to preserve custom validation - taxonomy = taxonomy.cast() - - # Closed taxonomies require a tag matching _value, - # and we'd rather match a closed taxonomy than an open one. - # So see if there's a matching tag available in this taxonomy. - tag = taxonomy.tag_set.filter(value=self.value).first() - - # Make sure this taxonomy will accept object tags like this. - self.taxonomy = taxonomy - self.tag = tag - if taxonomy.validate_object_tag(self): - changed = True - break - # If not, undo those changes and try the next one - else: - self.taxonomy = None - self.tag = None + # We used to have code here that would try to find a new taxonomy if the current taxonomy has been deleted. + # But for now that's removed, as it risks things like linking a tag to the wrong org's taxonomy. # Sync the stored _name with the taxonomy.name if self.taxonomy and self._name != self.taxonomy.name: @@ -794,6 +613,6 @@ def copy(self, object_tag: ObjectTag) -> Self: self.tag = object_tag.tag self.taxonomy = object_tag.taxonomy self.object_id = object_tag.object_id - self._value = object_tag._value - self._name = object_tag._name + self._value = object_tag._value # pylint: disable=protected-access + self._name = object_tag._name # pylint: disable=protected-access return self diff --git a/openedx_tagging/core/tagging/models/system_defined.py b/openedx_tagging/core/tagging/models/system_defined.py index c78e9aa83..23180f6e7 100644 --- a/openedx_tagging/core/tagging/models/system_defined.py +++ b/openedx_tagging/core/tagging/models/system_defined.py @@ -4,15 +4,15 @@ from __future__ import annotations import logging -from typing import Any from django.conf import settings from django.contrib.auth import get_user_model +from django.core.exceptions import ObjectDoesNotExist from django.db import models -from openedx_tagging.core.tagging.models.base import ObjectTag, Tag +from openedx_tagging.core.tagging.models.base import Tag -from .base import ObjectTag, Tag, Taxonomy +from .base import Tag, Taxonomy log = logging.getLogger(__name__) @@ -34,113 +34,16 @@ def system_defined(self) -> bool: return True -class ModelObjectTag(ObjectTag): - """ - Model-based ObjectTag, abstract class. - - Used by ModelSystemDefinedTaxonomy to maintain dynamic Tags which are associated with a configured Model instance. - """ - - class Meta: - proxy = True - - def __init__(self, *args: Any, **kwargs: Any) -> None: - """ - Checks if the `tag_class_model` is correct - """ - assert issubclass(self.tag_class_model, models.Model) - super().__init__(*args, **kwargs) - - @property - def tag_class_model(self) -> type[models.Model]: - """ - Subclasses must implement this method to return the Django.model - class referenced by these object tags. - """ - raise NotImplementedError - - @property - def tag_class_value(self) -> str: - """ - Returns the name of the tag_class_model field to use as the Tag.value when creating Tags for this taxonomy. - - Subclasses may override this method to use different fields. - """ - return "pk" - - def get_instance(self) -> models.Model | None: - """ - Returns the instance of tag_class_model associated with this object tag, or None if not found. - """ - instance_id = self.tag.external_id if self.tag else None - if instance_id: - try: - return self.tag_class_model.objects.get(pk=instance_id) - except ValueError as e: - log.exception(f"{self}: {str(e)}") - except self.tag_class_model.DoesNotExist: - log.exception( - f"{self}: {self.tag_class_model.__name__} pk={instance_id} does not exist." - ) - - return None - - def _resync_tag(self) -> bool: - """ - Resync our tag's value with the value from the instance. - - If the instance associated with the tag no longer exists, we unset our tag, because it's no longer valid. - - Returns True if the given tag was changed, False otherwise. - """ - instance = self.get_instance() - if instance: - value = getattr(instance, self.tag_class_value) - self.value = value - if self.tag and self.tag.value != value: - self.tag.value = value - self.tag.save() - return True - else: - self.tag = None - - return False - - @property - def tag_ref(self) -> str: - return (self.tag.external_id or self.tag.id) if self.tag else self._value - - @tag_ref.setter - def tag_ref(self, tag_ref: str): - """ - Sets the ObjectTag's Tag and/or value, depending on whether a valid Tag is found, or can be created. - - Creates a Tag for the given tag_ref value, if one containing that external_id not already exist. - """ - self.value = tag_ref - - if self.taxonomy: - try: - self.tag = self.taxonomy.tag_set.get( - external_id=tag_ref, - ) - except (ValueError, Tag.DoesNotExist): - # Creates a new Tag for this instance - self.tag = Tag( - taxonomy=self.taxonomy, - external_id=tag_ref, - ) - - self._resync_tag() - - class ModelSystemDefinedTaxonomy(SystemDefinedTaxonomy): """ Model based system taxonomy abstract class. - This type of taxonomy has an associated Django model in ModelObjectTag.tag_class_model(). - They are designed to create Tags when required for new ObjectTags, to maintain - their status as "closed" taxonomies. + This type of taxonomy has an associated Django model in + ModelSystemDefinedTaxonomy.tag_class_model. + + They are designed to create Tags when required for new ObjectTags, to + maintain their status as "closed" taxonomies. + The Tags are representations of the instances of the associated model. Tag.external_id stores an identifier from the instance (`pk` as default) @@ -155,41 +58,101 @@ class ModelSystemDefinedTaxonomy(SystemDefinedTaxonomy): class Meta: proxy = True - def __init__(self, *args: Any, **kwargs: Any) -> None: + @property + def tag_class_model(self) -> type[models.Model]: """ - Checks if the `object_tag_class` is a subclass of ModelObjectTag. + Define what Django model this taxonomy is associated with """ - assert issubclass(self.object_tag_class, ModelObjectTag) - super().__init__(*args, **kwargs) + raise NotImplementedError @property - def object_tag_class(self) -> type[ModelObjectTag]: + def tag_class_value_field(self) -> str: """ - Returns the ModelObjectTag subclass associated with this taxonomy. + The name of the tag_class_model field to use as the Tag.value when creating Tags for this taxonomy. - Model Taxonomy subclasses must implement this to provide a ModelObjectTag subclass. + Subclasses may override this method to use different fields. """ raise NotImplementedError - def _check_instance(self, object_tag: ObjectTag) -> bool: + @property + def tag_class_key_field(self) -> str: """ - Returns True if the instance exists + The name of the tag_class_model field to use as the Tag.external_id when creating Tags for this taxonomy. - Subclasses can override this method to perform their own instance validation checks. + This must be an immutable ID. """ - object_tag = self.object_tag_class.cast(object_tag) - return bool(object_tag.get_instance()) + return "pk" - def _check_tag(self, object_tag: ObjectTag) -> bool: - """ - Returns True if the instance is valid - """ - return super()._check_tag(object_tag) and self._check_instance(object_tag) + def validate_value(self, value: str): + """ + Check if 'value' is part of this Taxonomy, based on the specified model. + """ + try: + self.tag_class_model.objects.get(**{f"{self.tag_class_value_field}__iexact": value}) + return True + except ObjectDoesNotExist: + return False + + def tag_for_value(self, value: str): + """ + Get the Tag object for the given value. + """ + try: + # First we look up the instance by value. + # We specify 'iexact' but whether it's case sensitive or not on MySQL depends on the model's collation. + instance = self.tag_class_model.objects.get(**{f"{self.tag_class_value_field}__iexact": value}) + except ObjectDoesNotExist as exc: + raise Tag.DoesNotExist from exc + # Use the canonical value from here on (possibly with different case from the value given as a parameter) + value = getattr(instance, self.tag_class_value_field) + # We assume the value may change but the external_id is immutable. + # So look up keys using external_id. There may be a key with the same external_id but an out of date value. + external_id = str(getattr(instance, self.tag_class_key_field)) + tag, _created = self.tag_set.get_or_create(external_id=external_id, defaults={"value": value}) + if tag.value != value: + # Update the Tag to reflect the new cached 'value' + tag.value = value + tag.save() + return tag + + def validate_external_id(self, external_id: str): + """ + Check if 'external_id' is part of this Taxonomy. + """ + try: + self.tag_class_model.objects.get(**{f"{self.tag_class_key_field}__iexact": external_id}) + return True + except ObjectDoesNotExist: + return False + + def tag_for_external_id(self, external_id: str): + """ + Get the Tag object for the given external_id. + Some Taxonomies may auto-create the Tag at this point, e.g. a User + Taxonomy will create User Tags "just in time". + + Will raise Tag.DoesNotExist if the tag is not valid for this taxonomy. + """ + try: + # First we look up the instance by external_id + # We specify 'iexact' but whether it's case sensitive or not on MySQL depends on the model's collation. + instance = self.tag_class_model.objects.get(**{f"{self.tag_class_key_field}__iexact": external_id}) + except ObjectDoesNotExist as exc: + raise Tag.DoesNotExist from exc + value = getattr(instance, self.tag_class_value_field) + # Use the canonical external_id from here on (may differ in capitalization) + external_id = getattr(instance, self.tag_class_key_field) + tag, _created = self.tag_set.get_or_create(external_id=external_id, defaults={"value": value}) + if tag.value != value: + # Update the Tag to reflect the new cached 'value' + tag.value = value + tag.save() + return tag -class UserModelObjectTag(ModelObjectTag): +class UserSystemDefinedTaxonomy(ModelSystemDefinedTaxonomy): """ - ObjectTags for the UserSystemDefinedTaxonomy. + A Taxonomy that allows tagging objects using users. """ class Meta: @@ -198,12 +161,12 @@ class Meta: @property def tag_class_model(self) -> type[models.Model]: """ - Associate the user model + Define what Django model this taxonomy is associated with """ return get_user_model() @property - def tag_class_value(self) -> str: + def tag_class_value_field(self) -> str: """ Returns the name of the tag_class_model field to use as the Tag.value when creating Tags for this taxonomy. @@ -212,24 +175,6 @@ def tag_class_value(self) -> str: return "username" -class UserSystemDefinedTaxonomy(ModelSystemDefinedTaxonomy): - """ - User based system taxonomy class. - """ - - class Meta: - proxy = True - - @property - def object_tag_class(self): - """ - Returns the ObjectTag subclass associated with this taxonomy, which is ModelObjectTag by default. - - Model Taxonomy subclasses must implement this to provide a ModelObjectTag subclass. - """ - return UserModelObjectTag - - class LanguageTaxonomy(SystemDefinedTaxonomy): """ Language System-defined taxonomy @@ -278,27 +223,61 @@ def get_filtered_tags( search_in_all=search_in_all, ) + @classmethod def _get_available_languages(cls) -> set[str]: """ Get available languages from Django LANGUAGE. """ langs = set() for django_lang in settings.LANGUAGES: - # Split to get the language part - langs.add(django_lang[0].split("-")[0]) + langs.add(django_lang[0]) return langs - def _check_valid_language(self, object_tag: ObjectTag) -> bool: + def validate_value(self, value: str): """ - Returns True if the tag is on the available languages + Check if 'value' is part of this Taxonomy, based on the specified model. """ - available_langs = self._get_available_languages() - if not object_tag.tag: - raise AttributeError("Expected object_tag.tag to be set") - return object_tag.tag.external_id in available_langs + for _, lang_name in settings.LANGUAGES: + if lang_name == value: + return True + return False + + def tag_for_value(self, value: str): + """ + Get the Tag object for the given value. + """ + for lang_code, lang_name in settings.LANGUAGES: + if lang_name == value: + return self.tag_for_external_id(lang_code) + raise Tag.DoesNotExist - def _check_tag(self, object_tag: ObjectTag) -> bool: + def validate_external_id(self, external_id: str): """ - Returns True if the tag is on the available languages + Check if 'external_id' is part of this Taxonomy. + """ + lang_code = external_id.lower() + # Get settings.LANGUAGES (a list of tuples) as a dict. In LMS/CMS this is already cached as LANGUAGE_DICT + languages_as_dict = getattr(settings, "LANGUAGE_DICT", dict(settings.LANGUAGES)) + return lang_code in languages_as_dict + + def tag_for_external_id(self, external_id: str): + """ + Get the Tag object for the given external_id. + Some Taxonomies may auto-create the Tag at this point, e.g. a User + Taxonomy will create User Tags "just in time". + + Will raise Tag.DoesNotExist if the tag is not valid for this taxonomy. """ - return super()._check_tag(object_tag) and self._check_valid_language(object_tag) + lang_code = external_id.lower() + # Get settings.LANGUAGES (a list of tuples) as a dict. In LMS/CMS this is already cached as LANGUAGE_DICT + languages_as_dict = getattr(settings, "LANGUAGE_DICT", dict(settings.LANGUAGES)) + try: + lang_name = languages_as_dict[lang_code] + except KeyError as exc: + raise Tag.DoesNotExist from exc + tag, _created = self.tag_set.get_or_create(external_id=lang_code, defaults={"value": lang_name}) + if tag.value != lang_name: + # Update the Tag to reflect the new language name + tag.value = lang_name + tag.save() + return tag diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 756d4a793..22878fc76 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -53,8 +53,8 @@ class Meta: "name", "value", "taxonomy_id", - "tag_ref", - "is_valid", + # If the Tag or Taxonomy has been deleted, this ObjectTag shouldn't be shown to users. + "is_deleted", ] diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 91208a0f3..9dbb2cc01 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -318,6 +318,8 @@ def update(self, request, object_id, partial=False): tags = body.data.get("tags", []) try: tag_object(taxonomy, tags, object_id) + except Tag.DoesNotExist as e: + raise ValidationError(e) except ValueError as e: raise ValidationError(e) diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 422b15c1b..182f79c2f 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -6,6 +6,7 @@ from typing import Any import ddt # type: ignore[import] +import pytest from django.test import TestCase, override_settings import openedx_tagging.core.tagging.api as tagging_api @@ -81,7 +82,7 @@ def test_get_taxonomies(self) -> None: ] + self.dummy_taxonomies assert str(enabled[0]) == f" ({tax1.id}) Enabled" assert str(enabled[1]) == " (5) Import Taxonomy Test" - assert str(enabled[2]) == " (-1) Languages" + assert str(enabled[2]) == " (-1) Languages" assert str(enabled[3]) == " (1) Life on Earth" assert str(enabled[4]) == " (4) System defined taxonomy" @@ -174,226 +175,125 @@ def check_object_tag( assert object_tag.value == value def test_resync_object_tags(self) -> None: - missing_links = ObjectTag.objects.create( - object_id="abc", - _name=self.taxonomy.name, - _value=self.mammalia.value, - ) - changed_links = ObjectTag.objects.create( - object_id="def", - taxonomy=self.taxonomy, - tag=self.mammalia, - ) - changed_links.name = "Life" - changed_links.value = "Animals" - changed_links.save() - no_changes = ObjectTag.objects.create( - object_id="ghi", - taxonomy=self.taxonomy, - tag=self.mammalia, - ) - no_changes.name = self.taxonomy.name - no_changes.value = self.mammalia.value - no_changes.save() + self.taxonomy.allow_multiple = True + self.taxonomy.save() + open_taxonomy = Taxonomy.objects.create(name="Freetext Life", allow_free_text=True, allow_multiple=True) + + object_id = "obj1" + # Create some tags: + tagging_api.tag_object(self.taxonomy, [self.archaea.value, self.bacteria.value], object_id) # Regular tags + tagging_api.tag_object(open_taxonomy, ["foo", "bar"], object_id) # Free text tags + + # At first, none of these will be deleted: + assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + (self.archaea.value, False), + (self.bacteria.value, False), + ("foo", False), + ("bar", False), + ] - changed = tagging_api.resync_object_tags() - assert changed == 2 - for object_tag in (missing_links, changed_links, no_changes): - self.check_object_tag( - object_tag, self.taxonomy, self.mammalia, "Life on Earth", "Mammalia" - ) + # Delete "bacteria" from the taxonomy: + self.bacteria.delete() # TODO: add an API method for this - # Once all tags are resynced, they stay that way - changed = tagging_api.resync_object_tags() - assert changed == 0 + assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + (self.archaea.value, False), + (self.bacteria.value, True), # <--- deleted! But the value is preserved. + ("foo", False), + ("bar", False), + ] - # Resync will use the tag's taxonomy if possible - changed_links.taxonomy = None - changed_links.save() - changed = tagging_api.resync_object_tags() - assert changed == 1 - for object_tag in (missing_links, changed_links, no_changes): - self.check_object_tag( - object_tag, self.taxonomy, self.mammalia, "Life on Earth", "Mammalia" - ) + # Re-syncing the tags at this point does nothing: + tagging_api.resync_object_tags() - # Resync will use the taxonomy's tags if possible - changed_links.tag = None - changed_links.value = "Xenomorph" - changed_links.save() - changed = tagging_api.resync_object_tags() - assert changed == 0 - changed_links.value = "Mammalia" - changed_links.save() - - # ObjectTag value preserved even if linked tag is deleted - self.mammalia.delete() - for object_tag in (missing_links, changed_links, no_changes): - self.check_object_tag( - object_tag, self.taxonomy, None, "Life on Earth", "Mammalia" - ) - # Recreating the tag to test resyncing works - new_mammalia = Tag.objects.create( - value="Mammalia", - taxonomy=self.taxonomy, - ) + # Now re-create the tag + self.bacteria.save() + + # Then re-sync the tags: changed = tagging_api.resync_object_tags() - assert changed == 3 - for object_tag in (missing_links, changed_links, no_changes): - self.check_object_tag( - object_tag, self.taxonomy, new_mammalia, "Life on Earth", "Mammalia" - ) + assert changed == 1 - # ObjectTag name preserved even if linked taxonomy and its tags are deleted - self.taxonomy.delete() - for object_tag in (missing_links, changed_links, no_changes): - self.check_object_tag(object_tag, None, None, "Life on Earth", "Mammalia") + # Now the tag is not deleted: + assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + (self.archaea.value, False), + (self.bacteria.value, False), # <--- not deleted + ("foo", False), + ("bar", False), + ] - # Resyncing the tags for code coverage + # Re-syncing the tags now does nothing: changed = tagging_api.resync_object_tags() assert changed == 0 - # Recreate the taxonomy and resync some tags - first_taxonomy = tagging_api.create_taxonomy( - "Life on Earth", allow_free_text=True - ) - second_taxonomy = tagging_api.create_taxonomy("Life on Earth") - new_tag = Tag.objects.create( - value="Mammalia", - taxonomy=second_taxonomy, - ) - - # Ensure the resync prefers the closed taxonomy with the matching tag - changed = tagging_api.resync_object_tags( - ObjectTag.objects.filter(object_id__in=["abc", "def"]) - ) - assert changed == 2 - - for object_tag in (missing_links, changed_links): - self.check_object_tag( - object_tag, second_taxonomy, new_tag, "Life on Earth", "Mammalia" - ) - - # Ensure the omitted tag was not updated - self.check_object_tag(no_changes, None, None, "Life on Earth", "Mammalia") - - # Update that one too, to demonstrate the free-text tags are ok - no_changes.value = "Anamelia" - no_changes.save() - changed = tagging_api.resync_object_tags( - ObjectTag.objects.filter(id=no_changes.id) - ) - assert changed == 1 - self.check_object_tag( - no_changes, first_taxonomy, None, "Life on Earth", "Anamelia" - ) - - def test_tag_object(self) -> None: + def test_tag_object(self): self.taxonomy.allow_multiple = True - self.taxonomy.save() + test_tags = [ [ - self.archaea.id, - self.eubacteria.id, - self.chordata.id, + self.archaea, + self.eubacteria, + self.chordata, ], [ - self.chordata.id, - self.archaebacteria.id, + self.chordata, + self.archaebacteria, ], [ - self.archaebacteria.id, - self.archaea.id, + self.archaebacteria, + self.archaea, ], ] # Tag and re-tag the object, checking that the expected tags are returned and deleted for tag_list in test_tags: - object_tags = tagging_api.tag_object( + tagging_api.tag_object( self.taxonomy, - tag_list, + [t.value for t in tag_list], "biology101", ) - # Ensure the expected number of tags exist in the database - assert ( - list( - tagging_api.get_object_tags( - taxonomy_id=self.taxonomy.pk, - object_id="biology101", - ) - ) - == object_tags - ) + object_tags = tagging_api.get_object_tags("biology101", taxonomy_id=self.taxonomy.id) # And the expected number of tags were returned assert len(object_tags) == len(tag_list) for index, object_tag in enumerate(object_tags): - assert object_tag.tag_id == tag_list[index] - assert object_tag.is_valid() + object_tag.full_clean() # Should not raise any ValidationErrors + assert object_tag.tag_id == tag_list[index].id + assert object_tag._value == tag_list[index].value # pylint: disable=protected-access assert object_tag.taxonomy == self.taxonomy assert object_tag.name == self.taxonomy.name assert object_tag.object_id == "biology101" - # Delete the tags - tagging_api.delete_object_tags("biology101") - - # Ensure the tags were deleted - assert ( - len( - list( - tagging_api.get_object_tags( - object_id="biology101", - ) - ) - ) - == 0 - ) - - def test_tag_object_free_text(self) -> None: + def test_tag_object_free_text(self): self.taxonomy.allow_free_text = True - self.taxonomy.save() - object_tags = tagging_api.tag_object( + tagging_api.tag_object( self.taxonomy, ["Eukaryota Xenomorph"], "biology101", ) + object_tags = tagging_api.get_object_tags("biology101") assert len(object_tags) == 1 object_tag = object_tags[0] - assert object_tag.is_valid() + object_tag.full_clean() # Should not raise any ValidationErrors assert object_tag.taxonomy == self.taxonomy assert object_tag.name == self.taxonomy.name - assert object_tag.tag_ref == "Eukaryota Xenomorph" + assert object_tag._value == "Eukaryota Xenomorph" # pylint: disable=protected-access assert object_tag.get_lineage() == ["Eukaryota Xenomorph"] assert object_tag.object_id == "biology101" - def test_tag_object_no_multiple(self) -> None: - with self.assertRaises(ValueError) as exc: - tagging_api.tag_object( - self.taxonomy, - ["A", "B"], - "biology101", - ) - assert "only allows one tag per object" in str(exc.exception) + def test_tag_object_no_multiple(self): + with pytest.raises(ValueError) as excinfo: + tagging_api.tag_object(self.taxonomy, ["A", "B"], "biology101") + assert "only allows one tag per object" in str(excinfo.value) - def test_tag_object_required(self) -> None: + def test_tag_object_required(self): self.taxonomy.required = True - self.taxonomy.save() - with self.assertRaises(ValueError) as exc: - tagging_api.tag_object( - self.taxonomy, - [], - "biology101", - ) - assert "requires at least one tag per object" in str(exc.exception) + with pytest.raises(ValueError) as excinfo: + tagging_api.tag_object(self.taxonomy, [], "biology101") + assert "requires at least one tag per object" in str(excinfo.value) - def test_tag_object_invalid_tag(self) -> None: - with self.assertRaises(ValueError) as exc: - tagging_api.tag_object( - self.taxonomy, - ["Eukaryota Xenomorph"], - "biology101", - ) - assert "Invalid object tag for taxonomy (1): Eukaryota Xenomorph" in str(exc.exception) + def test_tag_object_invalid_tag(self): + with pytest.raises(tagging_api.TagDoesNotExist) as excinfo: + tagging_api.tag_object(self.taxonomy, ["Eukaryota Xenomorph"], "biology101") + assert "Tag matching query does not exist." in str(excinfo.value) def test_tag_object_string(self) -> None: with self.assertRaises(ValueError) as exc: @@ -414,17 +314,18 @@ def test_tag_object_integer(self) -> None: assert "Tags must be a list, not int." in str(exc.exception) def test_tag_object_same_id(self) -> None: - # Tag the object with the same id twice + # Tag the object with the same tag twice tagging_api.tag_object( self.taxonomy, - [self.eubacteria.id], + [self.eubacteria.value], "biology101", ) - object_tags = tagging_api.tag_object( + tagging_api.tag_object( self.taxonomy, - [self.eubacteria.id], + [self.eubacteria.value], "biology101", ) + object_tags = tagging_api.get_object_tags("biology101") assert len(object_tags) == 1 assert str(object_tags[0]) == " biology101: Life on Earth=Eubacteria" @@ -432,66 +333,24 @@ def test_tag_object_same_value(self) -> None: # Tag the object with the same value twice tagging_api.tag_object( self.taxonomy, - ["Eubacteria"], + [self.eubacteria.value, self.eubacteria.value], "biology101", ) - object_tags = tagging_api.tag_object( - self.taxonomy, - ["Eubacteria"], - "biology101", - ) - + object_tags = tagging_api.get_object_tags("biology101") assert len(object_tags) == 1 assert str(object_tags[0]) == " biology101: Life on Earth=Eubacteria" - def test_tag_object_same_mixed(self) -> None: - # Tag the object with the same id/value twice - tagging_api.tag_object( - self.taxonomy, - [self.eubacteria.id], - "biology101", - ) - object_tags = tagging_api.tag_object( - self.taxonomy, - ["Eubacteria"], - "biology101", - ) - - assert len(object_tags) == 1 - assert str(object_tags[0]) == " biology101: Life on Earth=Eubacteria" - - def test_tag_object_same_id_multiple(self) -> None: - self.taxonomy.allow_multiple = True - self.taxonomy.save() - # Tag the object with the same value twice - object_tags = tagging_api.tag_object( - self.taxonomy, - [self.eubacteria.id, self.eubacteria.id], - "biology101", - ) - assert len(object_tags) == 1 - - def test_tag_object_same_value_multiple(self) -> None: - self.taxonomy.allow_multiple = True - self.taxonomy.save() - # Tag the object with the same value twice - object_tags = tagging_api.tag_object( - self.taxonomy, - ["Eubacteria", "Eubacteria"], - "biology101", - ) - assert len(object_tags) == 1 - def test_tag_object_same_value_multiple_free(self) -> None: self.taxonomy.allow_multiple = True self.taxonomy.allow_free_text = True self.taxonomy.save() # Tag the object with the same value twice - object_tags = tagging_api.tag_object( + tagging_api.tag_object( self.taxonomy, ["tag1", "tag1"], "biology101", ) + object_tags = tagging_api.get_object_tags("biology101") assert len(object_tags) == 1 def test_tag_object_case_id(self) -> None: @@ -500,13 +359,13 @@ def test_tag_object_case_id(self) -> None: """ tagging_api.tag_object( self.taxonomy, - [self.eubacteria.id], + [self.eubacteria.value], "biology101", ) tagging_api.tag_object( self.taxonomy, - [self.archaea.id], + [self.archaea.value], "BIOLOGY101", ) @@ -529,48 +388,33 @@ def test_tag_object_case_id(self) -> None: @override_settings(LANGUAGES=test_languages) def test_tag_object_language_taxonomy(self) -> None: tags_list = [ - [get_tag("Azerbaijani").id], - [get_tag("English").id], + ["Azerbaijani"], + ["English"], ] for tags in tags_list: - object_tags = tagging_api.tag_object( - self.language_taxonomy, - tags, - "biology101", - ) + tagging_api.tag_object(self.language_taxonomy, tags, "biology101") # Ensure the expected number of tags exist in the database - assert ( - list( - tagging_api.get_object_tags( - taxonomy_id=self.language_taxonomy.pk, - object_id="biology101", - ) - ) - == object_tags - ) + object_tags = tagging_api.get_object_tags("biology101") # And the expected number of tags were returned assert len(object_tags) == len(tags) for index, object_tag in enumerate(object_tags): - assert object_tag.tag_id == tags[index] - assert object_tag.is_valid() + object_tag.full_clean() # Check full model validation + assert object_tag.value == tags[index] + assert not object_tag.is_deleted assert object_tag.taxonomy == self.language_taxonomy assert object_tag.name == self.language_taxonomy.name assert object_tag.object_id == "biology101" @override_settings(LANGUAGES=test_languages) - def test_tag_object_language_taxonomy_ivalid(self) -> None: - tags = [get_tag("Spanish").id] - with self.assertRaises(ValueError) as exc: + def test_tag_object_language_taxonomy_invalid(self) -> None: + with self.assertRaises(tagging_api.TagDoesNotExist): tagging_api.tag_object( self.language_taxonomy, - tags, + ["Spanish"], "biology101", ) - assert "Invalid object tag for taxonomy (-1): -40" in str( - exc.exception - ) def test_tag_object_model_system_taxonomy(self) -> None: users = [ @@ -579,45 +423,27 @@ def test_tag_object_model_system_taxonomy(self) -> None: ] for user in users: - tags = [user.id] - object_tags = tagging_api.tag_object( - self.user_taxonomy, - tags, - "biology101", - ) + tags = [user.username] + tagging_api.tag_object(self.user_taxonomy, tags, "biology101") # Ensure the expected number of tags exist in the database - assert ( - list( - tagging_api.get_object_tags( - taxonomy_id=self.user_taxonomy.pk, - object_id="biology101", - ) - ) - == object_tags - ) + object_tags = tagging_api.get_object_tags("biology101") # And the expected number of tags were returned assert len(object_tags) == len(tags) for object_tag in object_tags: + object_tag.full_clean() # Check full model validation assert object_tag.tag assert object_tag.tag.external_id == str(user.id) assert object_tag.tag.value == user.username - assert object_tag.is_valid() + assert not object_tag.is_deleted assert object_tag.taxonomy == self.user_taxonomy assert object_tag.name == self.user_taxonomy.name assert object_tag.object_id == "biology101" def test_tag_object_model_system_taxonomy_invalid(self) -> None: tags = ["Invalid id"] - with self.assertRaises(ValueError) as exc: - tagging_api.tag_object( - self.user_taxonomy, - tags, - "biology101", - ) - assert "Invalid object tag for taxonomy (3): Invalid id" in str( - exc.exception - ) + with self.assertRaises(tagging_api.TagDoesNotExist): + tagging_api.tag_object(self.user_taxonomy, tags, "biology101") def test_tag_object_limit(self) -> None: """ diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index aa7bca0a0..6c1d852ec 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -2,11 +2,14 @@ Test the tagging base models """ import ddt # type: ignore[import] +import pytest from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError from django.db import transaction from django.db.utils import IntegrityError from django.test.testcases import TestCase +from openedx_tagging.core.tagging import api from openedx_tagging.core.tagging.models import LanguageTaxonomy, ObjectTag, Tag, Taxonomy @@ -30,9 +33,7 @@ def setUp(self): self.system_taxonomy = Taxonomy.objects.get( name="System defined taxonomy" ) - self.language_taxonomy = Taxonomy.objects.get(name="Languages") - self.language_taxonomy.taxonomy_class = LanguageTaxonomy - self.language_taxonomy = self.language_taxonomy.cast() + self.language_taxonomy = LanguageTaxonomy.objects.get(name="Languages") self.user_taxonomy = Taxonomy.objects.get(name="User Authors").cast() self.archaea = get_tag("Archaea") self.archaebacteria = get_tag("Archaebacteria") @@ -177,7 +178,7 @@ class Meta: @ddt.ddt -class TestModelTagTaxonomy(TestTagTaxonomyMixin, TestCase): +class TestTagTaxonomy(TestTagTaxonomyMixin, TestCase): """ Test the Tag and Taxonomy models' properties and methods. """ @@ -326,7 +327,7 @@ def test_unique_tags(self): ).save() -class TestModelObjectTag(TestTagTaxonomyMixin, TestCase): +class TestObjectTag(TestTagTaxonomyMixin, TestCase): """ Test the ObjectTag model and the related Taxonomy methods and fields. """ @@ -410,144 +411,53 @@ def test_object_tag_lineage(self): object_tag.refresh_from_db() assert object_tag.get_lineage() == ["Another tag"] - def test_tag_ref(self): - object_tag = ObjectTag() - object_tag.tag_ref = 1 - object_tag.save() - assert object_tag.tag is None - assert object_tag.value == 1 - - def test_object_tag_is_valid(self): + def test_validate_value_free_text(self): open_taxonomy = Taxonomy.objects.create( name="Freetext Life", allow_free_text=True, ) + # An empty string or other non-string is not valid in a free-text taxonomy + assert open_taxonomy.validate_value("") is False + assert open_taxonomy.validate_value(None) is False + assert open_taxonomy.validate_value(True) is False + # But any other string value is valid: + assert open_taxonomy.validate_value("Any text we want") is True + + def test_validate_value_closed(self): + """ + Test validate_value() in a closed taxonomy + """ + assert self.taxonomy.validate_value("Eukaryota") is True + assert self.taxonomy.validate_value("Foobarensia") is False + assert self.taxonomy.tag_for_value("Eukaryota").value == "Eukaryota" + with pytest.raises(api.TagDoesNotExist): + self.taxonomy.tag_for_value("Foobarensia") - object_tag = ObjectTag( - taxonomy=self.taxonomy, - ) - # ObjectTag will only be valid for its taxonomy - assert not open_taxonomy.validate_object_tag(object_tag) - - # ObjectTags in a free-text taxonomy are valid with a value - assert not object_tag.is_valid() - object_tag.value = "Any text we want" - object_tag.taxonomy = open_taxonomy - assert not object_tag.is_valid() - object_tag.object_id = "object:id" - assert object_tag.is_valid() - + def test_clean(self): # ObjectTags in a closed taxonomy require a tag in that taxonomy - object_tag.taxonomy = self.taxonomy - object_tag.tag = Tag.objects.create( - taxonomy=self.system_taxonomy, + object_tag = ObjectTag(taxonomy=self.taxonomy, tag=Tag.objects.create( + taxonomy=self.system_taxonomy, # Different taxonomy value="PT", - ) - assert not object_tag.is_valid() + )) + with pytest.raises(ValidationError): + object_tag.full_clean() object_tag.tag = self.tag - assert object_tag.is_valid() - - def test_tag_object(self): - self.taxonomy.allow_multiple = True - - test_tags = [ - [ - self.archaea.id, - self.eubacteria.id, - self.chordata.id, - ], - [ - self.archaebacteria.id, - self.chordata.id, - ], - [ - self.archaea.id, - self.archaebacteria.id, - ], - ] - - # Tag and re-tag the object, checking that the expected tags are returned and deleted - for tag_list in test_tags: - object_tags = self.taxonomy.tag_object( - tag_list, - "biology101", - ) - - # Ensure the expected number of tags exist in the database - assert ObjectTag.objects.filter( - taxonomy=self.taxonomy, - object_id="biology101", - ).count() == len(tag_list) - # And the expected number of tags were returned - assert len(object_tags) == len(tag_list) - for index, object_tag in enumerate(object_tags): - assert object_tag.tag_id == tag_list[index] - assert object_tag.is_valid - assert object_tag.taxonomy == self.taxonomy - assert object_tag.name == self.taxonomy.name - assert object_tag.object_id == "biology101" - - def test_tag_object_free_text(self): - self.taxonomy.allow_free_text = True - object_tags = self.taxonomy.tag_object( - ["Eukaryota Xenomorph"], - "biology101", - ) - assert len(object_tags) == 1 - object_tag = object_tags[0] - assert object_tag.is_valid - assert object_tag.taxonomy == self.taxonomy - assert object_tag.name == self.taxonomy.name - assert object_tag.tag_ref == "Eukaryota Xenomorph" - assert object_tag.get_lineage() == ["Eukaryota Xenomorph"] - assert object_tag.object_id == "biology101" - - def test_tag_object_no_multiple(self): - with self.assertRaises(ValueError) as exc: - self.taxonomy.tag_object( - ["A", "B"], - "biology101", - ) - assert "only allows one tag per object" in str(exc.exception) - - def test_tag_object_required(self): - self.taxonomy.required = True - with self.assertRaises(ValueError) as exc: - self.taxonomy.tag_object( - [], - "biology101", - ) - assert "requires at least one tag per object" in str(exc.exception) - - def test_tag_object_invalid_tag(self): - with self.assertRaises(ValueError) as exc: - self.taxonomy.tag_object( - ["Eukaryota Xenomorph"], - "biology101", - ) - assert "Invalid object tag for taxonomy" in str(exc.exception) + object_tag._value = self.tag.value # pylint: disable=protected-access + object_tag.full_clean() def test_tag_case(self) -> None: """ Test that the object_id is case sensitive. """ # Tag with object_id with lower case - ObjectTag( - object_id="case:id:2", - taxonomy=self.taxonomy, - tag=self.domain_tags[0], - ).save() + api.tag_object(self.taxonomy, [self.domain_tags[0].value], object_id="case:id:2") # Tag with object_id with upper case should not trigger IntegrityError - ObjectTag( - object_id="CASE:id:2", - taxonomy=self.taxonomy, - tag=self.domain_tags[0], - ).save() + api.tag_object(self.taxonomy, [self.domain_tags[0].value], object_id="CASE:id:2") # Create another ObjectTag with lower case object_id should trigger IntegrityError with transaction.atomic(): - with self.assertRaises(IntegrityError): + with pytest.raises(IntegrityError): ObjectTag( object_id="case:id:2", taxonomy=self.taxonomy, @@ -556,9 +466,50 @@ def test_tag_case(self) -> None: # Create another ObjectTag with upper case object_id should trigger IntegrityError with transaction.atomic(): - with self.assertRaises(IntegrityError): + with pytest.raises(IntegrityError): ObjectTag( object_id="CASE:id:2", taxonomy=self.taxonomy, tag=self.domain_tags[0], ).save() + + def test_is_deleted(self): + self.taxonomy.allow_multiple = True + self.taxonomy.save() + open_taxonomy = Taxonomy.objects.create(name="Freetext Life", allow_free_text=True, allow_multiple=True) + + object_id = "obj1" + # Create some tags: + api.tag_object(self.taxonomy, [self.archaea.value, self.bacteria.value], object_id) # Regular tags + api.tag_object(open_taxonomy, ["foo", "bar", "tribble"], object_id) # Free text tags + + # At first, none of these will be deleted: + assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ + (self.archaea.value, False), + (self.bacteria.value, False), + ("foo", False), + ("bar", False), + ("tribble", False), + ] + + # Delete "bacteria" from the taxonomy: + self.bacteria.delete() # TODO: add an API method for this + + assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ + (self.archaea.value, False), + (self.bacteria.value, True), # <--- deleted! But the value is preserved. + ("foo", False), + ("bar", False), + ("tribble", False), + ] + + # Then delete the whole free text taxonomy + open_taxonomy.delete() + + assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ + (self.archaea.value, False), + (self.bacteria.value, True), # <--- deleted! But the value is preserved. + ("foo", True), # <--- Deleted, but the value is preserved + ("bar", True), # <--- Deleted, but the value is preserved + ("tribble", True), # <--- Deleted, but the value is preserved + ] diff --git a/tests/openedx_tagging/core/tagging/test_system_defined_models.py b/tests/openedx_tagging/core/tagging/test_system_defined_models.py index ff2d669be..09f7b2bcb 100644 --- a/tests/openedx_tagging/core/tagging/test_system_defined_models.py +++ b/tests/openedx_tagging/core/tagging/test_system_defined_models.py @@ -3,21 +3,22 @@ """ from __future__ import annotations +from datetime import datetime, timezone + import ddt # type: ignore[import] -from django.contrib.auth import get_user_model -from django.db.utils import IntegrityError +import pytest from django.test import TestCase, override_settings -from openedx_tagging.core.tagging.models.system_defined import ( - ModelObjectTag, - ModelSystemDefinedTaxonomy, - UserSystemDefinedTaxonomy, -) +from openedx_learning.core.publishing.models import LearningPackage +from openedx_tagging.core.tagging import api +from openedx_tagging.core.tagging.models import Taxonomy +from openedx_tagging.core.tagging.models.system_defined import ModelSystemDefinedTaxonomy, UserSystemDefinedTaxonomy from .test_models import TestTagTaxonomyMixin test_languages = [ ("en", "English"), + ("en-uk", "English (United Kingdom)"), ("az", "Azerbaijani"), ("id", "Indonesian"), ("qu", "Quechua"), @@ -31,29 +32,21 @@ class EmptyTestClass: """ -class InvalidModelTaxonomy(ModelSystemDefinedTaxonomy): +class TestLPTaxonomy(ModelSystemDefinedTaxonomy): """ - Model used for testing + Model used for testing - points to LearningPackage instances """ - @property - def object_tag_class(self): - return EmptyTestClass - - class Meta: - proxy = True - managed = False - app_label = "oel_tagging" - + def tag_class_model(self): + return LearningPackage -class TestModelTag(ModelObjectTag): - """ - Model used for testing - """ + @property + def tag_class_value_field(self) -> str: + return "key" @property - def tag_class_model(self): - return get_user_model() + def tag_class_key_field(self) -> str: + return "uuid" class Meta: proxy = True @@ -61,14 +54,17 @@ class Meta: app_label = "oel_tagging" -class TestModelTaxonomy(ModelSystemDefinedTaxonomy): +class CaseInsensitiveTitleLPTaxonomy(TestLPTaxonomy): """ - Model used for testing + Model that points to LearningPackage instances but uses 'title' as values """ - @property - def object_tag_class(self): - return TestModelTag + def tag_class_value_field(self) -> str: + # Title isn't unique, so wouldn't make a good 'value' in real usage, but title is case-insensitive so we use it + # here to test case insensitivity. (On MySQL, only columns with case-insensitive collation can be used with + # case-insensitive comparison operators. On SQLite you could just use the 'key' field for testing, and it works + # fine.) + return "title" class Meta: proxy = True @@ -82,122 +78,171 @@ class TestModelSystemDefinedTaxonomy(TestTagTaxonomyMixin, TestCase): Test for Model Model System defined taxonomy """ + @staticmethod + def _create_learning_pkg(**kwargs) -> LearningPackage: + timestamp = datetime.now(tz=timezone.utc) + return LearningPackage.objects.create(**kwargs, created=timestamp, updated=timestamp) + + @classmethod + def setUpClass(cls): + super().setUpClass() + # Create two learning packages and a taxonomy that can tag any object using learning packages as tags: + cls.learning_pkg_1 = cls._create_learning_pkg(key="p1", title="Learning Package 1") + cls.learning_pkg_2 = cls._create_learning_pkg(key="p2", title="Learning Package 2") + cls.lp_taxonomy = TestLPTaxonomy.objects.create( + taxonomy_class=TestLPTaxonomy, + name="LearningPackage Taxonomy", + allow_multiple=True, + ) + # Also create an "Author" taxonomy that can tag any object using user IDs/usernames: + cls.author_taxonomy = UserSystemDefinedTaxonomy.objects.create( + taxonomy_class=UserSystemDefinedTaxonomy, + name="Authors", + allow_multiple=True, + ) + + def test_lp_taxonomy_validation(self): + """ + Test that the validation methods of the Learning Package Taxonomy are working + """ + # Create a new LearningPackage - we know no Tag instances will exist for it yet. + valid_lp = self._create_learning_pkg(key="valid-lp", title="New Learning Packacge") + # The taxonomy can validate tags by value which we've defined as they 'key' of the LearningPackage: + assert self.lp_taxonomy.validate_value(self.learning_pkg_2.key) is True + assert self.lp_taxonomy.validate_value(self.learning_pkg_2.key) is True + assert self.lp_taxonomy.validate_value(valid_lp.key) is True + assert self.lp_taxonomy.validate_value("foo") is False + # The taxonomy can also validate tags by external_id, which we've defined as the UUID of the LearningPackage: + assert self.lp_taxonomy.validate_external_id(self.learning_pkg_2.uuid) is True + assert self.lp_taxonomy.validate_external_id(self.learning_pkg_2.uuid) is True + assert self.lp_taxonomy.validate_external_id(valid_lp.uuid) is True + assert self.lp_taxonomy.validate_external_id("ba11225e-9ec9-4a50-87ea-3155c7c20466") is False + + def test_author_taxonomy_validation(self): + """ + Test the validation methods of the Author Taxonomy (Author = User) + """ + assert self.author_taxonomy.validate_value(self.user_1.username) is True + assert self.author_taxonomy.validate_value(self.user_2.username) is True + assert self.author_taxonomy.validate_value("not a user") is False + # And we can validate by ID if we want: + assert self.author_taxonomy.validate_external_id(str(self.user_1.id)) is True + assert self.author_taxonomy.validate_external_id(str(self.user_2.id)) is True + assert self.author_taxonomy.validate_external_id("8742590") is False + @ddt.data( - (ModelSystemDefinedTaxonomy, NotImplementedError), - (ModelObjectTag, NotImplementedError), - (InvalidModelTaxonomy, AssertionError), - (UserSystemDefinedTaxonomy, None), + "validate_value", "tag_for_value", "validate_external_id", "tag_for_external_id", ) - @ddt.unpack - def test_implementation_error(self, taxonomy_cls, expected_exception): - if not expected_exception: - assert taxonomy_cls() - else: - with self.assertRaises(expected_exception): - taxonomy_cls() - - # FIXME: something is wrong with this test case. It's setting the string - # "tag_id" as the primary key (integer) of the Tag instance, and it mentions - # "parent validation" but there is nothing to do with parents here. - # - # @ddt.data( - # ("1", "tag_id", True), # Valid - # ("0", "tag_id", False), # Invalid user - # ("test_id", "tag_id", False), # Invalid user id - # ("1", None, False), # Testing parent validations - # ) - # @ddt.unpack - # def test_validations(self, tag_external_id: str, tag_id: str | None, expected: bool) -> None: - # tag = Tag( - # id=tag_id, - # taxonomy=self.user_taxonomy, - # value="_val", - # external_id=tag_external_id, - # ) - # object_tag = ObjectTag( - # object_id="id", - # tag=tag, - # ) - # - # assert self.user_taxonomy.validate_object_tag( - # object_tag=object_tag, - # check_object=False, - # check_taxonomy=False, - # check_tag=True, - # ) == expected - - def test_tag_object_invalid_user(self): - # Test user that doesn't exist - with self.assertRaises(ValueError): - self.user_taxonomy.tag_object(tags=[4], object_id="object_id") - - def _tag_object(self): - return self.user_taxonomy.tag_object( - tags=[self.user_1.id], object_id="object_id" + def test_warns_uncasted(self, method): + """ + Test that if we use a taxonomy directly without cast(), we get warned. + """ + base_taxonomy = Taxonomy.objects.get(pk=self.lp_taxonomy.pk) + with pytest.raises(TypeError) as excinfo: + # e.g. base_taxonomy.validate_value("foo") + getattr(base_taxonomy, method)("foo") + assert "Taxonomy was used incorrectly - without .cast()" in str(excinfo.value) + + def test_simple_tag_object(self): + """ + Test applying tags to an object. + """ + object1_id, object2_id = "obj1", "obj2" + api.tag_object(self.lp_taxonomy, ["p1"], object1_id) + api.tag_object(self.lp_taxonomy, ["p1", "p2"], object2_id) + assert [t.value for t in api.get_object_tags(object1_id)] == ["p1"] + assert [t.value for t in api.get_object_tags(object2_id)] == ["p1", "p2"] + + def test_invalid_tag(self): + """ + Trying to apply an invalid tag raises TagDoesNotExist + """ + with pytest.raises(api.TagDoesNotExist): + api.tag_object(self.lp_taxonomy, ["nonexistent"], "obj1") + + def test_case_insensitive_values(self): + """ + For now, values are case insensitive. We may change that in the future. + """ + object1_id, object2_id = "obj1", "obj2" + taxonomy = CaseInsensitiveTitleLPTaxonomy.objects.create( + taxonomy_class=CaseInsensitiveTitleLPTaxonomy, + name="LearningPackage Title Taxonomy", + allow_multiple=True, ) - - def test_tag_object_tag_creation(self): - # Test creation of a new Tag with user taxonomy - assert self.user_taxonomy.tag_set.count() == 0 - updated_tags = self._tag_object() - assert self.user_taxonomy.tag_set.count() == 1 - assert len(updated_tags) == 1 - assert updated_tags[0].tag.external_id == str(self.user_1.id) - assert updated_tags[0].tag.value == self.user_1.get_username() - - # Test parent functions - taxonomy = TestModelTaxonomy( - name="Test", - description="Test", + api.tag_object(taxonomy, ["LEARNING PACKAGE 1"], object1_id) + api.tag_object(taxonomy, ["Learning Package 1", "LEARNING PACKAGE 2"], object2_id) + # But they always get normalized to the case used on the actual model: + assert [t.value for t in api.get_object_tags(object1_id)] == ["Learning Package 1"] + assert [t.value for t in api.get_object_tags(object2_id)] == ["Learning Package 1", "Learning Package 2"] + + def test_multiple_taxonomies(self): + """ + Test using several different instances of a taxonomy to tag the same object + """ + reviewer_taxonomy = UserSystemDefinedTaxonomy.objects.create( + taxonomy_class=UserSystemDefinedTaxonomy, + name="Reviewer", + allow_multiple=True, ) - taxonomy.save() - assert taxonomy.tag_set.count() == 0 - updated_tags = taxonomy.tag_object(tags=[self.user_1.id], object_id="object_id") - assert taxonomy.tag_set.count() == 1 - assert taxonomy.tag_set.count() == 1 - assert len(updated_tags) == 1 - assert updated_tags[0].tag.external_id == str(self.user_1.id) - assert updated_tags[0].tag.value == str(self.user_1.id) - - def test_tag_object_existing_tag(self): - # Test add an existing Tag - self._tag_object() - assert self.user_taxonomy.tag_set.count() == 1 - with self.assertRaises(IntegrityError): - self._tag_object() + pr_1_id, pr_2_id = "pull_request_1", "pull_request_2" + + # Tag PR 1 as having "Author: user1, user2; Reviewer: user2" + api.tag_object(self.author_taxonomy, [self.user_1.username, self.user_2.username], pr_1_id) + api.tag_object(reviewer_taxonomy, [self.user_2.username], pr_1_id) + + # Tag PR 2 as having "Author: user2, reviewer: user1" + api.tag_object(self.author_taxonomy, [self.user_2.username], pr_2_id) + api.tag_object(reviewer_taxonomy, [self.user_1.username], pr_2_id) + + # Check the results: + assert [f"{t.taxonomy.name}:{t.value}" for t in api.get_object_tags(pr_1_id)] == [ + f"Authors:{self.user_1.username}", + f"Authors:{self.user_2.username}", + f"Reviewer:{self.user_2.username}", + ] + assert [f"{t.taxonomy.name}:{t.value}" for t in api.get_object_tags(pr_2_id)] == [ + f"Authors:{self.user_2.username}", + f"Reviewer:{self.user_1.username}", + ] def test_tag_object_resync(self): - self._tag_object() - - self.user_1.username = "new_username" + """ + If the value changes, we can use the new value to tag objects, and the + Tag will be updated automatically. + """ + # Tag two objects with "Author: user_1" + object1_id, object2_id, other_obj_id = "obj1", "obj2", "other" + api.tag_object(self.author_taxonomy, [self.user_1.username], object1_id) + api.tag_object(self.author_taxonomy, [self.user_1.username], object2_id) + initial_object_tags = api.get_object_tags(object1_id) + assert [t.value for t in initial_object_tags] == [self.user_1.username] + assert not list(api.get_object_tags(other_obj_id)) + # Change user_1's username: + new_username = "new_username" + self.user_1.username = new_username self.user_1.save() - updated_tags = self._tag_object() - assert self.user_taxonomy.tag_set.count() == 1 - assert len(updated_tags) == 1 - assert updated_tags[0].tag.external_id == str(self.user_1.id) - assert updated_tags[0].tag.value == self.user_1.get_username() + # Now we update the tags on just one of the objects: + api.tag_object(self.author_taxonomy, [new_username], object1_id) + assert [t.value for t in api.get_object_tags(object1_id)] == [new_username] + # But because this will have updated the shared Tag instance, object2 will also be updated as a side effect. + # This is good - all the objects throughout the system with this tag now show the new value. + assert [t.value for t in api.get_object_tags(object2_id)] == [new_username] + # And just to make sure there are no other random changes to other objects: + assert not list(api.get_object_tags(other_obj_id)) def test_tag_object_delete_user(self): + """ + Using a deleted model instance as a tag will raise TagDoesNotExist + """ + # Tag an object with "Author: user_1" + object_id = "obj123" + api.tag_object(self.author_taxonomy, [self.user_1.username], object_id) + assert [t.value for t in api.get_object_tags(object_id)] == [self.user_1.username] # Test after delete user - self._tag_object() - user_1_id = self.user_1.id self.user_1.delete() - with self.assertRaises(ValueError): - self.user_taxonomy.tag_object( - tags=[user_1_id], - object_id="object_id", - ) - - def test_tag_ref(self): - object_tag = TestModelTag() - object_tag.tag_ref = 1 - object_tag.save() - assert object_tag.tag is None - assert object_tag.value == 1 - - def test_get_instance(self): - object_tag = TestModelTag() - assert object_tag.get_instance() is None + with self.assertRaises(api.TagDoesNotExist): + api.tag_object(self.author_taxonomy, [self.user_1.username], object_id) @ddt.ddt @@ -207,37 +252,25 @@ class TestLanguageTaxonomy(TestTagTaxonomyMixin, TestCase): Test for Language taxonomy """ - # FIXME: something is wrong with this test case. It's setting the string - # "tag_id" as the primary key (integer) of the Tag instance, and it mentions - # "parent validation" but there is nothing to do with parents here. - # - # @ddt.data( - # ("en", "tag_id", True), # Valid - # ("es", "tag_id", False), # Not available lang - # ("en", None, False), # Test parent validations - # ) - # @ddt.unpack - # def test_validations(self, lang: str, tag_id: str | None, expected: bool): - # tag = Tag( - # id=tag_id, - # taxonomy=self.language_taxonomy, - # value="_val", - # external_id=lang, - # ) - # object_tag = ObjectTag( - # object_id="id", - # tag=tag, - # ) - # assert self.language_taxonomy.validate_object_tag( - # object_tag=object_tag, - # check_object=False, - # check_taxonomy=False, - # check_tag=True, - # ) == expected - - def test_get_tags(self): - tags = self.language_taxonomy.get_tags() - expected_langs = [lang[0] for lang in test_languages] - for tag in tags: - assert tag.external_id in expected_langs - assert tag.annotated_field == 0 + def test_validate_lang_ids(self): + """ + Whether or not languages are available as tags depends on the django settings + """ + assert self.language_taxonomy.validate_external_id("en") is True + assert self.language_taxonomy.tag_for_external_id("en").value == "English" + assert self.language_taxonomy.tag_for_external_id("en-uk").value == "English (United Kingdom)" + assert self.language_taxonomy.tag_for_external_id("id").value == "Indonesian" + + assert self.language_taxonomy.validate_external_id("xx") is False + with pytest.raises(api.TagDoesNotExist): + self.language_taxonomy.tag_for_external_id("xx") + + @override_settings(LANGUAGES=[("fr", "Français")]) + def test_minimal_languages(self): + """ + Whether or not languages are available as tags depends on the django settings + """ + assert self.language_taxonomy.validate_external_id("en") is False + with pytest.raises(api.TagDoesNotExist): + self.language_taxonomy.tag_for_external_id("en") + assert self.language_taxonomy.tag_for_external_id("fr").value == "Français" diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index b7b12cd31..effd6460b 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -533,7 +533,7 @@ def test_retrieve_object_tags_taxonomy_queryparam( if status.is_success(expected_status): assert len(response.data) == expected_count for object_tag in response.data: - assert object_tag.get("is_valid") is True + assert object_tag.get("is_deleted") is False assert object_tag.get("taxonomy_id") == self.enabled_taxonomy.pk @ddt.data( From a1cc499dbd9cf8bdbc7ddc79b1f3e5a287dff5ba Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 24 Sep 2023 20:44:24 -0400 Subject: [PATCH 048/282] chore: Updating Python Requirements --- requirements/base.txt | 45 +++++++------ requirements/ci.txt | 20 +++--- requirements/dev.txt | 135 ++++++++++++++++++++----------------- requirements/doc.txt | 86 ++++++++++++----------- requirements/pip-tools.txt | 19 ++++-- requirements/quality.txt | 104 +++++++++++++++------------- requirements/test.txt | 60 +++++++++-------- 7 files changed, 258 insertions(+), 211 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 3ad003afb..0afbf6540 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # make upgrade @@ -10,9 +10,13 @@ asgiref==3.7.2 # via django attrs==23.1.0 # via -r requirements/base.in +backports-zoneinfo[tzdata]==0.2.1 + # via + # celery + # kombu billiard==4.1.0 # via celery -celery==5.3.1 +celery==5.3.4 # via -r requirements/base.in certifi==2023.7.22 # via requests @@ -22,7 +26,7 @@ cffi==1.15.1 # pynacl charset-normalizer==3.2.0 # via requests -click==8.1.6 +click==8.1.7 # via # celery # click-didyoumean @@ -35,9 +39,9 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -cryptography==41.0.3 +cryptography==41.0.4 # via pyjwt -django==3.2.19 +django==3.2.21 # via # -c requirements/constraints.txt # -r requirements/base.in @@ -62,15 +66,15 @@ drf-jwt==1.19.2 # via edx-drf-extensions edx-django-utils==5.7.0 # via edx-drf-extensions -edx-drf-extensions==8.8.0 +edx-drf-extensions==8.10.0 # via -r requirements/base.in -edx-opaque-keys==2.4.0 +edx-opaque-keys==2.5.1 # via edx-drf-extensions idna==3.4 # via requests -kombu==5.3.1 +kombu==5.3.2 # via celery -newrelic==8.9.0 +newrelic==9.0.0 # via edx-django-utils pbr==5.11.1 # via stevedore @@ -89,10 +93,8 @@ pymongo==3.13.0 pynacl==1.5.0 # via edx-django-utils python-dateutil==2.8.2 - # via - # celery - # edx-drf-extensions -pytz==2023.3 + # via celery +pytz==2023.3.post1 # via # django # djangorestframework @@ -103,20 +105,23 @@ rules==3.3 semantic-version==2.10.0 # via edx-drf-extensions six==1.16.0 - # via - # edx-drf-extensions - # python-dateutil + # via python-dateutil sqlparse==0.4.4 # via django stevedore==5.1.0 # via # edx-django-utils # edx-opaque-keys -typing-extensions==4.6.3 - # via asgiref +typing-extensions==4.8.0 + # via + # asgiref + # edx-opaque-keys + # kombu tzdata==2023.3 - # via celery -urllib3==2.0.4 + # via + # backports-zoneinfo + # celery +urllib3==2.0.5 # via requests vine==5.0.0 # via diff --git a/requirements/ci.txt b/requirements/ci.txt index f47e116e0..c475896cb 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -1,26 +1,26 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # make upgrade # -click==8.1.6 +click==8.1.7 # via import-linter -distlib==0.3.6 +distlib==0.3.7 # via virtualenv -filelock==3.12.2 +filelock==3.12.4 # via # tox # virtualenv -grimp==2.4 +grimp==3.0 # via import-linter -import-linter==1.9.0 +import-linter==1.12.0 # via -r requirements/ci.in packaging==23.1 # via tox -platformdirs==3.6.0 +platformdirs==3.10.0 # via virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via tox py==1.11.0 # via tox @@ -34,9 +34,9 @@ tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.in -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # grimp # import-linter -virtualenv==20.23.1 +virtualenv==20.24.5 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 0fe66827f..41216c421 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # make upgrade @@ -12,26 +12,27 @@ asgiref==3.7.2 # via # -r requirements/quality.txt # django -astroid==2.15.5 +astroid==2.15.7 # via # -r requirements/quality.txt # pylint # pylint-celery attrs==23.1.0 # via -r requirements/quality.txt -billiard==4.1.0 +backports-zoneinfo[tzdata]==0.2.1 # via # -r requirements/quality.txt # celery -bleach==6.0.0 + # kombu +billiard==4.1.0 # via # -r requirements/quality.txt - # readme-renderer -build==0.10.0 + # celery +build==1.0.3 # via # -r requirements/pip-tools.txt # pip-tools -celery==5.3.1 +celery==5.3.4 # via -r requirements/quality.txt certifi==2023.7.22 # via @@ -42,13 +43,13 @@ cffi==1.15.1 # -r requirements/quality.txt # cryptography # pynacl -chardet==5.1.0 +chardet==5.2.0 # via diff-cover charset-normalizer==3.2.0 # via # -r requirements/quality.txt # requests -click==8.1.6 +click==8.1.7 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt @@ -79,32 +80,32 @@ click-repl==0.3.0 # via # -r requirements/quality.txt # celery -code-annotations==1.3.0 +code-annotations==1.5.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.2.7 +coverage[toml]==7.3.1 # via # -r requirements/quality.txt # pytest-cov -cryptography==41.0.3 +cryptography==41.0.4 # via # -r requirements/quality.txt # pyjwt # secretstorage ddt==1.6.0 # via -r requirements/quality.txt -diff-cover==7.6.0 +diff-cover==7.7.0 # via -r requirements/dev.in -dill==0.3.6 +dill==0.3.7 # via # -r requirements/quality.txt # pylint -distlib==0.3.6 +distlib==0.3.7 # via # -r requirements/ci.txt # virtualenv -django==3.2.19 +django==3.2.21 # via # -c requirements/constraints.txt # -r requirements/quality.txt @@ -122,11 +123,11 @@ django-crum==0.7.9 # via # -r requirements/quality.txt # edx-django-utils -django-debug-toolbar==4.1.0 +django-debug-toolbar==4.2.0 # via # -r requirements/dev.in # -r requirements/quality.txt -django-stubs==4.2.3 +django-stubs==4.2.4 # via # -r requirements/quality.txt # djangorestframework-stubs @@ -158,26 +159,26 @@ edx-django-utils==5.7.0 # via # -r requirements/quality.txt # edx-drf-extensions -edx-drf-extensions==8.8.0 +edx-drf-extensions==8.10.0 # via -r requirements/quality.txt -edx-i18n-tools==0.9.2 +edx-i18n-tools==1.2.0 # via -r requirements/dev.in edx-lint==5.3.4 # via -r requirements/quality.txt -edx-opaque-keys==2.4.0 +edx-opaque-keys==2.5.1 # via # -r requirements/quality.txt # edx-drf-extensions -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via # -r requirements/quality.txt # pytest -filelock==3.12.2 +filelock==3.12.4 # via # -r requirements/ci.txt # tox # virtualenv -grimp==2.4 +grimp==3.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -186,15 +187,21 @@ idna==3.4 # via # -r requirements/quality.txt # requests -import-linter==1.9.0 +import-linter==1.12.0 # via # -r requirements/ci.txt # -r requirements/quality.txt -importlib-metadata==6.7.0 +importlib-metadata==6.8.0 # via + # -r requirements/pip-tools.txt # -r requirements/quality.txt + # build # keyring # twine +importlib-resources==6.1.0 + # via + # -r requirements/quality.txt + # keyring iniconfig==2.0.0 # via # -r requirements/quality.txt @@ -203,7 +210,7 @@ isort==5.12.0 # via # -r requirements/quality.txt # pylint -jaraco-classes==3.2.3 +jaraco-classes==3.3.0 # via # -r requirements/quality.txt # keyring @@ -217,11 +224,11 @@ jinja2==3.1.2 # -r requirements/quality.txt # code-annotations # diff-cover -keyring==24.0.0 +keyring==24.2.0 # via # -r requirements/quality.txt # twine -kombu==5.3.1 +kombu==5.3.2 # via # -r requirements/quality.txt # celery @@ -245,9 +252,9 @@ mdurl==0.1.2 # via # -r requirements/quality.txt # markdown-it-py -mock==5.0.2 +mock==5.1.0 # via -r requirements/quality.txt -more-itertools==9.1.0 +more-itertools==10.1.0 # via # -r requirements/quality.txt # jaraco-classes @@ -260,12 +267,16 @@ mypy-extensions==1.0.0 # via # -r requirements/quality.txt # mypy -mysqlclient==2.1.1 +mysqlclient==2.2.0 # via -r requirements/quality.txt -newrelic==8.9.0 +newrelic==9.0.0 # via # -r requirements/quality.txt # edx-django-utils +nh3==0.2.14 + # via + # -r requirements/quality.txt + # readme-renderer packaging==23.1 # via # -r requirements/ci.txt @@ -274,25 +285,25 @@ packaging==23.1 # build # pytest # tox -path==16.6.0 +path==16.7.1 # via edx-i18n-tools pbr==5.11.1 # via # -r requirements/quality.txt # stevedore -pip-tools==6.13.0 +pip-tools==7.3.0 # via -r requirements/pip-tools.txt pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.6.0 +platformdirs==3.10.0 # via # -r requirements/ci.txt # -r requirements/quality.txt # pylint # virtualenv -pluggy==1.0.0 +pluggy==1.3.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -313,7 +324,7 @@ py==1.11.0 # via # -r requirements/ci.txt # tox -pycodestyle==2.10.0 +pycodestyle==2.11.0 # via -r requirements/quality.txt pycparser==2.21 # via @@ -321,7 +332,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.15.1 +pygments==2.16.1 # via # -r requirements/quality.txt # diff-cover @@ -332,7 +343,7 @@ pyjwt[crypto]==2.8.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -pylint==2.17.4 +pylint==2.17.5 # via # -r requirements/quality.txt # edx-lint @@ -364,7 +375,7 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.3.2 +pytest==7.4.2 # via # -r requirements/quality.txt # pytest-cov @@ -377,22 +388,21 @@ python-dateutil==2.8.2 # via # -r requirements/quality.txt # celery - # edx-drf-extensions python-slugify==8.0.1 # via # -r requirements/quality.txt # code-annotations -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/quality.txt # django # djangorestframework -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/quality.txt # code-annotations # edx-i18n-tools -readme-renderer==40.0 +readme-renderer==42.0 # via # -r requirements/quality.txt # twine @@ -411,7 +421,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.4.2 +rich==13.5.3 # via # -r requirements/quality.txt # twine @@ -429,8 +439,6 @@ six==1.16.0 # via # -r requirements/ci.txt # -r requirements/quality.txt - # bleach - # edx-drf-extensions # edx-lint # python-dateutil # tox @@ -463,11 +471,12 @@ tomli==2.0.1 # django-stubs # import-linter # mypy + # pip-tools # pylint # pyproject-hooks # pytest # tox -tomlkit==0.11.8 +tomlkit==0.12.1 # via # -r requirements/quality.txt # pylint @@ -476,20 +485,20 @@ tox==3.28.0 # -c requirements/constraints.txt # -r requirements/ci.txt # tox-battery -tox-battery==0.6.1 +tox-battery==0.6.2 # via -r requirements/dev.in twine==4.0.2 # via -r requirements/quality.txt -types-pytz==2023.3.0.1 +types-pytz==2023.3.1.1 # via # -r requirements/quality.txt # django-stubs -types-pyyaml==6.0.12.11 +types-pyyaml==6.0.12.12 # via # -r requirements/quality.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.2 +types-requests==2.31.0.4 # via # -r requirements/quality.txt # djangorestframework-stubs @@ -497,7 +506,7 @@ types-urllib3==1.26.25.14 # via # -r requirements/quality.txt # types-requests -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -506,15 +515,19 @@ typing-extensions==4.6.3 # django-stubs # django-stubs-ext # djangorestframework-stubs + # edx-opaque-keys # grimp # import-linter + # kombu # mypy # pylint + # rich tzdata==2023.3 # via # -r requirements/quality.txt + # backports-zoneinfo # celery -urllib3==2.0.4 +urllib3==2.0.5 # via # -r requirements/quality.txt # requests @@ -525,7 +538,7 @@ vine==5.0.0 # amqp # celery # kombu -virtualenv==20.23.1 +virtualenv==20.24.5 # via # -r requirements/ci.txt # tox @@ -533,11 +546,7 @@ wcwidth==0.2.6 # via # -r requirements/quality.txt # prompt-toolkit -webencodings==0.5.1 - # via - # -r requirements/quality.txt - # bleach -wheel==0.40.0 +wheel==0.41.2 # via # -r requirements/pip-tools.txt # pip-tools @@ -545,10 +554,12 @@ wrapt==1.15.0 # via # -r requirements/quality.txt # astroid -zipp==3.15.0 +zipp==3.17.0 # via + # -r requirements/pip-tools.txt # -r requirements/quality.txt # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/doc.txt b/requirements/doc.txt index 35ac24a0a..75db46673 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -22,15 +22,18 @@ babel==2.12.1 # via # pydata-sphinx-theme # sphinx +backports-zoneinfo[tzdata]==0.2.1 + # via + # -r requirements/test.txt + # celery + # kombu beautifulsoup4==4.12.2 # via pydata-sphinx-theme billiard==4.1.0 # via # -r requirements/test.txt # celery -bleach==6.0.0 - # via readme-renderer -celery==5.3.1 +celery==5.3.4 # via -r requirements/test.txt certifi==2023.7.22 # via @@ -45,7 +48,7 @@ charset-normalizer==3.2.0 # via # -r requirements/test.txt # requests -click==8.1.6 +click==8.1.7 # via # -r requirements/test.txt # celery @@ -67,24 +70,26 @@ click-repl==0.3.0 # via # -r requirements/test.txt # celery -code-annotations==1.3.0 +code-annotations==1.5.0 # via -r requirements/test.txt -coverage[toml]==7.2.7 +coverage[toml]==7.3.1 # via # -r requirements/test.txt # pytest-cov -cryptography==41.0.3 +cryptography==41.0.4 # via # -r requirements/test.txt # pyjwt ddt==1.6.0 # via -r requirements/test.txt -django==3.2.19 +django==3.2.21 # via # -c requirements/constraints.txt # -r requirements/test.txt # django-crum # django-debug-toolbar + # django-stubs + # django-stubs-ext # django-waffle # djangorestframework # drf-jwt @@ -95,9 +100,9 @@ django-crum==0.7.9 # via # -r requirements/test.txt # edx-django-utils -django-debug-toolbar==4.1.0 +django-debug-toolbar==4.2.0 # via -r requirements/test.txt -django-stubs==4.2.3 +django-stubs==4.2.4 # via # -r requirements/test.txt # djangorestframework-stubs @@ -134,17 +139,17 @@ edx-django-utils==5.7.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.8.0 +edx-drf-extensions==8.10.0 # via -r requirements/test.txt -edx-opaque-keys==2.4.0 +edx-opaque-keys==2.5.1 # via # -r requirements/test.txt # edx-drf-extensions -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via # -r requirements/test.txt # pytest -grimp==2.4 +grimp==3.0 # via # -r requirements/test.txt # import-linter @@ -154,8 +159,10 @@ idna==3.4 # requests imagesize==1.4.1 # via sphinx -import-linter==1.9.0 +import-linter==1.12.0 # via -r requirements/test.txt +importlib-metadata==6.8.0 + # via sphinx iniconfig==2.0.0 # via # -r requirements/test.txt @@ -165,7 +172,7 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -kombu==5.3.1 +kombu==5.3.2 # via # -r requirements/test.txt # celery @@ -173,7 +180,7 @@ markupsafe==2.1.3 # via # -r requirements/test.txt # jinja2 -mock==5.0.2 +mock==5.1.0 # via -r requirements/test.txt mypy==1.5.1 # via @@ -184,12 +191,14 @@ mypy-extensions==1.0.0 # via # -r requirements/test.txt # mypy -mysqlclient==2.1.1 +mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==8.9.0 +newrelic==9.0.0 # via # -r requirements/test.txt # edx-django-utils +nh3==0.2.14 + # via readme-renderer packaging==23.1 # via # -r requirements/test.txt @@ -200,7 +209,7 @@ pbr==5.11.1 # via # -r requirements/test.txt # stevedore -pluggy==1.0.0 +pluggy==1.3.0 # via # -r requirements/test.txt # pytest @@ -218,9 +227,9 @@ pycparser==2.21 # via # -r requirements/test.txt # cffi -pydata-sphinx-theme==0.13.3 +pydata-sphinx-theme==0.14.1 # via sphinx-book-theme -pygments==2.15.1 +pygments==2.16.1 # via # accessible-pygments # doc8 @@ -240,7 +249,7 @@ pynacl==1.5.0 # via # -r requirements/test.txt # edx-django-utils -pytest==7.3.2 +pytest==7.4.2 # via # -r requirements/test.txt # pytest-cov @@ -253,21 +262,21 @@ python-dateutil==2.8.2 # via # -r requirements/test.txt # celery - # edx-drf-extensions python-slugify==8.0.1 # via # -r requirements/test.txt # code-annotations -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/test.txt + # babel # django # djangorestframework -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/test.txt # code-annotations -readme-renderer==40.0 +readme-renderer==42.0 # via -r requirements/doc.in requests==2.31.0 # via @@ -286,12 +295,10 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/test.txt - # bleach - # edx-drf-extensions # python-dateutil snowballstemmer==2.2.0 # via sphinx -soupsieve==2.4.1 +soupsieve==2.5 # via beautifulsoup4 sphinx==6.2.1 # via @@ -305,7 +312,7 @@ sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-django==2.3 +sphinxcontrib-django==2.4 # via -r requirements/doc.in sphinxcontrib-htmlhelp==2.0.1 # via sphinx @@ -340,16 +347,16 @@ tomli==2.0.1 # import-linter # mypy # pytest -types-pytz==2023.3.0.1 +types-pytz==2023.3.1.1 # via # -r requirements/test.txt # django-stubs -types-pyyaml==6.0.12.11 +types-pyyaml==6.0.12.12 # via # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.2 +types-requests==2.31.0.4 # via # -r requirements/test.txt # djangorestframework-stubs @@ -357,22 +364,25 @@ types-urllib3==1.26.25.14 # via # -r requirements/test.txt # types-requests -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # -r requirements/test.txt # asgiref # django-stubs # django-stubs-ext # djangorestframework-stubs + # edx-opaque-keys # grimp # import-linter + # kombu # mypy # pydata-sphinx-theme tzdata==2023.3 # via # -r requirements/test.txt + # backports-zoneinfo # celery -urllib3==2.0.4 +urllib3==2.0.5 # via # -r requirements/test.txt # requests @@ -386,5 +396,5 @@ wcwidth==0.2.6 # via # -r requirements/test.txt # prompt-toolkit -webencodings==0.5.1 - # via bleach +zipp==3.17.0 + # via importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 9ce8ce227..894fa179a 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -1,23 +1,30 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.8 # by the following command: # # make upgrade # -build==0.10.0 +build==1.0.3 # via pip-tools -click==8.1.6 +click==8.1.7 # via pip-tools +importlib-metadata==6.8.0 + # via build packaging==23.1 # via build -pip-tools==6.13.0 +pip-tools==7.3.0 # via -r requirements/pip-tools.in pyproject-hooks==1.0.0 # via build tomli==2.0.1 - # via build -wheel==0.40.0 + # via + # build + # pip-tools + # pyproject-hooks +wheel==0.41.2 # via pip-tools +zipp==3.17.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # pip diff --git a/requirements/quality.txt b/requirements/quality.txt index a6a9044b3..beda673b0 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -12,19 +12,22 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -astroid==2.15.5 +astroid==2.15.7 # via # pylint # pylint-celery attrs==23.1.0 # via -r requirements/test.txt +backports-zoneinfo[tzdata]==0.2.1 + # via + # -r requirements/test.txt + # celery + # kombu billiard==4.1.0 # via # -r requirements/test.txt # celery -bleach==6.0.0 - # via readme-renderer -celery==5.3.1 +celery==5.3.4 # via -r requirements/test.txt certifi==2023.7.22 # via @@ -39,7 +42,7 @@ charset-normalizer==3.2.0 # via # -r requirements/test.txt # requests -click==8.1.6 +click==8.1.7 # via # -r requirements/test.txt # celery @@ -65,24 +68,24 @@ click-repl==0.3.0 # via # -r requirements/test.txt # celery -code-annotations==1.3.0 +code-annotations==1.5.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.2.7 +coverage[toml]==7.3.1 # via # -r requirements/test.txt # pytest-cov -cryptography==41.0.3 +cryptography==41.0.4 # via # -r requirements/test.txt # pyjwt # secretstorage ddt==1.6.0 # via -r requirements/test.txt -dill==0.3.6 +dill==0.3.7 # via pylint -django==3.2.19 +django==3.2.21 # via # -c requirements/constraints.txt # -r requirements/test.txt @@ -99,9 +102,9 @@ django-crum==0.7.9 # via # -r requirements/test.txt # edx-django-utils -django-debug-toolbar==4.1.0 +django-debug-toolbar==4.2.0 # via -r requirements/test.txt -django-stubs==4.2.3 +django-stubs==4.2.4 # via # -r requirements/test.txt # djangorestframework-stubs @@ -131,19 +134,19 @@ edx-django-utils==5.7.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.8.0 +edx-drf-extensions==8.10.0 # via -r requirements/test.txt edx-lint==5.3.4 # via -r requirements/quality.in -edx-opaque-keys==2.4.0 +edx-opaque-keys==2.5.1 # via # -r requirements/test.txt # edx-drf-extensions -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via # -r requirements/test.txt # pytest -grimp==2.4 +grimp==3.0 # via # -r requirements/test.txt # import-linter @@ -151,12 +154,14 @@ idna==3.4 # via # -r requirements/test.txt # requests -import-linter==1.9.0 +import-linter==1.12.0 # via -r requirements/test.txt -importlib-metadata==6.7.0 +importlib-metadata==6.8.0 # via # keyring # twine +importlib-resources==6.1.0 + # via keyring iniconfig==2.0.0 # via # -r requirements/test.txt @@ -165,7 +170,7 @@ isort==5.12.0 # via # -r requirements/quality.in # pylint -jaraco-classes==3.2.3 +jaraco-classes==3.3.0 # via keyring jeepney==0.8.0 # via @@ -175,9 +180,9 @@ jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==24.0.0 +keyring==24.2.0 # via twine -kombu==5.3.1 +kombu==5.3.2 # via # -r requirements/test.txt # celery @@ -193,9 +198,9 @@ mccabe==0.7.0 # via pylint mdurl==0.1.2 # via markdown-it-py -mock==5.0.2 +mock==5.1.0 # via -r requirements/test.txt -more-itertools==9.1.0 +more-itertools==10.1.0 # via jaraco-classes mypy==1.5.1 # via @@ -206,12 +211,14 @@ mypy-extensions==1.0.0 # via # -r requirements/test.txt # mypy -mysqlclient==2.1.1 +mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==8.9.0 +newrelic==9.0.0 # via # -r requirements/test.txt # edx-django-utils +nh3==0.2.14 + # via readme-renderer packaging==23.1 # via # -r requirements/test.txt @@ -222,9 +229,9 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.6.0 +platformdirs==3.10.0 # via pylint -pluggy==1.0.0 +pluggy==1.3.0 # via # -r requirements/test.txt # pytest @@ -236,7 +243,7 @@ psutil==5.9.5 # via # -r requirements/test.txt # edx-django-utils -pycodestyle==2.10.0 +pycodestyle==2.11.0 # via -r requirements/quality.in pycparser==2.21 # via @@ -244,7 +251,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.in -pygments==2.15.1 +pygments==2.16.1 # via # readme-renderer # rich @@ -253,7 +260,7 @@ pyjwt[crypto]==2.8.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -pylint==2.17.4 +pylint==2.17.5 # via # edx-lint # pylint-celery @@ -275,7 +282,7 @@ pynacl==1.5.0 # via # -r requirements/test.txt # edx-django-utils -pytest==7.3.2 +pytest==7.4.2 # via # -r requirements/test.txt # pytest-cov @@ -288,21 +295,20 @@ python-dateutil==2.8.2 # via # -r requirements/test.txt # celery - # edx-drf-extensions python-slugify==8.0.1 # via # -r requirements/test.txt # code-annotations -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/test.txt # django # djangorestframework -pyyaml==6.0 +pyyaml==6.0.1 # via # -r requirements/test.txt # code-annotations -readme-renderer==40.0 +readme-renderer==42.0 # via twine requests==2.31.0 # via @@ -315,7 +321,7 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.4.2 +rich==13.5.3 # via twine rules==3.3 # via -r requirements/test.txt @@ -328,8 +334,6 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/test.txt - # bleach - # edx-drf-extensions # edx-lint # python-dateutil snowballstemmer==2.2.0 @@ -358,20 +362,20 @@ tomli==2.0.1 # mypy # pylint # pytest -tomlkit==0.11.8 +tomlkit==0.12.1 # via pylint twine==4.0.2 # via -r requirements/quality.in -types-pytz==2023.3.0.1 +types-pytz==2023.3.1.1 # via # -r requirements/test.txt # django-stubs -types-pyyaml==6.0.12.11 +types-pyyaml==6.0.12.12 # via # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.2 +types-requests==2.31.0.4 # via # -r requirements/test.txt # djangorestframework-stubs @@ -379,7 +383,7 @@ types-urllib3==1.26.25.14 # via # -r requirements/test.txt # types-requests -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # -r requirements/test.txt # asgiref @@ -387,15 +391,19 @@ typing-extensions==4.6.3 # django-stubs # django-stubs-ext # djangorestframework-stubs + # edx-opaque-keys # grimp # import-linter + # kombu # mypy # pylint + # rich tzdata==2023.3 # via # -r requirements/test.txt + # backports-zoneinfo # celery -urllib3==2.0.4 +urllib3==2.0.5 # via # -r requirements/test.txt # requests @@ -410,9 +418,9 @@ wcwidth==0.2.6 # via # -r requirements/test.txt # prompt-toolkit -webencodings==0.5.1 - # via bleach wrapt==1.15.0 # via astroid -zipp==3.15.0 - # via importlib-metadata +zipp==3.17.0 + # via + # importlib-metadata + # importlib-resources diff --git a/requirements/test.txt b/requirements/test.txt index 51275c8f8..0601c09d7 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -14,11 +14,16 @@ asgiref==3.7.2 # django attrs==23.1.0 # via -r requirements/base.txt +backports-zoneinfo[tzdata]==0.2.1 + # via + # -r requirements/base.txt + # celery + # kombu billiard==4.1.0 # via # -r requirements/base.txt # celery -celery==5.3.1 +celery==5.3.4 # via -r requirements/base.txt certifi==2023.7.22 # via @@ -33,7 +38,7 @@ charset-normalizer==3.2.0 # via # -r requirements/base.txt # requests -click==8.1.6 +click==8.1.7 # via # -r requirements/base.txt # celery @@ -55,13 +60,13 @@ click-repl==0.3.0 # via # -r requirements/base.txt # celery -code-annotations==1.3.0 +code-annotations==1.5.0 # via -r requirements/test.in -coverage[toml]==7.2.7 +coverage[toml]==7.3.1 # via # -r requirements/test.in # pytest-cov -cryptography==41.0.3 +cryptography==41.0.4 # via # -r requirements/base.txt # pyjwt @@ -83,9 +88,9 @@ django-crum==0.7.9 # via # -r requirements/base.txt # edx-django-utils -django-debug-toolbar==4.1.0 +django-debug-toolbar==4.2.0 # via -r requirements/test.in -django-stubs==4.2.3 +django-stubs==4.2.4 # via # -r requirements/test.in # djangorestframework-stubs @@ -111,33 +116,33 @@ edx-django-utils==5.7.0 # via # -r requirements/base.txt # edx-drf-extensions -edx-drf-extensions==8.8.0 +edx-drf-extensions==8.10.0 # via -r requirements/base.txt -edx-opaque-keys==2.4.0 +edx-opaque-keys==2.5.1 # via # -r requirements/base.txt # edx-drf-extensions -exceptiongroup==1.1.1 +exceptiongroup==1.1.3 # via pytest -grimp==2.4 +grimp==3.0 # via import-linter idna==3.4 # via # -r requirements/base.txt # requests -import-linter==1.9.0 +import-linter==1.12.0 # via -r requirements/test.in iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -kombu==5.3.1 +kombu==5.3.2 # via # -r requirements/base.txt # celery markupsafe==2.1.3 # via jinja2 -mock==5.0.2 +mock==5.1.0 # via -r requirements/test.in mypy==1.5.1 # via @@ -146,9 +151,9 @@ mypy==1.5.1 # djangorestframework-stubs mypy-extensions==1.0.0 # via mypy -mysqlclient==2.1.1 +mysqlclient==2.2.0 # via -r requirements/test.in -newrelic==8.9.0 +newrelic==9.0.0 # via # -r requirements/base.txt # edx-django-utils @@ -158,7 +163,7 @@ pbr==5.11.1 # via # -r requirements/base.txt # stevedore -pluggy==1.0.0 +pluggy==1.3.0 # via pytest prompt-toolkit==3.0.39 # via @@ -185,7 +190,7 @@ pynacl==1.5.0 # via # -r requirements/base.txt # edx-django-utils -pytest==7.3.2 +pytest==7.4.2 # via # -r requirements/test.in # pytest-cov @@ -198,15 +203,14 @@ python-dateutil==2.8.2 # via # -r requirements/base.txt # celery - # edx-drf-extensions python-slugify==8.0.1 # via code-annotations -pytz==2023.3 +pytz==2023.3.post1 # via # -r requirements/base.txt # django # djangorestframework -pyyaml==6.0 +pyyaml==6.0.1 # via code-annotations requests==2.31.0 # via @@ -222,7 +226,6 @@ semantic-version==2.10.0 six==1.16.0 # via # -r requirements/base.txt - # edx-drf-extensions # python-dateutil sqlparse==0.4.4 # via @@ -244,31 +247,34 @@ tomli==2.0.1 # import-linter # mypy # pytest -types-pytz==2023.3.0.1 +types-pytz==2023.3.1.1 # via django-stubs -types-pyyaml==6.0.12.11 +types-pyyaml==6.0.12.12 # via # django-stubs # djangorestframework-stubs -types-requests==2.31.0.2 +types-requests==2.31.0.4 # via djangorestframework-stubs types-urllib3==1.26.25.14 # via types-requests -typing-extensions==4.6.3 +typing-extensions==4.8.0 # via # -r requirements/base.txt # asgiref # django-stubs # django-stubs-ext # djangorestframework-stubs + # edx-opaque-keys # grimp # import-linter + # kombu # mypy tzdata==2023.3 # via # -r requirements/base.txt + # backports-zoneinfo # celery -urllib3==2.0.4 +urllib3==2.0.5 # via # -r requirements/base.txt # requests From 8954d0748f8a1657482e40939a11d5a1c5e1a15a Mon Sep 17 00:00:00 2001 From: farhan Date: Wed, 4 Oct 2023 10:21:32 +0500 Subject: [PATCH 049/282] fix: Fix test cases --- openedx_learning/core/publishing/api.py | 2 +- .../core/tagging/models/system_defined.py | 20 +++++++++++++++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/openedx_learning/core/publishing/api.py b/openedx_learning/core/publishing/api.py index 6a425b8cc..b4b701a20 100644 --- a/openedx_learning/core/publishing/api.py +++ b/openedx_learning/core/publishing/api.py @@ -91,7 +91,7 @@ def create_publishable_entity_version( version_num=version_num, title=title, created=created, - created_by=created_by, + created_by_id=created_by, ) Draft.objects.create( entity_id=entity_id, diff --git a/openedx_tagging/core/tagging/models/system_defined.py b/openedx_tagging/core/tagging/models/system_defined.py index 23180f6e7..6851ab9d2 100644 --- a/openedx_tagging/core/tagging/models/system_defined.py +++ b/openedx_tagging/core/tagging/models/system_defined.py @@ -88,7 +88,10 @@ def validate_value(self, value: str): Check if 'value' is part of this Taxonomy, based on the specified model. """ try: - self.tag_class_model.objects.get(**{f"{self.tag_class_value_field}__iexact": value}) + # See https://github.com/typeddjango/django-stubs/issues/1684 for why we need to ignore this. + self.tag_class_model.objects.get( # type: ignore[attr-defined] + **{f"{self.tag_class_value_field}__iexact": value} + ) return True except ObjectDoesNotExist: return False @@ -100,7 +103,10 @@ def tag_for_value(self, value: str): try: # First we look up the instance by value. # We specify 'iexact' but whether it's case sensitive or not on MySQL depends on the model's collation. - instance = self.tag_class_model.objects.get(**{f"{self.tag_class_value_field}__iexact": value}) + # See https://github.com/typeddjango/django-stubs/issues/1684 for why we need to ignore this. + instance = self.tag_class_model.objects.get( # type: ignore[attr-defined] + **{f"{self.tag_class_value_field}__iexact": value} + ) except ObjectDoesNotExist as exc: raise Tag.DoesNotExist from exc # Use the canonical value from here on (possibly with different case from the value given as a parameter) @@ -120,7 +126,10 @@ def validate_external_id(self, external_id: str): Check if 'external_id' is part of this Taxonomy. """ try: - self.tag_class_model.objects.get(**{f"{self.tag_class_key_field}__iexact": external_id}) + # See https://github.com/typeddjango/django-stubs/issues/1684 for why we need to ignore this. + self.tag_class_model.objects.get( # type: ignore[attr-defined] + **{f"{self.tag_class_key_field}__iexact": external_id} + ) return True except ObjectDoesNotExist: return False @@ -136,7 +145,10 @@ def tag_for_external_id(self, external_id: str): try: # First we look up the instance by external_id # We specify 'iexact' but whether it's case sensitive or not on MySQL depends on the model's collation. - instance = self.tag_class_model.objects.get(**{f"{self.tag_class_key_field}__iexact": external_id}) + # See https://github.com/typeddjango/django-stubs/issues/1684 for why we need to ignore this. + instance = self.tag_class_model.objects.get( # type: ignore[attr-defined] + **{f"{self.tag_class_key_field}__iexact": external_id} + ) except ObjectDoesNotExist as exc: raise Tag.DoesNotExist from exc value = getattr(instance, self.tag_class_value_field) From 44f4fbed50755c950e169529aec38e7461ac67dd Mon Sep 17 00:00:00 2001 From: farhan Date: Wed, 4 Oct 2023 13:34:28 +0500 Subject: [PATCH 050/282] chore: Python Requirements Update --- requirements/base.txt | 10 +++++----- requirements/ci.txt | 4 ++-- requirements/dev.txt | 39 +++++++++++++------------------------- requirements/doc.txt | 25 +++++++++++------------- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 36 +++++++++++++---------------------- requirements/test.txt | 19 +++++++++---------- 7 files changed, 54 insertions(+), 81 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 0afbf6540..411aedc88 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -20,11 +20,11 @@ celery==5.3.4 # via -r requirements/base.in certifi==2023.7.22 # via requests -cffi==1.15.1 +cffi==1.16.0 # via # cryptography # pynacl -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 # via requests click==8.1.7 # via @@ -74,7 +74,7 @@ idna==3.4 # via requests kombu==5.3.2 # via celery -newrelic==9.0.0 +newrelic==9.1.0 # via edx-django-utils pbr==5.11.1 # via stevedore @@ -121,12 +121,12 @@ tzdata==2023.3 # via # backports-zoneinfo # celery -urllib3==2.0.5 +urllib3==2.0.6 # via requests vine==5.0.0 # via # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.8 # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index c475896cb..8e788a740 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -16,9 +16,9 @@ grimp==3.0 # via import-linter import-linter==1.12.0 # via -r requirements/ci.in -packaging==23.1 +packaging==23.2 # via tox -platformdirs==3.10.0 +platformdirs==3.11.0 # via virtualenv pluggy==1.3.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 41216c421..ca75b45c9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -12,7 +12,7 @@ asgiref==3.7.2 # via # -r requirements/quality.txt # django -astroid==2.15.7 +astroid==2.15.8 # via # -r requirements/quality.txt # pylint @@ -38,14 +38,14 @@ certifi==2023.7.22 # via # -r requirements/quality.txt # requests -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/quality.txt # cryptography # pynacl chardet==5.2.0 # via diff-cover -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 # via # -r requirements/quality.txt # requests @@ -84,7 +84,7 @@ code-annotations==1.5.0 # via # -r requirements/quality.txt # edx-lint -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/quality.txt # pytest-cov @@ -92,7 +92,6 @@ cryptography==41.0.4 # via # -r requirements/quality.txt # pyjwt - # secretstorage ddt==1.6.0 # via -r requirements/quality.txt diff-cover==7.7.0 @@ -214,11 +213,6 @@ jaraco-classes==3.3.0 # via # -r requirements/quality.txt # keyring -jeepney==0.8.0 - # via - # -r requirements/quality.txt - # keyring - # secretstorage jinja2==3.1.2 # via # -r requirements/quality.txt @@ -269,7 +263,7 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/quality.txt -newrelic==9.0.0 +newrelic==9.1.0 # via # -r requirements/quality.txt # edx-django-utils @@ -277,7 +271,7 @@ nh3==0.2.14 # via # -r requirements/quality.txt # readme-renderer -packaging==23.1 +packaging==23.2 # via # -r requirements/ci.txt # -r requirements/pip-tools.txt @@ -297,7 +291,7 @@ pkginfo==1.9.6 # via # -r requirements/quality.txt # twine -platformdirs==3.10.0 +platformdirs==3.11.0 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -343,7 +337,7 @@ pyjwt[crypto]==2.8.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -pylint==2.17.5 +pylint==2.17.7 # via # -r requirements/quality.txt # edx-lint @@ -421,16 +415,12 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.5.3 +rich==13.6.0 # via # -r requirements/quality.txt # twine rules==3.3 # via -r requirements/quality.txt -secretstorage==3.3.3 - # via - # -r requirements/quality.txt - # keyring semantic-version==2.10.0 # via # -r requirements/quality.txt @@ -498,14 +488,10 @@ types-pyyaml==6.0.12.12 # -r requirements/quality.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.4 +types-requests==2.31.0.7 # via # -r requirements/quality.txt # djangorestframework-stubs -types-urllib3==1.26.25.14 - # via - # -r requirements/quality.txt - # types-requests typing-extensions==4.8.0 # via # -r requirements/ci.txt @@ -527,11 +513,12 @@ tzdata==2023.3 # -r requirements/quality.txt # backports-zoneinfo # celery -urllib3==2.0.5 +urllib3==2.0.6 # via # -r requirements/quality.txt # requests # twine + # types-requests vine==5.0.0 # via # -r requirements/quality.txt @@ -542,7 +529,7 @@ virtualenv==20.24.5 # via # -r requirements/ci.txt # tox -wcwidth==0.2.6 +wcwidth==0.2.8 # via # -r requirements/quality.txt # prompt-toolkit diff --git a/requirements/doc.txt b/requirements/doc.txt index 75db46673..4a1b9379d 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -18,7 +18,7 @@ asgiref==3.7.2 # django attrs==23.1.0 # via -r requirements/test.txt -babel==2.12.1 +babel==2.13.0 # via # pydata-sphinx-theme # sphinx @@ -39,12 +39,12 @@ certifi==2023.7.22 # via # -r requirements/test.txt # requests -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 # via # -r requirements/test.txt # requests @@ -72,7 +72,7 @@ click-repl==0.3.0 # celery code-annotations==1.5.0 # via -r requirements/test.txt -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/test.txt # pytest-cov @@ -193,13 +193,13 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==9.0.0 +newrelic==9.1.0 # via # -r requirements/test.txt # edx-django-utils nh3==0.2.14 # via readme-renderer -packaging==23.1 +packaging==23.2 # via # -r requirements/test.txt # pydata-sphinx-theme @@ -312,7 +312,7 @@ sphinxcontrib-applehelp==1.0.4 # via sphinx sphinxcontrib-devhelp==1.0.2 # via sphinx -sphinxcontrib-django==2.4 +sphinxcontrib-django==2.5 # via -r requirements/doc.in sphinxcontrib-htmlhelp==2.0.1 # via sphinx @@ -356,14 +356,10 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.4 +types-requests==2.31.0.7 # via # -r requirements/test.txt # djangorestframework-stubs -types-urllib3==1.26.25.14 - # via - # -r requirements/test.txt - # types-requests typing-extensions==4.8.0 # via # -r requirements/test.txt @@ -382,17 +378,18 @@ tzdata==2023.3 # -r requirements/test.txt # backports-zoneinfo # celery -urllib3==2.0.5 +urllib3==2.0.6 # via # -r requirements/test.txt # requests + # types-requests vine==5.0.0 # via # -r requirements/test.txt # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.8 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 894fa179a..50d35f22e 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -10,7 +10,7 @@ click==8.1.7 # via pip-tools importlib-metadata==6.8.0 # via build -packaging==23.1 +packaging==23.2 # via build pip-tools==7.3.0 # via -r requirements/pip-tools.in diff --git a/requirements/quality.txt b/requirements/quality.txt index beda673b0..96b11057e 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -12,7 +12,7 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -astroid==2.15.7 +astroid==2.15.8 # via # pylint # pylint-celery @@ -33,12 +33,12 @@ certifi==2023.7.22 # via # -r requirements/test.txt # requests -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 # via # -r requirements/test.txt # requests @@ -72,7 +72,7 @@ code-annotations==1.5.0 # via # -r requirements/test.txt # edx-lint -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/test.txt # pytest-cov @@ -80,7 +80,6 @@ cryptography==41.0.4 # via # -r requirements/test.txt # pyjwt - # secretstorage ddt==1.6.0 # via -r requirements/test.txt dill==0.3.7 @@ -172,10 +171,6 @@ isort==5.12.0 # pylint jaraco-classes==3.3.0 # via keyring -jeepney==0.8.0 - # via - # keyring - # secretstorage jinja2==3.1.2 # via # -r requirements/test.txt @@ -213,13 +208,13 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==9.0.0 +newrelic==9.1.0 # via # -r requirements/test.txt # edx-django-utils nh3==0.2.14 # via readme-renderer -packaging==23.1 +packaging==23.2 # via # -r requirements/test.txt # pytest @@ -229,7 +224,7 @@ pbr==5.11.1 # stevedore pkginfo==1.9.6 # via twine -platformdirs==3.10.0 +platformdirs==3.11.0 # via pylint pluggy==1.3.0 # via @@ -260,7 +255,7 @@ pyjwt[crypto]==2.8.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -pylint==2.17.5 +pylint==2.17.7 # via # edx-lint # pylint-celery @@ -321,12 +316,10 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.5.3 +rich==13.6.0 # via twine rules==3.3 # via -r requirements/test.txt -secretstorage==3.3.3 - # via keyring semantic-version==2.10.0 # via # -r requirements/test.txt @@ -375,14 +368,10 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.4 +types-requests==2.31.0.7 # via # -r requirements/test.txt # djangorestframework-stubs -types-urllib3==1.26.25.14 - # via - # -r requirements/test.txt - # types-requests typing-extensions==4.8.0 # via # -r requirements/test.txt @@ -403,18 +392,19 @@ tzdata==2023.3 # -r requirements/test.txt # backports-zoneinfo # celery -urllib3==2.0.5 +urllib3==2.0.6 # via # -r requirements/test.txt # requests # twine + # types-requests vine==5.0.0 # via # -r requirements/test.txt # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.8 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/test.txt b/requirements/test.txt index 0601c09d7..3442d564e 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -29,12 +29,12 @@ certifi==2023.7.22 # via # -r requirements/base.txt # requests -cffi==1.15.1 +cffi==1.16.0 # via # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 # via # -r requirements/base.txt # requests @@ -62,7 +62,7 @@ click-repl==0.3.0 # celery code-annotations==1.5.0 # via -r requirements/test.in -coverage[toml]==7.3.1 +coverage[toml]==7.3.2 # via # -r requirements/test.in # pytest-cov @@ -153,11 +153,11 @@ mypy-extensions==1.0.0 # via mypy mysqlclient==2.2.0 # via -r requirements/test.in -newrelic==9.0.0 +newrelic==9.1.0 # via # -r requirements/base.txt # edx-django-utils -packaging==23.1 +packaging==23.2 # via pytest pbr==5.11.1 # via @@ -253,10 +253,8 @@ types-pyyaml==6.0.12.12 # via # django-stubs # djangorestframework-stubs -types-requests==2.31.0.4 +types-requests==2.31.0.7 # via djangorestframework-stubs -types-urllib3==1.26.25.14 - # via types-requests typing-extensions==4.8.0 # via # -r requirements/base.txt @@ -274,17 +272,18 @@ tzdata==2023.3 # -r requirements/base.txt # backports-zoneinfo # celery -urllib3==2.0.5 +urllib3==2.0.6 # via # -r requirements/base.txt # requests + # types-requests vine==5.0.0 # via # -r requirements/base.txt # amqp # celery # kombu -wcwidth==0.2.6 +wcwidth==0.2.8 # via # -r requirements/base.txt # prompt-toolkit From 909e3f8a8cd2324d05d26fdbc6a0f3ed20d93f20 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 5 Oct 2023 19:49:47 -0700 Subject: [PATCH 051/282] feat: Remove Taxonomy.required, make allow_multiple True by default [FC-0030] (#91) --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 9 +- .../tagging/fixtures/language_taxonomy.yaml | 1299 ----------------- .../tagging/management/commands/__init__.py | 0 .../commands/build_language_fixture.py | 48 - .../migrations/0005_language_taxonomy.py | 8 +- .../migrations/0011_remove_required.py | 22 + .../migrations/0012_language_taxonomy.py | 43 + openedx_tagging/core/tagging/models/base.py | 9 +- .../core/tagging/rest_api/v1/serializers.py | 1 - .../core/tagging/rest_api/v1/views.py | 8 +- .../core/fixtures/tagging.yaml | 4 - .../openedx_tagging/core/tagging/test_api.py | 61 +- .../core/tagging/test_models.py | 2 +- .../core/tagging/test_views.py | 29 +- 15 files changed, 127 insertions(+), 1418 deletions(-) delete mode 100644 openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml delete mode 100644 openedx_tagging/core/tagging/management/commands/__init__.py delete mode 100644 openedx_tagging/core/tagging/management/commands/build_language_fixture.py create mode 100644 openedx_tagging/core/tagging/migrations/0011_remove_required.py create mode 100644 openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 574c53b52..6f229c82d 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.2.0" +__version__ = "0.2.1" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 14595a1ae..f8106590c 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -26,8 +26,7 @@ def create_taxonomy( name: str, description: str | None = None, enabled=True, - required=False, - allow_multiple=False, + allow_multiple=True, allow_free_text=False, taxonomy_class: type[Taxonomy] | None = None, ) -> Taxonomy: @@ -38,7 +37,6 @@ def create_taxonomy( name=name, description=description or "", enabled=enabled, - required=required, allow_multiple=allow_multiple, allow_free_text=allow_free_text, ) @@ -213,11 +211,6 @@ def _check_new_tag_count(new_tag_count: int) -> None: if not taxonomy.allow_multiple and len(tags) > 1: raise ValueError(_(f"Taxonomy ({taxonomy.name}) only allows one tag per object.")) - if taxonomy.required and len(tags) == 0: - raise ValueError( - _(f"Taxonomy ({taxonomy.id}) requires at least one tag per object.") - ) - current_tags = list( ObjectTagClass.objects.filter(taxonomy=taxonomy, object_id=object_id) ) diff --git a/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml b/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml deleted file mode 100644 index b300d8ac6..000000000 --- a/openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml +++ /dev/null @@ -1,1299 +0,0 @@ -- model: oel_tagging.tag - pk: -1 - fields: - taxonomy: -1 - parent: null - value: Afar - external_id: aa -- model: oel_tagging.tag - pk: -2 - fields: - taxonomy: -1 - parent: null - value: Abkhazian - external_id: ab -- model: oel_tagging.tag - pk: -3 - fields: - taxonomy: -1 - parent: null - value: Avestan - external_id: ae -- model: oel_tagging.tag - pk: -4 - fields: - taxonomy: -1 - parent: null - value: Afrikaans - external_id: af -- model: oel_tagging.tag - pk: -5 - fields: - taxonomy: -1 - parent: null - value: Akan - external_id: ak -- model: oel_tagging.tag - pk: -6 - fields: - taxonomy: -1 - parent: null - value: Amharic - external_id: am -- model: oel_tagging.tag - pk: -7 - fields: - taxonomy: -1 - parent: null - value: Aragonese - external_id: an -- model: oel_tagging.tag - pk: -8 - fields: - taxonomy: -1 - parent: null - value: Arabic - external_id: ar -- model: oel_tagging.tag - pk: -9 - fields: - taxonomy: -1 - parent: null - value: Assamese - external_id: as -- model: oel_tagging.tag - pk: -10 - fields: - taxonomy: -1 - parent: null - value: Avaric - external_id: av -- model: oel_tagging.tag - pk: -11 - fields: - taxonomy: -1 - parent: null - value: Aymara - external_id: ay -- model: oel_tagging.tag - pk: -12 - fields: - taxonomy: -1 - parent: null - value: Azerbaijani - external_id: az -- model: oel_tagging.tag - pk: -13 - fields: - taxonomy: -1 - parent: null - value: Bashkir - external_id: ba -- model: oel_tagging.tag - pk: -14 - fields: - taxonomy: -1 - parent: null - value: Belarusian - external_id: be -- model: oel_tagging.tag - pk: -15 - fields: - taxonomy: -1 - parent: null - value: Bulgarian - external_id: bg -- model: oel_tagging.tag - pk: -16 - fields: - taxonomy: -1 - parent: null - value: Bihari languages - external_id: bh -- model: oel_tagging.tag - pk: -17 - fields: - taxonomy: -1 - parent: null - value: Bislama - external_id: bi -- model: oel_tagging.tag - pk: -18 - fields: - taxonomy: -1 - parent: null - value: Bambara - external_id: bm -- model: oel_tagging.tag - pk: -19 - fields: - taxonomy: -1 - parent: null - value: Bengali - external_id: bn -- model: oel_tagging.tag - pk: -20 - fields: - taxonomy: -1 - parent: null - value: Tibetan - external_id: bo -- model: oel_tagging.tag - pk: -21 - fields: - taxonomy: -1 - parent: null - value: Breton - external_id: br -- model: oel_tagging.tag - pk: -22 - fields: - taxonomy: -1 - parent: null - value: Bosnian - external_id: bs -- model: oel_tagging.tag - pk: -23 - fields: - taxonomy: -1 - parent: null - value: Catalan - external_id: ca -- model: oel_tagging.tag - pk: -24 - fields: - taxonomy: -1 - parent: null - value: Chechen - external_id: ce -- model: oel_tagging.tag - pk: -25 - fields: - taxonomy: -1 - parent: null - value: Chamorro - external_id: ch -- model: oel_tagging.tag - pk: -26 - fields: - taxonomy: -1 - parent: null - value: Corsican - external_id: co -- model: oel_tagging.tag - pk: -27 - fields: - taxonomy: -1 - parent: null - value: Cree - external_id: cr -- model: oel_tagging.tag - pk: -28 - fields: - taxonomy: -1 - parent: null - value: Czech - external_id: cs -- model: oel_tagging.tag - pk: -29 - fields: - taxonomy: -1 - parent: null - value: Church Slavic - external_id: cu -- model: oel_tagging.tag - pk: -30 - fields: - taxonomy: -1 - parent: null - value: Chuvash - external_id: cv -- model: oel_tagging.tag - pk: -31 - fields: - taxonomy: -1 - parent: null - value: Welsh - external_id: cy -- model: oel_tagging.tag - pk: -32 - fields: - taxonomy: -1 - parent: null - value: Danish - external_id: da -- model: oel_tagging.tag - pk: -33 - fields: - taxonomy: -1 - parent: null - value: German - external_id: de -- model: oel_tagging.tag - pk: -34 - fields: - taxonomy: -1 - parent: null - value: Divehi - external_id: dv -- model: oel_tagging.tag - pk: -35 - fields: - taxonomy: -1 - parent: null - value: Dzongkha - external_id: dz -- model: oel_tagging.tag - pk: -36 - fields: - taxonomy: -1 - parent: null - value: Ewe - external_id: ee -- model: oel_tagging.tag - pk: -37 - fields: - taxonomy: -1 - parent: null - value: Greek, Modern (1453-) - external_id: el -- model: oel_tagging.tag - pk: -38 - fields: - taxonomy: -1 - parent: null - value: English - external_id: en -- model: oel_tagging.tag - pk: -39 - fields: - taxonomy: -1 - parent: null - value: Esperanto - external_id: eo -- model: oel_tagging.tag - pk: -40 - fields: - taxonomy: -1 - parent: null - value: Spanish - external_id: es -- model: oel_tagging.tag - pk: -41 - fields: - taxonomy: -1 - parent: null - value: Estonian - external_id: et -- model: oel_tagging.tag - pk: -42 - fields: - taxonomy: -1 - parent: null - value: Basque - external_id: eu -- model: oel_tagging.tag - pk: -43 - fields: - taxonomy: -1 - parent: null - value: Persian - external_id: fa -- model: oel_tagging.tag - pk: -44 - fields: - taxonomy: -1 - parent: null - value: Fulah - external_id: ff -- model: oel_tagging.tag - pk: -45 - fields: - taxonomy: -1 - parent: null - value: Finnish - external_id: fi -- model: oel_tagging.tag - pk: -46 - fields: - taxonomy: -1 - parent: null - value: Fijian - external_id: fj -- model: oel_tagging.tag - pk: -47 - fields: - taxonomy: -1 - parent: null - value: Faroese - external_id: fo -- model: oel_tagging.tag - pk: -48 - fields: - taxonomy: -1 - parent: null - value: French - external_id: fr -- model: oel_tagging.tag - pk: -49 - fields: - taxonomy: -1 - parent: null - value: Western Frisian - external_id: fy -- model: oel_tagging.tag - pk: -50 - fields: - taxonomy: -1 - parent: null - value: Irish - external_id: ga -- model: oel_tagging.tag - pk: -51 - fields: - taxonomy: -1 - parent: null - value: Gaelic - external_id: gd -- model: oel_tagging.tag - pk: -52 - fields: - taxonomy: -1 - parent: null - value: Galician - external_id: gl -- model: oel_tagging.tag - pk: -53 - fields: - taxonomy: -1 - parent: null - value: Guarani - external_id: gn -- model: oel_tagging.tag - pk: -54 - fields: - taxonomy: -1 - parent: null - value: Gujarati - external_id: gu -- model: oel_tagging.tag - pk: -55 - fields: - taxonomy: -1 - parent: null - value: Manx - external_id: gv -- model: oel_tagging.tag - pk: -56 - fields: - taxonomy: -1 - parent: null - value: Hausa - external_id: ha -- model: oel_tagging.tag - pk: -57 - fields: - taxonomy: -1 - parent: null - value: Hebrew - external_id: he -- model: oel_tagging.tag - pk: -58 - fields: - taxonomy: -1 - parent: null - value: Hindi - external_id: hi -- model: oel_tagging.tag - pk: -59 - fields: - taxonomy: -1 - parent: null - value: Hiri Motu - external_id: ho -- model: oel_tagging.tag - pk: -60 - fields: - taxonomy: -1 - parent: null - value: Croatian - external_id: hr -- model: oel_tagging.tag - pk: -61 - fields: - taxonomy: -1 - parent: null - value: Haitian - external_id: ht -- model: oel_tagging.tag - pk: -62 - fields: - taxonomy: -1 - parent: null - value: Hungarian - external_id: hu -- model: oel_tagging.tag - pk: -63 - fields: - taxonomy: -1 - parent: null - value: Armenian - external_id: hy -- model: oel_tagging.tag - pk: -64 - fields: - taxonomy: -1 - parent: null - value: Herero - external_id: hz -- model: oel_tagging.tag - pk: -65 - fields: - taxonomy: -1 - parent: null - value: Interlingua (International Auxiliary Language Association) - external_id: ia -- model: oel_tagging.tag - pk: -66 - fields: - taxonomy: -1 - parent: null - value: Indonesian - external_id: id -- model: oel_tagging.tag - pk: -67 - fields: - taxonomy: -1 - parent: null - value: Interlingue - external_id: ie -- model: oel_tagging.tag - pk: -68 - fields: - taxonomy: -1 - parent: null - value: Igbo - external_id: ig -- model: oel_tagging.tag - pk: -69 - fields: - taxonomy: -1 - parent: null - value: Sichuan Yi - external_id: ii -- model: oel_tagging.tag - pk: -70 - fields: - taxonomy: -1 - parent: null - value: Inupiaq - external_id: ik -- model: oel_tagging.tag - pk: -71 - fields: - taxonomy: -1 - parent: null - value: Ido - external_id: io -- model: oel_tagging.tag - pk: -72 - fields: - taxonomy: -1 - parent: null - value: Icelandic - external_id: is -- model: oel_tagging.tag - pk: -73 - fields: - taxonomy: -1 - parent: null - value: Italian - external_id: it -- model: oel_tagging.tag - pk: -74 - fields: - taxonomy: -1 - parent: null - value: Inuktitut - external_id: iu -- model: oel_tagging.tag - pk: -75 - fields: - taxonomy: -1 - parent: null - value: Japanese - external_id: ja -- model: oel_tagging.tag - pk: -76 - fields: - taxonomy: -1 - parent: null - value: Javanese - external_id: jv -- model: oel_tagging.tag - pk: -77 - fields: - taxonomy: -1 - parent: null - value: Georgian - external_id: ka -- model: oel_tagging.tag - pk: -78 - fields: - taxonomy: -1 - parent: null - value: Kongo - external_id: kg -- model: oel_tagging.tag - pk: -79 - fields: - taxonomy: -1 - parent: null - value: Kikuyu - external_id: ki -- model: oel_tagging.tag - pk: -80 - fields: - taxonomy: -1 - parent: null - value: Kuanyama - external_id: kj -- model: oel_tagging.tag - pk: -81 - fields: - taxonomy: -1 - parent: null - value: Kazakh - external_id: kk -- model: oel_tagging.tag - pk: -82 - fields: - taxonomy: -1 - parent: null - value: Kalaallisut - external_id: kl -- model: oel_tagging.tag - pk: -83 - fields: - taxonomy: -1 - parent: null - value: Central Khmer - external_id: km -- model: oel_tagging.tag - pk: -84 - fields: - taxonomy: -1 - parent: null - value: Kannada - external_id: kn -- model: oel_tagging.tag - pk: -85 - fields: - taxonomy: -1 - parent: null - value: Korean - external_id: ko -- model: oel_tagging.tag - pk: -86 - fields: - taxonomy: -1 - parent: null - value: Kanuri - external_id: kr -- model: oel_tagging.tag - pk: -87 - fields: - taxonomy: -1 - parent: null - value: Kashmiri - external_id: ks -- model: oel_tagging.tag - pk: -88 - fields: - taxonomy: -1 - parent: null - value: Kurdish - external_id: ku -- model: oel_tagging.tag - pk: -89 - fields: - taxonomy: -1 - parent: null - value: Komi - external_id: kv -- model: oel_tagging.tag - pk: -90 - fields: - taxonomy: -1 - parent: null - value: Cornish - external_id: kw -- model: oel_tagging.tag - pk: -91 - fields: - taxonomy: -1 - parent: null - value: Kirghiz - external_id: ky -- model: oel_tagging.tag - pk: -92 - fields: - taxonomy: -1 - parent: null - value: Latin - external_id: la -- model: oel_tagging.tag - pk: -93 - fields: - taxonomy: -1 - parent: null - value: Luxembourgish - external_id: lb -- model: oel_tagging.tag - pk: -94 - fields: - taxonomy: -1 - parent: null - value: Ganda - external_id: lg -- model: oel_tagging.tag - pk: -95 - fields: - taxonomy: -1 - parent: null - value: Limburgan - external_id: li -- model: oel_tagging.tag - pk: -96 - fields: - taxonomy: -1 - parent: null - value: Lingala - external_id: ln -- model: oel_tagging.tag - pk: -97 - fields: - taxonomy: -1 - parent: null - value: Lao - external_id: lo -- model: oel_tagging.tag - pk: -98 - fields: - taxonomy: -1 - parent: null - value: Lithuanian - external_id: lt -- model: oel_tagging.tag - pk: -99 - fields: - taxonomy: -1 - parent: null - value: Luba-Katanga - external_id: lu -- model: oel_tagging.tag - pk: -100 - fields: - taxonomy: -1 - parent: null - value: Latvian - external_id: lv -- model: oel_tagging.tag - pk: -101 - fields: - taxonomy: -1 - parent: null - value: Malagasy - external_id: mg -- model: oel_tagging.tag - pk: -102 - fields: - taxonomy: -1 - parent: null - value: Marshallese - external_id: mh -- model: oel_tagging.tag - pk: -103 - fields: - taxonomy: -1 - parent: null - value: Maori - external_id: mi -- model: oel_tagging.tag - pk: -104 - fields: - taxonomy: -1 - parent: null - value: Macedonian - external_id: mk -- model: oel_tagging.tag - pk: -105 - fields: - taxonomy: -1 - parent: null - value: Malayalam - external_id: ml -- model: oel_tagging.tag - pk: -106 - fields: - taxonomy: -1 - parent: null - value: Mongolian - external_id: mn -- model: oel_tagging.tag - pk: -107 - fields: - taxonomy: -1 - parent: null - value: Marathi - external_id: mr -- model: oel_tagging.tag - pk: -108 - fields: - taxonomy: -1 - parent: null - value: Malay - external_id: ms -- model: oel_tagging.tag - pk: -109 - fields: - taxonomy: -1 - parent: null - value: Maltese - external_id: mt -- model: oel_tagging.tag - pk: -110 - fields: - taxonomy: -1 - parent: null - value: Burmese - external_id: my -- model: oel_tagging.tag - pk: -111 - fields: - taxonomy: -1 - parent: null - value: Nauru - external_id: na -- model: oel_tagging.tag - pk: -112 - fields: - taxonomy: -1 - parent: null - value: Bokmål, Norwegian - external_id: nb -- model: oel_tagging.tag - pk: -113 - fields: - taxonomy: -1 - parent: null - value: Ndebele, North - external_id: nd -- model: oel_tagging.tag - pk: -114 - fields: - taxonomy: -1 - parent: null - value: Nepali - external_id: ne -- model: oel_tagging.tag - pk: -115 - fields: - taxonomy: -1 - parent: null - value: Ndonga - external_id: ng -- model: oel_tagging.tag - pk: -116 - fields: - taxonomy: -1 - parent: null - value: Dutch - external_id: nl -- model: oel_tagging.tag - pk: -117 - fields: - taxonomy: -1 - parent: null - value: Norwegian Nynorsk - external_id: nn -- model: oel_tagging.tag - pk: -118 - fields: - taxonomy: -1 - parent: null - value: Norwegian - external_id: no -- model: oel_tagging.tag - pk: -119 - fields: - taxonomy: -1 - parent: null - value: Ndebele, South - external_id: nr -- model: oel_tagging.tag - pk: -120 - fields: - taxonomy: -1 - parent: null - value: Navajo - external_id: nv -- model: oel_tagging.tag - pk: -121 - fields: - taxonomy: -1 - parent: null - value: Chichewa - external_id: ny -- model: oel_tagging.tag - pk: -122 - fields: - taxonomy: -1 - parent: null - value: Occitan (post 1500) - external_id: oc -- model: oel_tagging.tag - pk: -123 - fields: - taxonomy: -1 - parent: null - value: Ojibwa - external_id: oj -- model: oel_tagging.tag - pk: -124 - fields: - taxonomy: -1 - parent: null - value: Oromo - external_id: om -- model: oel_tagging.tag - pk: -125 - fields: - taxonomy: -1 - parent: null - value: Oriya - external_id: or -- model: oel_tagging.tag - pk: -126 - fields: - taxonomy: -1 - parent: null - value: Ossetian - external_id: os -- model: oel_tagging.tag - pk: -127 - fields: - taxonomy: -1 - parent: null - value: Panjabi - external_id: pa -- model: oel_tagging.tag - pk: -128 - fields: - taxonomy: -1 - parent: null - value: Pali - external_id: pi -- model: oel_tagging.tag - pk: -129 - fields: - taxonomy: -1 - parent: null - value: Polish - external_id: pl -- model: oel_tagging.tag - pk: -130 - fields: - taxonomy: -1 - parent: null - value: Pushto - external_id: ps -- model: oel_tagging.tag - pk: -131 - fields: - taxonomy: -1 - parent: null - value: Portuguese - external_id: pt -- model: oel_tagging.tag - pk: -132 - fields: - taxonomy: -1 - parent: null - value: Quechua - external_id: qu -- model: oel_tagging.tag - pk: -133 - fields: - taxonomy: -1 - parent: null - value: Romansh - external_id: rm -- model: oel_tagging.tag - pk: -134 - fields: - taxonomy: -1 - parent: null - value: Rundi - external_id: rn -- model: oel_tagging.tag - pk: -135 - fields: - taxonomy: -1 - parent: null - value: Romanian - external_id: ro -- model: oel_tagging.tag - pk: -136 - fields: - taxonomy: -1 - parent: null - value: Russian - external_id: ru -- model: oel_tagging.tag - pk: -137 - fields: - taxonomy: -1 - parent: null - value: Kinyarwanda - external_id: rw -- model: oel_tagging.tag - pk: -138 - fields: - taxonomy: -1 - parent: null - value: Sanskrit - external_id: sa -- model: oel_tagging.tag - pk: -139 - fields: - taxonomy: -1 - parent: null - value: Sardinian - external_id: sc -- model: oel_tagging.tag - pk: -140 - fields: - taxonomy: -1 - parent: null - value: Sindhi - external_id: sd -- model: oel_tagging.tag - pk: -141 - fields: - taxonomy: -1 - parent: null - value: Northern Sami - external_id: se -- model: oel_tagging.tag - pk: -142 - fields: - taxonomy: -1 - parent: null - value: Sango - external_id: sg -- model: oel_tagging.tag - pk: -143 - fields: - taxonomy: -1 - parent: null - value: Sinhala - external_id: si -- model: oel_tagging.tag - pk: -144 - fields: - taxonomy: -1 - parent: null - value: Slovak - external_id: sk -- model: oel_tagging.tag - pk: -145 - fields: - taxonomy: -1 - parent: null - value: Slovenian - external_id: sl -- model: oel_tagging.tag - pk: -146 - fields: - taxonomy: -1 - parent: null - value: Samoan - external_id: sm -- model: oel_tagging.tag - pk: -147 - fields: - taxonomy: -1 - parent: null - value: Shona - external_id: sn -- model: oel_tagging.tag - pk: -148 - fields: - taxonomy: -1 - parent: null - value: Somali - external_id: so -- model: oel_tagging.tag - pk: -149 - fields: - taxonomy: -1 - parent: null - value: Albanian - external_id: sq -- model: oel_tagging.tag - pk: -150 - fields: - taxonomy: -1 - parent: null - value: Serbian - external_id: sr -- model: oel_tagging.tag - pk: -151 - fields: - taxonomy: -1 - parent: null - value: Swati - external_id: ss -- model: oel_tagging.tag - pk: -152 - fields: - taxonomy: -1 - parent: null - value: Sotho, Southern - external_id: st -- model: oel_tagging.tag - pk: -153 - fields: - taxonomy: -1 - parent: null - value: Sundanese - external_id: su -- model: oel_tagging.tag - pk: -154 - fields: - taxonomy: -1 - parent: null - value: Swedish - external_id: sv -- model: oel_tagging.tag - pk: -155 - fields: - taxonomy: -1 - parent: null - value: Swahili - external_id: sw -- model: oel_tagging.tag - pk: -156 - fields: - taxonomy: -1 - parent: null - value: Tamil - external_id: ta -- model: oel_tagging.tag - pk: -157 - fields: - taxonomy: -1 - parent: null - value: Telugu - external_id: te -- model: oel_tagging.tag - pk: -158 - fields: - taxonomy: -1 - parent: null - value: Tajik - external_id: tg -- model: oel_tagging.tag - pk: -159 - fields: - taxonomy: -1 - parent: null - value: Thai - external_id: th -- model: oel_tagging.tag - pk: -160 - fields: - taxonomy: -1 - parent: null - value: Tigrinya - external_id: ti -- model: oel_tagging.tag - pk: -161 - fields: - taxonomy: -1 - parent: null - value: Turkmen - external_id: tk -- model: oel_tagging.tag - pk: -162 - fields: - taxonomy: -1 - parent: null - value: Tagalog - external_id: tl -- model: oel_tagging.tag - pk: -163 - fields: - taxonomy: -1 - parent: null - value: Tswana - external_id: tn -- model: oel_tagging.tag - pk: -164 - fields: - taxonomy: -1 - parent: null - value: Tonga (Tonga Islands) - external_id: to -- model: oel_tagging.tag - pk: -165 - fields: - taxonomy: -1 - parent: null - value: Turkish - external_id: tr -- model: oel_tagging.tag - pk: -166 - fields: - taxonomy: -1 - parent: null - value: Tsonga - external_id: ts -- model: oel_tagging.tag - pk: -167 - fields: - taxonomy: -1 - parent: null - value: Tatar - external_id: tt -- model: oel_tagging.tag - pk: -168 - fields: - taxonomy: -1 - parent: null - value: Twi - external_id: tw -- model: oel_tagging.tag - pk: -169 - fields: - taxonomy: -1 - parent: null - value: Tahitian - external_id: ty -- model: oel_tagging.tag - pk: -170 - fields: - taxonomy: -1 - parent: null - value: Uighur - external_id: ug -- model: oel_tagging.tag - pk: -171 - fields: - taxonomy: -1 - parent: null - value: Ukrainian - external_id: uk -- model: oel_tagging.tag - pk: -172 - fields: - taxonomy: -1 - parent: null - value: Urdu - external_id: ur -- model: oel_tagging.tag - pk: -173 - fields: - taxonomy: -1 - parent: null - value: Uzbek - external_id: uz -- model: oel_tagging.tag - pk: -174 - fields: - taxonomy: -1 - parent: null - value: Venda - external_id: ve -- model: oel_tagging.tag - pk: -175 - fields: - taxonomy: -1 - parent: null - value: Vietnamese - external_id: vi -- model: oel_tagging.tag - pk: -176 - fields: - taxonomy: -1 - parent: null - value: Volapük - external_id: vo -- model: oel_tagging.tag - pk: -177 - fields: - taxonomy: -1 - parent: null - value: Walloon - external_id: wa -- model: oel_tagging.tag - pk: -178 - fields: - taxonomy: -1 - parent: null - value: Wolof - external_id: wo -- model: oel_tagging.tag - pk: -179 - fields: - taxonomy: -1 - parent: null - value: Xhosa - external_id: xh -- model: oel_tagging.tag - pk: -180 - fields: - taxonomy: -1 - parent: null - value: Yiddish - external_id: yi -- model: oel_tagging.tag - pk: -181 - fields: - taxonomy: -1 - parent: null - value: Yoruba - external_id: yo -- model: oel_tagging.tag - pk: -182 - fields: - taxonomy: -1 - parent: null - value: Zhuang - external_id: za -- model: oel_tagging.tag - pk: -183 - fields: - taxonomy: -1 - parent: null - value: Chinese - external_id: zh -- model: oel_tagging.tag - pk: -184 - fields: - taxonomy: -1 - parent: null - value: Zulu - external_id: zu -- model: oel_tagging.taxonomy - pk: -1 - fields: - name: Languages - description: ISO 639-1 Languages. Allows tags for any language configured for use on the instance - enabled: true - required: true - allow_multiple: false - allow_free_text: false - visible_to_authors: true - _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.LanguageTaxonomy diff --git a/openedx_tagging/core/tagging/management/commands/__init__.py b/openedx_tagging/core/tagging/management/commands/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/openedx_tagging/core/tagging/management/commands/build_language_fixture.py b/openedx_tagging/core/tagging/management/commands/build_language_fixture.py deleted file mode 100644 index 4901ca9fe..000000000 --- a/openedx_tagging/core/tagging/management/commands/build_language_fixture.py +++ /dev/null @@ -1,48 +0,0 @@ -""" -Script that downloads all the ISO 639-1 languages and processes them -to write the fixture for the Language system-defined taxonomy. - -This function is intended to be used only once, -but can be edited in the future if more data needs to be added to the fixture. -""" -import json -import urllib.request - -from django.core.management.base import BaseCommand - -endpoint = "https://pkgstore.datahub.io/core/language-codes/language-codes_json/data/97607046542b532c395cf83df5185246/language-codes_json.json" # noqa -output = "./openedx_tagging/core/tagging/fixtures/language_taxonomy.yaml" - - -class Command(BaseCommand): - def handle(self, **options): - json_data = self.download_json() - self.build_fixture(json_data) - - def download_json(self): - with urllib.request.urlopen(endpoint) as response: - json_data = response.read() - return json.loads(json_data) - - def build_fixture(self, json_data): - tag_pk = -1 - with open(output, "w") as output_file: - for lang_data in json_data: - lang_value = self.get_lang_value(lang_data) - lang_code = lang_data["alpha2"] - output_file.write("- model: oel_tagging.tag\n") - output_file.write(f" pk: {tag_pk}\n") - output_file.write(" fields:\n") - output_file.write(" taxonomy: -1\n") - output_file.write(" parent: null\n") - output_file.write(f" value: {lang_value}\n") - output_file.write(f" external_id: {lang_code}\n") - # System tags are identified with negative numbers to avoid clashing with user-created tags. - tag_pk -= 1 - - def get_lang_value(self, lang_data): - """ - Gets the lang value. Some languages has many values. - """ - lang_list = lang_data["English"].split(";") - return lang_list[0] diff --git a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py index b6758da11..84d02f8d4 100644 --- a/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py +++ b/openedx_tagging/core/tagging/migrations/0005_language_taxonomy.py @@ -8,15 +8,17 @@ def load_language_taxonomy(apps, schema_editor): """ Load language taxonomy and tags """ - call_command("loaddata", "--app=oel_tagging", "language_taxonomy.yaml") + # Due to changes in the data model this fixture was no longer working and is + # no longer needed anyways - the language taxonomy will be created in + # 0012_language_taxonomy. + pass def revert(apps, schema_editor): # pragma: no cover """ Deletes language taxonomy an tags """ - Taxonomy = apps.get_model("oel_tagging", "Taxonomy") - Taxonomy.objects.filter(id=-1).delete() + pass class Migration(migrations.Migration): diff --git a/openedx_tagging/core/tagging/migrations/0011_remove_required.py b/openedx_tagging/core/tagging/migrations/0011_remove_required.py new file mode 100644 index 000000000..6caf984ad --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0011_remove_required.py @@ -0,0 +1,22 @@ +# Generated by Django 3.2.19 on 2023-10-03 21:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_tagging', '0010_cleanups'), + ] + + operations = [ + migrations.RemoveField( + model_name='taxonomy', + name='required', + ), + migrations.AlterField( + model_name='taxonomy', + name='allow_multiple', + field=models.BooleanField(default=True, help_text='Indicates that multiple tags from this taxonomy may be added to an object.'), + ), + ] diff --git a/openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py b/openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py new file mode 100644 index 000000000..5576e83df --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.19 on 2023-07-28 13:33 + +from django.core.management import call_command +from django.db import migrations +from django.db.models import Count + + +def create_language_taxonomy(apps, schema_editor): + """ + Create language taxonomy + """ + # Make sure the language taxonomy exists: + Taxonomy = apps.get_model("oel_tagging", "Taxonomy") + lang_taxonomy, _created = Taxonomy.objects.get_or_create(id=-1, defaults={ + "name": "Languages", + "description": "Languages that are enabled on this system.", + "enabled": True, + "allow_multiple": False, + "allow_free_text": False, + "visible_to_authors": True, + "_taxonomy_class": "openedx_tagging.core.tagging.models.system_defined.LanguageTaxonomy", + }) + + # But delete any unused tags created by the old fixture: + lang_taxonomy.tag_set.annotate(usage_count=Count('objecttag')).filter(usage_count=0).delete() + + +def revert(apps, schema_editor): # pragma: no cover + """ + Deletes language taxonomy an tags + """ + Taxonomy = apps.get_model("oel_tagging", "Taxonomy") + Taxonomy.objects.filter(id=-1).delete() + + +class Migration(migrations.Migration): + dependencies = [ + ("oel_tagging", "0011_remove_required"), + ] + + operations = [ + migrations.RunPython(create_language_taxonomy, revert), + ] diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index eb4678bdb..d24d67d46 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -131,14 +131,8 @@ class Taxonomy(models.Model): default=True, help_text=_("Only enabled taxonomies will be shown to authors."), ) - required = models.BooleanField( - default=False, - help_text=_( - "Indicates that one or more tags from this taxonomy must be added to an object." - ), - ) allow_multiple = models.BooleanField( - default=False, # TODO: This should be true, or perhaps remove this property altogether + default=True, help_text=_( "Indicates that multiple tags from this taxonomy may be added to an object." ), @@ -260,7 +254,6 @@ def copy(self, taxonomy: Taxonomy) -> Taxonomy: self.name = taxonomy.name self.description = taxonomy.description self.enabled = taxonomy.enabled - self.required = taxonomy.required self.allow_multiple = taxonomy.allow_multiple self.allow_free_text = taxonomy.allow_free_text self.visible_to_authors = taxonomy.visible_to_authors diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 22878fc76..cf7d6304c 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -24,7 +24,6 @@ class Meta: "name", "description", "enabled", - "required", "allow_multiple", "allow_free_text", "system_defined", diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 9dbb2cc01..26fcad573 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -78,10 +78,8 @@ class TaxonomyView(ModelViewSet): applying tags from this taxonomy to an object. * enabled (optional): Only enabled taxonomies will be shown to authors (default: true). - * required (optional): Indicates that one or more tags from this - taxonomy must be added to an object (default: False). * allow_multiple (optional): Indicates that multiple tags from this - taxonomy may be added to an object (default: False). + taxonomy may be added to an object (default: True). * allow_free_text (optional): Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values (default: False). @@ -92,7 +90,6 @@ class TaxonomyView(ModelViewSet): "name": "Taxonomy Name", "description": "This is a description", "enabled": True, - "required": True, "allow_multiple": True, "allow_free_text": True, } @@ -110,8 +107,6 @@ class TaxonomyView(ModelViewSet): * description (optional): Provides extra information for the user when applying tags from this taxonomy to an object. * enabled (optional): Only enabled taxonomies will be shown to authors. - * required (optional): Indicates that one or more tags from this - taxonomy must be added to an object. * allow_multiple (optional): Indicates that multiple tags from this taxonomy may be added to an object. * allow_free_text (optional): Indicates that tags in this taxonomy need @@ -123,7 +118,6 @@ class TaxonomyView(ModelViewSet): "name": "Taxonomy New Name", "description": "This is a new description", "enabled": False, - "required": False, "allow_multiple": False, "allow_free_text": True, } diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index b3c070035..4715b667b 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -207,7 +207,6 @@ name: Life on Earth description: A taxonomy about life on earth. enabled: true - required: false allow_multiple: false allow_free_text: false - model: oel_tagging.taxonomy @@ -216,7 +215,6 @@ name: User Authors description: Allows tags for any User on the instance. enabled: true - required: false allow_multiple: false allow_free_text: false _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.UserSystemDefinedTaxonomy @@ -226,7 +224,6 @@ name: System defined taxonomy description: Generic System defined taxonomy enabled: true - required: false allow_multiple: false allow_free_text: false _taxonomy_class: openedx_tagging.core.tagging.models.system_defined.SystemDefinedTaxonomy @@ -236,7 +233,6 @@ name: Import Taxonomy Test description: "" enabled: true - required: false allow_multiple: false allow_free_text: false diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 182f79c2f..6ed0140a1 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -42,7 +42,6 @@ def test_create_taxonomy(self) -> None: # Note: we must specify '-> None' to op "name": "Difficulty", "description": "This taxonomy contains tags describing the difficulty of an activity", "enabled": False, - "required": True, "allow_multiple": True, "allow_free_text": True, } @@ -112,31 +111,61 @@ def test_get_tags(self) -> None: *self.phylum_tags, ] assert tagging_api.get_tags(self.system_taxonomy) == self.system_tags - tags = tagging_api.get_tags(self.language_taxonomy) - langs = [tag.external_id for tag in tags] - expected_langs = [lang[0] for lang in test_languages] - assert langs == expected_langs @override_settings(LANGUAGES=test_languages) def test_get_root_tags(self): assert tagging_api.get_root_tags(self.taxonomy) == self.domain_tags assert tagging_api.get_root_tags(self.system_taxonomy) == self.system_tags - tags = tagging_api.get_root_tags(self.language_taxonomy) - langs = [tag.external_id for tag in tags] - expected_langs = [lang[0] for lang in test_languages] - assert langs == expected_langs @override_settings(LANGUAGES=test_languages) + def test_get_root_language_tags(self): + """ + For the language taxonomy, listing and searching tags will only show + tags that have been used at least once. + """ + before_langs = [ + tag.external_id for tag in + tagging_api.get_root_tags(self.language_taxonomy) + ] + assert before_langs == ["en"] + # Use a few more tags: + for _lang_code, lang_value in test_languages: + tagging_api.tag_object(object_id="foo", taxonomy=self.language_taxonomy, tags=[lang_value]) + # now a search will return matching tags: + after_langs = [ + tag.external_id for tag in + tagging_api.get_root_tags(self.language_taxonomy) + ] + expected_langs = [lang_code for lang_code, _ in test_languages] + assert after_langs == expected_langs + def test_search_tags(self): assert tagging_api.search_tags( self.taxonomy, search_term='eU' ) == self.filtered_tags - tags = tagging_api.search_tags(self.language_taxonomy, search_term='IsH') - langs = [tag.external_id for tag in tags] - expected_langs = [lang[0] for lang in filtered_test_languages] - assert langs == expected_langs + @override_settings(LANGUAGES=test_languages) + def test_search_language_tags(self): + """ + For the language taxonomy, listing and searching tags will only show + tags that have been used at least once. + """ + before_langs = [ + tag.external_id for tag in + tagging_api.search_tags(self.language_taxonomy, search_term='IsH') + ] + assert before_langs == ["en"] + # Use a few more tags: + for _lang_code, lang_value in test_languages: + tagging_api.tag_object(object_id="foo", taxonomy=self.language_taxonomy, tags=[lang_value]) + # now a search will return matching tags: + after_langs = [ + tag.external_id for tag in + tagging_api.search_tags(self.language_taxonomy, search_term='IsH') + ] + expected_langs = [lang_code for lang_code, _ in filtered_test_languages] + assert after_langs == expected_langs def test_get_children_tags(self): assert tagging_api.get_children_tags( @@ -284,12 +313,6 @@ def test_tag_object_no_multiple(self): tagging_api.tag_object(self.taxonomy, ["A", "B"], "biology101") assert "only allows one tag per object" in str(excinfo.value) - def test_tag_object_required(self): - self.taxonomy.required = True - with pytest.raises(ValueError) as excinfo: - tagging_api.tag_object(self.taxonomy, [], "biology101") - assert "requires at least one tag per object" in str(excinfo.value) - def test_tag_object_invalid_tag(self): with pytest.raises(tagging_api.TagDoesNotExist) as excinfo: tagging_api.tag_object(self.taxonomy, ["Eukaryota Xenomorph"], "biology101") diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 6c1d852ec..ccc9e6c6d 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -43,7 +43,7 @@ def setUp(self): self.mammalia = get_tag("Mammalia") self.animalia = get_tag("Animalia") self.system_taxonomy_tag = get_tag("System Tag 1") - self.english_tag = get_tag("English") + self.english_tag = self.language_taxonomy.tag_for_external_id("en") self.user_1 = get_user_model()( id=1, username="test_user_1", diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index effd6460b..b843d2047 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -35,8 +35,7 @@ def check_taxonomy( name, description="", enabled=True, - required=False, - allow_multiple=False, + allow_multiple=True, allow_free_text=False, system_defined=False, visible_to_authors=True, @@ -48,7 +47,6 @@ def check_taxonomy( assert data["name"] == name assert data["description"] == description assert data["enabled"] == enabled - assert data["required"] == required assert data["allow_multiple"] == allow_multiple assert data["allow_free_text"] == allow_free_text assert data["system_defined"] == system_defined @@ -206,7 +204,6 @@ def test_create_taxonomy(self, user_attr: str | None, expected_status: int): "name": "taxonomy_data_2", "description": "This is a description", "enabled": False, - "required": True, "allow_multiple": True, } @@ -227,7 +224,6 @@ def test_create_taxonomy(self, user_attr: str | None, expected_status: int): @ddt.data( {}, - {"name": "Error taxonomy 2", "required": "Invalid value"}, {"name": "Error taxonomy 3", "enabled": "Invalid value"}, ) def test_create_taxonomy_error(self, create_data: dict[str, str]): @@ -260,7 +256,6 @@ def test_update_taxonomy(self, user_attr, expected_status): name="test update taxonomy", description="taxonomy description", enabled=True, - required=False, ) taxonomy.save() @@ -283,7 +278,6 @@ def test_update_taxonomy(self, user_attr, expected_status): "name": "new name", "description": "taxonomy description", "enabled": True, - "required": False, }, ) @@ -320,7 +314,7 @@ def test_update_taxonomy_404(self): ) @ddt.unpack def test_patch_taxonomy(self, user_attr, expected_status): - taxonomy = Taxonomy.objects.create(name="test patch taxonomy", enabled=False, required=True) + taxonomy = Taxonomy.objects.create(name="test patch taxonomy", enabled=False) taxonomy.save() url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) @@ -329,7 +323,7 @@ def test_patch_taxonomy(self, user_attr, expected_status): user = getattr(self, user_attr) self.client.force_authenticate(user=user) - response = self.client.patch(url, {"name": "new name", "required": False}, format="json") + response = self.client.patch(url, {"name": "new name", "enabled": True}, format="json") assert response.status_code == expected_status # If we were able to update the taxonomy, check if the name changed @@ -340,8 +334,7 @@ def test_patch_taxonomy(self, user_attr, expected_status): response.data["id"], **{ "name": "new name", - "enabled": False, - "required": False, + "enabled": True, }, ) @@ -439,8 +432,8 @@ def _object_permission(_user, object_id: str) -> bool: self.language_taxonomy = Taxonomy.objects.get(pk=LANGUAGE_TAXONOMY_ID) # Closed Taxonomies created by taxonomy admins, each with 20 ObjectTags - self.enabled_taxonomy = Taxonomy.objects.create(name="Enabled Taxonomy") - self.disabled_taxonomy = Taxonomy.objects.create(name="Disabled Taxonomy", enabled=False) + self.enabled_taxonomy = Taxonomy.objects.create(name="Enabled Taxonomy", allow_multiple=False) + self.disabled_taxonomy = Taxonomy.objects.create(name="Disabled Taxonomy", enabled=False, allow_multiple=False) self.multiple_taxonomy = Taxonomy.objects.create(name="Multiple Taxonomy", allow_multiple=True) for i in range(20): # Valid ObjectTags @@ -459,9 +452,11 @@ def _object_permission(_user, object_id: str) -> bool: # Free-Text Taxonomies created by taxonomy admins, each linked # to 10 ObjectTags - self.open_taxonomy_enabled = Taxonomy.objects.create(name="Enabled Free-Text Taxonomy", allow_free_text=True) + self.open_taxonomy_enabled = Taxonomy.objects.create( + name="Enabled Free-Text Taxonomy", allow_free_text=True, allow_multiple=False, + ) self.open_taxonomy_disabled = Taxonomy.objects.create( - name="Disabled Free-Text Taxonomy", allow_free_text=True, enabled=False + name="Disabled Free-Text Taxonomy", allow_free_text=True, enabled=False, allow_multiple=False, ) for i in range(10): ObjectTag.objects.create(object_id="abc", taxonomy=self.open_taxonomy_enabled, _value=f"Free Text {i}") @@ -683,10 +678,6 @@ def test_tag_object_invalid(self, user_attr, taxonomy_attr, tag_values, expected (None, "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), ("user", "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), ("staff", "open_taxonomy_disabled", [], status.HTTP_200_OK), - # Users and staff can't clear a taxonomy with required=True - (None, "language_taxonomy", [], status.HTTP_403_FORBIDDEN), - ("user", "language_taxonomy", [], status.HTTP_400_BAD_REQUEST), - ("staff", "language_taxonomy", [], status.HTTP_400_BAD_REQUEST), ) @ddt.unpack def test_tag_object_clear(self, user_attr, taxonomy_attr, tag_values, expected_status): From 334c11501815ea764993739e366fb0cc3bcc4e5c Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 8 Oct 2023 20:20:34 -0400 Subject: [PATCH 052/282] chore: Updating Python Requirements --- requirements/base.txt | 2 +- requirements/dev.txt | 20 ++++++++++++++++---- requirements/doc.txt | 6 +++--- requirements/quality.txt | 13 ++++++++++--- requirements/test.txt | 4 ++-- 5 files changed, 32 insertions(+), 13 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 411aedc88..5d4038c13 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -41,7 +41,7 @@ click-repl==0.3.0 # via celery cryptography==41.0.4 # via pyjwt -django==3.2.21 +django==3.2.22 # via # -c requirements/constraints.txt # -r requirements/base.in diff --git a/requirements/dev.txt b/requirements/dev.txt index ca75b45c9..157726f68 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -92,6 +92,7 @@ cryptography==41.0.4 # via # -r requirements/quality.txt # pyjwt + # secretstorage ddt==1.6.0 # via -r requirements/quality.txt diff-cover==7.7.0 @@ -104,7 +105,7 @@ distlib==0.3.7 # via # -r requirements/ci.txt # virtualenv -django==3.2.21 +django==3.2.22 # via # -c requirements/constraints.txt # -r requirements/quality.txt @@ -144,7 +145,7 @@ djangorestframework==3.14.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.2 +djangorestframework-stubs==3.14.3 # via -r requirements/quality.txt docutils==0.20.1 # via @@ -160,7 +161,7 @@ edx-django-utils==5.7.0 # edx-drf-extensions edx-drf-extensions==8.10.0 # via -r requirements/quality.txt -edx-i18n-tools==1.2.0 +edx-i18n-tools==1.3.0 # via -r requirements/dev.in edx-lint==5.3.4 # via -r requirements/quality.txt @@ -213,6 +214,11 @@ jaraco-classes==3.3.0 # via # -r requirements/quality.txt # keyring +jeepney==0.8.0 + # via + # -r requirements/quality.txt + # keyring + # secretstorage jinja2==3.1.2 # via # -r requirements/quality.txt @@ -230,6 +236,8 @@ lazy-object-proxy==1.9.0 # via # -r requirements/quality.txt # astroid +lxml==4.9.3 + # via edx-i18n-tools markdown-it-py==3.0.0 # via # -r requirements/quality.txt @@ -421,6 +429,10 @@ rich==13.6.0 # twine rules==3.3 # via -r requirements/quality.txt +secretstorage==3.3.3 + # via + # -r requirements/quality.txt + # keyring semantic-version==2.10.0 # via # -r requirements/quality.txt @@ -488,7 +500,7 @@ types-pyyaml==6.0.12.12 # -r requirements/quality.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.7 +types-requests==2.31.0.8 # via # -r requirements/quality.txt # djangorestframework-stubs diff --git a/requirements/doc.txt b/requirements/doc.txt index 4a1b9379d..ae9758731 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -82,7 +82,7 @@ cryptography==41.0.4 # pyjwt ddt==1.6.0 # via -r requirements/test.txt -django==3.2.21 +django==3.2.22 # via # -c requirements/constraints.txt # -r requirements/test.txt @@ -120,7 +120,7 @@ djangorestframework==3.14.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.2 +djangorestframework-stubs==3.14.3 # via -r requirements/test.txt doc8==1.1.1 # via -r requirements/doc.in @@ -356,7 +356,7 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.7 +types-requests==2.31.0.8 # via # -r requirements/test.txt # djangorestframework-stubs diff --git a/requirements/quality.txt b/requirements/quality.txt index 96b11057e..b29f7118e 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -80,11 +80,12 @@ cryptography==41.0.4 # via # -r requirements/test.txt # pyjwt + # secretstorage ddt==1.6.0 # via -r requirements/test.txt dill==0.3.7 # via pylint -django==3.2.21 +django==3.2.22 # via # -c requirements/constraints.txt # -r requirements/test.txt @@ -121,7 +122,7 @@ djangorestframework==3.14.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.2 +djangorestframework-stubs==3.14.3 # via -r requirements/test.txt docutils==0.20.1 # via readme-renderer @@ -171,6 +172,10 @@ isort==5.12.0 # pylint jaraco-classes==3.3.0 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage jinja2==3.1.2 # via # -r requirements/test.txt @@ -320,6 +325,8 @@ rich==13.6.0 # via twine rules==3.3 # via -r requirements/test.txt +secretstorage==3.3.3 + # via keyring semantic-version==2.10.0 # via # -r requirements/test.txt @@ -368,7 +375,7 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.7 +types-requests==2.31.0.8 # via # -r requirements/test.txt # djangorestframework-stubs diff --git a/requirements/test.txt b/requirements/test.txt index 3442d564e..b4db8b5ac 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -106,7 +106,7 @@ djangorestframework==3.14.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.2 +djangorestframework-stubs==3.14.3 # via -r requirements/test.in drf-jwt==1.19.2 # via @@ -253,7 +253,7 @@ types-pyyaml==6.0.12.12 # via # django-stubs # djangorestframework-stubs -types-requests==2.31.0.7 +types-requests==2.31.0.8 # via djangorestframework-stubs typing-extensions==4.8.0 # via From 0934450b518be6bc7fb8208935195809dc9ca3a1 Mon Sep 17 00:00:00 2001 From: Jillian Date: Mon, 9 Oct 2023 11:06:19 +1030 Subject: [PATCH 053/282] Adds missing __init__.py files [FC-0030] (#90) chore: adds missing __init__.py files and fixes resulting pylint issues * fixes a number of translation-of-non-string * fixes coverage for files in tests/ * fixes PytestCollectionWarnings --- openedx_tagging/core/__init__.py | 0 openedx_tagging/core/tagging/api.py | 8 +-- .../core/tagging/import_export/__init__.py | 3 ++ .../core/tagging/import_export/actions.py | 38 +++++++------- .../core/tagging/import_export/api.py | 15 +++--- .../core/tagging/import_export/exceptions.py | 51 ++++++++++--------- .../core/tagging/import_export/parsers.py | 10 ++-- .../core/tagging/models/import_export.py | 40 +++++++++++++++ .../core/tagging/rest_api/__init__.py | 0 .../core/tagging/rest_api/paginators.py | 4 ++ .../core/tagging/rest_api/v1/permissions.py | 15 ++++++ .../core/tagging/rest_api/v1/serializers.py | 27 ++++++++-- .../core/tagging/rest_api/v1/views.py | 10 ++-- .../openedx_tagging/core/tagging/test_api.py | 27 ++-------- .../core/tagging/test_models.py | 14 ++--- .../tagging/test_system_defined_models.py | 8 +-- .../core/tagging/test_views.py | 3 +- 17 files changed, 172 insertions(+), 101 deletions(-) create mode 100644 openedx_tagging/core/__init__.py create mode 100644 openedx_tagging/core/tagging/rest_api/__init__.py diff --git a/openedx_tagging/core/__init__.py b/openedx_tagging/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index f8106590c..e91409687 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -14,7 +14,7 @@ from django.db import transaction from django.db.models import F, QuerySet -from django.utils.translation import gettext_lazy as _ +from django.utils.translation import gettext as _ from .models import ObjectTag, Tag, Taxonomy @@ -196,11 +196,11 @@ def _check_new_tag_count(new_tag_count: int) -> None: if current_count + new_tag_count > 100: raise ValueError( - _(f"Cannot add more than 100 tags to ({object_id}).") + _("Cannot add more than 100 tags to ({object_id}).").format(object_id=object_id) ) if not isinstance(tags, list): - raise ValueError(_(f"Tags must be a list, not {type(tags).__name__}.")) + raise ValueError(_("Tags must be a list, not {type}.").format(type=type(tags).__name__)) ObjectTagClass = object_tag_class taxonomy = taxonomy.cast() # Make sure we're using the right subclass. This is a no-op if we are already. @@ -209,7 +209,7 @@ def _check_new_tag_count(new_tag_count: int) -> None: _check_new_tag_count(len(tags)) if not taxonomy.allow_multiple and len(tags) > 1: - raise ValueError(_(f"Taxonomy ({taxonomy.name}) only allows one tag per object.")) + raise ValueError(_("Taxonomy ({name}) only allows one tag per object.").format(name=taxonomy.name)) current_tags = list( ObjectTagClass.objects.filter(taxonomy=taxonomy, object_id=object_id) diff --git a/openedx_tagging/core/tagging/import_export/__init__.py b/openedx_tagging/core/tagging/import_export/__init__.py index 55bcdaebd..fb1088d44 100644 --- a/openedx_tagging/core/tagging/import_export/__init__.py +++ b/openedx_tagging/core/tagging/import_export/__init__.py @@ -1 +1,4 @@ +""" +Externally-facing import/export classes. +""" from .parsers import ParserFormat diff --git a/openedx_tagging/core/tagging/import_export/actions.py b/openedx_tagging/core/tagging/import_export/actions.py index 409194277..b968a6b0b 100644 --- a/openedx_tagging/core/tagging/import_export/actions.py +++ b/openedx_tagging/core/tagging/import_export/actions.py @@ -41,7 +41,7 @@ def __init__(self, taxonomy: Taxonomy, tag, index: int): self.index = index def __repr__(self) -> str: - return str(_(f"Action {self.name} (index={self.index},id={self.tag.id})")) + return str(_("Action {name} (index={index},id={id})").format(name=self.name, index=self.index, id=self.tag.id)) def __str__(self) -> str: return self.__repr__() @@ -104,11 +104,10 @@ def _validate_parent(self, indexed_actions) -> ImportActionError | None: ): return ImportActionError( action=self, - tag_id=self.tag.id, message=_( - f"Unknown parent tag ({self.tag.parent_id}). " + "Unknown parent tag ({parent_id}). " "You need to add parent before the child in your file." - ), + ).format(parent_id=self.tag.parent_id), ) return None @@ -122,10 +121,9 @@ def _validate_value(self, indexed_actions) -> ImportActionError | None: taxonomy_tag = self.taxonomy.tag_set.get(value=self.tag.value) return ImportActionError( action=self, - tag_id=self.tag.id, message=_( - f"Duplicated tag value with tag in database (external_id={taxonomy_tag.external_id})." - ), + "Duplicated tag value with tag in database (external_id={external_id})." + ).format(external_id=taxonomy_tag.external_id) ) except Tag.DoesNotExist: # Validates value duplication on create actions @@ -148,7 +146,6 @@ def _validate_value(self, indexed_actions) -> ImportActionError | None: if action: return ImportActionConflict( action=self, - tag_id=self.tag.id, conflict_action_index=action.index, message=_("Duplicated tag value."), ) @@ -175,9 +172,9 @@ def __str__(self) -> str: return str( _( "Create a new tag with values " - f"(external_id={self.tag.id}, value={self.tag.value}, " - f"parent_id={self.tag.parent_id})." - ) + "(external_id={external_id}, value={value}, " + "parent_id={parent_id})." + ).format(external_id=self.tag.id, value=self.tag.value, parent_id=self.tag.parent_id) ) @classmethod @@ -199,7 +196,6 @@ def _validate_id(self, indexed_actions) -> ImportActionError | None: if action: return ImportActionConflict( action=self, - tag_id=self.tag.id, conflict_action_index=action.index, message=_("Duplicated external_id tag."), ) @@ -263,13 +259,13 @@ def __str__(self) -> str: if not taxonomy_tag.parent: from_str = _("from empty parent") else: - from_str = _(f"from parent (external_id={taxonomy_tag.parent.external_id})") + from_str = _("from parent (external_id={external_id})").format(external_id=taxonomy_tag.parent.external_id) return str( _( - f"Update the parent of tag (external_id={taxonomy_tag.external_id}) " - f"{from_str} to parent (external_id={self.tag.parent_id})." - ) + "Update the parent of tag (external_id={external_id}) " + "{from_str} to parent (external_id={parent_id})." + ).format(external_id=taxonomy_tag.external_id, from_str=from_str, parent_id=self.tag.parent_id) ) @classmethod @@ -329,9 +325,9 @@ def __str__(self) -> str: taxonomy_tag = self._get_tag() return str( _( - f"Rename tag value of tag (external_id={taxonomy_tag.external_id}) " - f"from '{taxonomy_tag.value}' to '{self.tag.value}'" - ) + "Rename tag value of tag (external_id={external_id}) " + "from '{from_value}' to '{to_value}'" + ).format(external_id=taxonomy_tag.external_id, from_value=taxonomy_tag.value, to_value=self.tag.value) ) @classmethod @@ -378,7 +374,7 @@ class DeleteTag(ImportAction): def __str__(self) -> str: taxonomy_tag = self._get_tag() - return str(_(f"Delete tag (external_id={taxonomy_tag.external_id})")) + return str(_("Delete tag (external_id={external_id})").format(external_id=taxonomy_tag.external_id)) name = "delete" @@ -415,7 +411,7 @@ class WithoutChanges(ImportAction): name = "without_changes" def __str__(self) -> str: - return str(_(f"No changes needed for tag (external_id={self.tag.id})")) + return str(_("No changes needed for tag (external_id={external_id})").format(external_id=self.tag.id)) @classmethod def applies_for(cls, taxonomy: Taxonomy, tag) -> bool: diff --git a/openedx_tagging/core/tagging/import_export/api.py b/openedx_tagging/core/tagging/import_export/api.py index 77203bf5e..c90a78f4d 100644 --- a/openedx_tagging/core/tagging/import_export/api.py +++ b/openedx_tagging/core/tagging/import_export/api.py @@ -46,9 +46,10 @@ from io import BytesIO -from django.utils.translation import gettext_lazy as _ +from django.utils.translation import gettext as _ from ..models import TagImportTask, TagImportTaskState, Taxonomy +from .exceptions import TagImportError from .import_plan import TagImportPlan, TagImportTask from .parsers import ParserFormat, get_parser @@ -115,7 +116,7 @@ def import_tags( tag_import_plan.execute(task) task.end_success() return True - except Exception as exception: + except (TagImportError, ValueError) as exception: # Log any exception task.log_exception(exception) return False @@ -159,8 +160,10 @@ def _check_unique_import_task(taxonomy: Taxonomy) -> bool: if not last_task: return True return ( - last_task.status == TagImportTaskState.SUCCESS.value - or last_task.status == TagImportTaskState.ERROR.value + last_task.status in { + TagImportTaskState.SUCCESS.value, + TagImportTaskState.ERROR.value + } ) @@ -189,6 +192,6 @@ def _import_export_validations(taxonomy: Taxonomy): if taxonomy.system_defined: raise ValueError( _( - f"Invalid taxonomy ({taxonomy.id}): You cannot import/export a system-defined taxonomy." - ) + "Invalid taxonomy ({id}): You cannot import/export a system-defined taxonomy." + ).format(id=taxonomy.id) ) diff --git a/openedx_tagging/core/tagging/import_export/exceptions.py b/openedx_tagging/core/tagging/import_export/exceptions.py index 91a6b6f9b..5ec52cadb 100644 --- a/openedx_tagging/core/tagging/import_export/exceptions.py +++ b/openedx_tagging/core/tagging/import_export/exceptions.py @@ -16,9 +16,9 @@ class TagImportError(Exception): Base exception for import """ - def __init__(self, message: str = "", **kargs): + def __init__(self, message: str = ""): + super().__init__() self.message = message - super().__init__(message, **kargs) def __str__(self): return str(self.message) @@ -32,10 +32,9 @@ class TagParserError(TagImportError): Base exception for parsers """ - def __init__(self, tag: dict | None, **kargs): + def __init__(self, tag: dict | None, **kargs): # pylint: disable=unused-argument super().__init__() - self.tag = tag - self.message = _(f"Import parser error on {tag}") + self.message = _("Import parser error on {tag}").format(tag=tag) class ImportActionError(TagImportError): @@ -43,10 +42,11 @@ class ImportActionError(TagImportError): Base exception for actions """ - def __init__(self, action: ImportAction, tag_id: str, message: str, **kargs): + def __init__(self, action: ImportAction, message: str, **kargs): + super().__init__(**kargs) self.message = _( - f"Action error in '{action.name}' (#{action.index}): {message}" - ) + "Action error in '{name}' (#{index}): {message}" + ).format(name=action.name, index=action.index, message=message) class ImportActionConflict(ImportActionError): @@ -57,14 +57,19 @@ class ImportActionConflict(ImportActionError): def __init__( self, action: ImportAction, - tag_id: str, conflict_action_index: int, message: str, **kargs, ): + super().__init__(action, message, **kargs) self.message = _( - f"Conflict with '{action.name}' (#{action.index}) " - f"and action #{conflict_action_index}: {message}" + "Conflict with '{action_name}' (#{action_index}) " + "and action #{conflict_action_index}: {message}" + ).format( + action_name=action.name, + action_index=action.index, + conflict_action_index=conflict_action_index, + message=message, ) @@ -73,9 +78,9 @@ class InvalidFormat(TagParserError): Exception used when there is an error with the format """ - def __init__(self, tag: dict | None, format: str, message: str, **kargs): - super().__init__(tag) - self.message = _(f"Invalid '{format}' format: {message}") + def __init__(self, tag: dict | None, input_format: str, message: str, **kargs): + super().__init__(tag, **kargs) + self.message = _("Invalid '{format}' format: {message}").format(format=input_format, message=message) class FieldJSONError(TagParserError): @@ -83,9 +88,9 @@ class FieldJSONError(TagParserError): Exception used when missing a required field on the .json """ - def __init__(self, tag: dict | None, field, **kargs): - super().__init__(tag) - self.message = _(f"Missing '{field}' field on {tag}") + def __init__(self, tag: dict | None, field: str, **kargs): + super().__init__(tag, **kargs) + self.message = _("Missing '{field}' field on {tag}").format(field=field, tag=tag) class EmptyJSONField(TagParserError): @@ -93,9 +98,9 @@ class EmptyJSONField(TagParserError): Exception used when a required field is empty on the .json """ - def __init__(self, tag, field, **kargs): - self.tag = tag - self.message = _(f"Empty '{field}' field on {tag}") + def __init__(self, tag: dict | None, field: str, **kargs): + super().__init__(tag, **kargs) + self.message = _("Empty '{field}' field on {tag}").format(field=field, tag=tag) class EmptyCSVField(TagParserError): @@ -103,6 +108,6 @@ class EmptyCSVField(TagParserError): Exception used when a required field is empty on the .csv """ - def __init__(self, tag, field, row, **kargs): - self.tag = tag - self.message = _(f"Empty '{field}' field on the row {row}") + def __init__(self, tag: dict | None, field: str, row: int, **kargs): + super().__init__(tag, **kargs) + self.message = _("Empty '{field}' field on the row {row}").format(field=field, row=row) diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py index b0132f3d1..4413c04c4 100644 --- a/openedx_tagging/core/tagging/import_export/parsers.py +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -212,13 +212,13 @@ def _load_data(cls, file: BytesIO) -> tuple[list[dict], list[TagParserError]]: tags_data = json.load(file) except json.JSONDecodeError as error: return [], [ - InvalidFormat(tag=None, format=cls.format.value, message=str(error)) + InvalidFormat(tag=None, input_format=cls.format.value, message=str(error)) ] if "tags" not in tags_data: return [], [ InvalidFormat( tag=None, - format=cls.format.value, + input_format=cls.format.value, message=_("Missing 'tags' field on the .json file"), ) ] @@ -298,8 +298,8 @@ def _verify_header(cls, header_fields: list[str]) -> list[TagParserError]: errors.append( InvalidFormat( tag=None, - format=cls.format.value, - message=_(f"Missing '{req_field}' field on CSV headers"), + input_format=cls.format.value, + message=_("Missing '{req_field}' field on CSV headers").format(req_field=req_field), ) ) return errors @@ -319,4 +319,4 @@ def get_parser(parser_format: ParserFormat) -> type[Parser]: if parser_format == parser.format: return parser - raise ValueError(_(f"Parser not found for format {parser_format}")) + raise ValueError(_("Parser not found for format {parser_format}").format(parser_format=parser_format)) diff --git a/openedx_tagging/core/tagging/models/import_export.py b/openedx_tagging/core/tagging/models/import_export.py index c1b41b365..531a4ccaa 100644 --- a/openedx_tagging/core/tagging/models/import_export.py +++ b/openedx_tagging/core/tagging/models/import_export.py @@ -1,3 +1,7 @@ +""" +Models used by the Taxonomy import/export tasks. +""" + from datetime import datetime from enum import Enum @@ -9,6 +13,9 @@ class TagImportTaskState(Enum): + """ + Enumerates the states that a TagImportTask can be in. + """ LOADING_DATA = "loading_data" PLANNING = "planning" EXECUTING = "executing" @@ -48,6 +55,9 @@ class Meta: @classmethod def create(cls, taxonomy: Taxonomy): + """ + Creates and logs a new TagImportTask. + """ task = cls( taxonomy=taxonomy, status=TagImportTaskState.LOADING_DATA.value, @@ -58,6 +68,9 @@ def create(cls, taxonomy: Taxonomy): return task def add_log(self, message: str, save=True): + """ + Appends a log message to the task. + """ timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S") log_message = f"[{timestamp}] {message}\n" self.log += log_message @@ -65,44 +78,71 @@ def add_log(self, message: str, save=True): self.save() def log_exception(self, exception: Exception): + """ + Logs an exception and moves the task status to ERROR. + """ self.add_log(repr(exception), save=False) self.status = TagImportTaskState.ERROR.value self.save() def log_parser_start(self): + """ + Logs the parser start event. + """ self.add_log(_("Starting to load data from file")) def log_parser_end(self): + """ + Logs the parser finished event. + """ self.add_log(_("Load data finished")) def handle_parser_errors(self, errors): + """ + Handles parser errors by logging them and moving the task status to ERROR. + """ for error in errors: self.add_log(f"{str(error)}", save=False) self.status = TagImportTaskState.ERROR.value self.save() def log_start_planning(self): + """ + Starts task planning with a log message, and moves the task status to PLANNING. + """ self.add_log(_("Starting plan actions"), save=False) self.status = TagImportTaskState.PLANNING.value self.save() def log_plan(self, plan): + """ + Logs the task plan. + """ self.add_log(_("Plan finished")) plan_str = plan.plan() self.log += f"\n{plan_str}\n" self.save() def handle_plan_errors(self): + """ + Handles plan errors by moving the task status to ERROR. + """ # Error are logged with plan self.status = TagImportTaskState.ERROR.value self.save() def log_start_execute(self): + """ + Starts task execution with a log message, and moves the task status to EXECUTING. + """ self.add_log(_("Starting execute actions"), save=False) self.status = TagImportTaskState.EXECUTING.value self.save() def end_success(self): + """ + Completes task execution with a log message, and moves the task status to SUCCESS. + """ self.add_log(_("Execution finished"), save=False) self.status = TagImportTaskState.SUCCESS.value self.save() diff --git a/openedx_tagging/core/tagging/rest_api/__init__.py b/openedx_tagging/core/tagging/rest_api/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/openedx_tagging/core/tagging/rest_api/paginators.py b/openedx_tagging/core/tagging/rest_api/paginators.py index 6e2fd99f5..8848e5374 100644 --- a/openedx_tagging/core/tagging/rest_api/paginators.py +++ b/openedx_tagging/core/tagging/rest_api/paginators.py @@ -1,3 +1,7 @@ +""" +Paginators uses by the REST API +""" + from edx_rest_framework_extensions.paginators import DefaultPagination # type: ignore[import] # From this point, the tags begin to be paginated diff --git a/openedx_tagging/core/tagging/rest_api/v1/permissions.py b/openedx_tagging/core/tagging/rest_api/v1/permissions.py index 2e7f921c9..ed184549e 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/permissions.py +++ b/openedx_tagging/core/tagging/rest_api/v1/permissions.py @@ -6,6 +6,9 @@ class TaxonomyObjectPermissions(DjangoObjectPermissions): + """ + Maps each REST API methods to its corresponding Taxonomy permission. + """ perms_map = { "GET": ["%(app_label)s.view_%(model_name)s"], "OPTIONS": [], @@ -18,6 +21,9 @@ class TaxonomyObjectPermissions(DjangoObjectPermissions): class ObjectTagObjectPermissions(DjangoObjectPermissions): + """ + Maps each REST API methods to its corresponding ObjectTag permission. + """ perms_map = { "GET": ["%(app_label)s.view_%(model_name)s"], "OPTIONS": [], @@ -30,7 +36,13 @@ class ObjectTagObjectPermissions(DjangoObjectPermissions): class TagListPermissions(DjangoObjectPermissions): + """ + Permissions for Tag object views. + """ def has_permission(self, request, view): + """ + Returns True if the user on the given request is allowed the given view. + """ if not request.user or ( not request.user.is_authenticated and self.authenticated_users_only ): @@ -38,4 +50,7 @@ def has_permission(self, request, view): return True def has_object_permission(self, request, view, obj): + """ + Returns True if the user on the given request is allowed the given view for the given object. + """ return rules.has_perm("oel_tagging.list_tag", request.user, obj) diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index cf7d6304c..c5472afbf 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -8,7 +8,7 @@ from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy -class TaxonomyListQueryParamsSerializer(serializers.Serializer): +class TaxonomyListQueryParamsSerializer(serializers.Serializer): # pylint: disable=abstract-method """ Serializer for the query params for the GET view """ @@ -31,7 +31,7 @@ class Meta: ] -class ObjectTagListQueryParamsSerializer(serializers.Serializer): +class ObjectTagListQueryParamsSerializer(serializers.Serializer): # pylint: disable=abstract-method """ Serializer for the query params for the ObjectTag GET view """ @@ -57,7 +57,7 @@ class Meta: ] -class ObjectTagUpdateBodySerializer(serializers.Serializer): +class ObjectTagUpdateBodySerializer(serializers.Serializer): # pylint: disable=abstract-method """ Serializer of the body for the ObjectTag UPDATE view """ @@ -65,7 +65,7 @@ class ObjectTagUpdateBodySerializer(serializers.Serializer): tags = serializers.ListField(child=serializers.CharField(), required=True) -class ObjectTagUpdateQueryParamsSerializer(serializers.Serializer): +class ObjectTagUpdateQueryParamsSerializer(serializers.Serializer): # pylint: disable=abstract-method """ Serializer of the query params for the ObjectTag UPDATE view """ @@ -97,6 +97,9 @@ class Meta: ) def get_sub_tags_link(self, obj): + """ + Returns URL for the list of child tags of the current tag. + """ if obj.children.count(): query_params = f"?parent_tag_id={obj.id}" url = ( @@ -105,8 +108,12 @@ def get_sub_tags_link(self, obj): ) request = self.context.get("request") return request.build_absolute_uri(url) + return None def get_children_count(self, obj): + """ + Returns the number of child tags of the given tag. + """ return obj.children.count() @@ -131,6 +138,9 @@ class Meta: ) def get_sub_tags(self, obj): + """ + Returns a serialized list of child tags for the given tag. + """ serializer = TagsWithSubTagsSerializer( obj.children.all().order_by("value", "id"), many=True, @@ -139,6 +149,9 @@ def get_sub_tags(self, obj): return serializer.data def get_children_count(self, obj): + """ + Returns the number of child tags of the given tag. + """ return obj.children.count() @@ -150,8 +163,14 @@ class TagsForSearchSerializer(TagsWithSubTagsSerializer): """ def get_sub_tags(self, obj): + """ + Returns a serialized list of child tags for the given tag. + """ serializer = TagsWithSubTagsSerializer(obj.sub_tags, many=True, read_only=True) return serializer.data def get_children_count(self, obj): + """ + Returns the number of child tags of the given tag. + """ return len(obj.sub_tags) diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 26fcad573..f940add1c 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -244,7 +244,7 @@ def get_queryset(self) -> models.QuerySet: taxonomy_id = query_params.data.get("taxonomy", None) return get_object_tags(object_id, taxonomy_id) - def retrieve(self, request, object_id=None): + def retrieve(self, request, *args, **kwargs): """ Retrieve ObjectTags that belong to a given object_id @@ -259,7 +259,7 @@ def retrieve(self, request, object_id=None): serializer = ObjectTagSerializer(object_tags, many=True) return Response(serializer.data) - def update(self, request, object_id, partial=False): + def update(self, request, *args, **kwargs): """ Update ObjectTags that belong to a given object_id @@ -284,6 +284,7 @@ def update(self, request, object_id, partial=False): } """ + partial = kwargs.pop('partial', False) if partial: raise MethodNotAllowed("PATCH", detail="PATCH not allowed") @@ -296,6 +297,7 @@ def update(self, request, object_id, partial=False): perm = f"{taxonomy._meta.app_label}.change_objecttag" + object_id = kwargs.pop('object_id') perm_obj = ChangeObjectTagPermissionItem( taxonomy=taxonomy, object_id=object_id, @@ -313,9 +315,9 @@ def update(self, request, object_id, partial=False): try: tag_object(taxonomy, tags, object_id) except Tag.DoesNotExist as e: - raise ValidationError(e) + raise ValidationError from e except ValueError as e: - raise ValidationError(e) + raise ValidationError from e return self.retrieve(request, object_id) diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 6ed0140a1..f82a22016 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -10,7 +10,7 @@ from django.test import TestCase, override_settings import openedx_tagging.core.tagging.api as tagging_api -from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy +from openedx_tagging.core.tagging.models import ObjectTag, Taxonomy from .test_models import TestTagTaxonomyMixin, get_tag @@ -186,23 +186,6 @@ def test_get_children_tags(self): self.english_tag, ) - def check_object_tag( - self, - object_tag: ObjectTag, - taxonomy: Taxonomy | None, - tag: Tag | None, - name: str, - value: str, - ) -> None: - """ - Verifies that the properties of the given object_tag (once refreshed from the database) match those given. - """ - object_tag.refresh_from_db() - assert object_tag.taxonomy == taxonomy - assert object_tag.tag == tag - assert object_tag.name == name - assert object_tag.value == value - def test_resync_object_tags(self) -> None: self.taxonomy.allow_multiple = True self.taxonomy.save() @@ -487,8 +470,8 @@ def test_tag_object_limit(self) -> None: ["Eubacteria"], "object_1", ) - assert exc.exception - assert "Cannot add more than 100 tags to" in str(exc.exception) + assert exc.exception + assert "Cannot add more than 100 tags to" in str(exc.exception) # Updating existing tags should work for taxonomy in self.dummy_taxonomies: @@ -506,8 +489,8 @@ def test_tag_object_limit(self) -> None: ["New Dummy Tag 1", "New Dummy Tag 2"], "object_1", ) - assert exc.exception - assert "Cannot add more than 100 tags to" in str(exc.exception) + assert exc.exception + assert "Cannot add more than 100 tags to" in str(exc.exception) def test_get_object_tags(self) -> None: # Alpha tag has no taxonomy diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index ccc9e6c6d..2a7131f40 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -144,7 +144,7 @@ def setup_tag_depths(self): tag.depth = 2 -class TestTaxonomySubclassA(Taxonomy): +class TaxonomyTestSubclassA(Taxonomy): """ Model A for testing the taxonomy subclass casting. """ @@ -155,7 +155,7 @@ class Meta: app_label = "oel_tagging" -class TestTaxonomySubclassB(TestTaxonomySubclassA): +class TaxonomyTestSubclassB(TaxonomyTestSubclassA): """ Model B for testing the taxonomy subclass casting. """ @@ -166,7 +166,7 @@ class Meta: app_label = "oel_tagging" -class TestObjectTagSubclass(ObjectTag): +class ObjectTagTestSubclass(ObjectTag): """ Model for testing the ObjectTag copy. """ @@ -200,9 +200,9 @@ def test_representations(self): def test_taxonomy_cast(self): for subclass in ( - TestTaxonomySubclassA, + TaxonomyTestSubclassA, # Ensure that casting to a sub-subclass works as expected - TestTaxonomySubclassB, + TaxonomyTestSubclassB, # and that we can un-set the subclass None, ): @@ -349,11 +349,11 @@ def test_representations(self): ) def test_cast(self): - copy_tag = TestObjectTagSubclass.cast(self.object_tag) + copy_tag = ObjectTagTestSubclass.cast(self.object_tag) assert ( str(copy_tag) == repr(copy_tag) - == " object:id:1: Life on Earth=Bacteria" + == " object:id:1: Life on Earth=Bacteria" ) def test_object_tag_name(self): diff --git a/tests/openedx_tagging/core/tagging/test_system_defined_models.py b/tests/openedx_tagging/core/tagging/test_system_defined_models.py index 09f7b2bcb..3c0cb6763 100644 --- a/tests/openedx_tagging/core/tagging/test_system_defined_models.py +++ b/tests/openedx_tagging/core/tagging/test_system_defined_models.py @@ -32,7 +32,7 @@ class EmptyTestClass: """ -class TestLPTaxonomy(ModelSystemDefinedTaxonomy): +class LPTaxonomyTest(ModelSystemDefinedTaxonomy): """ Model used for testing - points to LearningPackage instances """ @@ -54,7 +54,7 @@ class Meta: app_label = "oel_tagging" -class CaseInsensitiveTitleLPTaxonomy(TestLPTaxonomy): +class CaseInsensitiveTitleLPTaxonomy(LPTaxonomyTest): """ Model that points to LearningPackage instances but uses 'title' as values """ @@ -89,8 +89,8 @@ def setUpClass(cls): # Create two learning packages and a taxonomy that can tag any object using learning packages as tags: cls.learning_pkg_1 = cls._create_learning_pkg(key="p1", title="Learning Package 1") cls.learning_pkg_2 = cls._create_learning_pkg(key="p2", title="Learning Package 2") - cls.lp_taxonomy = TestLPTaxonomy.objects.create( - taxonomy_class=TestLPTaxonomy, + cls.lp_taxonomy = LPTaxonomyTest.objects.create( + taxonomy_class=LPTaxonomyTest, name="LearningPackage Taxonomy", allow_multiple=True, ) diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index b843d2047..9183ee5f3 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -585,7 +585,8 @@ def test_object_tags_remaining_http_methods( response = self.client.post(url, {"test": "payload"}, format="json") elif http_method == "PATCH": response = self.client.patch(url, {"test": "payload"}, format="json") - elif http_method == "DELETE": + else: + assert http_method == "DELETE" response = self.client.delete(url) assert response.status_code == expected_status From b2767fa7825036ff925c5ef27c9096ab292aa648 Mon Sep 17 00:00:00 2001 From: Jillian Date: Mon, 9 Oct 2023 11:52:52 +1030 Subject: [PATCH 054/282] Taxonomy import templates [FC-0036] (#89) feat: adds view to download CSV and JSON taxonomy import template files to demonstrate their format and usage. Also alters the import code to ignore extra fields in import files. This allows us to add comments to our template import files, but more importantly, allows users to import tags from an external source which may have extra data we don't care about. --- MANIFEST.in | 2 +- openedx_learning/__init__.py | 2 +- .../core/tagging/import_export/parsers.py | 12 +- .../core/tagging/import_export/template.csv | 30 ++++ .../core/tagging/import_export/template.json | 158 ++++++++++++++++++ .../core/tagging/rest_api/v1/urls.py | 7 +- .../core/tagging/rest_api/v1/views_import.py | 51 ++++++ .../tagging/import_export/test_parsers.py | 11 ++ .../tagging/import_export/test_template.py | 90 ++++++++++ .../core/tagging/test_views_import.py | 39 +++++ 10 files changed, 394 insertions(+), 8 deletions(-) create mode 100644 openedx_tagging/core/tagging/import_export/template.csv create mode 100644 openedx_tagging/core/tagging/import_export/template.json create mode 100644 openedx_tagging/core/tagging/rest_api/v1/views_import.py create mode 100644 tests/openedx_tagging/core/tagging/import_export/test_template.py create mode 100644 tests/openedx_tagging/core/tagging/test_views_import.py diff --git a/MANIFEST.in b/MANIFEST.in index 0a2abd084..73b8b7ee4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,4 +3,4 @@ include LICENSE.txt include README.rst include requirements/base.in recursive-include openedx_learning *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py -recursive-include openedx_tagging *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py *.yaml +recursive-include openedx_tagging *.html *.png *.gif *.js *.css *.jpg *.jpeg *.svg *.py *.yaml *.json *.csv diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 6f229c82d..fe8603ba4 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.2.1" +__version__ = "0.2.2" diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py index 4413c04c4..c0b8207fc 100644 --- a/openedx_tagging/core/tagging/import_export/parsers.py +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -117,6 +117,7 @@ def _parse_tags( row = cls.inital_row for tag in tags_data: has_error = False + tag_data = {} # Verify the required fields for req_field in cls.required_fields: @@ -140,20 +141,21 @@ def _parse_tags( ) ) has_error = True + else: + tag_data[req_field] = tag[req_field] - tag["index"] = row + tag_data["index"] = row row += 1 # Skip parse if there is an error if has_error: continue - # Updating any empty optional field to None + # Optional fields default to None for opt_field in cls.optional_fields: - if opt_field in tag and not tag.get(opt_field): - tag[opt_field] = None + tag_data[opt_field] = tag.get(opt_field) or None - tags.append(TagItem(**tag)) + tags.append(TagItem(**tag_data)) return tags, errors diff --git a/openedx_tagging/core/tagging/import_export/template.csv b/openedx_tagging/core/tagging/import_export/template.csv new file mode 100644 index 000000000..cdee49849 --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/template.csv @@ -0,0 +1,30 @@ +id,value,parent_id,comments +WINDS,Wind instruments,,"This is an example Tag Import file, in CSV format." +PERCUSS,Percussion instruments,,"Only the 'id' and 'value' fields are required. They can be anything you like, but they must must be unique within the taxonomy. Existing tags matching 'id' will be updated on import." +ELECTRIC,Electronic instruments,,"Top-level tags have no 'parent_id', and you can have as many top-level tags as you wish." +STRINGS,String instruments,,"All other fields (like these 'comments') are ignored on import, and will not be included in subsequent tag exports." +BELLS,Idiophone,PERCUSS,"Providing a 'parent_id' creates a tag hierarchy." +DRUMS,Membranophone,PERCUSS,"The 'parent_id' must match an 'id' found earlier in the import file." +CAJÓN,Cajón,DRUMS,"Tag values may contain unicode characters." +PYLE,Pyle Stringed Jam Cajón,CAJÓN,"A tag hierarchy may contain as many as 3 levels. This tag is at level 4, and so it will not be shown to users." +THERAMIN,Theramin,ELECTRIC,"A tag hierarchy may contain uneven levels. Here, the Electronic branch has only 2 levels, while Percussion has 3." +CHORD,Chordophone,PERCUSS, +BRASS,Brass,WINDS, +WOODS,Woodwinds,WINDS, +FLUTE,Flute,WOODS, +PLUCK,Plucked strings,STRINGS, +MANDOLIN,Mandolin,PLUCK, +HARP,Harp,PLUCK, +BANJO,Banjo,PLUCK, +BOW,Bowed strings,STRINGS, +VIOLIN,Violin,BOW, +CELLO,Cello,BOW, +CLARINET,Clarinet,WOODS, +OBOE,Oboe,WOODS, +TRUMPET,Trumpet,BRASS, +TUBA,Tuba,BRASS, +SYNTH,Synthesizer,ELECTRIC, +CELESTA,Celesta,BELLS, +HI-HAT,Hi-hat,BELLS, +TABLA,Tabla,DRUMS, +PIANO,Piano,CHORD, diff --git a/openedx_tagging/core/tagging/import_export/template.json b/openedx_tagging/core/tagging/import_export/template.json new file mode 100644 index 000000000..666a34b9f --- /dev/null +++ b/openedx_tagging/core/tagging/import_export/template.json @@ -0,0 +1,158 @@ +{ + "tags": [ + { + "id": "WINDS", + "value": "Wind instruments", + "parent_id": "", + "comments": "This is an example Tag Import file, in JSON format." + }, + { + "id": "PERCUSS", + "value": "Percussion instruments", + "parent_id": "", + "comments": "Only the 'id' and 'value' fields are required. They can be anything you like, but they must must be unique within the taxonomy. Existing tags matching 'id' will be updated on import." + }, + { + "id": "ELECTRIC", + "value": "Electronic instruments", + "parent_id": "", + "comments": "Top-level tags have no 'parent_id', and you can have as many top-level tags as you wish." + }, + { + "id": "STRINGS", + "value": "String instruments", + "parent_id": "", + "comments": "All other fields (like these 'comments') are ignored on import, and will not be included in subsequent tag exports." + }, + { + "id": "BELLS", + "value": "Idiophone", + "parent_id": "PERCUSS", + "comments": "Providing a 'parent_id' creates a tag hierarchy." + }, + { + "id": "DRUMS", + "value": "Membranophone", + "parent_id": "PERCUSS", + "comments": "The 'parent_id' must match an 'id' found earlier in the import file." + }, + { + "id": "CAJÓN", + "value": "Cajón", + "parent_id": "DRUMS", + "comments": "Tag values may contain unicode characters." + }, + { + "id": "PYLE", + "value": "Pyle Stringed Jam Cajón", + "parent_id": "CAJÓN", + "comments": "A tag hierarchy may contain as many as 3 levels. This tag is at level 4, and so it will not be shown to users." + }, + { + "id": "THERAMIN", + "value": "Theramin", + "parent_id": "ELECTRIC", + "comments": "A tag hierarchy may contain uneven levels. Here, the Electronic branch has only 2 levels, while Percussion has 3." + }, + { + "id": "CHORD", + "value": "Chordophone", + "parent_id": "PERCUSS" + }, + { + "id": "BRASS", + "value": "Brass", + "parent_id": "WINDS" + }, + { + "id": "WOODS", + "value": "Woodwinds", + "parent_id": "WINDS" + }, + { + "id": "FLUTE", + "value": "Flute", + "parent_id": "WOODS" + }, + { + "id": "PLUCK", + "value": "Plucked strings", + "parent_id": "STRINGS" + }, + { + "id": "MANDOLIN", + "value": "Mandolin", + "parent_id": "PLUCK" + }, + { + "id": "HARP", + "value": "Harp", + "parent_id": "PLUCK" + }, + { + "id": "BANJO", + "value": "Banjo", + "parent_id": "PLUCK" + }, + { + "id": "BOW", + "value": "Bowed strings", + "parent_id": "STRINGS" + }, + { + "id": "VIOLIN", + "value": "Violin", + "parent_id": "BOW" + }, + { + "id": "CELLO", + "value": "Cello", + "parent_id": "BOW" + }, + { + "id": "CLARINET", + "value": "Clarinet", + "parent_id": "WOODS" + }, + { + "id": "OBOE", + "value": "Oboe", + "parent_id": "WOODS" + }, + { + "id": "TRUMPET", + "value": "Trumpet", + "parent_id": "BRASS" + }, + { + "id": "TUBA", + "value": "Tuba", + "parent_id": "BRASS" + }, + { + "id": "SYNTH", + "value": "Synthesizer", + "parent_id": "ELECTRIC" + }, + { + "id": "CELESTA", + "value": "Celesta", + "parent_id": "BELLS" + }, + { + "id": "HI-HAT", + "value": "Hi-hat", + "parent_id": "BELLS" + }, + { + "id": "TABLA", + "value": "Tabla", + "parent_id": "DRUMS" + }, + { + "id": "PIANO", + "value": "Piano", + "parent_id": "CHORD" + } + ] +} diff --git a/openedx_tagging/core/tagging/rest_api/v1/urls.py b/openedx_tagging/core/tagging/rest_api/v1/urls.py index 7b96cf98c..72ff87d0d 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/urls.py +++ b/openedx_tagging/core/tagging/rest_api/v1/urls.py @@ -5,7 +5,7 @@ from django.urls.conf import include, path from rest_framework.routers import DefaultRouter -from . import views +from . import views, views_import router = DefaultRouter() router.register("taxonomies", views.TaxonomyView, basename="taxonomy") @@ -18,4 +18,9 @@ views.TaxonomyTagsView.as_view(), name="taxonomy-tags", ), + path( + "import/template.", + views_import.TemplateView.as_view(), + name="taxonomy-import-template", + ), ] diff --git a/openedx_tagging/core/tagging/rest_api/v1/views_import.py b/openedx_tagging/core/tagging/rest_api/v1/views_import.py new file mode 100644 index 000000000..6c3a63398 --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/v1/views_import.py @@ -0,0 +1,51 @@ +""" +Taxonomy Import views +""" +from __future__ import annotations + +import os + +from django.http import FileResponse, Http404 +from rest_framework.request import Request +from rest_framework.views import APIView + + +class TemplateView(APIView): + """ + View which serves the static Taxonomy Import template files. + + **Example Requests** + GET /tagging/rest_api/v1/import/template.csv + GET /tagging/rest_api/v1/import/template.json + + **Query Returns** + * 200 - Success + * 404 - Template file not found + * 405 - Method not allowed + """ + http_method_names = ['get'] + + template_dir = os.path.join( + os.path.dirname(__file__), + "../../import_export/", + ) + allowed_ext_to_content_type = { + "csv": "text/csv", + "json": "application/json", + } + + def get(self, request: Request, file_ext: str, *args, **kwargs) -> FileResponse: + """ + Downloads the requested file as an attachment, + or raises 404 if not found. + """ + content_type = self.allowed_ext_to_content_type.get(file_ext) + if not content_type: + raise Http404 + + filename = f"template.{file_ext}" + content_disposition = f'attachment; filename="{filename}"' + fh = open(os.path.join(self.template_dir, filename), "rb") + response = FileResponse(fh, content_type=content_type) + response['Content-Disposition'] = content_disposition + return response diff --git a/tests/openedx_tagging/core/tagging/import_export/test_parsers.py b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py index 3bcd2667c..52a75afa1 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_parsers.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_parsers.py @@ -77,6 +77,12 @@ def test_load_data_errors(self) -> None: ) @ddt.data( + ( + {"tags": [ + {"id": "tag_1", "value": "Tag 1", "comment": "This field is ignored."}, # Valid + ]}, + [] + ), ( {"tags": [ {"id": "tag_1", "value": "Tag 1"}, # Valid @@ -209,6 +215,11 @@ class TestCSVParser(TestImportExportMixin, TestCase): # Valid "id,value\n", [] + ), + ( + # Valid + "id,value,ignored\n", + [] ) ) @ddt.unpack diff --git a/tests/openedx_tagging/core/tagging/import_export/test_template.py b/tests/openedx_tagging/core/tagging/import_export/test_template.py new file mode 100644 index 000000000..067a338c0 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/import_export/test_template.py @@ -0,0 +1,90 @@ +""" +Tests the import/export template files. +""" +from __future__ import annotations + +import os + +import ddt # type: ignore[import] +from django.test.testcases import TestCase + +from openedx_tagging.core.tagging.api import get_tags +from openedx_tagging.core.tagging.import_export import ParserFormat +from openedx_tagging.core.tagging.import_export import api as import_api + +from .mixins import TestImportExportMixin + + +@ddt.ddt +class TestImportTemplate(TestImportExportMixin, TestCase): + """ + Test the CSV import/export template. + """ + + def open_template_file(self, template_file): + """ + Returns an open file handler for the template file. + """ + dirname = os.path.dirname(__file__) + filename = os.path.join( + dirname, + '../../../../..', + template_file, + ) + return open(filename, "rb") + + @ddt.data( + ('openedx_tagging/core/tagging/import_export/template.csv', ParserFormat.CSV), + ('openedx_tagging/core/tagging/import_export/template.json', ParserFormat.JSON), + ) + @ddt.unpack + def test_import_template(self, template_file, parser_format): + with self.open_template_file(template_file) as import_file: + assert import_api.import_tags( + self.taxonomy, + import_file, + parser_format, + replace=True, + ), import_api.get_last_import_log(self.taxonomy) + + imported_tags = [ + { + "external_id": tag.external_id, + "value": tag.value, + "parent": tag.parent.external_id if tag.parent else None, + } + for tag in get_tags(self.taxonomy) + ] + assert imported_tags == [ + {'external_id': "ELECTRIC", 'parent': None, 'value': 'Electronic instruments'}, + {'external_id': 'PERCUSS', 'parent': None, 'value': 'Percussion instruments'}, + {'external_id': 'STRINGS', 'parent': None, 'value': 'String instruments'}, + {'external_id': 'WINDS', 'parent': None, 'value': 'Wind instruments'}, + {'external_id': 'SYNTH', 'parent': 'ELECTRIC', 'value': 'Synthesizer'}, + {'external_id': 'THERAMIN', 'parent': 'ELECTRIC', 'value': 'Theramin'}, + {'external_id': 'CHORD', 'parent': 'PERCUSS', 'value': 'Chordophone'}, + {'external_id': 'BELLS', 'parent': 'PERCUSS', 'value': 'Idiophone'}, + {'external_id': 'DRUMS', 'parent': 'PERCUSS', 'value': 'Membranophone'}, + {'external_id': 'BOW', 'parent': 'STRINGS', 'value': 'Bowed strings'}, + {'external_id': 'PLUCK', 'parent': 'STRINGS', 'value': 'Plucked strings'}, + {'external_id': 'BRASS', 'parent': 'WINDS', 'value': 'Brass'}, + {'external_id': 'WOODS', 'parent': 'WINDS', 'value': 'Woodwinds'}, + {'external_id': 'CELLO', 'parent': 'BOW', 'value': 'Cello'}, + {'external_id': 'VIOLIN', 'parent': 'BOW', 'value': 'Violin'}, + {'external_id': 'TRUMPET', 'parent': 'BRASS', 'value': 'Trumpet'}, + {'external_id': 'TUBA', 'parent': 'BRASS', 'value': 'Tuba'}, + {'external_id': 'PIANO', 'parent': 'CHORD', 'value': 'Piano'}, + # This tag is present in the import files, but it will be omitted from get_tags() + # because it sits beyond TAXONOMY_MAX_DEPTH. + # {'external_id': 'PYLE', 'parent': 'CAJÓN', 'value': 'Pyle Stringed Jam Cajón'}, + {'external_id': 'CELESTA', 'parent': 'BELLS', 'value': 'Celesta'}, + {'external_id': 'HI-HAT', 'parent': 'BELLS', 'value': 'Hi-hat'}, + {'external_id': 'CAJÓN', 'parent': 'DRUMS', 'value': 'Cajón'}, + {'external_id': 'TABLA', 'parent': 'DRUMS', 'value': 'Tabla'}, + {'external_id': 'BANJO', 'parent': 'PLUCK', 'value': 'Banjo'}, + {'external_id': 'HARP', 'parent': 'PLUCK', 'value': 'Harp'}, + {'external_id': 'MANDOLIN', 'parent': 'PLUCK', 'value': 'Mandolin'}, + {'external_id': 'CLARINET', 'parent': 'WOODS', 'value': 'Clarinet'}, + {'external_id': 'FLUTE', 'parent': 'WOODS', 'value': 'Flute'}, + {'external_id': 'OBOE', 'parent': 'WOODS', 'value': 'Oboe'}, + ] diff --git a/tests/openedx_tagging/core/tagging/test_views_import.py b/tests/openedx_tagging/core/tagging/test_views_import.py new file mode 100644 index 000000000..4c2d94c9a --- /dev/null +++ b/tests/openedx_tagging/core/tagging/test_views_import.py @@ -0,0 +1,39 @@ +""" +Tests import REST API views. +""" +from __future__ import annotations + +import ddt # type: ignore[import] +from rest_framework import status +from rest_framework.test import APITestCase + +TAXONOMY_TEMPLATE_URL = "/tagging/rest_api/v1/import/{filename}" + + +@ddt.ddt +class TestTemplateView(APITestCase): + """ + Tests the taxonomy template downloads. + """ + @ddt.data( + ("template.csv", "text/csv"), + ("template.json", "application/json"), + ) + @ddt.unpack + def test_download(self, filename, content_type): + url = TAXONOMY_TEMPLATE_URL.format(filename=filename) + response = self.client.get(url) + assert response.status_code == status.HTTP_200_OK + assert response.headers['Content-Type'] == content_type + assert response.headers['Content-Disposition'] == f'attachment; filename="{filename}"' + assert int(response.headers['Content-Length']) > 0 + + def test_download_not_found(self): + url = TAXONOMY_TEMPLATE_URL.format(filename="template.txt") + response = self.client.get(url) + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_download_method_not_allowed(self): + url = TAXONOMY_TEMPLATE_URL.format(filename="template.txt") + response = self.client.post(url) + assert response.status_code == status.HTTP_405_METHOD_NOT_ALLOWED From 8ba00437f1bc70c7baca898494435e68e1079653 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Mon, 9 Oct 2023 23:56:27 -0700 Subject: [PATCH 055/282] chore: Squash Tagging Migrations (#95) --- openedx_learning/__init__.py | 2 +- .../core/tagging/migrations/0001_squashed.py | 152 ++++++++++++++++++ .../migrations/0012_language_taxonomy.py | 2 +- 3 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 openedx_tagging/core/tagging/migrations/0001_squashed.py diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index fe8603ba4..fcfa685c9 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.2.2" +__version__ = "0.2.3" diff --git a/openedx_tagging/core/tagging/migrations/0001_squashed.py b/openedx_tagging/core/tagging/migrations/0001_squashed.py new file mode 100644 index 000000000..e4ee96cf9 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0001_squashed.py @@ -0,0 +1,152 @@ +# Generated by Django 3.2.19 on 2023-10-09 22:46 + +import django.db.models.deletion +from django.db import migrations, models + +import openedx_learning.lib.fields +import openedx_tagging.core.tagging.models.import_export + + +class Migration(migrations.Migration): + + replaces = [ + ('oel_tagging', '0001_initial'), + ('oel_tagging', '0002_auto_20230718_2026'), + ('oel_tagging', '0003_auto_20230721_1238'), + ('oel_tagging', '0004_auto_20230723_2001'), + ('oel_tagging', '0005_language_taxonomy'), + ('oel_tagging', '0006_alter_objecttag_unique_together'), + ('oel_tagging', '0006_auto_20230802_1631'), + ('oel_tagging', '0007_tag_import_task_log_null_fix'), + ('oel_tagging', '0008_taxonomy_description_not_null'), + ('oel_tagging', '0009_alter_objecttag_object_id'), + ('oel_tagging', '0010_cleanups'), + ('oel_tagging', '0011_remove_required'), + ] + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Taxonomy', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('name', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, db_index=True, help_text='User-facing label used when applying tags from this taxonomy to Open edX objects.', max_length=255)), + ('description', openedx_learning.lib.fields.MultiCollationTextField(blank=True, help_text='Provides extra information for the user when applying tags from this taxonomy to an object.')), + ('enabled', models.BooleanField(default=True, help_text='Only enabled taxonomies will be shown to authors.')), + ('allow_multiple', models.BooleanField(default=True, help_text='Indicates that multiple tags from this taxonomy may be added to an object.')), + ('allow_free_text', models.BooleanField(default=False, help_text='Indicates that tags in this taxonomy need not be predefined; authors may enter their own tag values.')), + ('visible_to_authors', models.BooleanField(default=True, editable=False, help_text='Indicates whether this taxonomy should be visible to object authors.')), + ('_taxonomy_class', models.CharField(help_text='Taxonomy subclass used to instantiate this instance; must be a fully-qualified module and class name. If the module/class cannot be imported, an error is logged and the base Taxonomy class is used instead.', max_length=255, null=True)), + ], + options={ + 'verbose_name_plural': 'Taxonomies', + }, + ), + migrations.CreateModel( + name='TagImportTask', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('log', models.TextField(blank=True, default=None, help_text='Action execution logs')), + ('status', models.CharField(choices=[(openedx_tagging.core.tagging.models.import_export.TagImportTaskState['LOADING_DATA'], 'loading_data'), (openedx_tagging.core.tagging.models.import_export.TagImportTaskState['PLANNING'], 'planning'), (openedx_tagging.core.tagging.models.import_export.TagImportTaskState['EXECUTING'], 'executing'), (openedx_tagging.core.tagging.models.import_export.TagImportTaskState['SUCCESS'], 'success'), (openedx_tagging.core.tagging.models.import_export.TagImportTaskState['ERROR'], 'error')], help_text='Task status', max_length=20)), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('taxonomy', models.ForeignKey(help_text='Taxonomy associated with this import', on_delete=django.db.models.deletion.CASCADE, to='oel_tagging.taxonomy')), + ], + ), + migrations.CreateModel( + name='Tag', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('value', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, help_text="Content of a given tag, occupying the 'value' part of the key:value pair.", max_length=500)), + ('external_id', openedx_learning.lib.fields.MultiCollationCharField(blank=True, db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, help_text='Used to link an Open edX Tag with a tag in an externally-defined taxonomy.', max_length=255, null=True)), + ('parent', models.ForeignKey(default=None, help_text='Tag that lives one level up from the current tag, forming a hierarchy.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='oel_tagging.tag')), + ('taxonomy', models.ForeignKey(default=None, help_text='Namespace and rules for using a given set of tags.', null=True, on_delete=django.db.models.deletion.CASCADE, to='oel_tagging.taxonomy')), + ], + ), + migrations.CreateModel( + name='ObjectTag', + fields=[ + ('id', models.BigAutoField(primary_key=True, serialize=False)), + ('object_id', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, db_index=True, editable=False, help_text='Identifier for the object being tagged', max_length=255)), + ('_name', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, help_text='User-facing label used for this tag, stored in case taxonomy is (or becomes) null. If the taxonomy field is set, then taxonomy.name takes precedence over this field.', max_length=255)), + ('_value', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_unicode_ci', 'sqlite': 'NOCASE'}, help_text='User-facing value used for this tag, stored in case tag is null, e.g if taxonomy is free text, or if it becomes null (e.g. if the Tag is deleted). If the tag field is set, then tag.value takes precedence over this field.', max_length=500)), + ('tag', models.ForeignKey(blank=True, default=None, help_text="Tag associated with this object tag. Provides the tag's 'value' if set.", null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_tagging.tag')), + ('taxonomy', models.ForeignKey(default=None, help_text="Taxonomy that this object tag belongs to. Used for validating the tag and provides the tag's 'name' if set.", null=True, on_delete=django.db.models.deletion.SET_NULL, to='oel_tagging.taxonomy')), + ], + ), + migrations.CreateModel( + name='SystemDefinedTaxonomy', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('oel_tagging.taxonomy',), + ), + migrations.AddIndex( + model_name='tagimporttask', + index=models.Index(fields=['taxonomy', '-creation_date'], name='oel_tagging_taxonom_5e948c_idx'), + ), + migrations.AddIndex( + model_name='tag', + index=models.Index(fields=['taxonomy', 'value'], name='oel_tagging_taxonom_89e779_idx'), + ), + migrations.AddIndex( + model_name='tag', + index=models.Index(fields=['taxonomy', 'external_id'], name='oel_tagging_taxonom_44e355_idx'), + ), + migrations.AlterUniqueTogether( + name='tag', + unique_together={('taxonomy', 'external_id'), ('taxonomy', 'value')}, + ), + migrations.AddIndex( + model_name='objecttag', + index=models.Index(fields=['taxonomy', 'object_id'], name='oel_tagging_taxonom_aa24e6_idx'), + ), + migrations.AddIndex( + model_name='objecttag', + index=models.Index(fields=['taxonomy', '_value'], name='oel_tagging_taxonom_3668ec_idx'), + ), + migrations.AlterUniqueTogether( + name='objecttag', + unique_together={('object_id', 'taxonomy', '_value'), ('object_id', 'taxonomy', 'tag_id')}, + ), + migrations.CreateModel( + name='LanguageTaxonomy', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('oel_tagging.systemdefinedtaxonomy',), + ), + migrations.CreateModel( + name='ModelSystemDefinedTaxonomy', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('oel_tagging.systemdefinedtaxonomy',), + ), + migrations.CreateModel( + name='UserSystemDefinedTaxonomy', + fields=[ + ], + options={ + 'proxy': True, + 'indexes': [], + 'constraints': [], + }, + bases=('oel_tagging.modelsystemdefinedtaxonomy',), + ), + ] diff --git a/openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py b/openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py index 5576e83df..132fcaaeb 100644 --- a/openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py +++ b/openedx_tagging/core/tagging/migrations/0012_language_taxonomy.py @@ -35,7 +35,7 @@ def revert(apps, schema_editor): # pragma: no cover class Migration(migrations.Migration): dependencies = [ - ("oel_tagging", "0011_remove_required"), + ("oel_tagging", "0001_squashed"), ] operations = [ From d66d6294c1b27fbb285cda365802925f528bfe39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Fri, 13 Oct 2023 11:43:51 -0500 Subject: [PATCH 056/282] feat: Authentication classes added to tagging API views (#98) * Authentication classes added to tagging API views * Cast taxonomies before serialize in TaxonomySerializer * Test updated --- .../core/tagging/rest_api/v1/serializers.py | 10 +++ .../core/tagging/rest_api/v1/utils.py | 24 +++++++ .../core/tagging/rest_api/v1/views.py | 4 ++ .../core/tagging/test_views.py | 70 +++++++++---------- 4 files changed, 73 insertions(+), 35 deletions(-) create mode 100644 openedx_tagging/core/tagging/rest_api/v1/utils.py diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index c5472afbf..388ee6cc4 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -17,6 +17,9 @@ class TaxonomyListQueryParamsSerializer(serializers.Serializer): # pylint: disa class TaxonomySerializer(serializers.ModelSerializer): + """ + Serializer for the Taxonomy model. + """ class Meta: model = Taxonomy fields = [ @@ -30,6 +33,13 @@ class Meta: "visible_to_authors", ] + def to_representation(self, instance): + """ + Cast the taxonomy before serialize + """ + instance = instance.cast() + return super().to_representation(instance) + class ObjectTagListQueryParamsSerializer(serializers.Serializer): # pylint: disable=abstract-method """ diff --git a/openedx_tagging/core/tagging/rest_api/v1/utils.py b/openedx_tagging/core/tagging/rest_api/v1/utils.py new file mode 100644 index 000000000..0667fb3b4 --- /dev/null +++ b/openedx_tagging/core/tagging/rest_api/v1/utils.py @@ -0,0 +1,24 @@ +""" +Utilities for the API +""" +from edx_rest_framework_extensions.auth.jwt.authentication import JwtAuthentication # type: ignore[import] +from edx_rest_framework_extensions.auth.session.authentication import ( # type: ignore[import] + SessionAuthenticationAllowInactiveUser, +) + + +def view_auth_classes(func_or_class): + """ + Function and class decorator that abstracts the authentication classes for api views. + """ + def _decorator(func_or_class): + """ + Requires either OAuth2 or Session-based authentication; + are the same authentication classes used on edx-platform + """ + func_or_class.authentication_classes = ( + JwtAuthentication, + SessionAuthenticationAllowInactiveUser, + ) + return func_or_class + return _decorator(func_or_class) diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index f940add1c..c00f78dc9 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -38,8 +38,10 @@ TaxonomyListQueryParamsSerializer, TaxonomySerializer, ) +from .utils import view_auth_classes +@view_auth_classes class TaxonomyView(ModelViewSet): """ View to list, create, retrieve, update, or delete Taxonomies. @@ -182,6 +184,7 @@ def perform_create(self, serializer) -> None: serializer.instance = create_taxonomy(**serializer.validated_data) +@view_auth_classes class ObjectTagView( mixins.RetrieveModelMixin, mixins.UpdateModelMixin, @@ -322,6 +325,7 @@ def update(self, request, *args, **kwargs): return self.retrieve(request, object_id) +@view_auth_classes class TaxonomyTagsView(ListAPIView): """ View to list tags of a taxonomy. diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 9183ee5f3..5092c414b 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -113,7 +113,7 @@ def test_list_taxonomy_queryparams(self, enabled, expected_status: int, expected assert len(response.data["results"]) == expected_count @ddt.data( - (None, status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), ("user", status.HTTP_200_OK), ("staff", status.HTTP_200_OK), ) @@ -161,8 +161,8 @@ def test_list_invalid_page(self) -> None: assert response.status_code == status.HTTP_404_NOT_FOUND @ddt.data( - (None, {"enabled": True}, status.HTTP_403_FORBIDDEN), - (None, {"enabled": False}, status.HTTP_403_FORBIDDEN), + (None, {"enabled": True}, status.HTTP_401_UNAUTHORIZED), + (None, {"enabled": False}, status.HTTP_401_UNAUTHORIZED), ("user", {"enabled": True}, status.HTTP_200_OK), ("user", {"enabled": False}, status.HTTP_404_NOT_FOUND), ("staff", {"enabled": True}, status.HTTP_200_OK), @@ -192,7 +192,7 @@ def test_detail_taxonomy_404(self) -> None: assert response.status_code == status.HTTP_404_NOT_FOUND @ddt.data( - (None, status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), ("user", status.HTTP_403_FORBIDDEN), ("staff", status.HTTP_201_CREATED), ) @@ -246,7 +246,7 @@ def test_create_taxonomy_system_defined(self, create_data): assert not response.data["system_defined"] @ddt.data( - (None, status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), ("user", status.HTTP_403_FORBIDDEN), ("staff", status.HTTP_200_OK), ) @@ -308,7 +308,7 @@ def test_update_taxonomy_404(self): assert response.status_code == status.HTTP_404_NOT_FOUND @ddt.data( - (None, status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), ("user", status.HTTP_403_FORBIDDEN), ("staff", status.HTTP_200_OK), ) @@ -365,7 +365,7 @@ def test_patch_taxonomy_404(self): assert response.status_code == status.HTTP_404_NOT_FOUND @ddt.data( - (None, status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), ("user", status.HTTP_403_FORBIDDEN), ("staff", status.HTTP_204_NO_CONTENT), ) @@ -481,10 +481,10 @@ def _object_permission(_user, object_id: str) -> bool: rules.set_perm("oel_tagging.change_objecttag_objectid", _object_permission) @ddt.data( - (None, "abc", status.HTTP_403_FORBIDDEN, None), + (None, "abc", status.HTTP_401_UNAUTHORIZED, None), ("user", "abc", status.HTTP_200_OK, 81), ("staff", "abc", status.HTTP_200_OK, 81), - (None, "non-existing-id", status.HTTP_403_FORBIDDEN, None), + (None, "non-existing-id", status.HTTP_401_UNAUTHORIZED, None), ("user", "non-existing-id", status.HTTP_200_OK, 0), ("staff", "non-existing-id", status.HTTP_200_OK, 0), ) @@ -506,7 +506,7 @@ def test_retrieve_object_tags(self, user_attr, object_id, expected_status, expec assert len(response.data) == expected_count @ddt.data( - (None, "abc", status.HTTP_403_FORBIDDEN, None), + (None, "abc", status.HTTP_401_UNAUTHORIZED, None), ("user", "abc", status.HTTP_200_OK, 20), ("staff", "abc", status.HTTP_200_OK, 20), ) @@ -532,7 +532,7 @@ def test_retrieve_object_tags_taxonomy_queryparam( assert object_tag.get("taxonomy_id") == self.enabled_taxonomy.pk @ddt.data( - (None, "abc", status.HTTP_403_FORBIDDEN), + (None, "abc", status.HTTP_401_UNAUTHORIZED), ("user", "abc", status.HTTP_400_BAD_REQUEST), ("staff", "abc", status.HTTP_400_BAD_REQUEST), ) @@ -552,9 +552,9 @@ def test_retrieve_object_tags_invalid_taxonomy_queryparam(self, user_attr, objec assert response.status_code == expected_status @ddt.data( - (None, "POST", status.HTTP_403_FORBIDDEN), - (None, "PATCH", status.HTTP_403_FORBIDDEN), - (None, "DELETE", status.HTTP_403_FORBIDDEN), + (None, "POST", status.HTTP_401_UNAUTHORIZED), + (None, "PATCH", status.HTTP_401_UNAUTHORIZED), + (None, "DELETE", status.HTTP_401_UNAUTHORIZED), ("user", "POST", status.HTTP_405_METHOD_NOT_ALLOWED), ("user", "PATCH", status.HTTP_405_METHOD_NOT_ALLOWED), ("user", "DELETE", status.HTTP_405_METHOD_NOT_ALLOWED), @@ -593,27 +593,27 @@ def test_object_tags_remaining_http_methods( @ddt.data( # Users and staff can add tags to a taxonomy - (None, "language_taxonomy", ["Portuguese"], status.HTTP_403_FORBIDDEN), + (None, "language_taxonomy", ["Portuguese"], status.HTTP_401_UNAUTHORIZED), ("user", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), ("staff", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), # Users and staff can clear add tags to a taxonomy - (None, "enabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + (None, "enabled_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), ("user", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), ("staff", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), # Only staff can add tag to a disabled taxonomy - (None, "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + (None, "disabled_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), ("staff", "disabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), # Users and staff can add a single tag to a allow_multiple=True taxonomy - (None, "multiple_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + (None, "multiple_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), ("staff", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), # Users and staff can add tags to an open taxonomy - (None, "open_taxonomy_enabled", ["tag1"], status.HTTP_403_FORBIDDEN), + (None, "open_taxonomy_enabled", ["tag1"], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), ("staff", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), # Only staff can add tags to a disabled open taxonomy - (None, "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), + (None, "open_taxonomy_disabled", ["tag1"], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), ("staff", "open_taxonomy_disabled", ["tag1"], status.HTTP_200_OK), ) @@ -635,17 +635,17 @@ def test_tag_object(self, user_attr, taxonomy_attr, tag_values, expected_status) @ddt.data( # Can't add invalid tags to a closed taxonomy - (None, "language_taxonomy", ["Invalid"], status.HTTP_403_FORBIDDEN), + (None, "language_taxonomy", ["Invalid"], status.HTTP_401_UNAUTHORIZED), ("user", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), ("staff", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), - (None, "enabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), + (None, "enabled_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), ("user", "enabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), ("staff", "enabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), - (None, "multiple_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), + (None, "multiple_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), ("staff", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), # Users can't edit tags from a disabled taxonomy. Staff can't add invalid tags to a closed taxonomy - (None, "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), + (None, "disabled_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), ("staff", "disabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), ) @@ -665,18 +665,18 @@ def test_tag_object_invalid(self, user_attr, taxonomy_attr, tag_values, expected @ddt.data( # Users and staff can clear tags from a taxonomy - (None, "enabled_taxonomy", [], status.HTTP_403_FORBIDDEN), + (None, "enabled_taxonomy", [], status.HTTP_401_UNAUTHORIZED), ("user", "enabled_taxonomy", [], status.HTTP_200_OK), ("staff", "enabled_taxonomy", [], status.HTTP_200_OK), # Users and staff can clear tags from a allow_multiple=True taxonomy - (None, "multiple_taxonomy", [], status.HTTP_403_FORBIDDEN), + (None, "multiple_taxonomy", [], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", [], status.HTTP_200_OK), ("staff", "multiple_taxonomy", [], status.HTTP_200_OK), # Only staff can clear tags from a disabled taxonomy - (None, "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), + (None, "disabled_taxonomy", [], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), ("staff", "disabled_taxonomy", [], status.HTTP_200_OK), - (None, "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), + (None, "open_taxonomy_disabled", [], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), ("staff", "open_taxonomy_disabled", [], status.HTTP_200_OK), ) @@ -698,22 +698,22 @@ def test_tag_object_clear(self, user_attr, taxonomy_attr, tag_values, expected_s @ddt.data( # Users and staff can add multiple tags to a allow_multiple=True taxonomy - (None, "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), + (None, "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), ("staff", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), - (None, "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_403_FORBIDDEN), + (None, "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), ("staff", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), # Users and staff can't add multple tags to a allow_multiple=False taxonomy - (None, "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), + (None, "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), ("user", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), ("staff", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), - (None, "language_taxonomy", ["Portuguese", "English"], status.HTTP_403_FORBIDDEN), + (None, "language_taxonomy", ["Portuguese", "English"], status.HTTP_401_UNAUTHORIZED), ("user", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), ("staff", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), # Users can't edit tags from a disabled taxonomy. Staff can't add multiple tags to # a taxonomy with allow_multiple=False - (None, "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), + (None, "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), ("staff", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), ) @@ -734,7 +734,7 @@ def test_tag_object_multiple(self, user_attr, taxonomy_attr, tag_values, expecte assert set(t["value"] for t in response.data) == set(tag_values) @ddt.data( - (None, status.HTTP_403_FORBIDDEN), + (None, status.HTTP_401_UNAUTHORIZED), ("user", status.HTTP_403_FORBIDDEN), ("staff", status.HTTP_403_FORBIDDEN), ) @@ -828,7 +828,7 @@ def test_invalid_taxonomy(self): def test_not_authorized_user(self): # Not authenticated user response = self.client.get(self.small_taxonomy_url) - assert response.status_code == status.HTTP_403_FORBIDDEN + assert response.status_code == status.HTTP_401_UNAUTHORIZED self.small_taxonomy.enabled = False self.small_taxonomy.save() From 1304904cb10a8235145b70fd60ea6bc33dd820da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Fri, 13 Oct 2023 13:49:22 -0300 Subject: [PATCH 057/282] feat: add export taxonomy rest api (#97) --- .../core/tagging/rest_api/v1/serializers.py | 8 ++ .../core/tagging/rest_api/v1/views.py | 58 +++++++++- openedx_tagging/core/tagging/rules.py | 1 + .../core/tagging/test_views.py | 102 ++++++++++++++++++ 4 files changed, 167 insertions(+), 2 deletions(-) diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 388ee6cc4..a4eb89ffa 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -16,6 +16,14 @@ class TaxonomyListQueryParamsSerializer(serializers.Serializer): # pylint: disa enabled = serializers.BooleanField(required=False) +class TaxonomyExportQueryParamsSerializer(serializers.Serializer): # pylint: disable=abstract-method + """ + Serializer for the query params for the GET view + """ + download = serializers.BooleanField(required=False, default=False) + output_format = serializers.RegexField(r"(?i)^(json|csv)$", allow_blank=False) + + class TaxonomySerializer(serializers.ModelSerializer): """ Serializer for the Taxonomy model. diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index c00f78dc9..b4774bafb 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -4,8 +4,9 @@ from __future__ import annotations from django.db import models -from django.http import Http404 +from django.http import Http404, HttpResponse from rest_framework import mixins +from rest_framework.decorators import action from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError from rest_framework.generics import ListAPIView from rest_framework.response import Response @@ -23,6 +24,8 @@ search_tags, tag_object, ) +from ...import_export.api import export_tags +from ...import_export.parsers import ParserFormat from ...models import Taxonomy from ...rules import ChangeObjectTagPermissionItem from ..paginators import SEARCH_TAGS_THRESHOLD, TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination @@ -35,6 +38,7 @@ TagsForSearchSerializer, TagsSerializer, TagsWithSubTagsSerializer, + TaxonomyExportQueryParamsSerializer, TaxonomyListQueryParamsSerializer, TaxonomySerializer, ) @@ -44,7 +48,7 @@ @view_auth_classes class TaxonomyView(ModelViewSet): """ - View to list, create, retrieve, update, or delete Taxonomies. + View to list, create, retrieve, update, delete or export Taxonomies. **List Query Parameters** * enabled (optional) - Filter by enabled status. Valid values: true, @@ -143,6 +147,23 @@ class TaxonomyView(ModelViewSet): * 404 - Taxonomy not found * 403 - Permission denied + **Export Query Parameters** + * output_format - Define the output format. Valid values: json, csv + * download (optional) - Add headers on the response to let the browser + automatically download the file. + + **Export Example Requests** + GET api/tagging/v1/taxonomy/:pk/export?output_format=csv - Export taxonomy as CSV + GET api/tagging/v1/taxonomy/:pk/export?output_format=json - Export taxonomy as JSON + GET api/tagging/v1/taxonomy/:pk/export?output_format=csv&download=1 - Export and downloads taxonomy as CSV + GET api/tagging/v1/taxonomy/:pk/export?output_format=json&download=1 - Export and downloads taxonomy as JSON + + **Export Query Returns** + * 200 - Success + * 400 - Invalid query parameter + * 403 - Permission denied + + """ serializer_class = TaxonomySerializer @@ -183,6 +204,39 @@ def perform_create(self, serializer) -> None: """ serializer.instance = create_taxonomy(**serializer.validated_data) + @action(detail=True, methods=["get"]) + def export(self, request, **_kwargs) -> HttpResponse: + """ + Export a taxonomy. + """ + taxonomy = self.get_object() + perm = "oel_tagging.export_taxonomy" + if not request.user.has_perm(perm, taxonomy): + raise PermissionDenied("You do not have permission to export this taxonomy.") + query_params = TaxonomyExportQueryParamsSerializer( + data=request.query_params.dict() + ) + query_params.is_valid(raise_exception=True) + output_format = query_params.data.get("output_format") + assert output_format is not None + if output_format.lower() == "json": + parser_format = ParserFormat.JSON + content_type = "application/json" + else: + parser_format = ParserFormat.CSV + if query_params.data.get("download"): + content_type = "text/csv" + else: + content_type = "text" + + tags = export_tags(taxonomy, parser_format) + if query_params.data.get("download"): + response = HttpResponse(tags.encode('utf-8'), content_type=content_type) + response["Content-Disposition"] = f'attachment; filename="{taxonomy.name}{parser_format.value}"' + return response + + return HttpResponse(tags, content_type=content_type) + @view_auth_classes class ObjectTagView( diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 00ec88116..878b1c901 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -109,6 +109,7 @@ def can_change_object_tag( rules.add_perm("oel_tagging.change_taxonomy", can_change_taxonomy) rules.add_perm("oel_tagging.delete_taxonomy", can_change_taxonomy) rules.add_perm("oel_tagging.view_taxonomy", can_view_taxonomy) +rules.add_perm("oel_tagging.export_taxonomy", can_view_taxonomy) # Tag rules.add_perm("oel_tagging.add_tag", can_change_tag) diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 5092c414b..98f2332b9 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -12,6 +12,8 @@ from rest_framework import status from rest_framework.test import APITestCase +from openedx_tagging.core.tagging.import_export import api as import_export_api +from openedx_tagging.core.tagging.import_export.parsers import ParserFormat from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy from openedx_tagging.core.tagging.rest_api.paginators import TagsPagination @@ -20,6 +22,7 @@ TAXONOMY_LIST_URL = "/tagging/rest_api/v1/taxonomies/" TAXONOMY_DETAIL_URL = "/tagging/rest_api/v1/taxonomies/{pk}/" +TAXONOMY_EXPORT_URL = "/tagging/rest_api/v1/taxonomies/{pk}/export/" TAXONOMY_TAGS_URL = "/tagging/rest_api/v1/taxonomies/{pk}/tags/" @@ -395,6 +398,105 @@ def test_delete_taxonomy_404(self): response = self.client.delete(url) assert response.status_code == status.HTTP_404_NOT_FOUND + @ddt.data( + ("csv", "text"), + ("json", "application/json") + ) + @ddt.unpack + def test_export_taxonomy(self, output_format, content_type): + """ + Tests if a user can export a taxonomy + """ + taxonomy = Taxonomy.objects.create(name="T1", enabled=True) + taxonomy.save() + for i in range(20): + # Valid ObjectTags + Tag.objects.create(taxonomy=taxonomy, value=f"Tag {i}").save() + + url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) + + self.client.force_authenticate(user=self.staff) + response = self.client.get(url, {"output_format": output_format}) + assert response.status_code == status.HTTP_200_OK + if output_format == "json": + expected_data = import_export_api.export_tags(taxonomy, ParserFormat.JSON) + else: + expected_data = import_export_api.export_tags(taxonomy, ParserFormat.CSV) + + assert response.headers['Content-Type'] == content_type + assert response.content == expected_data.encode("utf-8") + + @ddt.data( + ("csv", "text/csv"), + ("json", "application/json") + ) + @ddt.unpack + def test_export_taxonomy_download(self, output_format, content_type): + """ + Tests if a user can export a taxonomy with download option + """ + taxonomy = Taxonomy.objects.create(name="T1", enabled=True) + taxonomy.save() + for i in range(20): + # Valid ObjectTags + Tag.objects.create(taxonomy=taxonomy, value=f"Tag {i}").save() + + url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) + + self.client.force_authenticate(user=self.staff) + response = self.client.get(url, {"output_format": output_format, "download": True}) + assert response.status_code == status.HTTP_200_OK + if output_format == "json": + expected_data = import_export_api.export_tags(taxonomy, ParserFormat.JSON) + else: + expected_data = import_export_api.export_tags(taxonomy, ParserFormat.CSV) + + assert response.headers['Content-Type'] == content_type + assert response.headers['Content-Disposition'] == f'attachment; filename="{taxonomy.name}.{output_format}"' + assert response.content == expected_data.encode("utf-8") + + def test_export_taxonomy_invalid_param_output_format(self): + """ + Tests if a user can export a taxonomy using an invalid output_format param + """ + taxonomy = Taxonomy.objects.create(name="T1", enabled=True) + taxonomy.save() + + url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) + + self.client.force_authenticate(user=self.staff) + response = self.client.get(url, {"output_format": "html", "download": True}) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_export_taxonomy_invalid_param_download(self): + """ + Tests if a user can export a taxonomy using an invalid output_format param + """ + taxonomy = Taxonomy.objects.create(name="T1", enabled=True) + taxonomy.save() + + url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) + + self.client.force_authenticate(user=self.staff) + response = self.client.get(url, {"output_format": "json", "download": "invalid"}) + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_export_taxonomy_unauthorized(self): + """ + Tests if a user can export a taxonomy that he doesn't have authorization + """ + # Only staff can view a disabled taxonomy + taxonomy = Taxonomy.objects.create(name="T1", enabled=False) + taxonomy.save() + + url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) + + self.client.force_authenticate(user=self.user) + response = self.client.get(url, {"output_format": "json"}) + + # Return 404, because the user doesn't have permission to view the taxonomy + assert response.status_code == status.HTTP_404_NOT_FOUND + @ddt.ddt class TestObjectTagViewSet(APITestCase): From ca6bc859f69d5508d5457f42a61e8fd3263a7b53 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Fri, 13 Oct 2023 09:51:52 -0700 Subject: [PATCH 058/282] chore: version bump to 0.2.4 --- openedx_learning/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index fcfa685c9..19322ba20 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.2.3" +__version__ = "0.2.4" From 1bb59a4e283e334a6eeffccac4c0b59a1dbf071e Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 15 Oct 2023 20:20:50 -0400 Subject: [PATCH 059/282] chore: Updating Python Requirements --- requirements/base.txt | 4 ++-- requirements/ci.txt | 2 +- requirements/dev.txt | 14 +++++++------- requirements/doc.txt | 10 +++++----- requirements/quality.txt | 12 ++++++------ requirements/test.txt | 10 +++++----- 6 files changed, 26 insertions(+), 26 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 5d4038c13..01f645cd9 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -66,7 +66,7 @@ drf-jwt==1.19.2 # via edx-drf-extensions edx-django-utils==5.7.0 # via edx-drf-extensions -edx-drf-extensions==8.10.0 +edx-drf-extensions==8.11.1 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via edx-drf-extensions @@ -80,7 +80,7 @@ pbr==5.11.1 # via stevedore prompt-toolkit==3.0.39 # via click-repl -psutil==5.9.5 +psutil==5.9.6 # via edx-django-utils pycparser==2.21 # via cffi diff --git a/requirements/ci.txt b/requirements/ci.txt index 8e788a740..01a3f8b39 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -12,7 +12,7 @@ filelock==3.12.4 # via # tox # virtualenv -grimp==3.0 +grimp==3.1 # via import-linter import-linter==1.12.0 # via -r requirements/ci.in diff --git a/requirements/dev.txt b/requirements/dev.txt index 157726f68..37e215f46 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -95,7 +95,7 @@ cryptography==41.0.4 # secretstorage ddt==1.6.0 # via -r requirements/quality.txt -diff-cover==7.7.0 +diff-cover==8.0.0 # via -r requirements/dev.in dill==0.3.7 # via @@ -159,7 +159,7 @@ edx-django-utils==5.7.0 # via # -r requirements/quality.txt # edx-drf-extensions -edx-drf-extensions==8.10.0 +edx-drf-extensions==8.11.1 # via -r requirements/quality.txt edx-i18n-tools==1.3.0 # via -r requirements/dev.in @@ -178,7 +178,7 @@ filelock==3.12.4 # -r requirements/ci.txt # tox # virtualenv -grimp==3.0 +grimp==3.1 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -260,7 +260,7 @@ more-itertools==10.1.0 # via # -r requirements/quality.txt # jaraco-classes -mypy==1.5.1 +mypy==1.6.0 # via # -r requirements/quality.txt # django-stubs @@ -318,7 +318,7 @@ prompt-toolkit==3.0.39 # via # -r requirements/quality.txt # click-repl -psutil==5.9.5 +psutil==5.9.6 # via # -r requirements/quality.txt # edx-django-utils @@ -326,7 +326,7 @@ py==1.11.0 # via # -r requirements/ci.txt # tox -pycodestyle==2.11.0 +pycodestyle==2.11.1 # via -r requirements/quality.txt pycparser==2.21 # via @@ -500,7 +500,7 @@ types-pyyaml==6.0.12.12 # -r requirements/quality.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.8 +types-requests==2.31.0.9 # via # -r requirements/quality.txt # djangorestframework-stubs diff --git a/requirements/doc.txt b/requirements/doc.txt index ae9758731..edff4bda4 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -139,7 +139,7 @@ edx-django-utils==5.7.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.10.0 +edx-drf-extensions==8.11.1 # via -r requirements/test.txt edx-opaque-keys==2.5.1 # via @@ -149,7 +149,7 @@ exceptiongroup==1.1.3 # via # -r requirements/test.txt # pytest -grimp==3.0 +grimp==3.1 # via # -r requirements/test.txt # import-linter @@ -182,7 +182,7 @@ markupsafe==2.1.3 # jinja2 mock==5.1.0 # via -r requirements/test.txt -mypy==1.5.1 +mypy==1.6.0 # via # -r requirements/test.txt # django-stubs @@ -219,7 +219,7 @@ prompt-toolkit==3.0.39 # via # -r requirements/test.txt # click-repl -psutil==5.9.5 +psutil==5.9.6 # via # -r requirements/test.txt # edx-django-utils @@ -356,7 +356,7 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.8 +types-requests==2.31.0.9 # via # -r requirements/test.txt # djangorestframework-stubs diff --git a/requirements/quality.txt b/requirements/quality.txt index b29f7118e..c3c49a0ef 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -134,7 +134,7 @@ edx-django-utils==5.7.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.10.0 +edx-drf-extensions==8.11.1 # via -r requirements/test.txt edx-lint==5.3.4 # via -r requirements/quality.in @@ -146,7 +146,7 @@ exceptiongroup==1.1.3 # via # -r requirements/test.txt # pytest -grimp==3.0 +grimp==3.1 # via # -r requirements/test.txt # import-linter @@ -202,7 +202,7 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.1.0 # via jaraco-classes -mypy==1.5.1 +mypy==1.6.0 # via # -r requirements/test.txt # django-stubs @@ -239,11 +239,11 @@ prompt-toolkit==3.0.39 # via # -r requirements/test.txt # click-repl -psutil==5.9.5 +psutil==5.9.6 # via # -r requirements/test.txt # edx-django-utils -pycodestyle==2.11.0 +pycodestyle==2.11.1 # via -r requirements/quality.in pycparser==2.21 # via @@ -375,7 +375,7 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.8 +types-requests==2.31.0.9 # via # -r requirements/test.txt # djangorestframework-stubs diff --git a/requirements/test.txt b/requirements/test.txt index b4db8b5ac..549b890f6 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -116,7 +116,7 @@ edx-django-utils==5.7.0 # via # -r requirements/base.txt # edx-drf-extensions -edx-drf-extensions==8.10.0 +edx-drf-extensions==8.11.1 # via -r requirements/base.txt edx-opaque-keys==2.5.1 # via @@ -124,7 +124,7 @@ edx-opaque-keys==2.5.1 # edx-drf-extensions exceptiongroup==1.1.3 # via pytest -grimp==3.0 +grimp==3.1 # via import-linter idna==3.4 # via @@ -144,7 +144,7 @@ markupsafe==2.1.3 # via jinja2 mock==5.1.0 # via -r requirements/test.in -mypy==1.5.1 +mypy==1.6.0 # via # -r requirements/test.in # django-stubs @@ -169,7 +169,7 @@ prompt-toolkit==3.0.39 # via # -r requirements/base.txt # click-repl -psutil==5.9.5 +psutil==5.9.6 # via # -r requirements/base.txt # edx-django-utils @@ -253,7 +253,7 @@ types-pyyaml==6.0.12.12 # via # django-stubs # djangorestframework-stubs -types-requests==2.31.0.8 +types-requests==2.31.0.9 # via djangorestframework-stubs typing-extensions==4.8.0 # via From 38da66b74aae97bdfe43397ab706cb6fa8507bc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 16 Oct 2023 16:34:17 -0300 Subject: [PATCH 060/282] feat: add "can view object tags" permissions (#94) --- openedx_learning/__init__.py | 2 +- .../core/tagging/rest_api/v1/views.py | 40 +++++++-- openedx_tagging/core/tagging/rules.py | 62 ++++++++++++- .../core/tagging/test_rules.py | 10 +-- .../core/tagging/test_views.py | 87 +++++++++++-------- 5 files changed, 149 insertions(+), 52 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 19322ba20..d88e37a3d 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.2.4" +__version__ = "0.2.5" diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index b4774bafb..743f5bcd2 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -27,7 +27,7 @@ from ...import_export.api import export_tags from ...import_export.parsers import ParserFormat from ...models import Taxonomy -from ...rules import ChangeObjectTagPermissionItem +from ...rules import ObjectTagPermissionItem from ..paginators import SEARCH_TAGS_THRESHOLD, TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination from .permissions import ObjectTagObjectPermissions, TagListPermissions, TaxonomyObjectPermissions from .serializers import ( @@ -298,10 +298,30 @@ def get_queryset(self) -> models.QuerySet: data=self.request.query_params.dict() ) query_params.is_valid(raise_exception=True) - taxonomy_id = query_params.data.get("taxonomy", None) + taxonomy = query_params.validated_data.get("taxonomy", None) + taxonomy_id = None + if taxonomy: + taxonomy = taxonomy.cast() + taxonomy_id = taxonomy.id + + perm = "oel_tagging.view_objecttag" + perm_obj = ObjectTagPermissionItem( + taxonomy=taxonomy, + object_id=object_id, + ) + + if not self.request.user.has_perm( + perm, + # The obj arg expects a model, but we are passing an object + perm_obj, # type: ignore[arg-type] + ): + raise PermissionDenied( + "You do not have permission to view object tags for this taxonomy or object_id." + ) + return get_object_tags(object_id, taxonomy_id) - def retrieve(self, request, *args, **kwargs): + def retrieve(self, request, *args, **kwargs) -> Response: """ Retrieve ObjectTags that belong to a given object_id @@ -312,11 +332,11 @@ def retrieve(self, request, *args, **kwargs): path and returns a it as a single result however that is not behavior we want. """ - object_tags = self.get_queryset() + object_tags = self.filter_queryset(self.get_queryset()) serializer = ObjectTagSerializer(object_tags, many=True) return Response(serializer.data) - def update(self, request, *args, **kwargs): + def update(self, request, *args, **kwargs) -> Response: """ Update ObjectTags that belong to a given object_id @@ -352,15 +372,19 @@ def update(self, request, *args, **kwargs): taxonomy = query_params.validated_data.get("taxonomy", None) taxonomy = taxonomy.cast() - perm = f"{taxonomy._meta.app_label}.change_objecttag" + perm = "oel_tagging.change_objecttag" object_id = kwargs.pop('object_id') - perm_obj = ChangeObjectTagPermissionItem( + perm_obj = ObjectTagPermissionItem( taxonomy=taxonomy, object_id=object_id, ) - if not request.user.has_perm(perm, perm_obj): + if not request.user.has_perm( + perm, + # The obj arg expects a model, but we are passing an object + perm_obj, # type: ignore[arg-type] + ): raise PermissionDenied( "You do not have permission to change object tags for this taxonomy or object_id." ) diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 878b1c901..8249628f8 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -23,7 +23,7 @@ @define -class ChangeObjectTagPermissionItem: +class ObjectTagPermissionItem: """ Pair of taxonomy and object_id used for permission checking. """ @@ -65,6 +65,58 @@ def can_change_tag(user: UserType, tag: Tag | None = None) -> bool: ) +@rules.predicate +def can_view_object_tag_taxonomy(user: UserType, taxonomy: Taxonomy) -> bool: + """ + Only enabled taxonomy and users with permission to view this taxonomy can view object tags + from that taxonomy. + + This rule is different from can_view_taxonomy because it checks if the taxonomy is enabled. + """ + if not taxonomy: + return True + + return taxonomy.cast().enabled and can_view_taxonomy(user, taxonomy) + + +@rules.predicate +def can_view_object_tag_objectid(_user: UserType, _object_id: str) -> bool: + """ + Everybody can view object tags from any objects. + + This rule could be defined in other apps for proper permission checking. + """ + return True + + +@rules.predicate +def can_view_object_tag( + user: UserType, perm_obj: ObjectTagPermissionItem | None = None +) -> bool: + """ + Checks if the user has permissions to view tags on the given taxonomy and object_id. + """ + + # The following code allows METHOD permission (GET) in the viewset for everyone + if perm_obj is None: + return True + + # Checks the permission for the taxonomy + taxonomy_perm = user.has_perm( + "oel_tagging.view_objecttag_taxonomy", perm_obj.taxonomy + ) + if not taxonomy_perm: + return False + + # Checks the permission for the object_id + objectid_perm = user.has_perm( + "oel_tagging.view_objecttag_objectid", + # The obj arg expects an object, but we are passing a string + perm_obj.object_id, # type: ignore[arg-type] + ) + return objectid_perm + + @rules.predicate def can_change_object_tag_objectid(_user: UserType, _object_id: str) -> bool: """ @@ -77,7 +129,7 @@ def can_change_object_tag_objectid(_user: UserType, _object_id: str) -> bool: @rules.predicate def can_change_object_tag( - user: UserType, perm_obj: ChangeObjectTagPermissionItem | None = None + user: UserType, perm_obj: ObjectTagPermissionItem | None = None ) -> bool: """ Checks if the user has permissions to create or modify tags on the given taxonomy and object_id. @@ -122,8 +174,10 @@ def can_change_object_tag( rules.add_perm("oel_tagging.add_objecttag", can_change_object_tag) rules.add_perm("oel_tagging.change_objecttag", can_change_object_tag) rules.add_perm("oel_tagging.delete_objecttag", can_change_object_tag) -rules.add_perm("oel_tagging.view_objecttag", rules.always_allow) +rules.add_perm("oel_tagging.view_objecttag", can_view_object_tag) # Users can tag objects using tags from any taxonomy that they have permission to view -rules.add_perm("oel_tagging.change_objecttag_taxonomy", can_view_taxonomy) +rules.add_perm("oel_tagging.view_objecttag_objectid", can_view_object_tag_objectid) +rules.add_perm("oel_tagging.view_objecttag_taxonomy", can_view_object_tag_taxonomy) +rules.add_perm("oel_tagging.change_objecttag_taxonomy", can_view_object_tag_taxonomy) rules.add_perm("oel_tagging.change_objecttag_objectid", can_change_object_tag_objectid) diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py index d49f8b997..f30cbd27c 100644 --- a/tests/openedx_tagging/core/tagging/test_rules.py +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -7,7 +7,7 @@ from django.test.testcases import TestCase from openedx_tagging.core.tagging.models import ObjectTag -from openedx_tagging.core.tagging.rules import ChangeObjectTagPermissionItem +from openedx_tagging.core.tagging.rules import ObjectTagPermissionItem from .test_models import TestTagTaxonomyMixin @@ -188,7 +188,7 @@ def test_add_change_object_tag(self, perm): """ Everyone can create/edit an ObjectTag with an enabled Taxonomy """ - obj_perm = ChangeObjectTagPermissionItem( + obj_perm = ObjectTagPermissionItem( taxonomy=self.object_tag.taxonomy, object_id=self.object_tag.object_id, ) @@ -210,12 +210,12 @@ def test_object_tag_disabled_taxonomy(self, perm): """ self.taxonomy.enabled = False self.taxonomy.save() - obj_perm = ChangeObjectTagPermissionItem( + obj_perm = ObjectTagPermissionItem( taxonomy=self.object_tag.taxonomy, object_id=self.object_tag.object_id, ) assert self.superuser.has_perm(perm, obj_perm) - assert self.staff.has_perm(perm, obj_perm) + assert not self.staff.has_perm(perm, obj_perm) assert not self.learner.has_perm(perm, obj_perm) @ddt.data( @@ -229,7 +229,7 @@ def test_object_tag_without_object_permission(self, perm): """ self.taxonomy.enabled = False self.taxonomy.save() - obj_perm = ChangeObjectTagPermissionItem( + obj_perm = ObjectTagPermissionItem( taxonomy=self.object_tag.taxonomy, object_id="not abc", ) diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 98f2332b9..d97296ba0 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -17,6 +17,7 @@ from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy from openedx_tagging.core.tagging.rest_api.paginators import TagsPagination +from openedx_tagging.core.tagging.rules import can_change_object_tag_objectid, can_view_object_tag_objectid User = get_user_model() @@ -506,11 +507,23 @@ class TestObjectTagViewSet(APITestCase): def setUp(self): - def _object_permission(_user, object_id: str) -> bool: + def _change_object_permission(user, object_id: str) -> bool: """ - Everyone have object permission on object_id "abc" + For testing, let everyone have edit object permission on object_id "abc" and "limit_tag_count" """ - return object_id in ("abc", "limit_tag_count") + if object_id in ("abc", "limit_tag_count"): + return True + + return can_change_object_tag_objectid(user, object_id) + + def _view_object_permission(user, object_id: str) -> bool: + """ + For testing, let everyone have view object permission on all objects but "unauthorized_id" + """ + if object_id == "unauthorized_id": + return False + + return can_view_object_tag_objectid(user, object_id) super().setUp() @@ -580,22 +593,20 @@ def _object_permission(_user, object_id: str) -> bool: self.dummy_taxonomies.append(taxonomy) # Override the object permission for the test - rules.set_perm("oel_tagging.change_objecttag_objectid", _object_permission) + rules.set_perm("oel_tagging.change_objecttag_objectid", _change_object_permission) + rules.set_perm("oel_tagging.view_objecttag_objectid", _view_object_permission) @ddt.data( - (None, "abc", status.HTTP_401_UNAUTHORIZED, None), - ("user", "abc", status.HTTP_200_OK, 81), - ("staff", "abc", status.HTTP_200_OK, 81), - (None, "non-existing-id", status.HTTP_401_UNAUTHORIZED, None), - ("user", "non-existing-id", status.HTTP_200_OK, 0), - ("staff", "non-existing-id", status.HTTP_200_OK, 0), + (None, status.HTTP_401_UNAUTHORIZED, None), + ("user", status.HTTP_200_OK, 81), + ("staff", status.HTTP_200_OK, 81), ) @ddt.unpack - def test_retrieve_object_tags(self, user_attr, object_id, expected_status, expected_count): + def test_retrieve_object_tags(self, user_attr, expected_status, expected_count): """ Test retrieving object tags """ - url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id="abc") if user_attr: user = getattr(self, user_attr) @@ -607,6 +618,15 @@ def test_retrieve_object_tags(self, user_attr, object_id, expected_status, expec if status.is_success(expected_status): assert len(response.data) == expected_count + def test_retrieve_object_tags_unauthorized(self): + """ + Test retrieving object tags from an unauthorized object_id + """ + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id="unauthorized_id") + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + assert response.status_code == status.HTTP_403_FORBIDDEN + @ddt.data( (None, "abc", status.HTTP_401_UNAUTHORIZED, None), ("user", "abc", status.HTTP_200_OK, 20), @@ -694,30 +714,30 @@ def test_object_tags_remaining_http_methods( assert response.status_code == expected_status @ddt.data( - # Users and staff can add tags to a taxonomy + # Users and staff can add tags (None, "language_taxonomy", ["Portuguese"], status.HTTP_401_UNAUTHORIZED), ("user", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), ("staff", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), - # Users and staff can clear add tags to a taxonomy + # Users and staff can clear add tags (None, "enabled_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), ("user", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), ("staff", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), - # Only staff can add tag to a disabled taxonomy + # Nobody can add tag using a disabled taxonomy (None, "disabled_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), - # Users and staff can add a single tag to a allow_multiple=True taxonomy + ("staff", "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), + # Users and staff can add a single tag using a allow_multiple=True taxonomy (None, "multiple_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), ("staff", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), - # Users and staff can add tags to an open taxonomy + # Users and staff can add tags using an open taxonomy (None, "open_taxonomy_enabled", ["tag1"], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), ("staff", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), - # Only staff can add tags to a disabled open taxonomy + # Nobody can add tags using a disabled open taxonomy (None, "open_taxonomy_disabled", ["tag1"], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), - ("staff", "open_taxonomy_disabled", ["tag1"], status.HTTP_200_OK), + ("staff", "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), ) @ddt.unpack def test_tag_object(self, user_attr, taxonomy_attr, tag_values, expected_status): @@ -736,7 +756,7 @@ def test_tag_object(self, user_attr, taxonomy_attr, tag_values, expected_status) assert set(t["value"] for t in response.data) == set(tag_values) @ddt.data( - # Can't add invalid tags to a closed taxonomy + # Can't add invalid tags using a closed taxonomy (None, "language_taxonomy", ["Invalid"], status.HTTP_401_UNAUTHORIZED), ("user", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), ("staff", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), @@ -746,10 +766,10 @@ def test_tag_object(self, user_attr, taxonomy_attr, tag_values, expected_status) (None, "multiple_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), ("staff", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), - # Users can't edit tags from a disabled taxonomy. Staff can't add invalid tags to a closed taxonomy + # Nobody can edit object tags using a disabled taxonomy. (None, "disabled_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), + ("staff", "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), ) @ddt.unpack def test_tag_object_invalid(self, user_attr, taxonomy_attr, tag_values, expected_status): @@ -766,21 +786,21 @@ def test_tag_object_invalid(self, user_attr, taxonomy_attr, tag_values, expected assert not status.is_success(expected_status) # No success cases here @ddt.data( - # Users and staff can clear tags from a taxonomy + # Users and staff can clear tags (None, "enabled_taxonomy", [], status.HTTP_401_UNAUTHORIZED), ("user", "enabled_taxonomy", [], status.HTTP_200_OK), ("staff", "enabled_taxonomy", [], status.HTTP_200_OK), - # Users and staff can clear tags from a allow_multiple=True taxonomy + # Users and staff can clear object tags using a allow_multiple=True taxonomy (None, "multiple_taxonomy", [], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", [], status.HTTP_200_OK), ("staff", "multiple_taxonomy", [], status.HTTP_200_OK), - # Only staff can clear tags from a disabled taxonomy + # Nobody can clear tags using a disabled taxonomy (None, "disabled_taxonomy", [], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", [], status.HTTP_200_OK), + ("staff", "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), (None, "open_taxonomy_disabled", [], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), - ("staff", "open_taxonomy_disabled", [], status.HTTP_200_OK), + ("staff", "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), ) @ddt.unpack def test_tag_object_clear(self, user_attr, taxonomy_attr, tag_values, expected_status): @@ -799,25 +819,24 @@ def test_tag_object_clear(self, user_attr, taxonomy_attr, tag_values, expected_s assert set(t["value"] for t in response.data) == set(tag_values) @ddt.data( - # Users and staff can add multiple tags to a allow_multiple=True taxonomy + # Users and staff can add multiple tags using a allow_multiple=True taxonomy (None, "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), ("user", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), ("staff", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), (None, "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_401_UNAUTHORIZED), ("user", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), ("staff", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), - # Users and staff can't add multple tags to a allow_multiple=False taxonomy + # Users and staff can't add multple tags using a allow_multiple=False taxonomy (None, "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), ("user", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), ("staff", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), (None, "language_taxonomy", ["Portuguese", "English"], status.HTTP_401_UNAUTHORIZED), ("user", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), ("staff", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), - # Users can't edit tags from a disabled taxonomy. Staff can't add multiple tags to - # a taxonomy with allow_multiple=False + # Nobody can edit tags using a disabled taxonomy. (None, "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), ("user", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), + ("staff", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), ) @ddt.unpack def test_tag_object_multiple(self, user_attr, taxonomy_attr, tag_values, expected_status): @@ -846,7 +865,7 @@ def test_tag_object_without_permission(self, user_attr, expected_status): user = getattr(self, user_attr) self.client.force_authenticate(user=user) - url = OBJECT_TAGS_UPDATE_URL.format(object_id="not abc", taxonomy_id=self.enabled_taxonomy.pk) + url = OBJECT_TAGS_UPDATE_URL.format(object_id="view_only", taxonomy_id=self.enabled_taxonomy.pk) response = self.client.put(url, {"tags": ["Tag 1"]}, format="json") assert response.status_code == expected_status From 1081ea7b897625d24c8ceda209bcf809c6b7269c Mon Sep 17 00:00:00 2001 From: Yusuf Musleh Date: Mon, 23 Oct 2023 02:07:25 +0300 Subject: [PATCH 061/282] feat: CRUD API for Taxonomy Tags [FC-0036] (#96) * Implements add, update, delete taxonomy tag api/rest + tests * resyncs ObjectTags when tag added or updated. * Updates Taxonomy Tags permissions to use Taxonomy permissions * Bumps version to 0.2.6 --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 55 ++ openedx_tagging/core/tagging/models/base.py | 99 ++++ .../core/tagging/rest_api/v1/permissions.py | 26 +- .../core/tagging/rest_api/v1/serializers.py | 37 +- .../core/tagging/rest_api/v1/views.py | 195 ++++++- openedx_tagging/core/tagging/rules.py | 28 +- .../core/tagging/test_rules.py | 4 +- .../core/tagging/test_views.py | 542 +++++++++++++++++- 9 files changed, 950 insertions(+), 38 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index d88e37a3d..b602ca309 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.2.5" +__version__ = "0.2.6" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index e91409687..3316b32bd 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -135,6 +135,7 @@ def resync_object_tags(object_tags: QuerySet | None = None) -> int: if changed: object_tag.save() num_changed += 1 + return num_changed @@ -315,3 +316,57 @@ def autocomplete_tags( # remove repeats .distinct() ) + + +def add_tag_to_taxonomy( + taxonomy: Taxonomy, + tag: str, + parent_tag_value: str | None = None, + external_id: str | None = None +) -> Tag: + """ + Adds a new Tag to provided Taxonomy. If a Tag already exists in the + Taxonomy, an exception is raised, otherwise the newly created + Tag is returned + """ + taxonomy = taxonomy.cast() + new_tag = taxonomy.add_tag(tag, parent_tag_value, external_id) + + # Resync all related ObjectTags after creating new Tag to + # to ensure any existing ObjectTags with the same value will + # be linked to the new Tag + object_tags = taxonomy.objecttag_set.all() + resync_object_tags(object_tags) + + return new_tag + + +def update_tag_in_taxonomy(taxonomy: Taxonomy, tag: str, new_value: str): + """ + Update a Tag that belongs to a Taxonomy. The related ObjectTags are + updated accordingly. + + Currently only supports updating the Tag value. + """ + taxonomy = taxonomy.cast() + updated_tag = taxonomy.update_tag(tag, new_value) + + # Resync all related ObjectTags to update to the new Tag value + object_tags = taxonomy.objecttag_set.all() + resync_object_tags(object_tags) + + return updated_tag + + +def delete_tags_from_taxonomy( + taxonomy: Taxonomy, + tags: list[str], + with_subtags: bool +): + """ + Delete Tags that belong to a Taxonomy. If any of the Tags have children and + the `with_subtags` is not set to `True` it will fail, otherwise + the sub-tags will be deleted as well. + """ + taxonomy = taxonomy.cast() + taxonomy.delete_tags(tags, with_subtags) diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index d24d67d46..4ecf52c5f 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -342,6 +342,105 @@ def get_filtered_tags( return tag_set.order_by("value", "id") + def add_tag( + self, + tag_value: str, + parent_tag_value: str | None = None, + external_id: str | None = None + ) -> Tag: + """ + Add new Tag to Taxonomy. If an existing Tag with the `tag_value` already + exists in the Taxonomy, an exception is raised, otherwise the newly + created Tag is returned + """ + self.check_casted() + + if self.allow_free_text: + raise ValueError( + "add_tag() doesn't work for free text taxonomies. They don't use Tag instances." + ) + + if self.system_defined: + raise ValueError( + "add_tag() doesn't work for system defined taxonomies. They cannot be modified." + ) + + if self.tag_set.filter(value__iexact=tag_value).exists(): + raise ValueError(f"Tag with value '{tag_value}' already exists for taxonomy.") + + parent = None + if parent_tag_value: + # Get parent tag from taxonomy, raises Tag.DoesNotExist if doesn't + # belong to taxonomy + parent = self.tag_set.get(value__iexact=parent_tag_value) + + tag = Tag.objects.create( + taxonomy=self, value=tag_value, parent=parent, external_id=external_id + ) + + return tag + + def update_tag(self, tag: str, new_value: str) -> Tag: + """ + Update an existing Tag in Taxonomy and return it. Currently only + supports updating the Tag's value. + """ + self.check_casted() + + if self.allow_free_text: + raise ValueError( + "update_tag() doesn't work for free text taxonomies. They don't use Tag instances." + ) + + if self.system_defined: + raise ValueError( + "update_tag() doesn't work for system defined taxonomies. They cannot be modified." + ) + + # Update Tag instance with new value, raises Tag.DoesNotExist if + # tag doesn't belong to taxonomy + tag_to_update = self.tag_set.get(value__iexact=tag) + tag_to_update.value = new_value + tag_to_update.save() + return tag_to_update + + def delete_tags(self, tags: List[str], with_subtags: bool = False): + """ + Delete the Taxonomy Tags provided. If any of them have children and + the `with_subtags` is not set to `True` it will fail, otherwise + the sub-tags will be deleted as well. + """ + self.check_casted() + + if self.allow_free_text: + raise ValueError( + "delete_tags() doesn't work for free text taxonomies. They don't use Tag instances." + ) + + if self.system_defined: + raise ValueError( + "delete_tags() doesn't work for system defined taxonomies. They cannot be modified." + ) + + tags_to_delete = self.tag_set.filter(value__in=tags) + + if tags_to_delete.count() != len(tags): + # If they do not match that means there is one or more Tag ID(s) + # provided that do not belong to this Taxonomy + raise ValueError("Invalid tag id provided or tag id does not belong to taxonomy") + + # Check if any Tag contains subtags (children) + contains_children = tags_to_delete.filter(children__isnull=False).distinct().exists() + + if contains_children and not with_subtags: + raise ValueError( + "Tag(s) contain children, `with_subtags` must be `True` for " + "all Tags and their subtags (children) to be deleted." + ) + + # Delete the Tags with their subtags if any + tags_to_delete.delete() + def validate_value(self, value: str) -> bool: """ Check if 'value' is part of this Taxonomy. diff --git a/openedx_tagging/core/tagging/rest_api/v1/permissions.py b/openedx_tagging/core/tagging/rest_api/v1/permissions.py index ed184549e..b63a6b7ee 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/permissions.py +++ b/openedx_tagging/core/tagging/rest_api/v1/permissions.py @@ -4,6 +4,8 @@ import rules # type: ignore[import] from rest_framework.permissions import DjangoObjectPermissions +from ...models import Tag + class TaxonomyObjectPermissions(DjangoObjectPermissions): """ @@ -35,22 +37,24 @@ class ObjectTagObjectPermissions(DjangoObjectPermissions): } -class TagListPermissions(DjangoObjectPermissions): +class TagObjectPermissions(DjangoObjectPermissions): """ - Permissions for Tag object views. + Maps each REST API methods to its corresponding Tag permission. """ - def has_permission(self, request, view): - """ - Returns True if the user on the given request is allowed the given view. - """ - if not request.user or ( - not request.user.is_authenticated and self.authenticated_users_only - ): - return False - return True + perms_map = { + "GET": ["%(app_label)s.view_%(model_name)s"], + "OPTIONS": [], + "HEAD": ["%(app_label)s.view_%(model_name)s"], + "POST": ["%(app_label)s.add_%(model_name)s"], + "PUT": ["%(app_label)s.change_%(model_name)s"], + "PATCH": ["%(app_label)s.change_%(model_name)s"], + "DELETE": ["%(app_label)s.delete_%(model_name)s"], + } + # This is to handle the special case for GET list of Taxonomy Tags def has_object_permission(self, request, view, obj): """ Returns True if the user on the given request is allowed the given view for the given object. """ + obj = obj.taxonomy if isinstance(obj, Tag) else obj return rules.has_perm("oel_tagging.list_tag", request.user, obj) diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index a4eb89ffa..c34c15dad 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -1,7 +1,6 @@ """ API Serializers for taxonomies """ - from rest_framework import serializers from rest_framework.reverse import reverse @@ -110,6 +109,7 @@ class Meta: "value", "taxonomy_id", "parent_id", + "external_id", "sub_tags_link", "children_count", ) @@ -120,11 +120,12 @@ def get_sub_tags_link(self, obj): """ if obj.children.count(): query_params = f"?parent_tag_id={obj.id}" + request = self.context.get("request") + url_namespace = request.resolver_match.namespace # get the namespace, usually "oel_tagging" url = ( - reverse("oel_tagging:taxonomy-tags", args=[str(obj.taxonomy_id)]) + reverse(f"{url_namespace}:taxonomy-tags", args=[str(obj.taxonomy_id)]) + query_params ) - request = self.context.get("request") return request.build_absolute_uri(url) return None @@ -192,3 +193,33 @@ def get_children_count(self, obj): Returns the number of child tags of the given tag. """ return len(obj.sub_tags) + + +class TaxonomyTagCreateBodySerializer(serializers.Serializer): # pylint: disable=abstract-method + """ + Serializer of the body for the Taxonomy Tags CREATE request + """ + + tag = serializers.CharField(required=True) + parent_tag_value = serializers.CharField(required=False) + external_id = serializers.CharField(required=False) + + +class TaxonomyTagUpdateBodySerializer(serializers.Serializer): # pylint: disable=abstract-method + """ + Serializer of the body for the Taxonomy Tags UPDATE request + """ + + tag = serializers.CharField(required=True) + updated_tag_value = serializers.CharField(required=True) + + +class TaxonomyTagDeleteBodySerializer(serializers.Serializer): # pylint: disable=abstract-method + """ + Serializer of the body for the Taxonomy Tags DELETE request + """ + + tags = serializers.ListField( + child=serializers.CharField(), required=True + ) + with_subtags = serializers.BooleanField(required=False) diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 743f5bcd2..6428043c0 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -5,17 +5,20 @@ from django.db import models from django.http import Http404, HttpResponse -from rest_framework import mixins +from rest_framework import mixins, status from rest_framework.decorators import action from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError -from rest_framework.generics import ListAPIView +from rest_framework.generics import ListAPIView, RetrieveUpdateDestroyAPIView from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet, ModelViewSet from openedx_tagging.core.tagging.models.base import Tag from ...api import ( + TagDoesNotExist, + add_tag_to_taxonomy, create_taxonomy, + delete_tags_from_taxonomy, get_children_tags, get_object_tags, get_root_tags, @@ -23,13 +26,14 @@ get_taxonomy, search_tags, tag_object, + update_tag_in_taxonomy, ) from ...import_export.api import export_tags from ...import_export.parsers import ParserFormat from ...models import Taxonomy from ...rules import ObjectTagPermissionItem from ..paginators import SEARCH_TAGS_THRESHOLD, TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination -from .permissions import ObjectTagObjectPermissions, TagListPermissions, TaxonomyObjectPermissions +from .permissions import ObjectTagObjectPermissions, TagObjectPermissions, TaxonomyObjectPermissions from .serializers import ( ObjectTagListQueryParamsSerializer, ObjectTagSerializer, @@ -41,6 +45,9 @@ TaxonomyExportQueryParamsSerializer, TaxonomyListQueryParamsSerializer, TaxonomySerializer, + TaxonomyTagCreateBodySerializer, + TaxonomyTagDeleteBodySerializer, + TaxonomyTagUpdateBodySerializer, ) from .utils import view_auth_classes @@ -395,7 +402,7 @@ def update(self, request, *args, **kwargs) -> Response: tags = body.data.get("tags", []) try: tag_object(taxonomy, tags, object_id) - except Tag.DoesNotExist as e: + except TagDoesNotExist as e: raise ValidationError from e except ValueError as e: raise ValidationError from e @@ -404,28 +411,96 @@ def update(self, request, *args, **kwargs) -> Response: @view_auth_classes -class TaxonomyTagsView(ListAPIView): +class TaxonomyTagsView(ListAPIView, RetrieveUpdateDestroyAPIView): """ - View to list tags of a taxonomy. + View to list/create/update/delete tags of a taxonomy. **List Query Parameters** - * pk (required) - The pk of the taxonomy to retrieve tags. + * id (required) - The ID of the taxonomy to retrieve tags. * parent_tag_id (optional) - Id of the tag to retrieve children tags. * page (optional) - Page number (default: 1) * page_size (optional) - Number of items per page (default: 10) **List Example Requests** - GET api/tagging/v1/taxonomy/:pk/tags - Get tags of taxonomy - GET api/tagging/v1/taxonomy/:pk/tags?parent_tag_id=30 - Get children tags of tag + GET api/tagging/v1/taxonomy/:id/tags - Get tags of taxonomy + GET api/tagging/v1/taxonomy/:id/tags?parent_tag_id=30 - Get children tags of tag **List Query Returns** * 200 - Success * 400 - Invalid query parameter * 403 - Permission denied * 404 - Taxonomy not found + + **Create Query Parameters** + * id (required) - The ID of the taxonomy to create a Tag for + + **Create Request Body** + * tag (required): The value of the Tag that should be added to + the Taxonomy + * parent_tag_value (optional): The value of the parent tag that the new + Tag should fall under + * extenal_id (optional): The external id for the new Tag + + **Create Example Requests** + POST api/tagging/v1/taxonomy/:id/tags - Create a Tag in taxonomy + { + "value": "New Tag", + "parent_tag_value": "Parent Tag" + "external_id": "abc123", + } + + **Create Query Returns** + * 201 - Success + * 400 - Invalid parameters provided + * 403 - Permission denied + * 404 - Taxonomy not found + + **Update Query Parameters** + * id (required) - The ID of the taxonomy to update a Tag in + + **Update Request Body** + * tag (required): The value (identifier) of the Tag to be updated + * updated_tag_value (required): The updated value of the Tag + + **Update Example Requests** + PATCH api/tagging/v1/taxonomy/:id/tags - Update a Tag in Taxonomy + { + "tag": "Tag 1", + "updated_tag_value": "Updated Tag Value" + } + + **Update Query Returns** + * 200 - Success + * 400 - Invalid parameters provided + * 403 - Permission denied + * 404 - Taxonomy, Tag or Parent Tag not found + + **Delete Query Parameters** + * id (required) - The ID of the taxonomy to Delete Tag(s) in + + **Delete Request Body** + * tags (required): The values (identifiers) of Tags that should be + deleted from Taxonomy + * with_subtags (optional): If a Tag in the provided ids contains + children (subtags), deletion will fail unless + set to `True`. Defaults to `False`. + + **Delete Example Requests** + DELETE api/tagging/v1/taxonomy/:id/tags - Delete Tag(s) in Taxonomy + { + "tags": ["Tag 1", "Tag 2", "Tag 3"], + "with_subtags": True + } + + **Delete Query Returns** + * 200 - Success + * 400 - Invalid parameters provided + * 403 - Permission denied + * 404 - Taxonomy not found + """ - permission_classes = [TagListPermissions] + permission_classes = [TagObjectPermissions] pagination_enabled = True def __init__(self): @@ -558,7 +633,7 @@ def get_matching_tags( return result - def get_queryset(self) -> list[Tag]: # type: ignore[override] + def get_queryset(self) -> models.QuerySet[Tag]: # type: ignore[override] """ Builds and returns the queryset to be paginated. @@ -576,7 +651,103 @@ def get_queryset(self) -> list[Tag]: # type: ignore[override] search_term=search_term, ) + # Convert the results back to a QuerySet for permissions to apply + # Due to the conversion we lose the populated `sub_tags` attribute, + # in the case of using the special search serializer so we + # need to repopulate it again + if self.serializer_class == TagsForSearchSerializer: + results_dict = {tag.id: tag for tag in result} + + result_queryset = Tag.objects.filter(id__in=results_dict.keys()) + + for tag in result_queryset: + sub_tags = results_dict[tag.id].sub_tags # type: ignore[attr-defined] + tag.sub_tags = sub_tags # type: ignore[attr-defined] + + else: + result_queryset = Tag.objects.filter(id__in=[tag.id for tag in result]) + # This function is not called automatically self.pagination_class = self.get_pagination_class() - return result + return result_queryset + + def post(self, request, *args, **kwargs): + """ + Creates new Tag in Taxonomy and returns the newly created Tag. + """ + pk = self.kwargs.get("pk") + taxonomy = self.get_taxonomy(pk) + + body = TaxonomyTagCreateBodySerializer(data=request.data) + body.is_valid(raise_exception=True) + + tag = body.data.get("tag") + parent_tag_value = body.data.get("parent_tag_value", None) + external_id = body.data.get("external_id", None) + + try: + new_tag = add_tag_to_taxonomy( + taxonomy, tag, parent_tag_value, external_id + ) + except TagDoesNotExist as e: + raise Http404("Parent Tag not found") from e + except ValueError as e: + raise ValidationError(e) from e + + self.serializer_class = TagsSerializer + serializer_context = self.get_serializer_context() + return Response( + self.serializer_class(new_tag, context=serializer_context).data, + status=status.HTTP_201_CREATED + ) + + def update(self, request, *args, **kwargs): + """ + Updates a Tag that belongs to the Taxonomy and returns it. + Currently only updating the Tag value is supported. + """ + pk = self.kwargs.get("pk") + taxonomy = self.get_taxonomy(pk) + + body = TaxonomyTagUpdateBodySerializer(data=request.data) + body.is_valid(raise_exception=True) + + tag = body.data.get("tag") + updated_tag_value = body.data.get("updated_tag_value") + + try: + updated_tag = update_tag_in_taxonomy(taxonomy, tag, updated_tag_value) + except TagDoesNotExist as e: + raise Http404("Tag not found") from e + except ValueError as e: + raise ValidationError(e) from e + + self.serializer_class = TagsSerializer + serializer_context = self.get_serializer_context() + return Response( + self.serializer_class(updated_tag, context=serializer_context).data, + status=status.HTTP_200_OK + ) + + def delete(self, request, *args, **kwargs): + """ + Deletes Tag(s) in Taxonomy. If any of the Tags have children and + the `with_subtags` is not set to `True` it will fail, otherwise + the sub-tags will be deleted as well. + """ + pk = self.kwargs.get("pk") + taxonomy = self.get_taxonomy(pk) + + body = TaxonomyTagDeleteBodySerializer(data=request.data) + body.is_valid(raise_exception=True) + + tags = body.data.get("tags") + with_subtags = body.data.get("with_subtags") + + try: + delete_tags_from_taxonomy(taxonomy, tags, with_subtags) + except ValueError as e: + raise ValidationError(e) from e + + return Response(status=status.HTTP_200_OK) diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 8249628f8..97ca56de2 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -51,17 +51,27 @@ def can_change_taxonomy(user: UserType, taxonomy: Taxonomy | None = None) -> boo ) +@rules.predicate +def can_view_tag(user: UserType, tag: Tag | None = None) -> bool: + """ + User can view tags for any taxonomy they can view. + """ + taxonomy = tag.taxonomy.cast() if (tag and tag.taxonomy) else None + return user.has_perm( + "oel_tagging.view_taxonomy", + taxonomy, + ) + + @rules.predicate def can_change_tag(user: UserType, tag: Tag | None = None) -> bool: """ - Even taxonomy admins cannot add tags to system taxonomies (their tags are system-defined), or free-text taxonomies - (these don't have predefined tags). + Users can change tags for any taxonomy they can modify. """ taxonomy = tag.taxonomy.cast() if (tag and tag.taxonomy) else None - return is_taxonomy_admin(user) and ( - not tag - or not taxonomy - or (taxonomy and not taxonomy.allow_free_text and not taxonomy.system_defined) + return user.has_perm( + "oel_tagging.change_taxonomy", + taxonomy, ) @@ -166,8 +176,10 @@ def can_change_object_tag( # Tag rules.add_perm("oel_tagging.add_tag", can_change_tag) rules.add_perm("oel_tagging.change_tag", can_change_tag) -rules.add_perm("oel_tagging.delete_tag", is_taxonomy_admin) -rules.add_perm("oel_tagging.view_tag", rules.always_allow) +rules.add_perm("oel_tagging.delete_tag", can_change_tag) +rules.add_perm("oel_tagging.view_tag", can_view_tag) +# Special Case for listing Tags, we check if we can view the Taxonomy since +# that is what is passed in rather than a Tag object rules.add_perm("oel_tagging.list_tag", can_view_taxonomy) # ObjectTag diff --git a/tests/openedx_tagging/core/tagging/test_rules.py b/tests/openedx_tagging/core/tagging/test_rules.py index f30cbd27c..11cc2ced9 100644 --- a/tests/openedx_tagging/core/tagging/test_rules.py +++ b/tests/openedx_tagging/core/tagging/test_rules.py @@ -141,12 +141,12 @@ def test_add_change_tag(self, perm): ) def test_tag_free_text_taxonomy(self, perm): """ - Taxonomy administrators cannot modify tags on a free-text Taxonomy + Taxonomy administrators can modify any Tag, even those associated with a free-text Taxonomy """ self.taxonomy.allow_free_text = True self.taxonomy.save() assert self.superuser.has_perm(perm, self.bacteria) - assert not self.staff.has_perm(perm, self.bacteria) + assert self.staff.has_perm(perm, self.bacteria) assert not self.learner.has_perm(perm, self.bacteria) @ddt.data( diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index d97296ba0..b52c0e64d 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -897,7 +897,7 @@ def test_tag_object_count_limit(self): class TestTaxonomyTagsView(TestTaxonomyViewMixin): """ - Tests the list tags of taxonomy view + Tests the list/create/update/delete tags of taxonomy view """ fixtures = ["tests/openedx_tagging/core/fixtures/tagging.yaml"] @@ -1209,3 +1209,543 @@ def test_next_children(self): assert data.get("count") == self.children_tags_count[0] assert data.get("num_pages") == 2 assert data.get("current_page") == 2 + + def test_create_tag_in_taxonomy_while_loggedout(self): + new_tag_value = "New Tag" + + create_data = { + "tag": new_tag_value + } + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + def test_create_tag_in_taxonomy_without_permission(self): + self.client.force_authenticate(user=self.user) + new_tag_value = "New Tag" + + create_data = { + "tag": new_tag_value + } + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_create_tag_in_taxonomy(self): + self.client.force_authenticate(user=self.staff) + new_tag_value = "New Tag" + + create_data = { + "tag": new_tag_value + } + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_201_CREATED + + data = response.data + + self.assertIsNotNone(data.get("id")) + self.assertEqual(data.get("value"), new_tag_value) + self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) + self.assertIsNone(data.get("parent_id")) + self.assertIsNone(data.get("external_id")) + self.assertIsNone(data.get("sub_tags_link")) + self.assertEqual(data.get("children_count"), 0) + + def test_create_tag_in_taxonomy_with_parent(self): + self.client.force_authenticate(user=self.staff) + parent_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + new_tag_value = "New Child Tag" + new_external_id = "extId" + + create_data = { + "tag": new_tag_value, + "parent_tag_value": parent_tag.value, + "external_id": new_external_id + } + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_201_CREATED + + data = response.data + + self.assertIsNotNone(data.get("id")) + self.assertEqual(data.get("value"), new_tag_value) + self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) + self.assertEqual(data.get("parent_id"), parent_tag.id) + self.assertEqual(data.get("external_id"), new_external_id) + self.assertIsNone(data.get("sub_tags_link")) + self.assertEqual(data.get("children_count"), 0) + + def test_create_tag_in_invalid_taxonomy(self): + self.client.force_authenticate(user=self.staff) + new_tag_value = "New Tag" + + create_data = { + "tag": new_tag_value + } + + invalid_taxonomy_url = TAXONOMY_TAGS_URL.format(pk=919191) + response = self.client.post( + invalid_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_create_tag_in_free_text_taxonomy(self): + self.client.force_authenticate(user=self.staff) + new_tag_value = "New Tag" + + create_data = { + "tag": new_tag_value + } + + # Setting free text flag on taxonomy + self.small_taxonomy.allow_free_text = True + self.small_taxonomy.save() + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_create_tag_in_system_defined_taxonomy(self): + self.client.force_authenticate(user=self.staff) + new_tag_value = "New Tag" + + create_data = { + "tag": new_tag_value + } + + # Setting taxonomy to be system defined + self.small_taxonomy.taxonomy_class = SystemDefinedTaxonomy + self.small_taxonomy.save() + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_create_tag_in_taxonomy_with_invalid_parent_tag(self): + self.client.force_authenticate(user=self.staff) + invalid_parent_tag = "Invalid Tag" + new_tag_value = "New Child Tag" + + create_data = { + "tag": new_tag_value, + "parent_tag_value": invalid_parent_tag, + } + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_create_tag_in_taxonomy_with_parent_tag_in_other_taxonomy(self): + self.client.force_authenticate(user=self.staff) + tag_in_other_taxonomy = Tag.objects.get(id=1) + new_tag_value = "New Child Tag" + + create_data = { + "tag": new_tag_value, + "parent_tag_value": tag_in_other_taxonomy.value, + } + + response = self.client.post( + self.large_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_create_tag_in_taxonomy_with_already_existing_value(self): + self.client.force_authenticate(user=self.staff) + new_tag_value = "New Tag" + + create_data = { + "tag": new_tag_value + } + + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_201_CREATED + + # Make request again with the same Tag value after it was created + response = self.client.post( + self.small_taxonomy_url, create_data, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_update_tag_in_taxonomy_while_loggedout(self): + updated_tag_value = "Updated Tag" + + # Existing Tag that will be updated + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + update_data = { + "tag": existing_tag.value, + "updated_tag_value": updated_tag_value + } + + # Test updating using the PUT method + response = self.client.put( + self.small_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + def test_update_tag_in_taxonomy_without_permission(self): + self.client.force_authenticate(user=self.user) + updated_tag_value = "Updated Tag" + + # Existing Tag that will be updated + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + update_data = { + "tag": existing_tag.value, + "updated_tag_value": updated_tag_value + } + + # Test updating using the PUT method + response = self.client.put( + self.small_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_update_tag_in_taxonomy_with_different_methods(self): + self.client.force_authenticate(user=self.staff) + updated_tag_value = "Updated Tag" + updated_tag_value_2 = "Updated Tag 2" + + # Existing Tag that will be updated + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + update_data = { + "tag": existing_tag.value, + "updated_tag_value": updated_tag_value + } + + # Test updating using the PUT method + response = self.client.put( + self.small_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + data = response.data + + # Check that Tag value got updated + self.assertEqual(data.get("id"), existing_tag.id) + self.assertEqual(data.get("value"), updated_tag_value) + self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) + self.assertEqual(data.get("parent_id"), existing_tag.parent) + self.assertEqual(data.get("external_id"), existing_tag.external_id) + + # Test updating using the PATCH method + update_data["tag"] = updated_tag_value # Since the value changed + update_data["updated_tag_value"] = updated_tag_value_2 + response = self.client.patch( + self.small_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + data = response.data + + # Check the Tag value got updated again + self.assertEqual(data.get("id"), existing_tag.id) + self.assertEqual(data.get("value"), updated_tag_value_2) + self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) + self.assertEqual(data.get("parent_id"), existing_tag.parent) + self.assertEqual(data.get("external_id"), existing_tag.external_id) + + def test_update_tag_in_taxonomy_reflects_changes_in_object_tags(self): + self.client.force_authenticate(user=self.staff) + + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + # Setup ObjectTags + # _value=existing_tag.value + object_tag_1 = ObjectTag.objects.create( + object_id="abc", taxonomy=self.small_taxonomy, tag=existing_tag + ) + object_tag_2 = ObjectTag.objects.create( + object_id="def", taxonomy=self.small_taxonomy, tag=existing_tag + ) + object_tag_3 = ObjectTag.objects.create( + object_id="ghi", taxonomy=self.small_taxonomy, tag=existing_tag + ) + + assert object_tag_1.value == existing_tag.value + assert object_tag_2.value == existing_tag.value + assert object_tag_3.value == existing_tag.value + + updated_tag_value = "Updated Tag" + update_data = { + "tag": existing_tag.value, + "updated_tag_value": updated_tag_value + } + + # Test updating using the PUT method + response = self.client.put( + self.small_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + data = response.data + + # Check that Tag value got updated + self.assertEqual(data.get("id"), existing_tag.id) + self.assertEqual(data.get("value"), updated_tag_value) + self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) + self.assertEqual(data.get("parent_id"), existing_tag.parent) + self.assertEqual(data.get("external_id"), existing_tag.external_id) + + # Check that the ObjectTags got updated as well + object_tag_1.refresh_from_db() + self.assertEqual(object_tag_1.value, updated_tag_value) + object_tag_2.refresh_from_db() + self.assertEqual(object_tag_2.value, updated_tag_value) + object_tag_3.refresh_from_db() + self.assertEqual(object_tag_3.value, updated_tag_value) + + def test_update_tag_in_taxonomy_with_invalid_tag(self): + self.client.force_authenticate(user=self.staff) + updated_tag_value = "Updated Tag" + + update_data = { + "tag": 919191, + "updated_tag_value": updated_tag_value + } + + response = self.client.put( + self.small_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_update_tag_in_taxonomy_with_tag_in_other_taxonomy(self): + self.client.force_authenticate(user=self.staff) + updated_tag_value = "Updated Tag" + tag_in_other_taxonomy = Tag.objects.get(id=1) + + update_data = { + "tag": tag_in_other_taxonomy.value, + "updated_tag_value": updated_tag_value + } + + response = self.client.put( + self.large_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_update_tag_in_taxonomy_with_no_tag_value_provided(self): + self.client.force_authenticate(user=self.staff) + + # Existing Tag that will be updated + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + update_data = { + "tag": existing_tag.value + } + + response = self.client.put( + self.small_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_update_tag_in_invalid_taxonomy(self): + self.client.force_authenticate(user=self.staff) + + # Existing Tag that will be updated + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + updated_tag_value = "Updated Tag" + update_data = { + "tag": existing_tag.value, + "updated_tag_value": updated_tag_value + } + + invalid_taxonomy_url = TAXONOMY_TAGS_URL.format(pk=919191) + response = self.client.put( + invalid_taxonomy_url, update_data, format="json" + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_delete_single_tag_from_taxonomy_while_loggedout(self): + # Get Tag that will be deleted + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + delete_data = { + "tags": [existing_tag.value], + "with_subtags": True + } + + response = self.client.delete( + self.small_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_401_UNAUTHORIZED + + def test_delete_single_tag_from_taxonomy_without_permission(self): + self.client.force_authenticate(user=self.user) + # Get Tag that will be deleted + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + delete_data = { + "tags": [existing_tag.value], + "with_subtags": True + } + + response = self.client.delete( + self.small_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_403_FORBIDDEN + + def test_delete_single_tag_from_taxonomy(self): + self.client.force_authenticate(user=self.staff) + + # Get Tag that will be deleted + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + delete_data = { + "tags": [existing_tag.value], + "with_subtags": True + } + + response = self.client.delete( + self.small_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + # Check that Tag no longer exists + with self.assertRaises(Tag.DoesNotExist): + existing_tag.refresh_from_db() + + def test_delete_multiple_tags_from_taxonomy(self): + self.client.force_authenticate(user=self.staff) + + # Get Tags that will be deleted + existing_tags = self.small_taxonomy.tag_set.filter(parent=None)[:3] + + delete_data = { + "tags": [existing_tag.value for existing_tag in existing_tags], + "with_subtags": True + } + + response = self.client.delete( + self.small_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + # Check that Tags no longer exists + for existing_tag in existing_tags: + with self.assertRaises(Tag.DoesNotExist): + existing_tag.refresh_from_db() + + def test_delete_tag_with_subtags_should_fail_without_flag_passed(self): + self.client.force_authenticate(user=self.staff) + + # Get Tag that will be deleted + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + delete_data = { + "tags": [existing_tag.value] + } + + response = self.client.delete( + self.small_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_delete_tag_in_invalid_taxonomy(self): + self.client.force_authenticate(user=self.staff) + + # Get Tag that will be deleted + existing_tag = self.small_taxonomy.tag_set.filter(parent=None).first() + + delete_data = { + "tags": [existing_tag.value] + } + + invalid_taxonomy_url = TAXONOMY_TAGS_URL.format(pk=919191) + response = self.client.delete( + invalid_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_404_NOT_FOUND + + def test_delete_tag_in_taxonomy_with_invalid_tag(self): + self.client.force_authenticate(user=self.staff) + + delete_data = { + "tags": ["Invalid Tag"] + } + + response = self.client.delete( + self.small_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_delete_tag_with_tag_in_other_taxonomy(self): + self.client.force_authenticate(user=self.staff) + + # Get Tag in other Taxonomy + tag_in_other_taxonomy = self.small_taxonomy.tag_set.filter(parent=None).first() + + delete_data = { + "tags": [tag_in_other_taxonomy.value] + } + + response = self.client.delete( + self.large_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + + def test_delete_tag_in_taxonomy_without_subtags(self): + self.client.force_authenticate(user=self.staff) + + # Get Tag that will be deleted + existing_tag = self.small_taxonomy.tag_set.filter(children__isnull=True).first() + + delete_data = { + "tags": [existing_tag.value] + } + + response = self.client.delete( + self.small_taxonomy_url, delete_data, format="json" + ) + + assert response.status_code == status.HTTP_200_OK + + # Check that Tag no longer exists + with self.assertRaises(Tag.DoesNotExist): + existing_tag.refresh_from_db() From 9b5635dacf0ec917df19c03a49440c67def87c89 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 22 Oct 2023 20:19:54 -0400 Subject: [PATCH 062/282] chore: Updating Python Requirements --- requirements/base.txt | 9 +++++---- requirements/dev.txt | 35 +++++++++++++++-------------------- requirements/doc.txt | 21 ++++++++++++--------- requirements/quality.txt | 31 +++++++++++++++---------------- requirements/test.txt | 20 +++++++++++--------- 5 files changed, 58 insertions(+), 58 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 01f645cd9..c8af9b44e 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -24,7 +24,7 @@ cffi==1.16.0 # via # cryptography # pynacl -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via requests click==8.1.7 # via @@ -66,7 +66,7 @@ drf-jwt==1.19.2 # via edx-drf-extensions edx-django-utils==5.7.0 # via edx-drf-extensions -edx-drf-extensions==8.11.1 +edx-drf-extensions==8.12.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via edx-drf-extensions @@ -74,7 +74,7 @@ idna==3.4 # via requests kombu==5.3.2 # via celery -newrelic==9.1.0 +newrelic==9.1.1 # via edx-django-utils pbr==5.11.1 # via stevedore @@ -88,6 +88,7 @@ pyjwt[crypto]==2.8.0 # via # drf-jwt # edx-drf-extensions + # pyjwt pymongo==3.13.0 # via edx-opaque-keys pynacl==1.5.0 @@ -121,7 +122,7 @@ tzdata==2023.3 # via # backports-zoneinfo # celery -urllib3==2.0.6 +urllib3==2.0.7 # via requests vine==5.0.0 # via diff --git a/requirements/dev.txt b/requirements/dev.txt index 37e215f46..97b58609d 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -12,7 +12,7 @@ asgiref==3.7.2 # via # -r requirements/quality.txt # django -astroid==2.15.8 +astroid==3.0.1 # via # -r requirements/quality.txt # pylint @@ -22,6 +22,7 @@ attrs==23.1.0 backports-zoneinfo[tzdata]==0.2.1 # via # -r requirements/quality.txt + # backports-zoneinfo # celery # kombu billiard==4.1.0 @@ -45,7 +46,7 @@ cffi==1.16.0 # pynacl chardet==5.2.0 # via diff-cover -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via # -r requirements/quality.txt # requests @@ -87,6 +88,7 @@ code-annotations==1.5.0 coverage[toml]==7.3.2 # via # -r requirements/quality.txt + # coverage # pytest-cov cryptography==41.0.4 # via @@ -127,11 +129,11 @@ django-debug-toolbar==4.2.0 # via # -r requirements/dev.in # -r requirements/quality.txt -django-stubs==4.2.4 +django-stubs==4.2.5 # via # -r requirements/quality.txt # djangorestframework-stubs -django-stubs-ext==4.2.2 +django-stubs-ext==4.2.5 # via # -r requirements/quality.txt # django-stubs @@ -145,7 +147,7 @@ djangorestframework==3.14.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.3 +djangorestframework-stubs==3.14.4 # via -r requirements/quality.txt docutils==0.20.1 # via @@ -159,7 +161,7 @@ edx-django-utils==5.7.0 # via # -r requirements/quality.txt # edx-drf-extensions -edx-drf-extensions==8.11.1 +edx-drf-extensions==8.12.0 # via -r requirements/quality.txt edx-i18n-tools==1.3.0 # via -r requirements/dev.in @@ -232,10 +234,6 @@ kombu==5.3.2 # via # -r requirements/quality.txt # celery -lazy-object-proxy==1.9.0 - # via - # -r requirements/quality.txt - # astroid lxml==4.9.3 # via edx-i18n-tools markdown-it-py==3.0.0 @@ -260,7 +258,7 @@ more-itertools==10.1.0 # via # -r requirements/quality.txt # jaraco-classes -mypy==1.6.0 +mypy==1.6.1 # via # -r requirements/quality.txt # django-stubs @@ -271,7 +269,7 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/quality.txt -newrelic==9.1.0 +newrelic==9.1.1 # via # -r requirements/quality.txt # edx-django-utils @@ -345,7 +343,8 @@ pyjwt[crypto]==2.8.0 # -r requirements/quality.txt # drf-jwt # edx-drf-extensions -pylint==2.17.7 + # pyjwt +pylint==3.0.2 # via # -r requirements/quality.txt # edx-lint @@ -356,7 +355,7 @@ pylint-celery==0.3 # via # -r requirements/quality.txt # edx-lint -pylint-django==2.5.3 +pylint-django==2.5.4 # via # -r requirements/quality.txt # edx-lint @@ -500,7 +499,7 @@ types-pyyaml==6.0.12.12 # -r requirements/quality.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.9 +types-requests==2.31.0.10 # via # -r requirements/quality.txt # djangorestframework-stubs @@ -525,7 +524,7 @@ tzdata==2023.3 # -r requirements/quality.txt # backports-zoneinfo # celery -urllib3==2.0.6 +urllib3==2.0.7 # via # -r requirements/quality.txt # requests @@ -549,10 +548,6 @@ wheel==0.41.2 # via # -r requirements/pip-tools.txt # pip-tools -wrapt==1.15.0 - # via - # -r requirements/quality.txt - # astroid zipp==3.17.0 # via # -r requirements/pip-tools.txt diff --git a/requirements/doc.txt b/requirements/doc.txt index edff4bda4..345c626ef 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -25,6 +25,7 @@ babel==2.13.0 backports-zoneinfo[tzdata]==0.2.1 # via # -r requirements/test.txt + # backports-zoneinfo # celery # kombu beautifulsoup4==4.12.2 @@ -44,7 +45,7 @@ cffi==1.16.0 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via # -r requirements/test.txt # requests @@ -75,6 +76,7 @@ code-annotations==1.5.0 coverage[toml]==7.3.2 # via # -r requirements/test.txt + # coverage # pytest-cov cryptography==41.0.4 # via @@ -102,11 +104,11 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.2.0 # via -r requirements/test.txt -django-stubs==4.2.4 +django-stubs==4.2.5 # via # -r requirements/test.txt # djangorestframework-stubs -django-stubs-ext==4.2.2 +django-stubs-ext==4.2.5 # via # -r requirements/test.txt # django-stubs @@ -120,7 +122,7 @@ djangorestframework==3.14.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.3 +djangorestframework-stubs==3.14.4 # via -r requirements/test.txt doc8==1.1.1 # via -r requirements/doc.in @@ -139,7 +141,7 @@ edx-django-utils==5.7.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.11.1 +edx-drf-extensions==8.12.0 # via -r requirements/test.txt edx-opaque-keys==2.5.1 # via @@ -182,7 +184,7 @@ markupsafe==2.1.3 # jinja2 mock==5.1.0 # via -r requirements/test.txt -mypy==1.6.0 +mypy==1.6.1 # via # -r requirements/test.txt # django-stubs @@ -193,7 +195,7 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==9.1.0 +newrelic==9.1.1 # via # -r requirements/test.txt # edx-django-utils @@ -241,6 +243,7 @@ pyjwt[crypto]==2.8.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions + # pyjwt pymongo==3.13.0 # via # -r requirements/test.txt @@ -356,7 +359,7 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.9 +types-requests==2.31.0.10 # via # -r requirements/test.txt # djangorestframework-stubs @@ -378,7 +381,7 @@ tzdata==2023.3 # -r requirements/test.txt # backports-zoneinfo # celery -urllib3==2.0.6 +urllib3==2.0.7 # via # -r requirements/test.txt # requests diff --git a/requirements/quality.txt b/requirements/quality.txt index c3c49a0ef..f5b88b43c 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -12,7 +12,7 @@ asgiref==3.7.2 # via # -r requirements/test.txt # django -astroid==2.15.8 +astroid==3.0.1 # via # pylint # pylint-celery @@ -21,6 +21,7 @@ attrs==23.1.0 backports-zoneinfo[tzdata]==0.2.1 # via # -r requirements/test.txt + # backports-zoneinfo # celery # kombu billiard==4.1.0 @@ -38,7 +39,7 @@ cffi==1.16.0 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via # -r requirements/test.txt # requests @@ -75,6 +76,7 @@ code-annotations==1.5.0 coverage[toml]==7.3.2 # via # -r requirements/test.txt + # coverage # pytest-cov cryptography==41.0.4 # via @@ -104,11 +106,11 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.2.0 # via -r requirements/test.txt -django-stubs==4.2.4 +django-stubs==4.2.5 # via # -r requirements/test.txt # djangorestframework-stubs -django-stubs-ext==4.2.2 +django-stubs-ext==4.2.5 # via # -r requirements/test.txt # django-stubs @@ -122,7 +124,7 @@ djangorestframework==3.14.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.3 +djangorestframework-stubs==3.14.4 # via -r requirements/test.txt docutils==0.20.1 # via readme-renderer @@ -134,7 +136,7 @@ edx-django-utils==5.7.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.11.1 +edx-drf-extensions==8.12.0 # via -r requirements/test.txt edx-lint==5.3.4 # via -r requirements/quality.in @@ -186,8 +188,6 @@ kombu==5.3.2 # via # -r requirements/test.txt # celery -lazy-object-proxy==1.9.0 - # via astroid markdown-it-py==3.0.0 # via rich markupsafe==2.1.3 @@ -202,7 +202,7 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.1.0 # via jaraco-classes -mypy==1.6.0 +mypy==1.6.1 # via # -r requirements/test.txt # django-stubs @@ -213,7 +213,7 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==9.1.0 +newrelic==9.1.1 # via # -r requirements/test.txt # edx-django-utils @@ -260,7 +260,8 @@ pyjwt[crypto]==2.8.0 # -r requirements/test.txt # drf-jwt # edx-drf-extensions -pylint==2.17.7 + # pyjwt +pylint==3.0.2 # via # edx-lint # pylint-celery @@ -268,7 +269,7 @@ pylint==2.17.7 # pylint-plugin-utils pylint-celery==0.3 # via edx-lint -pylint-django==2.5.3 +pylint-django==2.5.4 # via edx-lint pylint-plugin-utils==0.8.2 # via @@ -375,7 +376,7 @@ types-pyyaml==6.0.12.12 # -r requirements/test.txt # django-stubs # djangorestframework-stubs -types-requests==2.31.0.9 +types-requests==2.31.0.10 # via # -r requirements/test.txt # djangorestframework-stubs @@ -399,7 +400,7 @@ tzdata==2023.3 # -r requirements/test.txt # backports-zoneinfo # celery -urllib3==2.0.6 +urllib3==2.0.7 # via # -r requirements/test.txt # requests @@ -415,8 +416,6 @@ wcwidth==0.2.8 # via # -r requirements/test.txt # prompt-toolkit -wrapt==1.15.0 - # via astroid zipp==3.17.0 # via # importlib-metadata diff --git a/requirements/test.txt b/requirements/test.txt index 549b890f6..60ebf2389 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -17,6 +17,7 @@ attrs==23.1.0 backports-zoneinfo[tzdata]==0.2.1 # via # -r requirements/base.txt + # backports-zoneinfo # celery # kombu billiard==4.1.0 @@ -34,7 +35,7 @@ cffi==1.16.0 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 # via # -r requirements/base.txt # requests @@ -90,11 +91,11 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.2.0 # via -r requirements/test.in -django-stubs==4.2.4 +django-stubs==4.2.5 # via # -r requirements/test.in # djangorestframework-stubs -django-stubs-ext==4.2.2 +django-stubs-ext==4.2.5 # via django-stubs django-waffle==4.0.0 # via @@ -106,7 +107,7 @@ djangorestframework==3.14.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions -djangorestframework-stubs==3.14.3 +djangorestframework-stubs==3.14.4 # via -r requirements/test.in drf-jwt==1.19.2 # via @@ -116,7 +117,7 @@ edx-django-utils==5.7.0 # via # -r requirements/base.txt # edx-drf-extensions -edx-drf-extensions==8.11.1 +edx-drf-extensions==8.12.0 # via -r requirements/base.txt edx-opaque-keys==2.5.1 # via @@ -144,7 +145,7 @@ markupsafe==2.1.3 # via jinja2 mock==5.1.0 # via -r requirements/test.in -mypy==1.6.0 +mypy==1.6.1 # via # -r requirements/test.in # django-stubs @@ -153,7 +154,7 @@ mypy-extensions==1.0.0 # via mypy mysqlclient==2.2.0 # via -r requirements/test.in -newrelic==9.1.0 +newrelic==9.1.1 # via # -r requirements/base.txt # edx-django-utils @@ -182,6 +183,7 @@ pyjwt[crypto]==2.8.0 # -r requirements/base.txt # drf-jwt # edx-drf-extensions + # pyjwt pymongo==3.13.0 # via # -r requirements/base.txt @@ -253,7 +255,7 @@ types-pyyaml==6.0.12.12 # via # django-stubs # djangorestframework-stubs -types-requests==2.31.0.9 +types-requests==2.31.0.10 # via djangorestframework-stubs typing-extensions==4.8.0 # via @@ -272,7 +274,7 @@ tzdata==2023.3 # -r requirements/base.txt # backports-zoneinfo # celery -urllib3==2.0.6 +urllib3==2.0.7 # via # -r requirements/base.txt # requests From 8909421d3c8811aee95f772f1926f519c02727f5 Mon Sep 17 00:00:00 2001 From: farhan Date: Mon, 23 Oct 2023 14:06:30 +0500 Subject: [PATCH 063/282] fix: fix quality error, add type annotation --- openedx_learning/core/components/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx_learning/core/components/models.py b/openedx_learning/core/components/models.py index 35910ba21..2a24beb65 100644 --- a/openedx_learning/core/components/models.py +++ b/openedx_learning/core/components/models.py @@ -157,7 +157,7 @@ class ComponentVersion(PublishableEntityVersionMixin): # The raw_contents hold the actual interesting data associated with this # ComponentVersion. - raw_contents = models.ManyToManyField( + raw_contents: models.ManyToManyField[RawContent, ComponentVersionRawContent] = models.ManyToManyField( RawContent, through="ComponentVersionRawContent", related_name="component_versions", From d59056fcc6c3a4ec3227984deaee804fb2993243 Mon Sep 17 00:00:00 2001 From: Feanil Patel Date: Mon, 23 Oct 2023 09:50:44 -0400 Subject: [PATCH 064/282] docs: Update the security e-mail address. This repository is now managed by the Axim Collaborative and security issues with it should be reported to security@openedx.org instead of security@edx.org This work is being done as a part of https://github.com/openedx/wg-security/issues/16 --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 5ad362c6b..5fad583c3 100644 --- a/README.rst +++ b/README.rst @@ -128,7 +128,7 @@ This repo is in a very experimental state. Discussion using GitHub Issues is wel Reporting Security Issues ------------------------- -Please do not report security issues in public. Please email security@edx.org. +Please do not report security issues in public. Please email security@openedx.org. Help ---- From be5d527ac773970561e4922a9b6282cbf97ff899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 23 Oct 2023 15:08:42 -0300 Subject: [PATCH 065/282] fix: add `lookup_value_regex` for taxonomy rest api (#101) --- openedx_tagging/core/tagging/rest_api/v1/views.py | 1 + tests/openedx_tagging/core/tagging/test_views.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 6428043c0..01b7779f6 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -173,6 +173,7 @@ class TaxonomyView(ModelViewSet): """ + lookup_value_regex = r"\d+" serializer_class = TaxonomySerializer permission_classes = [TaxonomyObjectPermissions] diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index b52c0e64d..1755d97bb 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -195,6 +195,13 @@ def test_detail_taxonomy_404(self) -> None: response = self.client.get(url) assert response.status_code == status.HTTP_404_NOT_FOUND + def test_detail_taxonomy_invalud_pk(self) -> None: + url = TAXONOMY_DETAIL_URL.format(pk="invalid") + + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + assert response.status_code == status.HTTP_404_NOT_FOUND + @ddt.data( (None, status.HTTP_401_UNAUTHORIZED), ("user", status.HTTP_403_FORBIDDEN), From a8337ddde0deffe12fb4c6cee49644c24abd46ae Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 26 Oct 2023 10:27:34 -0700 Subject: [PATCH 066/282] feat!: Tagging: More powerful, flexible implementation of get_filtered_tags() (#92) --- mysql_test_settings.py | 8 + openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 126 ++--- openedx_tagging/core/tagging/data.py | 39 ++ .../core/tagging/import_export/parsers.py | 15 +- openedx_tagging/core/tagging/models/base.py | 281 +++++++--- .../core/tagging/models/system_defined.py | 47 -- openedx_tagging/core/tagging/models/utils.py | 24 + .../core/tagging/rest_api/v1/serializers.py | 114 ++-- .../core/tagging/rest_api/v1/views.py | 217 ++------ .../core/fixtures/tagging.yaml | 21 + .../tagging/import_export/test_template.py | 69 ++- .../openedx_tagging/core/tagging/test_api.py | 319 ++++++----- .../core/tagging/test_models.py | 515 +++++++++++++----- .../core/tagging/test_views.py | 326 +++++++---- tests/openedx_tagging/core/tagging/utils.py | 26 + tox.ini | 2 +- 17 files changed, 1271 insertions(+), 880 deletions(-) create mode 100644 openedx_tagging/core/tagging/data.py create mode 100644 openedx_tagging/core/tagging/models/utils.py create mode 100644 tests/openedx_tagging/core/tagging/utils.py diff --git a/mysql_test_settings.py b/mysql_test_settings.py index ae8e37d3c..f519c5407 100644 --- a/mysql_test_settings.py +++ b/mysql_test_settings.py @@ -7,6 +7,14 @@ The tox targets for py38-django32 and py38-django42 will use this settings file. For the most part, you can use test_settings.py instead (that's the default if you just run "pytest" with no arguments). + +If you need a compatible MySQL server running locally, spin one up with: +docker run --rm \ + -e MYSQL_DATABASE=test_oel_db \ + -e MYSQL_USER=test_oel_user \ + -e MYSQL_PASSWORD=test_oel_pass \ + -e MYSQL_RANDOM_ROOT_PASSWORD=true \ + -p 3306:3306 mysql:8 """ from test_settings import * diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index b602ca309..8872149d9 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.2.6" +__version__ = "0.3.0" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 3316b32bd..a655cd783 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -13,9 +13,10 @@ from __future__ import annotations from django.db import transaction -from django.db.models import F, QuerySet +from django.db.models import QuerySet from django.utils.translation import gettext as _ +from .data import TagDataQuerySet from .models import ObjectTag, Tag, Taxonomy # Export this as part of the API @@ -70,54 +71,66 @@ def get_taxonomies(enabled=True) -> QuerySet[Taxonomy]: return queryset.filter(enabled=enabled) -def get_tags(taxonomy: Taxonomy) -> list[Tag]: +def get_tags(taxonomy: Taxonomy) -> TagDataQuerySet: """ - Returns a list of predefined tags for the given taxonomy. + Returns a QuerySet of all the tags in the given taxonomy. - Note that if the taxonomy allows free-text tags, then the returned list will be empty. + Note that if the taxonomy is dynamic or free-text, only tags that have + already been applied to some object will be returned. """ - return taxonomy.cast().get_tags() + return taxonomy.cast().get_filtered_tags() -def get_root_tags(taxonomy: Taxonomy) -> list[Tag]: +def get_root_tags(taxonomy: Taxonomy) -> TagDataQuerySet: """ Returns a list of the root tags for the given taxonomy. Note that if the taxonomy allows free-text tags, then the returned list will be empty. """ - return list(taxonomy.cast().get_filtered_tags()) + return taxonomy.cast().get_filtered_tags(depth=1) -def search_tags(taxonomy: Taxonomy, search_term: str) -> list[Tag]: +def search_tags( + taxonomy: Taxonomy, + search_term: str, + exclude_object_id: str | None = None, + include_counts: bool = False, +) -> TagDataQuerySet: """ - Returns a list of all tags that contains `search_term` of the given taxonomy. + Returns a list of all tags that contains `search_term` of the given + taxonomy, as well as their ancestors (so they can be displayed in a tree). - Note that if the taxonomy allows free-text tags, then the returned list will be empty. + If exclude_object_id is set, any tags applied to that object will be + excluded from the results, e.g. to power an autocomplete search when adding + additional tags to an object. """ - return list( - taxonomy.cast().get_filtered_tags( - search_term=search_term, - search_in_all=True, + excluded_values = None + if exclude_object_id: + # Fetch tags that the object already has to exclude them from the result. + # Note: this adds a fair bit of complexity. In the future, maybe we can just do this filtering on the frontend? + excluded_values = list( + taxonomy.objecttag_set.filter(object_id=exclude_object_id).values_list( + "_value", flat=True + ) ) + qs = taxonomy.cast().get_filtered_tags( + search_term=search_term, + excluded_values=excluded_values, + include_counts=include_counts, ) + return qs def get_children_tags( taxonomy: Taxonomy, - parent_tag_id: int, - search_term: str | None = None, -) -> list[Tag]: + parent_tag_value: str, +) -> TagDataQuerySet: """ - Returns a list of children tags for the given parent tag. + Returns a QuerySet of children tags for the given parent tag. Note that if the taxonomy allows free-text tags, then the returned list will be empty. """ - return list( - taxonomy.cast().get_filtered_tags( - parent_tag_id=parent_tag_id, - search_term=search_term, - ) - ) + return taxonomy.cast().get_filtered_tags(parent_tag_value=parent_tag_value, depth=1) def resync_object_tags(object_tags: QuerySet | None = None) -> int: @@ -253,71 +266,6 @@ def _check_new_tag_count(new_tag_count: int) -> None: object_tag.save() -# TODO: return tags from closed taxonomies as well as the count of how many times each is used. -def autocomplete_tags( - taxonomy: Taxonomy, - search: str, - object_id: str | None = None, - object_tags_only=True, -) -> QuerySet: - """ - Provides auto-complete suggestions by matching the `search` string against existing - ObjectTags linked to the given taxonomy. A case-insensitive search is used in order - to return the highest number of relevant tags. - - If `object_id` is provided, then object tag values already linked to this object - are omitted from the returned suggestions. (ObjectTag values must be unique for a - given object + taxonomy, and so omitting these suggestions helps users avoid - duplication errors.). - - Returns a QuerySet of dictionaries containing distinct `value` (string) and - `tag` (numeric ID) values, sorted alphabetically by `value`. - The `value` is what should be shown as a suggestion to users, - and if it's a free-text taxonomy, `tag` will be `None`: we include the `tag` ID - in anticipation of the second use case listed below. - - Use cases: - * This method is useful for reducing tag variation in free-text taxonomies by showing - users tags that are similar to what they're typing. E.g., if the `search` string "dn" - shows that other objects have been tagged with "DNA", "DNA electrophoresis", and "DNA fingerprinting", - this encourages users to use those existing tags if relevant, instead of creating new ones that - look similar (e.g. "dna finger-printing"). - * It could also be used to assist tagging for closed taxonomies with a list of possible tags which is too - large to return all at once, e.g. a user model taxonomy that dynamically creates tags on request for any - registered user in the database. (Note that this is not implemented yet, but may be as part of a future change.) - """ - if not object_tags_only: - raise NotImplementedError( - _( - "Using this would return a query set of tags instead of object tags." - "For now we recommend fetching all of the taxonomy's tags " - "using get_tags() and filtering them on the frontend." - ) - ) - # Fetch tags that the object already has to exclude them from the result - excluded_tags: list[str] = [] - if object_id: - excluded_tags = list( - taxonomy.objecttag_set.filter(object_id=object_id).values_list( - "_value", flat=True - ) - ) - return ( - # Fetch object tags from this taxonomy whose value contains the search - taxonomy.objecttag_set.filter(_value__icontains=search) - # omit any tags whose values match the tags on the given object - .exclude(_value__in=excluded_tags) - # alphabetical ordering - .order_by("_value") - # Alias the `_value` field to `value` to make it nicer for users - .annotate(value=F("_value")) - # obtain tag values - .values("value", "tag_id") - # remove repeats - .distinct() - ) - - def add_tag_to_taxonomy( taxonomy: Taxonomy, tag: str, diff --git a/openedx_tagging/core/tagging/data.py b/openedx_tagging/core/tagging/data.py new file mode 100644 index 000000000..1ab8e9876 --- /dev/null +++ b/openedx_tagging/core/tagging/data.py @@ -0,0 +1,39 @@ +""" +Data models used by openedx-tagging +""" +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, TypedDict + +from django.db.models import QuerySet +from typing_extensions import NotRequired, TypeAlias + + +class TagData(TypedDict): + """ + Data about a single tag. Many of the tagging API methods return Django + QuerySets that resolve to these dictionaries. + + Even though the data will be in this same format, it will not necessarily + be an instance of this class but rather a plain dictionary. This is more a + type than a class. + """ + value: str + external_id: str | None + child_count: int + depth: int + parent_value: str | None + # Note: usage_count may or may not be present, depending on the request. + usage_count: NotRequired[int] + # Internal database ID, if any. Generally should not be used; prefer 'value' which is unique within each taxonomy. + _id: int | None + + +if TYPE_CHECKING: + from django_stubs_ext import ValuesQuerySet + TagDataQuerySet: TypeAlias = ValuesQuerySet[Any, TagData] + # The following works better for pyright (provides proper VS Code autocompletions), + # but I can't find any way to specify different types for pyright vs mypy :/ + # TagDataQuerySet: TypeAlias = QuerySet[TagData] +else: + TagDataQuerySet = QuerySet[TagData] diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py index c0b8207fc..90268ff6a 100644 --- a/openedx_tagging/core/tagging/import_export/parsers.py +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -10,7 +10,6 @@ from django.utils.translation import gettext as _ -from ..api import get_tags from ..models import Taxonomy from .exceptions import EmptyCSVField, EmptyJSONField, FieldJSONError, InvalidFormat, TagParserError from .import_plan import TagItem @@ -166,17 +165,19 @@ def _load_tags_for_export(cls, taxonomy: Taxonomy) -> list[dict]: with required and optional fields The tags are ordered by hierarchy, first, parents and then children. - `get_tags` is in charge of returning this in a hierarchical way. + `get_filtered_tags` is in charge of returning this in a hierarchical + way. """ - tags = get_tags(taxonomy) + tags = taxonomy.get_filtered_tags().all() result = [] for tag in tags: result_tag = { - "id": tag.external_id or tag.id, - "value": tag.value, + "id": tag["external_id"] or tag["_id"], + "value": tag["value"], } - if tag.parent: - result_tag["parent_id"] = tag.parent.external_id or tag.parent.id + if tag["parent_value"]: + parent_tag = next(t for t in tags if t["value"] == tag["parent_value"]) + result_tag["parent_id"] = parent_tag["external_id"] or parent_tag["_id"] result.append(result_tag) return result diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index 4ecf52c5f..71318a13b 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -8,12 +8,18 @@ from django.core.exceptions import ValidationError from django.db import models +from django.db.models import F, Q, Value +from django.db.models.functions import Concat, Lower +from django.utils.functional import cached_property from django.utils.module_loading import import_string from django.utils.translation import gettext_lazy as _ from typing_extensions import Self # Until we upgrade to python 3.11 from openedx_learning.lib.fields import MultiCollationTextField, case_insensitive_char_field, case_sensitive_char_field +from ..data import TagDataQuerySet +from .utils import ConcatNull + log = logging.getLogger(__name__) @@ -106,6 +112,43 @@ def get_lineage(self) -> Lineage: depth -= 1 return lineage + @cached_property + def depth(self) -> int: + """ + How many ancestors this Tag has. Zero for root tags. + """ + depth = 0 + tag = self + while tag.parent: + depth += 1 + tag = tag.parent + return depth + + @staticmethod + def annotate_depth(qs: models.QuerySet) -> models.QuerySet: + """ + Given a query that loads Tag objects, annotate it with the depth of + each tag. + """ + return qs.annotate(depth=models.Case( + models.When(parent_id=None, then=0), + models.When(parent__parent_id=None, then=1), + models.When(parent__parent__parent_id=None, then=2), + models.When(parent__parent__parent__parent_id=None, then=3), + # If the depth is 4 or more, currently we just "collapse" the depth + # to 4 in order not to add too many joins to this query in general. + default=4, + )) + + @cached_property + def child_count(self) -> int: + """ + How many child tags this tag has in the taxonomy. + """ + if self.taxonomy and not self.taxonomy.allow_free_text: + return self.taxonomy.tag_set.filter(parent=self).count() + return 0 + class Taxonomy(models.Model): """ @@ -260,87 +303,201 @@ def copy(self, taxonomy: Taxonomy) -> Taxonomy: self._taxonomy_class = taxonomy._taxonomy_class # pylint: disable=protected-access return self - def get_tags( + def get_filtered_tags( self, - tag_set: models.QuerySet[Tag] | None = None, - ) -> list[Tag]: + depth: int | None = TAXONOMY_MAX_DEPTH, + parent_tag_value: str | None = None, + search_term: str | None = None, + include_counts: bool = False, + excluded_values: list[str] | None = None, + ) -> TagDataQuerySet: """ - Returns a list of all Tags in the current taxonomy, from the root(s) - down to TAXONOMY_MAX_DEPTH tags, in tree order. + Returns a filtered QuerySet of tag values. + For free text or dynamic taxonomies, this will only return tag values + that have actually been used. - Use `tag_set` to do an initial filtering of the tags. + By default returns all the tags of the given taxonomy - Annotates each returned Tag with its ``depth`` in the tree (starting at - 0). + Use `depth=1` to return a single level of tags, without any child + tags included. Use `depth=None` or `depth=TAXONOMY_MAX_DEPTH` to return + all descendants of the tags, up to our maximum supported depth. - Performance note: may perform as many as TAXONOMY_MAX_DEPTH select - queries. - """ - tags: list[Tag] = [] - if self.allow_free_text: - return tags + Use `parent_tag_value` to return only the children/descendants of a specific tag. - if tag_set is None: - tag_set = self.tag_set.all() + Use `search_term` to filter the results by values that contains `search_term`. - parents = None + Use `excluded_values` to exclude tags with that value (and their parents, if applicable) from the results. - for depth in range(TAXONOMY_MAX_DEPTH): - filtered_tags = tag_set.prefetch_related("parent") - if parents is None: - filtered_tags = filtered_tags.filter(parent=None) + Note: This is mostly an 'internal' API and generally code outside of openedx_tagging + should use the APIs in openedx_tagging.api which in turn use this. + """ + if self.allow_free_text: + if parent_tag_value is not None: + raise ValueError("Cannot specify a parent tag ID for free text taxonomies") + result = self._get_filtered_tags_free_text(search_term=search_term, include_counts=include_counts) + if excluded_values: + return result.exclude(value__in=excluded_values) else: - filtered_tags = filtered_tags.filter(parent__in=parents) - next_parents = list( - filtered_tags.annotate( - annotated_field=models.Value( - depth, output_field=models.IntegerField() - ) - ) - .order_by("parent__value", "value", "id") - .all() + return result + elif depth == 1: + result = self._get_filtered_tags_one_level( + parent_tag_value=parent_tag_value, + search_term=search_term, + include_counts=include_counts, + ) + if excluded_values: + return result.exclude(value__in=excluded_values) + else: + return result + elif depth is None or depth == TAXONOMY_MAX_DEPTH: + return self._get_filtered_tags_deep( + parent_tag_value=parent_tag_value, + search_term=search_term, + include_counts=include_counts, + excluded_values=excluded_values, ) - tags.extend(next_parents) - parents = next_parents - if not parents: - break - return tags + else: + raise ValueError("Unsupported depth value for get_filtered_tags()") - def get_filtered_tags( + def _get_filtered_tags_free_text( self, - tag_set: models.QuerySet[Tag] | None = None, - parent_tag_id: int | None = None, - search_term: str | None = None, - search_in_all: bool = False, - ) -> models.QuerySet[Tag]: + search_term: str | None, + include_counts: bool, + ) -> TagDataQuerySet: """ - Returns a filtered QuerySet of tags. - By default returns the root tags of the given taxonomy - - Use `parent_tag_id` to return the children of a tag. - - Use `search_term` to filter the results by values that contains `search_term`. - - Set `search_in_all` to True to make the search in all tags on the given taxonomy. + Implementation of get_filtered_tags() for free text taxonomies. + """ + assert self.allow_free_text + qs: models.QuerySet = self.objecttag_set.all() + if search_term: + qs = qs.filter(_value__icontains=search_term) + # Rename "_value" to "value" + qs = qs.annotate(value=F('_value')) + # Add in all these fixed fields that don't really apply to free text tags, but we include for consistency: + qs = qs.annotate( + depth=Value(0), + child_count=Value(0), + external_id=Value(None, output_field=models.CharField()), + parent_value=Value(None, output_field=models.CharField()), + _id=Value(None, output_field=models.CharField()), + ) + qs = qs.values("value", "child_count", "depth", "parent_value", "external_id", "_id").order_by("value") + if include_counts: + return qs.annotate(usage_count=models.Count("value")) + else: + return qs.distinct() - Note: This is mostly an 'internal' API and generally code outside of openedx_tagging - should use the APIs in openedx_tagging.api which in turn use this. + def _get_filtered_tags_one_level( + self, + parent_tag_value: str | None, + search_term: str | None, + include_counts: bool, + ) -> TagDataQuerySet: + """ + Implementation of get_filtered_tags() for closed taxonomies, where + depth=1. When depth=1, we're only looking at a single "level" of the + taxononomy, like all root tags or all children of a specific tag. """ - if tag_set is None: - tag_set = self.tag_set.all() + # A closed, and possibly hierarchical taxonomy. We're just fetching a single "level" of tags. + if parent_tag_value: + parent_tag = self.tag_for_value(parent_tag_value) + qs: models.QuerySet = self.tag_set.filter(parent_id=parent_tag.pk) + qs = qs.annotate(depth=Value(parent_tag.depth + 1)) + # Use parent_tag.value not parent_tag_value because they may differ in case + qs = qs.annotate(parent_value=Value(parent_tag.value)) + else: + qs = self.tag_set.filter(parent=None).annotate(depth=Value(0)) + qs = qs.annotate(parent_value=Value(None, output_field=models.CharField())) + qs = qs.annotate(child_count=models.Count("children")) + # Filter by search term: + if search_term: + qs = qs.filter(value__icontains=search_term) + qs = qs.annotate(_id=F("id")) # ID has an underscore to encourage use of 'value' rather than this internal ID + qs = qs.values("value", "child_count", "depth", "parent_value", "external_id", "_id").order_by("value") + if include_counts: + # We need to include the count of how many times this tag is used to tag objects. + # You'd think we could just use: + # qs = qs.annotate(usage_count=models.Count("objecttag__pk")) + # but that adds another join which starts creating a cross product and the children and usage_count become + # intertwined and multiplied with each other. So we use a subquery. + obj_tags = ObjectTag.objects.filter(tag_id=models.OuterRef("pk")).order_by().annotate( + # We need to use Func() to get Count() without GROUP BY - see https://stackoverflow.com/a/69031027 + count=models.Func(F('id'), function='Count') + ) + qs = qs.annotate(usage_count=models.Subquery(obj_tags.values('count'))) + return qs - if self.allow_free_text: - return tag_set.none() + def _get_filtered_tags_deep( + self, + parent_tag_value: str | None, + search_term: str | None, + include_counts: bool, + excluded_values: list[str] | None, + ) -> TagDataQuerySet: + """ + Implementation of get_filtered_tags() for closed taxonomies, where + we're including tags from multiple levels of the hierarchy. + """ + # All tags (possibly below a certain tag?) in the closed taxonomy, up to depth TAXONOMY_MAX_DEPTH + if parent_tag_value: + main_parent_id = self.tag_for_value(parent_tag_value).pk + else: + main_parent_id = None - if not search_in_all: - # If not search in all taxonomy, then apply parent filter. - tag_set = tag_set.filter(parent=parent_tag_id) + assert TAXONOMY_MAX_DEPTH == 3 # If we change TAXONOMY_MAX_DEPTH we need to change this query code: + qs: models.QuerySet = self.tag_set.filter( + Q(parent_id=main_parent_id) | + Q(parent__parent_id=main_parent_id) | + Q(parent__parent__parent_id=main_parent_id) + ) if search_term: - # Apply search filter - tag_set = tag_set.filter(value__icontains=search_term) - - return tag_set.order_by("value", "id") + # We need to do an additional query to find all the tags that match the search term, then limit the + # search to those tags and their ancestors. + matching_tags = qs.filter(value__icontains=search_term).values( + 'id', 'parent_id', 'parent__parent_id', 'parent__parent__parent_id' + ) + if excluded_values: + matching_tags = matching_tags.exclude(value__in=excluded_values) + matching_ids = [] + for row in matching_tags: + for pk in row.values(): + if pk is not None: + matching_ids.append(pk) + qs = qs.filter(pk__in=matching_ids) + elif excluded_values: + raise NotImplementedError("Using excluded_values without search_term is not currently supported.") + # We could implement this in the future but I'd prefer to get rid of the "excluded_values" API altogether. + # It remains to be seen if it's useful to do that on the backend, or if we can do it better/simpler on the + # frontend. + + qs = qs.annotate(child_count=models.Count("children")) + # Add the "depth" to each tag: + qs = Tag.annotate_depth(qs) + # Add the "lineage" as a field called "sort_key" to sort them in order correctly: + qs = qs.annotate(sort_key=Lower(Concat( + # For a root tag, we want sort_key="RootValue" and for a depth=1 tag + # we want sort_key="RootValue\tValue". The following does that, since + # ConcatNull(...) returns NULL if any argument is NULL. + ConcatNull(F("parent__parent__parent__value"), Value("\t")), + ConcatNull(F("parent__parent__value"), Value("\t")), + ConcatNull(F("parent__value"), Value("\t")), + F("value"), + Value("\t"), # We also need the '\t' separator character at the end, or MySQL will sort things wrong + output_field=models.CharField(), + ))) + # Add the parent value + qs = qs.annotate(parent_value=F("parent__value")) + qs = qs.annotate(_id=F("id")) # ID has an underscore to encourage use of 'value' rather than this internal ID + qs = qs.values("value", "child_count", "depth", "parent_value", "external_id", "_id").order_by("sort_key") + if include_counts: + # Including the counts is a bit tricky; see the comment above in _get_filtered_tags_one_level() + obj_tags = ObjectTag.objects.filter(tag_id=models.OuterRef("pk")).order_by().annotate( + # We need to use Func() to get Count() without GROUP BY - see https://stackoverflow.com/a/69031027 + count=models.Func(F('id'), function='Count') + ) + qs = qs.annotate(usage_count=models.Subquery(obj_tags.values('count'))) + return qs def add_tag( self, diff --git a/openedx_tagging/core/tagging/models/system_defined.py b/openedx_tagging/core/tagging/models/system_defined.py index 6851ab9d2..3efb68f22 100644 --- a/openedx_tagging/core/tagging/models/system_defined.py +++ b/openedx_tagging/core/tagging/models/system_defined.py @@ -198,53 +198,6 @@ class LanguageTaxonomy(SystemDefinedTaxonomy): class Meta: proxy = True - def get_tags( - self, - tag_set: models.QuerySet[Tag] | None = None, - ) -> list[Tag]: - """ - Returns a list of all the available Language Tags, annotated with ``depth`` = 0. - """ - available_langs = self._get_available_languages() - tag_set = self.tag_set.filter(external_id__in=available_langs) - return super().get_tags(tag_set=tag_set) - - def get_filtered_tags( - self, - tag_set: models.QuerySet[Tag] | None = None, - parent_tag_id: int | None = None, - search_term: str | None = None, - search_in_all: bool = False, - ) -> models.QuerySet[Tag]: - """ - Returns a filtered QuerySet of available Language Tags. - By default returns all the available Language Tags. - - `parent_tag_id` returns an empty result because all Language tags are root tags. - - Use `search_term` to filter the results by values that contains `search_term`. - """ - if parent_tag_id: - return self.tag_set.none() - - available_langs = self._get_available_languages() - tag_set = self.tag_set.filter(external_id__in=available_langs) - return super().get_filtered_tags( - tag_set=tag_set, - search_term=search_term, - search_in_all=search_in_all, - ) - - @classmethod - def _get_available_languages(cls) -> set[str]: - """ - Get available languages from Django LANGUAGE. - """ - langs = set() - for django_lang in settings.LANGUAGES: - langs.add(django_lang[0]) - return langs - def validate_value(self, value: str): """ Check if 'value' is part of this Taxonomy, based on the specified model. diff --git a/openedx_tagging/core/tagging/models/utils.py b/openedx_tagging/core/tagging/models/utils.py new file mode 100644 index 000000000..1f7dcb920 --- /dev/null +++ b/openedx_tagging/core/tagging/models/utils.py @@ -0,0 +1,24 @@ +""" +Utilities for tagging and taxonomy models +""" + +from django.db.models.expressions import Func + + +class ConcatNull(Func): # pylint: disable=abstract-method + """ + Concatenate two arguments together. Like normal SQL but unlike Django's + "Concat", if either argument is NULL, the result will be NULL. + """ + + function = "CONCAT" + + def as_sqlite(self, compiler, connection, **extra_context): + """ SQLite doesn't have CONCAT() but has a concatenation operator """ + return super().as_sql( + compiler, + connection, + template="%(expressions)s", + arg_joiner=" || ", + **extra_context, + ) diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index c34c15dad..4fe62c72a 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -1,9 +1,12 @@ """ API Serializers for taxonomies """ +from __future__ import annotations + from rest_framework import serializers from rest_framework.reverse import reverse +from openedx_tagging.core.tagging.data import TagData from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy @@ -92,107 +95,56 @@ class ObjectTagUpdateQueryParamsSerializer(serializers.Serializer): # pylint: d ) -class TagsSerializer(serializers.ModelSerializer): +class TagDataSerializer(serializers.Serializer): """ - Serializer for Tags + Serializer for TagData dicts. Also can serialize Tag instances. Adds a link to get the sub tags """ + value = serializers.CharField() + external_id = serializers.CharField(allow_null=True) + child_count = serializers.IntegerField() + depth = serializers.IntegerField() + parent_value = serializers.CharField(allow_null=True) + usage_count = serializers.IntegerField(required=False) + # Internal database ID, if any. Generally should not be used; prefer 'value' which is unique within each taxonomy. + # Free text taxonomies never have '_id' for their tags. + _id = serializers.IntegerField(allow_null=True) - sub_tags_link = serializers.SerializerMethodField() - children_count = serializers.SerializerMethodField() - - class Meta: - model = Tag - fields = ( - "id", - "value", - "taxonomy_id", - "parent_id", - "external_id", - "sub_tags_link", - "children_count", - ) + sub_tags_url = serializers.SerializerMethodField() - def get_sub_tags_link(self, obj): + def get_sub_tags_url(self, obj: TagData | Tag): """ Returns URL for the list of child tags of the current tag. """ - if obj.children.count(): - query_params = f"?parent_tag_id={obj.id}" - request = self.context.get("request") + child_count = obj.child_count if isinstance(obj, Tag) else obj["child_count"] + if child_count > 0 and "taxonomy_id" in self.context: + value = obj.value if isinstance(obj, Tag) else obj["value"] + query_params = f"?parent_tag={value}" + request = self.context["request"] url_namespace = request.resolver_match.namespace # get the namespace, usually "oel_tagging" url = ( - reverse(f"{url_namespace}:taxonomy-tags", args=[str(obj.taxonomy_id)]) + reverse(f"{url_namespace}:taxonomy-tags", args=[str(self.context["taxonomy_id"])]) + query_params ) return request.build_absolute_uri(url) return None - def get_children_count(self, obj): + def to_representation(self, instance: TagData | Tag) -> dict: """ - Returns the number of child tags of the given tag. + Convert this TagData (or Tag model instance) to the serialized dictionary """ - return obj.children.count() - - -class TagsWithSubTagsSerializer(serializers.ModelSerializer): - """ - Serializer for Tags. - - Represents a tree with a list of sub tags - """ - - sub_tags = serializers.SerializerMethodField() - children_count = serializers.SerializerMethodField() + data = super().to_representation(instance) + if isinstance(instance, Tag): + data["_id"] = instance.pk # The ID field won't otherwise be detected. + data["parent_value"] = instance.parent.value if instance.parent else None + return data - class Meta: - model = Tag - fields = ( - "id", - "value", - "taxonomy_id", - "sub_tags", - "children_count", - ) + def update(self, instance, validated_data): + raise RuntimeError('`update()` is not supported by the TagData serializer.') - def get_sub_tags(self, obj): - """ - Returns a serialized list of child tags for the given tag. - """ - serializer = TagsWithSubTagsSerializer( - obj.children.all().order_by("value", "id"), - many=True, - read_only=True, - ) - return serializer.data - - def get_children_count(self, obj): - """ - Returns the number of child tags of the given tag. - """ - return obj.children.count() - - -class TagsForSearchSerializer(TagsWithSubTagsSerializer): - """ - Serializer for Tags - - Used to filter sub tags of a given tag - """ - - def get_sub_tags(self, obj): - """ - Returns a serialized list of child tags for the given tag. - """ - serializer = TagsWithSubTagsSerializer(obj.sub_tags, many=True, read_only=True) - return serializer.data - - def get_children_count(self, obj): - """ - Returns the number of child tags of the given tag. - """ - return len(obj.sub_tags) + def create(self, validated_data): + raise RuntimeError('`create()` is not supported by the TagData serializer.') class TaxonomyTagCreateBodySerializer(serializers.Serializer): # pylint: disable=abstract-method diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 01b7779f6..951c8e6f2 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -12,36 +12,30 @@ from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet, ModelViewSet -from openedx_tagging.core.tagging.models.base import Tag - from ...api import ( TagDoesNotExist, add_tag_to_taxonomy, create_taxonomy, delete_tags_from_taxonomy, - get_children_tags, get_object_tags, - get_root_tags, get_taxonomies, get_taxonomy, - search_tags, tag_object, update_tag_in_taxonomy, ) +from ...data import TagDataQuerySet from ...import_export.api import export_tags from ...import_export.parsers import ParserFormat from ...models import Taxonomy from ...rules import ObjectTagPermissionItem -from ..paginators import SEARCH_TAGS_THRESHOLD, TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination +from ..paginators import TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination from .permissions import ObjectTagObjectPermissions, TagObjectPermissions, TaxonomyObjectPermissions from .serializers import ( ObjectTagListQueryParamsSerializer, ObjectTagSerializer, ObjectTagUpdateBodySerializer, ObjectTagUpdateQueryParamsSerializer, - TagsForSearchSerializer, - TagsSerializer, - TagsWithSubTagsSerializer, + TagDataSerializer, TaxonomyExportQueryParamsSerializer, TaxonomyListQueryParamsSerializer, TaxonomySerializer, @@ -416,15 +410,28 @@ class TaxonomyTagsView(ListAPIView, RetrieveUpdateDestroyAPIView): """ View to list/create/update/delete tags of a taxonomy. + If you specify ?root_only or ?parent_tag_value=..., only one "level" of the + hierachy will be returned. Otherwise, several levels will be returned, in + tree order, up to the maximum supported depth. Additional levels/depth can + be retrieved by using ?parent_tag_value to load more data. + + Note: If the taxonomy is particularly large (> 1,000 tags), ?root_only is + automatically set true by default and cannot be disabled. This way, users + can more easily select which tags they want to expand in the tree, and load + just that subset of the tree as needed. This may be changed in the future. + **List Query Parameters** * id (required) - The ID of the taxonomy to retrieve tags. - * parent_tag_id (optional) - Id of the tag to retrieve children tags. + * parent_tag (optional) - Retrieve children of the tag with this value. + * root_only (optional) - If specified, only root tags are returned. + * include_counts (optional) - Include the count of how many times each + tag has been used. * page (optional) - Page number (default: 1) * page_size (optional) - Number of items per page (default: 10) **List Example Requests** GET api/tagging/v1/taxonomy/:id/tags - Get tags of taxonomy - GET api/tagging/v1/taxonomy/:id/tags?parent_tag_id=30 - Get children tags of tag + GET api/tagging/v1/taxonomy/:id/tags?parent_tag=Physics&include_counts - Get child tags of tag **List Query Returns** * 200 - Success @@ -502,22 +509,8 @@ class TaxonomyTagsView(ListAPIView, RetrieveUpdateDestroyAPIView): """ permission_classes = [TagObjectPermissions] - pagination_enabled = True - - def __init__(self): - # Initialized here to avoid errors on type hints - self.serializer_class = TagsSerializer - - def get_pagination_class(self): - """ - Get the corresponding class depending if the pagination is enabled. - - It is necessary to call this function before returning the data. - """ - if self.pagination_enabled: - return TagsPagination - else: - return DisabledTagsPagination + pagination_class = TagsPagination + serializer_class = TagDataSerializer def get_taxonomy(self, pk: int) -> Taxonomy: """ @@ -529,150 +522,50 @@ def get_taxonomy(self, pk: int) -> Taxonomy: self.check_object_permissions(self.request, taxonomy) return taxonomy - def _build_search_tree(self, tags: list[Tag]) -> list[Tag]: - """ - Builds a tree with the result tags for a search. - - The retult is a pruned tree that contains - the path from root tags to tags that match the search. - """ - tag_ids = [tag.id for tag in tags] - - # Get missing parents. - # Not all parents are in the search result. - # This occurs when a child tag is on the search result, but its parent not, - # we need to add the parent to show the tree from the root to the child. - for tag in tags: - if tag.parent and tag.parent_id and tag.parent_id not in tag_ids: - tag_ids.append(tag.parent_id) - tags.append(tag.parent) # Our loop will iterate over this new parent tag too. - - groups: dict[int, list[Tag]] = {} - roots: list[Tag] = [] - - # Group tags by parent - for tag in tags: - if tag.parent_id is not None: - if tag.parent_id not in groups: - groups[tag.parent_id] = [] - groups[tag.parent_id].append(tag) - else: - roots.append(tag) - - for tag in tags: - # Used to serialize searched childrens - tag.sub_tags = groups.get(tag.id, []) # type: ignore[attr-defined] - - return roots + def get_serializer_context(self): + context = super().get_serializer_context() + context.update({ + "request": self.request, + "taxonomy_id": int(self.kwargs["pk"]), + }) + return context - def get_matching_tags( - self, - taxonomy_id: int, - parent_tag_id: str | None = None, - search_term: str | None = None, - ) -> list[Tag]: + def get_queryset(self) -> TagDataQuerySet: """ - Returns a list of tags for the given taxonomy. - - The pagination can be enabled or disabled depending of the taxonomy size. - You can read the desicion '0014_*' to more info about this logic. - Also, determines the serializer to be used. - - Use `parent_tag_id` to get the children of the given tag. - - Use `search_term` to filter tags values that contains the given term. + Builds and returns the queryset to be paginated. """ + taxonomy_id = int(self.kwargs.get("pk")) taxonomy = self.get_taxonomy(taxonomy_id) - if parent_tag_id: - # Get children of a tag. - - # If you need to get the children, then the roots are - # paginated, so we need to paginate the childrens too. - self.pagination_enabled = True - - # Normal serializer, with children link. - self.serializer_class = TagsSerializer - return get_children_tags( - taxonomy, - int(parent_tag_id), - search_term=search_term, - ) + parent_tag_value = self.request.query_params.get("parent_tag", None) + root_only = "root_only" in self.request.query_params + include_counts = "include_counts" in self.request.query_params + search_term = self.request.query_params.get("search_term", None) + + if parent_tag_value: + # Fetching tags below a certain parent is always paginated and only returns the direct children + depth = 1 + if root_only: + raise ValidationError("?root_only and ?parent_tag cannot be used together") else: - if search_term: - # Search tags - result = search_tags( - taxonomy, - search_term, - ) - # Checks the result size to determine whether - # to turn pagination on or off. - self.pagination_enabled = len(result) > SEARCH_TAGS_THRESHOLD - - # Use the special serializer to only show the tree - # of the search result. - self.serializer_class = TagsForSearchSerializer - - result = self._build_search_tree(result) + if root_only: + depth = 1 # User Explicitly requested to load only the root tags for now + elif search_term: + depth = None # For search, default to maximum depth but use normal pagination + elif taxonomy.tag_set.count() > TAGS_THRESHOLD: + # This is a very large taxonomy. Only load the root tags at first, so users can choose what to load. + depth = 1 else: - # Get root tags of taxonomy - - # Checks the taxonomy size to determine whether - # to turn pagination on or off. - self.pagination_enabled = taxonomy.tag_set.count() > TAGS_THRESHOLD - - if self.pagination_enabled: - # If pagination is enabled, use the normal serializer - # with children link. - self.serializer_class = TagsSerializer - else: - # If pagination is disabled, use the special serializer - # to show children. In this case, we return all taxonomy tags - # in a tree structure. - self.serializer_class = TagsWithSubTagsSerializer - - result = get_root_tags(taxonomy) - - return result - - def get_queryset(self) -> models.QuerySet[Tag]: # type: ignore[override] - """ - Builds and returns the queryset to be paginated. - - The return type is not a QuerySet because the tagging python api functions - return lists, and on this point convert the list to a query set - is an unnecesary operation. - """ - pk = self.kwargs.get("pk") - parent_tag_id = self.request.query_params.get("parent_tag_id", None) - search_term = self.request.query_params.get("search_term", None) + # We can load and display all the tags in the taxonomy at once: + self.pagination_class = DisabledTagsPagination + depth = None # Maximum depth - result = self.get_matching_tags( - pk, - parent_tag_id=parent_tag_id, + return taxonomy.get_filtered_tags( + parent_tag_value=parent_tag_value, search_term=search_term, + depth=depth, + include_counts=include_counts, ) - # Convert the results back to a QuerySet for permissions to apply - # Due to the conversion we lose the populated `sub_tags` attribute, - # in the case of using the special search serializer so we - # need to repopulate it again - if self.serializer_class == TagsForSearchSerializer: - results_dict = {tag.id: tag for tag in result} - - result_queryset = Tag.objects.filter(id__in=results_dict.keys()) - - for tag in result_queryset: - sub_tags = results_dict[tag.id].sub_tags # type: ignore[attr-defined] - tag.sub_tags = sub_tags # type: ignore[attr-defined] - - else: - result_queryset = Tag.objects.filter(id__in=[tag.id for tag in result]) - - # This function is not called automatically - self.pagination_class = self.get_pagination_class() - - return result_queryset - def post(self, request, *args, **kwargs): """ Creates new Tag in Taxonomy and returns the newly created Tag. @@ -696,7 +589,6 @@ def post(self, request, *args, **kwargs): except ValueError as e: raise ValidationError(e) from e - self.serializer_class = TagsSerializer serializer_context = self.get_serializer_context() return Response( self.serializer_class(new_tag, context=serializer_context).data, @@ -724,7 +616,6 @@ def update(self, request, *args, **kwargs): except ValueError as e: raise ValidationError(e) from e - self.serializer_class = TagsSerializer serializer_context = self.get_serializer_context() return Response( self.serializer_class(updated_tag, context=serializer_context).data, diff --git a/tests/openedx_tagging/core/fixtures/tagging.yaml b/tests/openedx_tagging/core/fixtures/tagging.yaml index 4715b667b..164b93990 100644 --- a/tests/openedx_tagging/core/fixtures/tagging.yaml +++ b/tests/openedx_tagging/core/fixtures/tagging.yaml @@ -1,3 +1,24 @@ +# - Bacteria +# |- Archaebacteria +# |- Eubacteria +# - Archaea +# |- DPANN +# |- Euryarchaeida +# |- Proteoarchaeota +# - Eukaryota +# |- Animalia +# | |- Arthropoda +# | |- Chordata +# | | |- Mammalia +# | |- Cnidaria +# | |- Ctenophora +# | |- Gastrotrich +# | |- Placozoa +# | |- Porifera +# |- Fungi +# |- Monera +# |- Plantae +# |- Protista - model: oel_tagging.tag pk: 1 fields: diff --git a/tests/openedx_tagging/core/tagging/import_export/test_template.py b/tests/openedx_tagging/core/tagging/import_export/test_template.py index 067a338c0..9947de6bd 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_template.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_template.py @@ -12,6 +12,7 @@ from openedx_tagging.core.tagging.import_export import ParserFormat from openedx_tagging.core.tagging.import_export import api as import_api +from ..utils import pretty_format_tags from .mixins import TestImportExportMixin @@ -47,44 +48,36 @@ def test_import_template(self, template_file, parser_format): replace=True, ), import_api.get_last_import_log(self.taxonomy) - imported_tags = [ - { - "external_id": tag.external_id, - "value": tag.value, - "parent": tag.parent.external_id if tag.parent else None, - } - for tag in get_tags(self.taxonomy) - ] - assert imported_tags == [ - {'external_id': "ELECTRIC", 'parent': None, 'value': 'Electronic instruments'}, - {'external_id': 'PERCUSS', 'parent': None, 'value': 'Percussion instruments'}, - {'external_id': 'STRINGS', 'parent': None, 'value': 'String instruments'}, - {'external_id': 'WINDS', 'parent': None, 'value': 'Wind instruments'}, - {'external_id': 'SYNTH', 'parent': 'ELECTRIC', 'value': 'Synthesizer'}, - {'external_id': 'THERAMIN', 'parent': 'ELECTRIC', 'value': 'Theramin'}, - {'external_id': 'CHORD', 'parent': 'PERCUSS', 'value': 'Chordophone'}, - {'external_id': 'BELLS', 'parent': 'PERCUSS', 'value': 'Idiophone'}, - {'external_id': 'DRUMS', 'parent': 'PERCUSS', 'value': 'Membranophone'}, - {'external_id': 'BOW', 'parent': 'STRINGS', 'value': 'Bowed strings'}, - {'external_id': 'PLUCK', 'parent': 'STRINGS', 'value': 'Plucked strings'}, - {'external_id': 'BRASS', 'parent': 'WINDS', 'value': 'Brass'}, - {'external_id': 'WOODS', 'parent': 'WINDS', 'value': 'Woodwinds'}, - {'external_id': 'CELLO', 'parent': 'BOW', 'value': 'Cello'}, - {'external_id': 'VIOLIN', 'parent': 'BOW', 'value': 'Violin'}, - {'external_id': 'TRUMPET', 'parent': 'BRASS', 'value': 'Trumpet'}, - {'external_id': 'TUBA', 'parent': 'BRASS', 'value': 'Tuba'}, - {'external_id': 'PIANO', 'parent': 'CHORD', 'value': 'Piano'}, + assert pretty_format_tags(get_tags(self.taxonomy), external_id=True) == [ + 'Electronic instruments (ELECTRIC) (None) (children: 2)', + ' Synthesizer (SYNTH) (Electronic instruments) (children: 0)', + ' Theramin (THERAMIN) (Electronic instruments) (children: 0)', + 'Percussion instruments (PERCUSS) (None) (children: 3)', + ' Chordophone (CHORD) (Percussion instruments) (children: 1)', + ' Piano (PIANO) (Chordophone) (children: 0)', + ' Idiophone (BELLS) (Percussion instruments) (children: 2)', + ' Celesta (CELESTA) (Idiophone) (children: 0)', + ' Hi-hat (HI-HAT) (Idiophone) (children: 0)', + ' Membranophone (DRUMS) (Percussion instruments) (children: 2)', + ' Cajón (CAJÓN) (Membranophone) (children: 1)', # This tag is present in the import files, but it will be omitted from get_tags() # because it sits beyond TAXONOMY_MAX_DEPTH. - # {'external_id': 'PYLE', 'parent': 'CAJÓN', 'value': 'Pyle Stringed Jam Cajón'}, - {'external_id': 'CELESTA', 'parent': 'BELLS', 'value': 'Celesta'}, - {'external_id': 'HI-HAT', 'parent': 'BELLS', 'value': 'Hi-hat'}, - {'external_id': 'CAJÓN', 'parent': 'DRUMS', 'value': 'Cajón'}, - {'external_id': 'TABLA', 'parent': 'DRUMS', 'value': 'Tabla'}, - {'external_id': 'BANJO', 'parent': 'PLUCK', 'value': 'Banjo'}, - {'external_id': 'HARP', 'parent': 'PLUCK', 'value': 'Harp'}, - {'external_id': 'MANDOLIN', 'parent': 'PLUCK', 'value': 'Mandolin'}, - {'external_id': 'CLARINET', 'parent': 'WOODS', 'value': 'Clarinet'}, - {'external_id': 'FLUTE', 'parent': 'WOODS', 'value': 'Flute'}, - {'external_id': 'OBOE', 'parent': 'WOODS', 'value': 'Oboe'}, + # Pyle Stringed Jam Cajón (PYLE) (Cajón) (children: 0) + ' Tabla (TABLA) (Membranophone) (children: 0)', + 'String instruments (STRINGS) (None) (children: 2)', + ' Bowed strings (BOW) (String instruments) (children: 2)', + ' Cello (CELLO) (Bowed strings) (children: 0)', + ' Violin (VIOLIN) (Bowed strings) (children: 0)', + ' Plucked strings (PLUCK) (String instruments) (children: 3)', + ' Banjo (BANJO) (Plucked strings) (children: 0)', + ' Harp (HARP) (Plucked strings) (children: 0)', + ' Mandolin (MANDOLIN) (Plucked strings) (children: 0)', + 'Wind instruments (WINDS) (None) (children: 2)', + ' Brass (BRASS) (Wind instruments) (children: 2)', + ' Trumpet (TRUMPET) (Brass) (children: 0)', + ' Tuba (TUBA) (Brass) (children: 0)', + ' Woodwinds (WOODS) (Wind instruments) (children: 3)', + ' Clarinet (CLARINET) (Woodwinds) (children: 0)', + ' Flute (FLUTE) (Woodwinds) (children: 0)', + ' Oboe (OBOE) (Woodwinds) (children: 0)', ] diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index f82a22016..909495a4e 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -13,6 +13,7 @@ from openedx_tagging.core.tagging.models import ObjectTag, Taxonomy from .test_models import TestTagTaxonomyMixin, get_tag +from .utils import pretty_format_tags test_languages = [ ("az", "Azerbaijani"), @@ -30,6 +31,17 @@ ("pl", "Polish"), ] +tag_values_for_autocomplete_test = [ + 'Archaea', + 'Archaebacteria', + 'Animalia', + 'Arthropoda', + 'Plantae', + 'Monera', + 'Gastrotrich', + 'Placozoa', +] + @ddt.ddt class TestApiTagging(TestTagTaxonomyMixin, TestCase): @@ -102,20 +114,56 @@ def test_get_taxonomies(self) -> None: self.user_taxonomy, ] + self.dummy_taxonomies - @override_settings(LANGUAGES=test_languages) def test_get_tags(self) -> None: - self.setup_tag_depths() - assert tagging_api.get_tags(self.taxonomy) == [ - *self.domain_tags, - *self.kingdom_tags, - *self.phylum_tags, + assert pretty_format_tags(tagging_api.get_tags(self.taxonomy), parent=False) == [ + "Archaea (children: 3)", + " DPANN (children: 0)", + " Euryarchaeida (children: 0)", + " Proteoarchaeota (children: 0)", + "Bacteria (children: 2)", + " Archaebacteria (children: 0)", + " Eubacteria (children: 0)", + "Eukaryota (children: 5)", + " Animalia (children: 7)", + " Arthropoda (children: 0)", + " Chordata (children: 1)", # The child of this is excluded due to depth limit + " Cnidaria (children: 0)", + " Ctenophora (children: 0)", + " Gastrotrich (children: 0)", + " Placozoa (children: 0)", + " Porifera (children: 0)", + " Fungi (children: 0)", + " Monera (children: 0)", + " Plantae (children: 0)", + " Protista (children: 0)", ] - assert tagging_api.get_tags(self.system_taxonomy) == self.system_tags @override_settings(LANGUAGES=test_languages) + def test_get_tags_system(self) -> None: + assert pretty_format_tags(tagging_api.get_tags(self.system_taxonomy), parent=False) == [ + "System Tag 1 (children: 0)", + "System Tag 2 (children: 0)", + "System Tag 3 (children: 0)", + "System Tag 4 (children: 0)", + ] + def test_get_root_tags(self): - assert tagging_api.get_root_tags(self.taxonomy) == self.domain_tags - assert tagging_api.get_root_tags(self.system_taxonomy) == self.system_tags + root_life_on_earth_tags = tagging_api.get_root_tags(self.taxonomy) + assert pretty_format_tags(root_life_on_earth_tags, parent=False) == [ + 'Archaea (children: 3)', + 'Bacteria (children: 2)', + 'Eukaryota (children: 5)', + ] + + @override_settings(LANGUAGES=test_languages) + def test_get_root_tags_system(self): + result = tagging_api.get_root_tags(self.system_taxonomy) + assert pretty_format_tags(result, parent=False) == [ + 'System Tag 1 (children: 0)', + 'System Tag 2 (children: 0)', + 'System Tag 3 (children: 0)', + 'System Tag 4 (children: 0)', + ] @override_settings(LANGUAGES=test_languages) def test_get_root_language_tags(self): @@ -124,7 +172,7 @@ def test_get_root_language_tags(self): tags that have been used at least once. """ before_langs = [ - tag.external_id for tag in + tag["external_id"] for tag in tagging_api.get_root_tags(self.language_taxonomy) ] assert before_langs == ["en"] @@ -133,17 +181,21 @@ def test_get_root_language_tags(self): tagging_api.tag_object(object_id="foo", taxonomy=self.language_taxonomy, tags=[lang_value]) # now a search will return matching tags: after_langs = [ - tag.external_id for tag in + tag["external_id"] for tag in tagging_api.get_root_tags(self.language_taxonomy) ] expected_langs = [lang_code for lang_code, _ in test_languages] assert after_langs == expected_langs - def test_search_tags(self): - assert tagging_api.search_tags( - self.taxonomy, - search_term='eU' - ) == self.filtered_tags + def test_search_tags(self) -> None: + result = tagging_api.search_tags(self.taxonomy, search_term='eU') + assert pretty_format_tags(result, parent=False) == [ + 'Archaea (children: 3)', # Doesn't match 'eU' but is included because a child is included + ' Euryarchaeida (children: 0)', + 'Bacteria (children: 2)', # Doesn't match 'eU' but is included because a child is included + ' Eubacteria (children: 0)', + 'Eukaryota (children: 5)', + ] @override_settings(LANGUAGES=test_languages) def test_search_language_tags(self): @@ -152,7 +204,7 @@ def test_search_language_tags(self): tags that have been used at least once. """ before_langs = [ - tag.external_id for tag in + tag["external_id"] for tag in tagging_api.search_tags(self.language_taxonomy, search_term='IsH') ] assert before_langs == ["en"] @@ -161,30 +213,43 @@ def test_search_language_tags(self): tagging_api.tag_object(object_id="foo", taxonomy=self.language_taxonomy, tags=[lang_value]) # now a search will return matching tags: after_langs = [ - tag.external_id for tag in + tag["external_id"] for tag in tagging_api.search_tags(self.language_taxonomy, search_term='IsH') ] expected_langs = [lang_code for lang_code, _ in filtered_test_languages] assert after_langs == expected_langs - def test_get_children_tags(self): - assert tagging_api.get_children_tags( - self.taxonomy, - self.animalia.id, - ) == self.phylum_tags - assert tagging_api.get_children_tags( - self.taxonomy, - self.animalia.id, - search_term='dA', - ) == self.filtered_phylum_tags - assert not tagging_api.get_children_tags( - self.system_taxonomy, - self.system_taxonomy_tag.id, - ) - assert not tagging_api.get_children_tags( - self.language_taxonomy, - self.english_tag, - ) + def test_get_children_tags(self) -> None: + """ + Test getting the children of a particular tag in a closed taxonomy. + """ + result1 = tagging_api.get_children_tags(self.taxonomy, "Animalia") + assert pretty_format_tags(result1, parent=False) == [ + ' Arthropoda (children: 0)', + ' Chordata (children: 1)', + # Mammalia is a child of Chordata but excluded here. + ' Cnidaria (children: 0)', + ' Ctenophora (children: 0)', + ' Gastrotrich (children: 0)', + ' Placozoa (children: 0)', + ' Porifera (children: 0)', + ] + + def test_get_children_tags_invalid_taxonomy(self) -> None: + """ + Calling get_children_tags on free text taxonomies gives an error. + """ + free_text_taxonomy = Taxonomy.objects.create(allow_free_text=True, name="FreeText") + tagging_api.tag_object(object_id="obj1", taxonomy=free_text_taxonomy, tags=["some_tag"]) + with self.assertRaises(ValueError) as exc: + tagging_api.get_children_tags(free_text_taxonomy, "some_tag") + assert "Cannot specify a parent tag ID for free text taxonomies" in str(exc.exception) + + def test_get_children_tags_no_children(self) -> None: + """ + Trying to get children of a system tag that has no children yields an empty result: + """ + assert not list(tagging_api.get_children_tags(self.system_taxonomy, self.system_taxonomy_tag.value)) def test_resync_object_tags(self) -> None: self.taxonomy.allow_multiple = True @@ -506,9 +571,7 @@ def test_get_object_tags(self) -> None: # Fetch all the tags for a given object ID assert list( - tagging_api.get_object_tags( - object_id="abc", - ) + tagging_api.get_object_tags(object_id="abc") ) == [ alpha, beta, @@ -516,59 +579,68 @@ def test_get_object_tags(self) -> None: # Fetch all the tags for a given object ID + taxonomy assert list( - tagging_api.get_object_tags( - object_id="abc", - taxonomy_id=self.taxonomy.pk, - ) + tagging_api.get_object_tags(object_id="abc", taxonomy_id=self.taxonomy.pk) ) == [ beta, ] @ddt.data( - ("ChA", ["Archaea", "Archaebacteria"], [2, 5]), - ("ar", ['Archaea', 'Archaebacteria', 'Arthropoda'], [2, 5, 14]), - ("aE", ['Archaea', 'Archaebacteria', 'Plantae'], [2, 5, 10]), - ( - "a", - [ - 'Animalia', - 'Archaea', - 'Archaebacteria', - 'Arthropoda', - 'Gastrotrich', - 'Monera', - 'Placozoa', - 'Plantae', - ], - [9, 2, 5, 14, 16, 13, 19, 10], - ), + ("ChA", [ + "Archaea (used: 1, children: 3)", + " Euryarchaeida (used: 0, children: 0)", + " Proteoarchaeota (used: 0, children: 0)", + "Bacteria (used: 0, children: 2)", # does not contain "cha" but a child does + " Archaebacteria (used: 1, children: 0)", + ]), + ("ar", [ + "Archaea (used: 1, children: 3)", + " Euryarchaeida (used: 0, children: 0)", + " Proteoarchaeota (used: 0, children: 0)", + "Bacteria (used: 0, children: 2)", # does not contain "ar" but a child does + " Archaebacteria (used: 1, children: 0)", + "Eukaryota (used: 0, children: 5)", + " Animalia (used: 1, children: 7)", # does not contain "ar" but a child does + " Arthropoda (used: 1, children: 0)", + " Cnidaria (used: 0, children: 0)", + ]), + ("aE", [ + "Archaea (used: 1, children: 3)", + " Euryarchaeida (used: 0, children: 0)", + " Proteoarchaeota (used: 0, children: 0)", + "Bacteria (used: 0, children: 2)", # does not contain "ae" but a child does + " Archaebacteria (used: 1, children: 0)", + "Eukaryota (used: 0, children: 5)", # does not contain "ae" but a child does + " Plantae (used: 1, children: 0)", + ]), + ("a", [ + "Archaea (used: 1, children: 3)", + " DPANN (used: 0, children: 0)", + " Euryarchaeida (used: 0, children: 0)", + " Proteoarchaeota (used: 0, children: 0)", + "Bacteria (used: 0, children: 2)", + " Archaebacteria (used: 1, children: 0)", + " Eubacteria (used: 0, children: 0)", + "Eukaryota (used: 0, children: 5)", + " Animalia (used: 1, children: 7)", + " Arthropoda (used: 1, children: 0)", + " Chordata (used: 0, children: 1)", + " Cnidaria (used: 0, children: 0)", + " Ctenophora (used: 0, children: 0)", + " Gastrotrich (used: 1, children: 0)", + " Placozoa (used: 1, children: 0)", + " Porifera (used: 0, children: 0)", + " Monera (used: 1, children: 0)", + " Plantae (used: 1, children: 0)", + " Protista (used: 0, children: 0)", + ]), ) @ddt.unpack - def test_autocomplete_tags(self, search: str, expected_values: list[str], expected_ids: list[int | None]) -> None: - tags = [ - 'Archaea', - 'Archaebacteria', - 'Animalia', - 'Arthropoda', - 'Plantae', - 'Monera', - 'Gastrotrich', - 'Placozoa', - ] + expected_values # To create repeats + def test_autocomplete_tags_closed(self, search: str, expected: list[str]) -> None: + """ + Test autocompletion/search for tags using a closed taxonomy. + """ closed_taxonomy = self.taxonomy - open_taxonomy = tagging_api.create_taxonomy( - "Free_Text_Taxonomy", - allow_free_text=True, - ) - - for index, value in enumerate(tags): - # Creating ObjectTags for open taxonomy - ObjectTag( - object_id=f"object_id_{index}", - taxonomy=open_taxonomy, - _value=value, - ).save() - + for index, value in enumerate(tag_values_for_autocomplete_test): # Creating ObjectTags for closed taxonomy tag = get_tag(value) ObjectTag( @@ -578,72 +650,21 @@ def test_autocomplete_tags(self, search: str, expected_values: list[str], expect _value=value, ).save() - # Test for open taxonomy - self._validate_autocomplete_tags( - open_taxonomy, - search, - expected_values, - [None] * len(expected_ids), - ) + result = tagging_api.search_tags(closed_taxonomy, search, include_counts=True) + assert pretty_format_tags(result, parent=False) == expected - # Test for closed taxonomy - self._validate_autocomplete_tags( - closed_taxonomy, - search, - expected_values, - expected_ids, - ) - - def test_autocompleate_not_implemented(self) -> None: - with self.assertRaises(NotImplementedError): - tagging_api.autocomplete_tags(self.taxonomy, 'test', None, object_tags_only=False) - - def _get_tag_values(self, tags) -> list[str]: - """ - Get tag values from tagging_api.autocomplete_tags() result - """ - return [tag.get("value") for tag in tags] - - def _get_tag_ids(self, tags) -> list[int]: - """ - Get tag ids from tagging_api.autocomplete_tags() result + def test_autocomplete_tags_closed_omit_object(self) -> None: """ - return [tag.get("tag_id") for tag in tags] - - def _validate_autocomplete_tags( - self, - taxonomy: Taxonomy, - search: str, - expected_values: list[str], - expected_ids: list[int | None], - ) -> None: + Test autocomplete search that omits the tags from a specific object """ - Validate autocomplete tags - """ - - # Normal search - result = tagging_api.autocomplete_tags(taxonomy, search) - tag_values = self._get_tag_values(result) - for value in tag_values: - assert search.lower() in value.lower() - - assert tag_values == expected_values - assert self._get_tag_ids(result) == expected_ids - - # Create ObjectTag to simulate the content tagging - tag_model = None - if not taxonomy.allow_free_text: - tag_model = get_tag(tag_values[0]) - object_id = 'new_object_id' - ObjectTag( - object_id=object_id, - taxonomy=taxonomy, - tag=tag_model, - _value=tag_values[0], - ).save() - - # Search with object - result = tagging_api.autocomplete_tags(taxonomy, search, object_id) - assert self._get_tag_values(result) == expected_values[1:] - assert self._get_tag_ids(result) == expected_ids[1:] + tagging_api.tag_object(object_id=object_id, taxonomy=self.taxonomy, tags=["Archaebacteria"]) + result = tagging_api.search_tags(self.taxonomy, "ChA", exclude_object_id=object_id) + assert pretty_format_tags(result, parent=False) == [ + "Archaea (children: 3)", + " Euryarchaeida (children: 0)", + " Proteoarchaeota (children: 0)", + # These results are no longer included because of exclude_object_id: + # "Bacteria (children: 2)", # does not contain "cha" but a child does + # " Archaebacteria (children: 0)", + ] diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 2a7131f40..8c006a14c 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -1,6 +1,8 @@ """ Test the tagging base models """ +from __future__ import annotations + import ddt # type: ignore[import] import pytest from django.contrib.auth import get_user_model @@ -12,6 +14,8 @@ from openedx_tagging.core.tagging import api from openedx_tagging.core.tagging.models import LanguageTaxonomy, ObjectTag, Tag, Taxonomy +from .utils import pretty_format_tags + def get_tag(value): """ @@ -55,54 +59,6 @@ def setUp(self): ) self.user_2.save() - # Domain tags (depth=0) - # https://en.wikipedia.org/wiki/Domain_(biology) - self.domain_tags = [ - get_tag("Archaea"), - get_tag("Bacteria"), - get_tag("Eukaryota"), - ] - # Domain tags that contains 'ar' - self.filtered_domain_tags = [ - get_tag("Archaea"), - get_tag("Eukaryota"), - ] - - # Kingdom tags (depth=1) - self.kingdom_tags = [ - # Kingdoms of https://en.wikipedia.org/wiki/Archaea - get_tag("DPANN"), - get_tag("Euryarchaeida"), - get_tag("Proteoarchaeota"), - # Kingdoms of https://en.wikipedia.org/wiki/Bacterial_taxonomy - get_tag("Archaebacteria"), - get_tag("Eubacteria"), - # Kingdoms of https://en.wikipedia.org/wiki/Eukaryote - get_tag("Animalia"), - get_tag("Fungi"), - get_tag("Monera"), - get_tag("Plantae"), - get_tag("Protista"), - ] - - # Phylum tags (depth=2) - self.phylum_tags = [ - # Some phyla of https://en.wikipedia.org/wiki/Animalia - get_tag("Arthropoda"), - get_tag("Chordata"), - get_tag("Cnidaria"), - get_tag("Ctenophora"), - get_tag("Gastrotrich"), - get_tag("Placozoa"), - get_tag("Porifera"), - ] - # Phylum tags that contains 'da' - self.filtered_phylum_tags = [ - get_tag("Arthropoda"), - get_tag("Chordata"), - get_tag("Cnidaria"), - ] - # Biology tags that contains 'eu' self.filtered_tags = [ get_tag("Eubacteria"), @@ -132,17 +88,6 @@ def setUp(self): ) self.dummy_taxonomies.append(taxonomy) - def setup_tag_depths(self): - """ - Annotate our tags with depth so we can compare them. - """ - for tag in self.domain_tags: - tag.depth = 0 - for tag in self.kingdom_tags: - tag.depth = 1 - for tag in self.phylum_tags: - tag.depth = 2 - class TaxonomyTestSubclassA(Taxonomy): """ @@ -237,6 +182,20 @@ def test_taxonomy_cast_bad_value(self): self.taxonomy.taxonomy_class = str assert " must be a subclass of Taxonomy" in str(exc.exception) + def test_unique_tags(self): + # Creating new tag + Tag( + taxonomy=self.taxonomy, + value='New value' + ).save() + + # Creating repeated tag + with self.assertRaises(IntegrityError): + Tag( + taxonomy=self.taxonomy, + value=self.archaea.value, + ).save() + @ddt.data( # Root tags just return their own value ("bacteria", ["Bacteria"]), @@ -251,80 +210,378 @@ def test_taxonomy_cast_bad_value(self): def test_get_lineage(self, tag_attr, lineage): assert getattr(self, tag_attr).get_lineage() == lineage - def test_get_tags(self): - self.setup_tag_depths() - assert self.taxonomy.get_tags() == [ - *self.domain_tags, - *self.kingdom_tags, - *self.phylum_tags, + +@ddt.ddt +class TestFilteredTagsClosedTaxonomy(TestTagTaxonomyMixin, TestCase): + """ + Test the the get_filtered_tags() method of closed taxonomies + """ + def test_get_root(self) -> None: + """ + Test basic retrieval of root tags in the closed taxonomy, using + get_filtered_tags(). Without counts included. + """ + result = list(self.taxonomy.get_filtered_tags(depth=1, include_counts=False)) + common_fields = {"depth": 0, "parent_value": None, "external_id": None} + for r in result: + del r["_id"] # Remove the internal database IDs; they aren't interesting here and a other tests check them + assert result == [ + # These are the root tags, in alphabetical order: + {"value": "Archaea", "child_count": 3, **common_fields}, + {"value": "Bacteria", "child_count": 2, **common_fields}, + {"value": "Eukaryota", "child_count": 5, **common_fields}, + ] + + def test_get_child_tags_one_level(self) -> None: + """ + Test basic retrieval of tags one level below the "Eukaryota" root tag in + the closed taxonomy, using get_filtered_tags(). With counts included. + """ + result = list(self.taxonomy.get_filtered_tags(depth=1, parent_tag_value="Eukaryota", include_counts=True)) + common_fields = {"depth": 1, "parent_value": "Eukaryota", "usage_count": 0, "external_id": None} + for r in result: + del r["_id"] # Remove the internal database IDs; they aren't interesting here and a other tests check them + assert result == [ + # These are the Eukaryota tags, in alphabetical order: + {"value": "Animalia", "child_count": 7, **common_fields}, + {"value": "Fungi", "child_count": 0, **common_fields}, + {"value": "Monera", "child_count": 0, **common_fields}, + {"value": "Plantae", "child_count": 0, **common_fields}, + {"value": "Protista", "child_count": 0, **common_fields}, ] - def test_get_root_tags(self): - assert list(self.taxonomy.get_filtered_tags()) == self.domain_tags - assert list( - self.taxonomy.get_filtered_tags(search_term='aR') - ) == self.filtered_domain_tags - - def test_get_tags_free_text(self): - self.taxonomy.allow_free_text = True - with self.assertNumQueries(0): - assert self.taxonomy.get_tags() == [] - - def test_get_children_tags(self): - assert list( - self.taxonomy.get_filtered_tags(parent_tag_id=self.animalia.id) - ) == self.phylum_tags - assert list( - self.taxonomy.get_filtered_tags( - parent_tag_id=self.animalia.id, - search_term='dA', - ) - ) == self.filtered_phylum_tags - assert not list( - self.system_taxonomy.get_filtered_tags( - parent_tag_id=self.system_taxonomy_tag.id - ) - ) + def test_get_grandchild_tags_one_level(self) -> None: + """ + Test basic retrieval of a single level of tags at two level belows the + "Eukaryota" root tag in the closed taxonomy, using get_filtered_tags(). + """ + result = list(self.taxonomy.get_filtered_tags(depth=1, parent_tag_value="Animalia")) + common_fields = {"depth": 2, "parent_value": "Animalia", "external_id": None} + for r in result: + del r["_id"] # Remove the internal database IDs; they aren't interesting here and a other tests check them + assert result == [ + # These are the Eukaryota tags, in alphabetical order: + {"value": "Arthropoda", "child_count": 0, **common_fields}, + {"value": "Chordata", "child_count": 1, **common_fields}, + {"value": "Cnidaria", "child_count": 0, **common_fields}, + {"value": "Ctenophora", "child_count": 0, **common_fields}, + {"value": "Gastrotrich", "child_count": 0, **common_fields}, + {"value": "Placozoa", "child_count": 0, **common_fields}, + {"value": "Porifera", "child_count": 0, **common_fields}, + ] - def test_get_children_tags_free_text(self): - self.taxonomy.allow_free_text = True - assert not list(self.taxonomy.get_filtered_tags( - parent_tag_id=self.animalia.id - )) - assert not list(self.taxonomy.get_filtered_tags( - parent_tag_id=self.animalia.id, - search_term='dA', - )) + def test_get_depth_1_search_term(self) -> None: + """ + Filter the root tags to only those that match a search term + """ + result = list(self.taxonomy.get_filtered_tags(depth=1, search_term="ARCH", include_counts=True)) + assert result == [ + { + "value": "Archaea", + "child_count": 3, + "depth": 0, + "usage_count": 0, + "parent_value": None, + "external_id": None, + "_id": 2, # These IDs are hard-coded in the test fixture file + }, + ] + # Note that other tags in the taxonomy match "ARCH" but are excluded because of the depth=1 search - def test_search_tags(self): - assert list(self.taxonomy.get_filtered_tags( - search_term='eU', - search_in_all=True - )) == self.filtered_tags - - def test_get_tags_shallow_taxonomy(self): - taxonomy = Taxonomy.objects.create(name="Difficulty") - tags = [ - Tag.objects.create(taxonomy=taxonomy, value="1. Easy"), - Tag.objects.create(taxonomy=taxonomy, value="2. Moderate"), - Tag.objects.create(taxonomy=taxonomy, value="3. Hard"), + def test_get_depth_1_child_search_term(self) -> None: + """ + Filter the child tags of "Bacteria" to only those that match a search term + """ + result = list(self.taxonomy.get_filtered_tags(depth=1, search_term="ARCH", parent_tag_value="Bacteria")) + assert result == [ + { + "value": "Archaebacteria", + "child_count": 0, + "depth": 1, + "parent_value": "Bacteria", + "external_id": None, + "_id": 5, # These IDs are hard-coded in the test fixture file + }, ] + # Note that other tags in the taxonomy match "ARCH" but are excluded because of the depth=1 search + + def test_depth_1_queries(self) -> None: + """ + Test the number of queries used by get_filtered_tags() with closed + taxonomies when depth=1. This should be a constant, not O(n). + """ + with self.assertNumQueries(1): + self.test_get_root() + with self.assertNumQueries(1): + self.test_get_depth_1_search_term() + # When listing the tags below a specific tag, there is one additional query to load each ancestor tag: + with self.assertNumQueries(2): + self.test_get_child_tags_one_level() with self.assertNumQueries(2): - assert taxonomy.get_tags() == tags + self.test_get_depth_1_child_search_term() + with self.assertNumQueries(3): + self.test_get_grandchild_tags_one_level() - def test_unique_tags(self): - # Creating new tag - Tag( - taxonomy=self.taxonomy, - value='New value' - ).save() + ################## - # Creating repeated tag - with self.assertRaises(IntegrityError): - Tag( - taxonomy=self.taxonomy, - value=self.archaea.value, - ).save() + def test_get_all(self) -> None: + """ + Test getting all of the tags in the taxonomy, using get_filtered_tags() + """ + result = pretty_format_tags(self.taxonomy.get_filtered_tags()) + assert result == [ + "Archaea (None) (children: 3)", + " DPANN (Archaea) (children: 0)", + " Euryarchaeida (Archaea) (children: 0)", + " Proteoarchaeota (Archaea) (children: 0)", + "Bacteria (None) (children: 2)", + " Archaebacteria (Bacteria) (children: 0)", + " Eubacteria (Bacteria) (children: 0)", + "Eukaryota (None) (children: 5)", + " Animalia (Eukaryota) (children: 7)", + " Arthropoda (Animalia) (children: 0)", + " Chordata (Animalia) (children: 1)", # note this has a child but the child is not included + " Cnidaria (Animalia) (children: 0)", + " Ctenophora (Animalia) (children: 0)", + " Gastrotrich (Animalia) (children: 0)", + " Placozoa (Animalia) (children: 0)", + " Porifera (Animalia) (children: 0)", + " Fungi (Eukaryota) (children: 0)", + " Monera (Eukaryota) (children: 0)", + " Plantae (Eukaryota) (children: 0)", + " Protista (Eukaryota) (children: 0)", + ] + + def test_search(self) -> None: + """ + Search the whole taxonomy (up to max depth) for a given term. Should + return all tags that match the term as well as their ancestors. + """ + result = pretty_format_tags(self.taxonomy.get_filtered_tags(search_term="ARCH")) + assert result == [ + "Archaea (None) (children: 3)", # Matches the value of this root tag, ARCHaea + " Euryarchaeida (Archaea) (children: 0)", # Matches the value of this child tag + " Proteoarchaeota (Archaea) (children: 0)", # Matches the value of this child tag + "Bacteria (None) (children: 2)", # Does not match this tag but matches a descendant: + " Archaebacteria (Bacteria) (children: 0)", # Matches the value of this child tag + ] + + def test_search_2(self) -> None: + """ + Another search test, that matches a tag deeper in the taxonomy to check + that all its ancestors are returned by the search. + """ + result = pretty_format_tags(self.taxonomy.get_filtered_tags(search_term="chordata")) + assert result == [ + "Eukaryota (None) (children: 5)", + " Animalia (Eukaryota) (children: 7)", + " Chordata (Animalia) (children: 1)", # this is the matching tag. + ] + + def test_tags_deep(self) -> None: + """ + Test getting a deep tag in the taxonomy + """ + result = list(self.taxonomy.get_filtered_tags(parent_tag_value="Chordata", include_counts=True)) + assert result == [ + { + "value": "Mammalia", + "parent_value": "Chordata", + "depth": 3, + "usage_count": 0, + "child_count": 0, + "external_id": None, + "_id": 21, # These IDs are hard-coded in the test fixture file + } + ] + + def test_deep_queries(self) -> None: + """ + Test the number of queries used by get_filtered_tags() with closed + taxonomies when depth=None. This should be a constant, not O(n). + """ + with self.assertNumQueries(1): + self.test_get_all() + # Searching below a specific tag requires an additional query to load that tag: + with self.assertNumQueries(2): + self.test_tags_deep() + # Keyword search requires an additional query: + with self.assertNumQueries(2): + self.test_search() + with self.assertNumQueries(2): + self.test_search_2() + + def test_get_external_id(self) -> None: + """ + Test that if our tags have external IDs, those external IDs are returned + """ + self.bacteria.external_id = "bct001" + self.bacteria.save() + result = list(self.taxonomy.get_filtered_tags(search_term="Eubacteria")) + assert result[0]["value"] == "Bacteria" + assert result[0]["external_id"] == "bct001" + + def test_usage_count(self) -> None: + """ + Test that the usage count in the results is right + """ + api.tag_object(object_id="obj01", taxonomy=self.taxonomy, tags=["Bacteria"]) + api.tag_object(object_id="obj02", taxonomy=self.taxonomy, tags=["Bacteria"]) + api.tag_object(object_id="obj03", taxonomy=self.taxonomy, tags=["Bacteria"]) + api.tag_object(object_id="obj04", taxonomy=self.taxonomy, tags=["Eubacteria"]) + # Now the API should reflect these usage counts: + result = pretty_format_tags(self.taxonomy.get_filtered_tags(search_term="bacteria", include_counts=True)) + assert result == [ + "Bacteria (None) (used: 3, children: 2)", + " Archaebacteria (Bacteria) (used: 0, children: 0)", + " Eubacteria (Bacteria) (used: 1, children: 0)", + ] + # Same with depth=1, which uses a different query internally: + result1 = pretty_format_tags( + self.taxonomy.get_filtered_tags(search_term="bacteria", include_counts=True, depth=1) + ) + assert result1 == [ + "Bacteria (None) (used: 3, children: 2)", + ] + + def test_pathological_tree_sort(self) -> None: + """ + Check for bugs in how tree sorting happens, if the tag names are very + similar. + """ + # pylint: disable=unused-variable + taxonomy = api.create_taxonomy("Sort Test") + root1 = Tag.objects.create(taxonomy=taxonomy, value="1") + child1_1 = Tag.objects.create(taxonomy=taxonomy, value="11", parent=root1) + child1_2 = Tag.objects.create(taxonomy=taxonomy, value="2", parent=root1) + child1_3 = Tag.objects.create(taxonomy=taxonomy, value="1 A", parent=root1) + child1_4 = Tag.objects.create(taxonomy=taxonomy, value="11111", parent=root1) + grandchild1_4_1 = Tag.objects.create(taxonomy=taxonomy, value="1111-grandchild", parent=child1_4) + root2 = Tag.objects.create(taxonomy=taxonomy, value="111") + child2_1 = Tag.objects.create(taxonomy=taxonomy, value="11111111", parent=root2) + child2_2 = Tag.objects.create(taxonomy=taxonomy, value="123", parent=root2) + result = pretty_format_tags(taxonomy.get_filtered_tags()) + assert result == [ + "1 (None) (children: 4)", + " 1 A (1) (children: 0)", + " 11 (1) (children: 0)", + " 11111 (1) (children: 1)", + " 1111-grandchild (11111) (children: 0)", + " 2 (1) (children: 0)", + "111 (None) (children: 2)", + " 11111111 (111) (children: 0)", + " 123 (111) (children: 0)", + ] + + def test_case_insensitive_sort(self) -> None: + """ + Make sure the sorting is case-insensitive + """ + # pylint: disable=unused-variable + taxonomy = api.create_taxonomy("Sort Test") + root1 = Tag.objects.create(taxonomy=taxonomy, value="ALPHABET") + child1_1 = Tag.objects.create(taxonomy=taxonomy, value="Android", parent=root1) + child1_2 = Tag.objects.create(taxonomy=taxonomy, value="abacus", parent=root1) + child1_2 = Tag.objects.create(taxonomy=taxonomy, value="azure", parent=root1) + child1_3 = Tag.objects.create(taxonomy=taxonomy, value="aardvark", parent=root1) + child1_4 = Tag.objects.create(taxonomy=taxonomy, value="ANVIL", parent=root1) + + root2 = Tag.objects.create(taxonomy=taxonomy, value="abstract") + child2_1 = Tag.objects.create(taxonomy=taxonomy, value="Andes", parent=root2) + child2_2 = Tag.objects.create(taxonomy=taxonomy, value="azores islands", parent=root2) + + result = pretty_format_tags(taxonomy.get_filtered_tags()) + assert result == [ + "abstract (None) (children: 2)", + " Andes (abstract) (children: 0)", + " azores islands (abstract) (children: 0)", + "ALPHABET (None) (children: 5)", + " aardvark (ALPHABET) (children: 0)", + " abacus (ALPHABET) (children: 0)", + " Android (ALPHABET) (children: 0)", + " ANVIL (ALPHABET) (children: 0)", + " azure (ALPHABET) (children: 0)", + ] + + # And it's case insensitive when getting only a single level: + result = pretty_format_tags(taxonomy.get_filtered_tags(parent_tag_value="ALPHABET", depth=1)) + assert result == [ + " aardvark (ALPHABET) (children: 0)", + " abacus (ALPHABET) (children: 0)", + " Android (ALPHABET) (children: 0)", + " ANVIL (ALPHABET) (children: 0)", + " azure (ALPHABET) (children: 0)", + ] + + +class TestFilteredTagsFreeTextTaxonomy(TestCase): + """ + Tests for listing/autocompleting/searching for tags in a free text taxonomy. + + Free text taxonomies only return tags that are actually used. + """ + + def setUp(self): + super().setUp() + self.taxonomy = Taxonomy.objects.create(allow_free_text=True, name="FreeText") + # The "triple" tag will be applied to three objects, "double" to two, and "solo" to one: + api.tag_object(object_id="obj1", taxonomy=self.taxonomy, tags=["triple"]) + api.tag_object(object_id="obj2", taxonomy=self.taxonomy, tags=["triple", "double"]) + api.tag_object(object_id="obj3", taxonomy=self.taxonomy, tags=["triple", "double"]) + api.tag_object(object_id="obj4", taxonomy=self.taxonomy, tags=["solo"]) + + def test_get_filtered_tags(self): + """ + Test basic retrieval of all tags in the taxonomy. + Without counts included. + """ + result = list(self.taxonomy.get_filtered_tags(include_counts=False)) + common_fields = {"child_count": 0, "depth": 0, "parent_value": None, "external_id": None, "_id": None} + assert result == [ + # These should appear in alphabetical order: + {"value": "double", **common_fields}, + {"value": "solo", **common_fields}, + {"value": "triple", **common_fields}, + ] + + def test_get_filtered_tags_with_count(self): + """ + Test basic retrieval of all tags in the taxonomy. + Without counts included. + """ + result = list(self.taxonomy.get_filtered_tags(include_counts=True)) + common_fields = {"child_count": 0, "depth": 0, "parent_value": None, "external_id": None, "_id": None} + assert result == [ + # These should appear in alphabetical order: + {"value": "double", "usage_count": 2, **common_fields}, + {"value": "solo", "usage_count": 1, **common_fields}, + {"value": "triple", "usage_count": 3, **common_fields}, + ] + + def test_get_filtered_tags_num_queries(self): + """ + Test that the number of queries used by get_filtered_tags() is fixed + and not O(n) or worse. + """ + with self.assertNumQueries(1): + self.test_get_filtered_tags() + with self.assertNumQueries(1): + self.test_get_filtered_tags_with_count() + + def test_get_filtered_tags_with_search(self) -> None: + """ + Test basic retrieval of only matching tags. + """ + result1 = list(self.taxonomy.get_filtered_tags(search_term="le", include_counts=True)) + common_fields = {"child_count": 0, "depth": 0, "parent_value": None, "external_id": None, "_id": None} + assert result1 == [ + # These should appear in alphabetical order: + {"value": "double", "usage_count": 2, **common_fields}, + {"value": "triple", "usage_count": 3, **common_fields}, + ] + # And it should be case insensitive: + result2 = list(self.taxonomy.get_filtered_tags(search_term="LE", include_counts=True)) + assert result1 == result2 class TestObjectTag(TestTagTaxonomyMixin, TestCase): @@ -450,10 +707,10 @@ def test_tag_case(self) -> None: Test that the object_id is case sensitive. """ # Tag with object_id with lower case - api.tag_object(self.taxonomy, [self.domain_tags[0].value], object_id="case:id:2") + api.tag_object(self.taxonomy, [self.chordata.value], object_id="case:id:2") # Tag with object_id with upper case should not trigger IntegrityError - api.tag_object(self.taxonomy, [self.domain_tags[0].value], object_id="CASE:id:2") + api.tag_object(self.taxonomy, [self.chordata.value], object_id="CASE:id:2") # Create another ObjectTag with lower case object_id should trigger IntegrityError with transaction.atomic(): @@ -461,7 +718,7 @@ def test_tag_case(self) -> None: ObjectTag( object_id="case:id:2", taxonomy=self.taxonomy, - tag=self.domain_tags[0], + tag=self.chordata, ).save() # Create another ObjectTag with upper case object_id should trigger IntegrityError @@ -470,7 +727,7 @@ def test_tag_case(self) -> None: ObjectTag( object_id="CASE:id:2", taxonomy=self.taxonomy, - tag=self.domain_tags[0], + tag=self.chordata, ).save() def test_is_deleted(self): diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 1755d97bb..6a3125464 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -3,7 +3,7 @@ """ from __future__ import annotations -from urllib.parse import parse_qs, urlparse +from urllib.parse import parse_qs, quote, quote_plus, urlparse import ddt # type: ignore[import] # typing support in rules depends on https://github.com/dfunckt/django-rules/pull/177 @@ -19,6 +19,8 @@ from openedx_tagging.core.tagging.rest_api.paginators import TagsPagination from openedx_tagging.core.tagging.rules import can_change_object_tag_objectid, can_view_object_tag_objectid +from .utils import pretty_format_tags + User = get_user_model() TAXONOMY_LIST_URL = "/tagging/rest_api/v1/taxonomies/" @@ -965,9 +967,12 @@ def test_not_authorized_user(self): assert response.status_code == status.HTTP_403_FORBIDDEN - def test_small_taxonomy(self): + def test_small_taxonomy_root(self): + """ + Test explicitly requesting only the root tags of a small taxonomy. + """ self.client.force_authenticate(user=self.staff) - response = self.client.get(self.small_taxonomy_url) + response = self.client.get(self.small_taxonomy_url + "?root_only&include_counts") assert response.status_code == status.HTTP_200_OK data = response.data @@ -977,13 +982,20 @@ def test_small_taxonomy(self): root_count = self.small_taxonomy.tag_set.filter(parent=None).count() assert len(results) == root_count - # Checking tag fields - root_tag = self.small_taxonomy.tag_set.get(id=results[0].get("id")) - root_children_count = root_tag.children.count() + # Checking tag fields on the first tag returned: + root_tag = self.small_taxonomy.tag_set.get(id=results[0].get("_id")) assert results[0].get("value") == root_tag.value - assert results[0].get("taxonomy_id") == self.small_taxonomy.id - assert results[0].get("children_count") == root_children_count - assert len(results[0].get("sub_tags")) == root_children_count + assert results[0].get("child_count") == root_tag.children.count() + assert results[0].get("depth") == 0 # root tags always have depth 0 + assert results[0].get("parent_value") is None + assert results[0].get("usage_count") == 0 + + # Check that we can load sub-tags of that tag: + sub_tags_response = self.client.get(results[0]["sub_tags_url"]) + assert sub_tags_response.status_code == status.HTTP_200_OK + sub_tags_result = sub_tags_response.data["results"] + assert len(sub_tags_result) == root_tag.children.count() + assert set(t["value"] for t in sub_tags_result) == set(t.value for t in root_tag.children.all()) # Checking pagination values assert data.get("next") is None @@ -992,7 +1004,86 @@ def test_small_taxonomy(self): assert data.get("num_pages") == 1 assert data.get("current_page") == 1 + def test_small_taxonomy(self): + """ + Test loading all the tags of a small taxonomy at once. + """ + self.client.force_authenticate(user=self.staff) + response = self.client.get(self.small_taxonomy_url) + assert response.status_code == status.HTTP_200_OK + + data = response.data + results = data.get("results", []) + assert pretty_format_tags(results) == [ + "Archaea (None) (children: 3)", + " DPANN (Archaea) (children: 0)", + " Euryarchaeida (Archaea) (children: 0)", + " Proteoarchaeota (Archaea) (children: 0)", + "Bacteria (None) (children: 2)", + " Archaebacteria (Bacteria) (children: 0)", + " Eubacteria (Bacteria) (children: 0)", + "Eukaryota (None) (children: 5)", + " Animalia (Eukaryota) (children: 7)", + " Arthropoda (Animalia) (children: 0)", + " Chordata (Animalia) (children: 1)", + " Cnidaria (Animalia) (children: 0)", + " Ctenophora (Animalia) (children: 0)", + " Gastrotrich (Animalia) (children: 0)", + " Placozoa (Animalia) (children: 0)", + " Porifera (Animalia) (children: 0)", + " Fungi (Eukaryota) (children: 0)", + " Monera (Eukaryota) (children: 0)", + " Plantae (Eukaryota) (children: 0)", + " Protista (Eukaryota) (children: 0)", + ] + + # Checking pagination values + assert data.get("next") is None + assert data.get("previous") is None + assert data.get("count") == len(results) + assert data.get("num_pages") == 1 + assert data.get("current_page") == 1 + + def test_small_taxonomy_paged(self): + """ + Test loading only the first few of the tags of a small taxonomy. + """ + self.client.force_authenticate(user=self.staff) + response = self.client.get(self.small_taxonomy_url + "?page_size=5") + assert response.status_code == status.HTTP_200_OK + data = response.data + assert pretty_format_tags(data["results"]) == [ + "Archaea (None) (children: 3)", + " DPANN (Archaea) (children: 0)", + " Euryarchaeida (Archaea) (children: 0)", + " Proteoarchaeota (Archaea) (children: 0)", + "Bacteria (None) (children: 2)", + ] + + # Checking pagination values + assert data.get("next") is not None + assert data.get("previous") is None + assert data.get("count") == 20 + assert data.get("num_pages") == 4 + assert data.get("current_page") == 1 + + # Get the next page: + next_response = self.client.get(data.get("next")) + assert next_response.status_code == status.HTTP_200_OK + next_data = next_response.data + assert pretty_format_tags(next_data["results"]) == [ + " Archaebacteria (Bacteria) (children: 0)", + " Eubacteria (Bacteria) (children: 0)", + "Eukaryota (None) (children: 5)", + " Animalia (Eukaryota) (children: 7)", + " Arthropoda (Animalia) (children: 0)", + ] + assert next_data.get("current_page") == 2 + def test_small_search(self): + """ + Test performing a search + """ search_term = 'eU' url = f"{self.small_taxonomy_url}?search_term={search_term}" self.client.force_authenticate(user=self.staff) @@ -1000,45 +1091,65 @@ def test_small_search(self): assert response.status_code == status.HTTP_200_OK data = response.data - results = data.get("results", []) - - assert len(results) == 3 + assert pretty_format_tags(data["results"], parent=False) == [ + "Archaea (children: 3)", # No match in this tag, but a child matches so it's included + " Euryarchaeida (children: 0)", + "Bacteria (children: 2)", # No match in this tag, but a child matches so it's included + " Eubacteria (children: 0)", + "Eukaryota (children: 5)", + ] # Checking pagination values assert data.get("next") is None assert data.get("previous") is None - assert data.get("count") == 3 + assert data.get("count") == 5 assert data.get("num_pages") == 1 assert data.get("current_page") == 1 def test_large_taxonomy(self): + """ + Test listing the tags in a large taxonomy (~7,000 tags). + """ self._build_large_taxonomy() self.client.force_authenticate(user=self.staff) - response = self.client.get(self.large_taxonomy_url) + response = self.client.get(self.large_taxonomy_url + "?include_counts") assert response.status_code == status.HTTP_200_OK data = response.data - results = data.get("results", []) + results = data["results"] + + # Even though we didn't specify root_only, only the root tags will have + # been returned, because of the taxonomy's size. + assert pretty_format_tags(results) == [ + "Tag 0 (None) (used: 0, children: 12)", + "Tag 1099 (None) (used: 0, children: 12)", + "Tag 1256 (None) (used: 0, children: 12)", + "Tag 1413 (None) (used: 0, children: 12)", + "Tag 157 (None) (used: 0, children: 12)", + "Tag 1570 (None) (used: 0, children: 12)", + "Tag 1727 (None) (used: 0, children: 12)", + "Tag 1884 (None) (used: 0, children: 12)", + "Tag 2041 (None) (used: 0, children: 12)", + "Tag 2198 (None) (used: 0, children: 12)", + # ... there are 41 more root tags but they're excluded from this first result page. + ] # Count of paginated root tags assert len(results) == self.page_size - # Checking tag fields - root_tag = self.large_taxonomy.tag_set.get(id=results[0].get("id")) - assert results[0].get("value") == root_tag.value - assert results[0].get("taxonomy_id") == self.large_taxonomy.id - assert results[0].get("parent_id") == root_tag.parent_id - assert results[0].get("children_count") == root_tag.children.count() - assert results[0].get("sub_tags_link") == ( + # Checking some other tag fields not covered by the pretty-formatted string above: + root_tag = self.large_taxonomy.tag_set.get(value=results[0].get("value")) + assert results[0].get("_id") == root_tag.id + assert results[0].get("sub_tags_url") == ( "http://testserver/tagging/" f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?parent_tag_id={root_tag.id}" + f"/tags/?parent_tag={quote(results[0]['value'])}" ) # Checking pagination values assert data.get("next") == ( "http://testserver/tagging/" - f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/?page=2" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/?include_counts=&page=2" ) assert data.get("previous") is None assert data.get("count") == self.root_tags_count @@ -1072,62 +1183,48 @@ def test_next_page_large_taxonomy(self): assert data.get("current_page") == 2 def test_large_search(self): + """ + Test searching in a large taxonomy + """ self._build_large_taxonomy() - search_term = '1' + search_term = '11' url = f"{self.large_taxonomy_url}?search_term={search_term}" self.client.force_authenticate(user=self.staff) response = self.client.get(url) assert response.status_code == status.HTTP_200_OK data = response.data - results = data.get("results", []) - - # Count of paginated root tags - assert len(results) == self.page_size - - # Checking pagination values - assert data.get("next") == ( - "http://testserver/tagging/" - f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?page=2&search_term={search_term}" - ) - assert data.get("previous") is None - assert data.get("count") == 51 - assert data.get("num_pages") == 6 + results = data["results"] + assert pretty_format_tags(results) == [ + "Tag 0 (None) (children: 12)", # First 2 results don't match but have children that match + " Tag 1 (Tag 0) (children: 12)", + " Tag 11 (Tag 1) (children: 0)", + " Tag 105 (Tag 0) (children: 12)", # Non-match but children match + " Tag 110 (Tag 105) (children: 0)", + " Tag 111 (Tag 105) (children: 0)", + " Tag 112 (Tag 105) (children: 0)", + " Tag 113 (Tag 105) (children: 0)", + " Tag 114 (Tag 105) (children: 0)", + " Tag 115 (Tag 105) (children: 0)", + ] + assert data.get("count") == 362 + assert data.get("num_pages") == 37 assert data.get("current_page") == 1 - - def test_next_large_search(self): - self._build_large_taxonomy() - search_term = '1' - url = f"{self.large_taxonomy_url}?search_term={search_term}" - - # Get first page of the search - self.client.force_authenticate(user=self.staff) - response = self.client.get(url) - - # Get next page - response = self.client.get(response.data.get("next")) - - data = response.data - results = data.get("results", []) - - # Count of paginated root tags - assert len(results) == self.page_size - - # Checking pagination values - assert data.get("next") == ( - "http://testserver/tagging/" - f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?page=3&search_term={search_term}" - ) - assert data.get("previous") == ( - "http://testserver/tagging/" - f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?search_term={search_term}" - ) - assert data.get("count") == 51 - assert data.get("num_pages") == 6 - assert data.get("current_page") == 2 + # Get the next page: + next_response = self.client.get(response.data.get("next")) + assert next_response.status_code == status.HTTP_200_OK + assert pretty_format_tags(next_response.data["results"]) == [ + " Tag 116 (Tag 105) (children: 0)", + " Tag 117 (Tag 105) (children: 0)", + " Tag 118 (Tag 0) (children: 12)", + " Tag 119 (Tag 118) (children: 0)", + "Tag 1099 (None) (children: 12)", + " Tag 1100 (Tag 1099) (children: 12)", + " Tag 1101 (Tag 1100) (children: 0)", + " Tag 1102 (Tag 1100) (children: 0)", + " Tag 1103 (Tag 1100) (children: 0)", + " Tag 1104 (Tag 1100) (children: 0)", + ] def test_get_children(self): self._build_large_taxonomy() @@ -1138,7 +1235,7 @@ def test_get_children(self): results = response.data.get("results", []) # Get children tags - response = self.client.get(results[0].get("sub_tags_link")) + response = self.client.get(results[0].get("sub_tags_url")) assert response.status_code == status.HTTP_200_OK data = response.data @@ -1148,22 +1245,21 @@ def test_get_children(self): assert len(results) == self.page_size # Checking tag fields - tag = self.large_taxonomy.tag_set.get(id=results[0].get("id")) + tag = self.large_taxonomy.tag_set.get(id=results[0].get("_id")) assert results[0].get("value") == tag.value - assert results[0].get("taxonomy_id") == self.large_taxonomy.id - assert results[0].get("parent_id") == tag.parent_id - assert results[0].get("children_count") == tag.children.count() - assert results[0].get("sub_tags_link") == ( + assert results[0].get("parent_value") == tag.parent.value + assert results[0].get("child_count") == tag.children.count() + assert results[0].get("sub_tags_url") == ( "http://testserver/tagging/" f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?parent_tag_id={tag.id}" + f"/tags/?parent_tag={quote(tag.value)}" ) # Checking pagination values assert data.get("next") == ( "http://testserver/tagging/" f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?page=2&parent_tag_id={tag.parent_id}" + f"/tags/?page=2&parent_tag={quote_plus(tag.parent.value)}" ) assert data.get("previous") is None assert data.get("count") == self.children_tags_count[0] @@ -1176,17 +1272,21 @@ def test_get_leaves(self): parent_tag = Tag.objects.get(value="Animalia") # Build url to get tags depth=2 - url = f"{self.small_taxonomy_url}?parent_tag_id={parent_tag.id}" + url = f"{self.small_taxonomy_url}?parent_tag={parent_tag.value}" response = self.client.get(url) - results = response.data.get("results", []) - - # Checking tag fields - tag = self.small_taxonomy.tag_set.get(id=results[0].get("id")) - assert results[0].get("value") == tag.value - assert results[0].get("taxonomy_id") == self.small_taxonomy.id - assert results[0].get("parent_id") == tag.parent_id - assert results[0].get("children_count") == tag.children.count() - assert results[0].get("sub_tags_link") is None + results = response.data["results"] + + # Even though we didn't specify root_only, only the root tags will have + # been returned, because of the taxonomy's size. + assert pretty_format_tags(results) == [ + " Arthropoda (Animalia) (children: 0)", + " Chordata (Animalia) (children: 1)", + " Cnidaria (Animalia) (children: 0)", + " Ctenophora (Animalia) (children: 0)", + " Gastrotrich (Animalia) (children: 0)", + " Placozoa (Animalia) (children: 0)", + " Porifera (Animalia) (children: 0)", + ] def test_next_children(self): self._build_large_taxonomy() @@ -1196,22 +1296,27 @@ def test_next_children(self): response = self.client.get(self.large_taxonomy_url) results = response.data.get("results", []) - # Get children to obtain next link - response = self.client.get(results[0].get("sub_tags_link")) + # Get the URL that gives us the children of the first root tag + first_root_tag = results[0] + response = self.client.get(first_root_tag.get("sub_tags_url")) - # Get next children + # Get next page of children response = self.client.get(response.data.get("next")) assert response.status_code == status.HTTP_200_OK data = response.data - results = data.get("results", []) - tag = self.large_taxonomy.tag_set.get(id=results[0].get("id")) + results = data["results"] + assert pretty_format_tags(results) == [ + # There are 12 child tags total, so on this second page, we see only 2 (10 were on the first page): + " Tag 79 (Tag 0) (children: 12)", + " Tag 92 (Tag 0) (children: 12)", + ] # Checking pagination values assert data.get("next") is None assert data.get("previous") == ( "http://testserver/tagging/" - f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/?parent_tag_id={tag.parent_id}" + f"rest_api/v1/taxonomies/{self.large_taxonomy.id}/tags/?parent_tag={quote_plus(first_root_tag['value'])}" ) assert data.get("count") == self.children_tags_count[0] assert data.get("num_pages") == 2 @@ -1260,13 +1365,12 @@ def test_create_tag_in_taxonomy(self): data = response.data - self.assertIsNotNone(data.get("id")) + self.assertIsNotNone(data.get("_id")) self.assertEqual(data.get("value"), new_tag_value) - self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) - self.assertIsNone(data.get("parent_id")) + self.assertIsNone(data.get("parent_value")) self.assertIsNone(data.get("external_id")) self.assertIsNone(data.get("sub_tags_link")) - self.assertEqual(data.get("children_count"), 0) + self.assertEqual(data.get("child_count"), 0) def test_create_tag_in_taxonomy_with_parent(self): self.client.force_authenticate(user=self.staff) @@ -1288,13 +1392,12 @@ def test_create_tag_in_taxonomy_with_parent(self): data = response.data - self.assertIsNotNone(data.get("id")) + self.assertIsNotNone(data.get("_id")) self.assertEqual(data.get("value"), new_tag_value) - self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) - self.assertEqual(data.get("parent_id"), parent_tag.id) + self.assertEqual(data.get("parent_value"), parent_tag.value) self.assertEqual(data.get("external_id"), new_external_id) self.assertIsNone(data.get("sub_tags_link")) - self.assertEqual(data.get("children_count"), 0) + self.assertEqual(data.get("child_count"), 0) def test_create_tag_in_invalid_taxonomy(self): self.client.force_authenticate(user=self.staff) @@ -1460,10 +1563,9 @@ def test_update_tag_in_taxonomy_with_different_methods(self): data = response.data # Check that Tag value got updated - self.assertEqual(data.get("id"), existing_tag.id) + self.assertEqual(data.get("_id"), existing_tag.id) self.assertEqual(data.get("value"), updated_tag_value) - self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) - self.assertEqual(data.get("parent_id"), existing_tag.parent) + self.assertEqual(data.get("parent_value"), existing_tag.parent) self.assertEqual(data.get("external_id"), existing_tag.external_id) # Test updating using the PATCH method @@ -1478,10 +1580,9 @@ def test_update_tag_in_taxonomy_with_different_methods(self): data = response.data # Check the Tag value got updated again - self.assertEqual(data.get("id"), existing_tag.id) + self.assertEqual(data.get("_id"), existing_tag.id) self.assertEqual(data.get("value"), updated_tag_value_2) - self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) - self.assertEqual(data.get("parent_id"), existing_tag.parent) + self.assertEqual(data.get("parent_value"), existing_tag.parent) self.assertEqual(data.get("external_id"), existing_tag.external_id) def test_update_tag_in_taxonomy_reflects_changes_in_object_tags(self): @@ -1521,10 +1622,9 @@ def test_update_tag_in_taxonomy_reflects_changes_in_object_tags(self): data = response.data # Check that Tag value got updated - self.assertEqual(data.get("id"), existing_tag.id) + self.assertEqual(data.get("_id"), existing_tag.id) self.assertEqual(data.get("value"), updated_tag_value) - self.assertEqual(data.get("taxonomy_id"), self.small_taxonomy.pk) - self.assertEqual(data.get("parent_id"), existing_tag.parent) + self.assertEqual(data.get("parent_value"), None) self.assertEqual(data.get("external_id"), existing_tag.external_id) # Check that the ObjectTags got updated as well diff --git a/tests/openedx_tagging/core/tagging/utils.py b/tests/openedx_tagging/core/tagging/utils.py new file mode 100644 index 000000000..0070f12c0 --- /dev/null +++ b/tests/openedx_tagging/core/tagging/utils.py @@ -0,0 +1,26 @@ +""" +Useful utilities for testing tagging and taxonomy code. +""" +from __future__ import annotations + + +def pretty_format_tags(result, parent=True, external_id=False) -> list[str]: + """ + Format the result of get_filtered_tags() to be more human readable. + + Also works with other wrappers around get_filtered_tags, like api.get_tags() + Also works with serialized TagData from the REST API. + """ + pretty_results = [] + for t in result: + line = f"{t['depth'] * ' '}{t['value']} " + if external_id: + line += f"({t['external_id']}) " + if parent: + line += f"({t['parent_value']}) " + line += "(" + if "usage_count" in t: + line += f"used: {t['usage_count']}, " + line += f"children: {t['child_count']})" + pretty_results.append(line) + return pretty_results diff --git a/tox.ini b/tox.ini index c42101f58..284b213fc 100644 --- a/tox.ini +++ b/tox.ini @@ -77,7 +77,7 @@ commands = mypy pycodestyle openedx_learning openedx_tagging tests manage.py setup.py pydocstyle openedx_learning openedx_tagging tests manage.py setup.py - isort --check-only --diff --recursive tests test_utils openedx_learning openedx_tagging manage.py setup.py test_settings.py + isort --check-only --diff tests test_utils openedx_learning openedx_tagging manage.py setup.py test_settings.py make selfcheck [testenv:pii_check] From 9b264bbaddec6503a3803dd7da4a3d95079f37a5 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Tue, 17 Oct 2023 15:47:06 -0400 Subject: [PATCH 067/282] fix: allow LearningPackages with content to be deleted --- openedx_learning/core/publishing/admin.py | 12 +++++--- openedx_learning/core/publishing/api.py | 11 +++++-- .../migrations/0002_alter_fk_on_delete.py | 30 +++++++++++++++++++ openedx_learning/core/publishing/models.py | 8 +++-- 4 files changed, 52 insertions(+), 9 deletions(-) create mode 100644 openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py diff --git a/openedx_learning/core/publishing/admin.py b/openedx_learning/core/publishing/admin.py index 56c59d28f..3cff2ebb8 100644 --- a/openedx_learning/core/publishing/admin.py +++ b/openedx_learning/core/publishing/admin.py @@ -73,7 +73,7 @@ class PublishLogAdmin(ReadOnlyModelAdmin): @admin.register(PublishableEntity) -class PublishableEntityAdmin(ReadOnlyModelAdmin): +class APublishableEntityAdmin(ReadOnlyModelAdmin): """ Read-only admin view for Publishable Entities """ @@ -94,14 +94,18 @@ class PublishableEntityAdmin(ReadOnlyModelAdmin): def get_queryset(self, request): queryset = super().get_queryset(request) return queryset.select_related( - "learning_package", "published__version", "draft__version" + "learning_package", "published__version", ) def draft_version(self, entity): - return entity.draft.version.version_num + if entity.draft.version: + return entity.draft.version.version_num + return None def published_version(self, entity): - return entity.published.version.version_num + if entity.published.version: + return entity.published.version.version_num + return None @admin.register(Published) diff --git a/openedx_learning/core/publishing/api.py b/openedx_learning/core/publishing/api.py index b4b701a20..b97222339 100644 --- a/openedx_learning/core/publishing/api.py +++ b/openedx_learning/core/publishing/api.py @@ -93,9 +93,9 @@ def create_publishable_entity_version( created=created, created_by_id=created_by, ) - Draft.objects.create( + Draft.objects.update_or_create( entity_id=entity_id, - version=version, + defaults=dict(version=version), ) return version @@ -180,6 +180,13 @@ def publish_from_drafts( return publish_log +def delete_drafts( + learning_package_id: int, + draft_qset: QuerySet, +): + pass + + def register_content_models( content_model_cls: type[PublishableEntityMixin], content_version_model_cls: type[PublishableEntityVersionMixin], diff --git a/openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py b/openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py new file mode 100644 index 000000000..844a2a4cb --- /dev/null +++ b/openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.21 on 2023-10-13 14:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + """ + Make it so that Draft and Published cascade deletes from PublishableEntity. + + This makes it so that deleting a LearningPackage properly cleans up these + models as well. + """ + + dependencies = [ + ('oel_publishing', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='draft', + name='entity', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentity'), + ), + migrations.AlterField( + model_name='published', + name='entity', + field=models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='oel_publishing.publishableentity'), + ), + ] diff --git a/openedx_learning/core/publishing/models.py b/openedx_learning/core/publishing/models.py index 7cbeef87c..1eb6803c8 100644 --- a/openedx_learning/core/publishing/models.py +++ b/openedx_learning/core/publishing/models.py @@ -308,10 +308,12 @@ class Draft(models.Model): are updated, not when publishing happens. The Published model only changes when something is published. """ - + # If we're removing a PublishableEntity entirely, also remove the Draft + # entry for it. This isn't a normal operation, but can happen if you're + # deleting an entire LearningPackage. entity = models.OneToOneField( PublishableEntity, - on_delete=models.RESTRICT, + on_delete=models.CASCADE, primary_key=True, ) version = models.OneToOneField( @@ -425,7 +427,7 @@ class Published(models.Model): entity = models.OneToOneField( PublishableEntity, - on_delete=models.RESTRICT, + on_delete=models.CASCADE, primary_key=True, ) version = models.OneToOneField( From 52b32f0b4489547a2df0f346290053295cad6013 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 23 Oct 2023 12:15:55 -0400 Subject: [PATCH 068/282] feat: admin helper for finding third-party 1:1 models Created the one_to_one_related_model_html helper to let us link 1:1 models from lower-level models to higher-level ones. So for instance, "publishing" is a lower level app than "components", meaning that components can call into publishing, but publishing should have no awareness of components. But it's really convenient to be able to click on a PublishableEntity and go to the corresponding Component. This is even more important as people add more models, potentially even in edx-platform plugins. --- openedx_learning/core/publishing/admin.py | 19 ++++-- openedx_learning/lib/admin_utils.py | 73 +++++++++++++++++++++++ 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/openedx_learning/core/publishing/admin.py b/openedx_learning/core/publishing/admin.py index 3cff2ebb8..2045164df 100644 --- a/openedx_learning/core/publishing/admin.py +++ b/openedx_learning/core/publishing/admin.py @@ -5,7 +5,10 @@ from django.contrib import admin -from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin +from openedx_learning.lib.admin_utils import ( + one_to_one_related_model_html, + ReadOnlyModelAdmin, +) from .models import LearningPackage, PublishableEntity, Published, PublishLog, PublishLogRecord @@ -73,11 +76,11 @@ class PublishLogAdmin(ReadOnlyModelAdmin): @admin.register(PublishableEntity) -class APublishableEntityAdmin(ReadOnlyModelAdmin): +class PublishableEntityAdmin(ReadOnlyModelAdmin): """ Read-only admin view for Publishable Entities """ - fields = ( + list_display = [ "key", "draft_version", "published_version", @@ -85,10 +88,11 @@ class APublishableEntityAdmin(ReadOnlyModelAdmin): "learning_package", "created", "created_by", - ) - readonly_fields = fields - list_display = fields + ] list_filter = ["learning_package"] + + fields = list_display + ["see_also"] + readonly_fields = fields search_fields = ["key", "uuid"] def get_queryset(self, request): @@ -97,6 +101,9 @@ def get_queryset(self, request): "learning_package", "published__version", ) + def see_also(self, entity): + return one_to_one_related_model_html(entity) + def draft_version(self, entity): if entity.draft.version: return entity.draft.version.version_num diff --git a/openedx_learning/lib/admin_utils.py b/openedx_learning/lib/admin_utils.py index f7509726f..b1662d259 100644 --- a/openedx_learning/lib/admin_utils.py +++ b/openedx_learning/lib/admin_utils.py @@ -2,6 +2,10 @@ Convenience utilities for the Django Admin. """ from django.contrib import admin +from django.db.models.fields.reverse_related import OneToOneRel +from django.urls import reverse, NoReverseMatch +from django.utils.html import format_html, format_html_join +from django.utils.safestring import mark_safe class ReadOnlyModelAdmin(admin.ModelAdmin): @@ -26,3 +30,72 @@ def has_change_permission(self, request, obj=None): def has_delete_permission(self, request, obj=None): return False + + +def one_to_one_related_model_html(model_obj): + """ + HTML for clickable list of a models that are 1:1-related to ``model_obj``. + + Our design pattern encourages people to hang models off of our lower-level + core lib models. For example, Component has a OneToOneField that references + PublishableEntity. It would be really convenient to have PublishableEntity's + admin page display the link to Component, but the ``publishable`` app is + intended to be a lower-level app than ``components`` and isn't supposed to + be aware of it. The same situation occurs for third-party apps that might + want to extend Component. + + So instead of creating a circular dependency by having ``publishing`` + referencing ``components``, we use Django model introspection to iterate + over all models that have a OneToOneField to the passe din``model_obj``. + This allows us to preserve our dependency boundaries within openedx-learning + and accomodate any third party apps that might further extend these models. + + This will output a list with one entry for each related field. + + * If the field's value is None, we output f"{field_name}: -" + * If the field has a value but no "change" admin page, we output the string + representation of the model obj referenced by that field, i.e. + f{"field_name: {related_model_obj}"}. + * If the field has a value and an admin page, we output the same as above, + but we make the related model object's string representation a link to its + "change" admin page. + """ + one_to_one_field_names = [ + field.name + for field in model_obj._meta.related_objects + if isinstance(field, OneToOneRel) + ] + text = [] + for field_name in one_to_one_field_names: + related_model_obj = getattr(model_obj, field_name, None) + + # No instance of the related model was found, so just use "-" + if related_model_obj is None: + text.append(f"{field_name}: -") + continue + + app_label = related_model_obj._meta.app_label + model_name = related_model_obj._meta.model_name + try: + details_url = reverse( + f"admin:{app_label}_{model_name}_change", + args=(related_model_obj.pk,) + ) + except NoReverseMatch: + # No Admin URL available, so just put the str representation of the + # related model instance. + text.append(f"{field_name}: {related_model_obj}") + continue + + # If we go this far, there is a related model instance and it has a + # "change" admin page (even though it's probably read-only via + # permissions). + html = format_html( + '{}: {}', + field_name, + details_url, + related_model_obj, + ) + text.append(html) + + return format_html_join("\n", "
  • {}
    • ", ((t,) for t in text)) From 21be1ee745586cd5d7844e83db8f40695c9efa8e Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 23 Oct 2023 13:31:42 -0400 Subject: [PATCH 069/282] feat: added get/set_draft_version to publishing API --- openedx_learning/core/publishing/api.py | 39 ++++++++++++-- .../core/publishing/test_api.py | 52 ++++++++++++++++--- 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/openedx_learning/core/publishing/api.py b/openedx_learning/core/publishing/api.py index b97222339..624425d9a 100644 --- a/openedx_learning/core/publishing/api.py +++ b/openedx_learning/core/publishing/api.py @@ -180,11 +180,40 @@ def publish_from_drafts( return publish_log -def delete_drafts( - learning_package_id: int, - draft_qset: QuerySet, -): - pass +def get_draft_version(publishable_entity_id: int) -> PublishableEntityVersion | None: + """ + Return current draft PublishableEntityVersion for this PublishableEntity. + + This function will return None if there is no current draft. + """ + try: + draft = Draft.objects.select_related("version").get( + entity_id=publishable_entity_id + ) + except Draft.DoesNotExist: + # No draft was ever created. + return None + + # draft.version could be None if it was set that way by set_draft_version. + # Setting the Draft.version to None is how we show that we've "deleted" the + # content in Studio. + return draft.version + + +def set_draft_version(publishable_entity_id: int, publishable_entity_version_pk: int | None) -> None: + """ + Modify the Draft of a PublishableEntity to be a PublishableEntityVersion. + + This would most commonly be used to set the Draft to point to a newly + created PublishableEntityVersion that was created in Studio (because someone + edited some content). Setting a Draft's version to None is like deleting it + from Studio's editing point of view. We don't actually delete the Draft row + because we'll need that for publishing purposes (i.e. to delete content from + the published branch). + """ + draft = Draft.objects.get(entity_id=publishable_entity_id) + draft.version_id = publishable_entity_version_pk + draft.save() def register_content_models( diff --git a/tests/openedx_learning/core/publishing/test_api.py b/tests/openedx_learning/core/publishing/test_api.py index 2a4fdced3..7f880ba18 100644 --- a/tests/openedx_learning/core/publishing/test_api.py +++ b/tests/openedx_learning/core/publishing/test_api.py @@ -8,8 +8,8 @@ from django.core.exceptions import ValidationError from django.test import TestCase -from openedx_learning.core.publishing.api import create_learning_package - +from openedx_learning.core.publishing import api as publishing_api +from openedx_learning.core.publishing.models import Draft class CreateLearningPackageTestCase(TestCase): """ @@ -22,7 +22,7 @@ def test_normal(self) -> None: # Note: we must specify '-> None' to opt in to t key = "my_key" title = "My Excellent Title with Emoji 🔥" created = datetime(2023, 4, 2, 15, 9, 0, tzinfo=timezone.utc) - package = create_learning_package(key, title, created) + package = publishing_api.create_learning_package(key, title, created) assert package.key == "my_key" assert package.title == "My Excellent Title with Emoji 🔥" @@ -41,7 +41,7 @@ def test_auto_datetime(self) -> None: """ key = "my_key" title = "My Excellent Title with Emoji 🔥" - package = create_learning_package(key, title) + package = publishing_api.create_learning_package(key, title) assert package.key == "my_key" assert package.title == "My Excellent Title with Emoji 🔥" @@ -62,7 +62,7 @@ def test_non_utc_time(self) -> None: Require UTC timezone for created. """ with pytest.raises(ValidationError) as excinfo: - create_learning_package("my_key", "A Title", datetime(2023, 4, 2)) + publishing_api.create_learning_package("my_key", "A Title", datetime(2023, 4, 2)) message_dict = excinfo.value.message_dict # Both datetime fields should be marked as invalid @@ -73,8 +73,46 @@ def test_already_exists(self) -> None: """ Raises ValidationError for duplicate keys. """ - create_learning_package("my_key", "Original") + publishing_api.create_learning_package("my_key", "Original") with pytest.raises(ValidationError) as excinfo: - create_learning_package("my_key", "Duplicate") + publishing_api.create_learning_package("my_key", "Duplicate") message_dict = excinfo.value.message_dict assert "key" in message_dict + + +class DraftTestCase(TestCase): + + def test_draft_lifecycle(self): + """ + Test basic lifecycle of a Draft. + """ + created = datetime(2023, 4, 2, 15, 9, 0, tzinfo=timezone.utc) + package = publishing_api.create_learning_package( + "my_package_key", + "Draft Testing LearningPackage 🔥", + created=created, + ) + entity = publishing_api.create_publishable_entity( + package.id, + "my_entity", + created, + created_by=None, + ) + # Drafts are NOT created when a PublishableEntity is created, only when + # its first PublisahbleEntityVersion is. + assert publishing_api.get_draft_version(entity.id) is None + + entity_version = publishing_api.create_publishable_entity_version( + entity_id=entity.id, + version_num=1, + title="An Entity 🌴", + created=created, + created_by=None, + ) + assert entity_version == publishing_api.get_draft_version(entity.id) + + # We never really remove rows from the table holding Drafts. We just + # mark the version as None. + publishing_api.set_draft_version(entity.id, None) + entity_version = publishing_api.get_draft_version(entity.id) + assert entity_version is None From 6ac624e40a8a6c000be77bff81fdb4fba1c9f407 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 23 Oct 2023 14:35:38 -0400 Subject: [PATCH 070/282] chore: linting fixes --- openedx_learning/core/publishing/admin.py | 29 ++++++++++++++----- openedx_learning/core/publishing/api.py | 6 ++-- .../migrations/0002_alter_fk_on_delete.py | 2 +- openedx_learning/lib/admin_utils.py | 5 ++-- .../core/publishing/test_api.py | 5 +++- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/openedx_learning/core/publishing/admin.py b/openedx_learning/core/publishing/admin.py index 2045164df..90f72bcc8 100644 --- a/openedx_learning/core/publishing/admin.py +++ b/openedx_learning/core/publishing/admin.py @@ -5,10 +5,7 @@ from django.contrib import admin -from openedx_learning.lib.admin_utils import ( - one_to_one_related_model_html, - ReadOnlyModelAdmin, -) +from openedx_learning.lib.admin_utils import ReadOnlyModelAdmin, one_to_one_related_model_html from .models import LearningPackage, PublishableEntity, Published, PublishLog, PublishLogRecord @@ -90,11 +87,29 @@ class PublishableEntityAdmin(ReadOnlyModelAdmin): "created_by", ] list_filter = ["learning_package"] - - fields = list_display + ["see_also"] - readonly_fields = fields search_fields = ["key", "uuid"] + fields = [ + "key", + "draft_version", + "published_version", + "uuid", + "learning_package", + "created", + "created_by", + "see_also", + ] + readonly_fields = [ + "key", + "draft_version", + "published_version", + "uuid", + "learning_package", + "created", + "created_by", + "see_also", + ] + def get_queryset(self, request): queryset = super().get_queryset(request) return queryset.select_related( diff --git a/openedx_learning/core/publishing/api.py b/openedx_learning/core/publishing/api.py index 624425d9a..ab4099ad2 100644 --- a/openedx_learning/core/publishing/api.py +++ b/openedx_learning/core/publishing/api.py @@ -95,7 +95,7 @@ def create_publishable_entity_version( ) Draft.objects.update_or_create( entity_id=entity_id, - defaults=dict(version=version), + defaults={"version": version}, ) return version @@ -193,10 +193,10 @@ def get_draft_version(publishable_entity_id: int) -> PublishableEntityVersion | except Draft.DoesNotExist: # No draft was ever created. return None - + # draft.version could be None if it was set that way by set_draft_version. # Setting the Draft.version to None is how we show that we've "deleted" the - # content in Studio. + # content in Studio. return draft.version diff --git a/openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py b/openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py index 844a2a4cb..c77a20400 100644 --- a/openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py +++ b/openedx_learning/core/publishing/migrations/0002_alter_fk_on_delete.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.21 on 2023-10-13 14:25 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/openedx_learning/lib/admin_utils.py b/openedx_learning/lib/admin_utils.py index b1662d259..ece0e3a45 100644 --- a/openedx_learning/lib/admin_utils.py +++ b/openedx_learning/lib/admin_utils.py @@ -3,9 +3,8 @@ """ from django.contrib import admin from django.db.models.fields.reverse_related import OneToOneRel -from django.urls import reverse, NoReverseMatch +from django.urls import NoReverseMatch, reverse from django.utils.html import format_html, format_html_join -from django.utils.safestring import mark_safe class ReadOnlyModelAdmin(admin.ModelAdmin): @@ -49,7 +48,7 @@ def one_to_one_related_model_html(model_obj): over all models that have a OneToOneField to the passe din``model_obj``. This allows us to preserve our dependency boundaries within openedx-learning and accomodate any third party apps that might further extend these models. - + This will output a list with one entry for each related field. * If the field's value is None, we output f"{field_name}: -" diff --git a/tests/openedx_learning/core/publishing/test_api.py b/tests/openedx_learning/core/publishing/test_api.py index 7f880ba18..9327a1008 100644 --- a/tests/openedx_learning/core/publishing/test_api.py +++ b/tests/openedx_learning/core/publishing/test_api.py @@ -9,7 +9,7 @@ from django.test import TestCase from openedx_learning.core.publishing import api as publishing_api -from openedx_learning.core.publishing.models import Draft + class CreateLearningPackageTestCase(TestCase): """ @@ -81,6 +81,9 @@ def test_already_exists(self) -> None: class DraftTestCase(TestCase): + """ + Test basic operations with Drafts. + """ def test_draft_lifecycle(self): """ From 13b9909090216119d42e941685a3c905482af075 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Fri, 27 Oct 2023 10:04:01 -0400 Subject: [PATCH 071/282] chore: bump version to 0.3.1 --- openedx_learning/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 8872149d9..38d376f15 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.3.0" +__version__ = "0.3.1" From fcc57cdad4b18ffdd938557b18e0fcbef1d87df5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Fri, 27 Oct 2023 14:02:57 -0300 Subject: [PATCH 072/282] fix: allow export of system defined and open taxonomies (#107) --- .../core/tagging/import_export/api.py | 16 ++++++++-------- .../core/tagging/import_export/test_api.py | 17 +---------------- 2 files changed, 9 insertions(+), 24 deletions(-) diff --git a/openedx_tagging/core/tagging/import_export/api.py b/openedx_tagging/core/tagging/import_export/api.py index c90a78f4d..6afa9096a 100644 --- a/openedx_tagging/core/tagging/import_export/api.py +++ b/openedx_tagging/core/tagging/import_export/api.py @@ -75,7 +75,7 @@ def import_tags( in the file (regardless of action), then `tag_2` and `tag_3` will be deleted if `replace=True` """ - _import_export_validations(taxonomy) + _import_validations(taxonomy) # Checks that exists only one task import in progress at a time per taxonomy if not _check_unique_import_task(taxonomy): @@ -146,7 +146,6 @@ def export_tags(taxonomy: Taxonomy, output_format: ParserFormat) -> str: """ Returns a string with all tag data of the given taxonomy """ - _import_export_validations(taxonomy) parser = get_parser(output_format) return parser.export(taxonomy) @@ -178,20 +177,21 @@ def _get_last_import_task(taxonomy: Taxonomy) -> TagImportTask | None: ) -def _import_export_validations(taxonomy: Taxonomy): +def _import_validations(taxonomy: Taxonomy): """ - Validates if the taxonomy is allowed to import or export tags + Validates if the taxonomy is allowed to import tags """ taxonomy = taxonomy.cast() if taxonomy.allow_free_text: - raise NotImplementedError( + raise ValueError( _( - "Import/export for free-form taxonomies will be implemented in the future." - ) + "Invalid taxonomy ({id}): You cannot import a free-form taxonomy." + ).format(id=taxonomy.id) ) + if taxonomy.system_defined: raise ValueError( _( - "Invalid taxonomy ({id}): You cannot import/export a system-defined taxonomy." + "Invalid taxonomy ({id}): You cannot import a system-defined taxonomy." ).format(id=taxonomy.id) ) diff --git a/tests/openedx_tagging/core/tagging/import_export/test_api.py b/tests/openedx_tagging/core/tagging/import_export/test_api.py index 3fa1d46cf..9721fec44 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_api.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_api.py @@ -74,7 +74,7 @@ def test_invalid_import_tags(self) -> None: def test_import_export_validations(self) -> None: # Check that import is invalid with open taxonomy - with self.assertRaises(NotImplementedError): + with self.assertRaises(ValueError): import_export_api.import_tags( self.open_taxonomy, self.file, @@ -174,21 +174,6 @@ def test_start_task_after_success(self) -> None: self.parser_format, ) - def test_export_validations(self) -> None: - # Check that import is invalid with open taxonomy - with self.assertRaises(NotImplementedError): - import_export_api.export_tags( - self.open_taxonomy, - self.parser_format, - ) - - # Check that import is invalid with system taxonomy - with self.assertRaises(ValueError): - import_export_api.export_tags( - self.system_taxonomy, - self.parser_format, - ) - def test_import_with_export_output(self) -> None: for parser_format in ParserFormat: output = import_export_api.export_tags( From 7e2d6638903074a83f8a7b134e91ff8ee75763d4 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 29 Oct 2023 20:20:06 -0400 Subject: [PATCH 073/282] chore: Updating Python Requirements --- requirements/base.txt | 2 +- requirements/ci.txt | 4 ++-- requirements/dev.txt | 15 +++++++-------- requirements/doc.txt | 11 +++++------ requirements/quality.txt | 11 +++++------ requirements/test.txt | 7 +++---- 6 files changed, 23 insertions(+), 27 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index c8af9b44e..ecea8617b 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -39,7 +39,7 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -cryptography==41.0.4 +cryptography==41.0.5 # via pyjwt django==3.2.22 # via diff --git a/requirements/ci.txt b/requirements/ci.txt index 01a3f8b39..436463d18 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,7 +8,7 @@ click==8.1.7 # via import-linter distlib==0.3.7 # via virtualenv -filelock==3.12.4 +filelock==3.13.0 # via # tox # virtualenv @@ -38,5 +38,5 @@ typing-extensions==4.8.0 # via # grimp # import-linter -virtualenv==20.24.5 +virtualenv==20.24.6 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 97b58609d..8540c2e3b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -90,7 +90,7 @@ coverage[toml]==7.3.2 # -r requirements/quality.txt # coverage # pytest-cov -cryptography==41.0.4 +cryptography==41.0.5 # via # -r requirements/quality.txt # pyjwt @@ -129,7 +129,7 @@ django-debug-toolbar==4.2.0 # via # -r requirements/dev.in # -r requirements/quality.txt -django-stubs==4.2.5 +django-stubs==4.2.6 # via # -r requirements/quality.txt # djangorestframework-stubs @@ -165,7 +165,7 @@ edx-drf-extensions==8.12.0 # via -r requirements/quality.txt edx-i18n-tools==1.3.0 # via -r requirements/dev.in -edx-lint==5.3.4 +edx-lint==5.3.6 # via -r requirements/quality.txt edx-opaque-keys==2.5.1 # via @@ -175,7 +175,7 @@ exceptiongroup==1.1.3 # via # -r requirements/quality.txt # pytest -filelock==3.12.4 +filelock==3.13.0 # via # -r requirements/ci.txt # tox @@ -261,7 +261,6 @@ more-itertools==10.1.0 mypy==1.6.1 # via # -r requirements/quality.txt - # django-stubs # djangorestframework-stubs mypy-extensions==1.0.0 # via @@ -355,7 +354,7 @@ pylint-celery==0.3 # via # -r requirements/quality.txt # edx-lint -pylint-django==2.5.4 +pylint-django==2.5.5 # via # -r requirements/quality.txt # edx-lint @@ -376,7 +375,7 @@ pyproject-hooks==1.0.0 # via # -r requirements/pip-tools.txt # build -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements/quality.txt # pytest-cov @@ -536,7 +535,7 @@ vine==5.0.0 # amqp # celery # kombu -virtualenv==20.24.5 +virtualenv==20.24.6 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index 345c626ef..e5663a3c1 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -18,7 +18,7 @@ asgiref==3.7.2 # django attrs==23.1.0 # via -r requirements/test.txt -babel==2.13.0 +babel==2.13.1 # via # pydata-sphinx-theme # sphinx @@ -78,7 +78,7 @@ coverage[toml]==7.3.2 # -r requirements/test.txt # coverage # pytest-cov -cryptography==41.0.4 +cryptography==41.0.5 # via # -r requirements/test.txt # pyjwt @@ -104,7 +104,7 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.2.0 # via -r requirements/test.txt -django-stubs==4.2.5 +django-stubs==4.2.6 # via # -r requirements/test.txt # djangorestframework-stubs @@ -187,7 +187,6 @@ mock==5.1.0 mypy==1.6.1 # via # -r requirements/test.txt - # django-stubs # djangorestframework-stubs mypy-extensions==1.0.0 # via @@ -229,7 +228,7 @@ pycparser==2.21 # via # -r requirements/test.txt # cffi -pydata-sphinx-theme==0.14.1 +pydata-sphinx-theme==0.14.2 # via sphinx-book-theme pygments==2.16.1 # via @@ -252,7 +251,7 @@ pynacl==1.5.0 # via # -r requirements/test.txt # edx-django-utils -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements/test.txt # pytest-cov diff --git a/requirements/quality.txt b/requirements/quality.txt index f5b88b43c..5bc7af81b 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -78,7 +78,7 @@ coverage[toml]==7.3.2 # -r requirements/test.txt # coverage # pytest-cov -cryptography==41.0.4 +cryptography==41.0.5 # via # -r requirements/test.txt # pyjwt @@ -106,7 +106,7 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.2.0 # via -r requirements/test.txt -django-stubs==4.2.5 +django-stubs==4.2.6 # via # -r requirements/test.txt # djangorestframework-stubs @@ -138,7 +138,7 @@ edx-django-utils==5.7.0 # edx-drf-extensions edx-drf-extensions==8.12.0 # via -r requirements/test.txt -edx-lint==5.3.4 +edx-lint==5.3.6 # via -r requirements/quality.in edx-opaque-keys==2.5.1 # via @@ -205,7 +205,6 @@ more-itertools==10.1.0 mypy==1.6.1 # via # -r requirements/test.txt - # django-stubs # djangorestframework-stubs mypy-extensions==1.0.0 # via @@ -269,7 +268,7 @@ pylint==3.0.2 # pylint-plugin-utils pylint-celery==0.3 # via edx-lint -pylint-django==2.5.4 +pylint-django==2.5.5 # via edx-lint pylint-plugin-utils==0.8.2 # via @@ -283,7 +282,7 @@ pynacl==1.5.0 # via # -r requirements/test.txt # edx-django-utils -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements/test.txt # pytest-cov diff --git a/requirements/test.txt b/requirements/test.txt index 60ebf2389..05af47ff2 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -67,7 +67,7 @@ coverage[toml]==7.3.2 # via # -r requirements/test.in # pytest-cov -cryptography==41.0.4 +cryptography==41.0.5 # via # -r requirements/base.txt # pyjwt @@ -91,7 +91,7 @@ django-crum==0.7.9 # edx-django-utils django-debug-toolbar==4.2.0 # via -r requirements/test.in -django-stubs==4.2.5 +django-stubs==4.2.6 # via # -r requirements/test.in # djangorestframework-stubs @@ -148,7 +148,6 @@ mock==5.1.0 mypy==1.6.1 # via # -r requirements/test.in - # django-stubs # djangorestframework-stubs mypy-extensions==1.0.0 # via mypy @@ -192,7 +191,7 @@ pynacl==1.5.0 # via # -r requirements/base.txt # edx-django-utils -pytest==7.4.2 +pytest==7.4.3 # via # -r requirements/test.in # pytest-cov From a248f35ac6df1bfedd15c6e690fa1d40d8018994 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 2 Nov 2023 11:22:05 -0700 Subject: [PATCH 074/282] feat!: Improve the "get object tags" API (an unstable REST API) (#111) --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/api.py | 22 +- .../migrations/0013_tag_parent_blank.py | 19 + openedx_tagging/core/tagging/models/base.py | 49 +- .../core/tagging/rest_api/v1/serializers.py | 49 +- .../core/tagging/rest_api/v1/urls.py | 1 + .../core/tagging/rest_api/v1/views.py | 92 ++- .../openedx_tagging/core/tagging/test_api.py | 45 +- .../core/tagging/test_models.py | 171 +++--- .../core/tagging/test_views.py | 563 ++++++++++-------- 10 files changed, 646 insertions(+), 367 deletions(-) create mode 100644 openedx_tagging/core/tagging/migrations/0013_tag_parent_blank.py diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 38d376f15..711ab06c2 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.3.1" +__version__ = "0.3.2" diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index a655cd783..8c5ed7648 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -12,12 +12,14 @@ """ from __future__ import annotations -from django.db import transaction -from django.db.models import QuerySet +from django.db import models, transaction +from django.db.models import F, QuerySet, Value +from django.db.models.functions import Coalesce, Concat, Lower from django.utils.translation import gettext as _ from .data import TagDataQuerySet from .models import ObjectTag, Tag, Taxonomy +from .models.utils import ConcatNull # Export this as part of the API TagDoesNotExist = Tag.DoesNotExist @@ -165,8 +167,20 @@ def get_object_tags( filters = {"taxonomy_id": taxonomy_id} if taxonomy_id else {} tags = ( object_tag_class.objects.filter(object_id=object_id, **filters) - .select_related("tag", "taxonomy") - .order_by("id") + # Preload related objects, including data for the "get_lineage" method on ObjectTag/Tag: + .select_related("taxonomy", "tag", "tag__parent", "tag__parent__parent") + # Sort the tags within each taxonomy in "tree order". See Taxonomy._get_filtered_tags_deep for details on this: + .annotate(sort_key=Lower(Concat( + ConcatNull(F("tag__parent__parent__parent__value"), Value("\t")), + ConcatNull(F("tag__parent__parent__value"), Value("\t")), + ConcatNull(F("tag__parent__value"), Value("\t")), + Coalesce(F("tag__value"), F("_value")), + Value("\t"), + output_field=models.CharField(), + ))) + .annotate(taxonomy_name=Coalesce(F("taxonomy__name"), F("_name"))) + # Sort first by taxonomy name, then by tag value in tree order: + .order_by("taxonomy_name", "sort_key") ) return tags diff --git a/openedx_tagging/core/tagging/migrations/0013_tag_parent_blank.py b/openedx_tagging/core/tagging/migrations/0013_tag_parent_blank.py new file mode 100644 index 000000000..d947bbd31 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0013_tag_parent_blank.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.22 on 2023-10-30 21:51 + +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_tagging', '0012_language_taxonomy'), + ] + + operations = [ + migrations.AlterField( + model_name='tag', + name='parent', + field=models.ForeignKey(blank=True, default=None, help_text='Tag that lives one level up from the current tag, forming a hierarchy.', null=True, on_delete=django.db.models.deletion.CASCADE, related_name='children', to='oel_tagging.tag'), + ), + ] diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index 71318a13b..86f948770 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -51,6 +51,7 @@ class Tag(models.Model): parent = models.ForeignKey( "self", null=True, + blank=True, default=None, on_delete=models.CASCADE, related_name="children", @@ -100,18 +101,30 @@ def get_lineage(self) -> Lineage: Queries and returns the lineage of the current tag as a list of Tag.value strings. The root Tag.value is first, followed by its child.value, and on down to self.value. - - Performance note: may perform as many as TAXONOMY_MAX_DEPTH select queries. """ - lineage: Lineage = [] - tag: Tag | None = self - depth = TAXONOMY_MAX_DEPTH - while tag and depth > 0: - lineage.insert(0, tag.value) - tag = tag.parent - depth -= 1 + lineage: Lineage = [self.value] + next_ancestor = self.get_next_ancestor() + while next_ancestor: + lineage.insert(0, next_ancestor.value) + next_ancestor = next_ancestor.get_next_ancestor() return lineage + def get_next_ancestor(self) -> Tag | None: + """ + Fetch the parent of this Tag. + + While doing so, preload several ancestors at the same time, so we can + use fewer database queries than the basic approach of iterating through + parent.parent.parent... + """ + if self.parent_id is None: + return None + if not Tag.parent.is_cached(self): # pylint: disable=no-member + # Parent is not yet loaded. Retrieve our parent, grandparent, and great-grandparent in one query. + # This is not actually changing the parent, just loading it and caching it. + self.parent = Tag.objects.select_related("parent", "parent__parent").get(pk=self.parent_id) + return self.parent + @cached_property def depth(self) -> int: """ @@ -149,6 +162,20 @@ def child_count(self) -> int: return self.taxonomy.tag_set.filter(parent=self).count() return 0 + def clean(self): + """ + Validate this tag before saving + """ + # Don't allow leading or trailing whitespace: + self.value = self.value.strip() + if self.external_id: + self.external_id = self.external_id.strip() + # Don't allow \t (tab) character at all, as we use it for lineage in database queries + if "\t" in self.value: + raise ValidationError("Tags in a taxonomy cannot contain a TAB character.") + if self.external_id and "\t" in self.external_id: + raise ValidationError("Tag external ID cannot contain a TAB character.") + class Taxonomy(models.Model): """ @@ -534,6 +561,7 @@ def add_tag( tag = Tag.objects.create( taxonomy=self, value=tag_value, parent=parent, external_id=external_id ) + tag.full_clean() return tag @@ -802,6 +830,9 @@ def clean(self): raise ValidationError("Invalid _value - empty string") if self.taxonomy and self.taxonomy.name != self._name: raise ValidationError("ObjectTag's _name is out of sync with Taxonomy.name") + if "," in self.object_id or "*" in self.object_id: + # Some APIs may use these characters to allow wildcard matches or multiple matches in the future. + raise ValidationError("Object ID contains invalid characters") def get_lineage(self) -> Lineage: """ diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 4fe62c72a..3281af3f0 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -3,6 +3,8 @@ """ from __future__ import annotations +from typing import Any + from rest_framework import serializers from rest_framework.reverse import reverse @@ -61,22 +63,61 @@ class ObjectTagListQueryParamsSerializer(serializers.Serializer): # pylint: dis ) -class ObjectTagSerializer(serializers.ModelSerializer): +class ObjectTagMinimalSerializer(serializers.ModelSerializer): """ - Serializer for the ObjectTag model. + Minimal serializer for the ObjectTag model. """ class Meta: model = ObjectTag - fields = [ + fields = ["value", "lineage"] + + lineage = serializers.ListField(child=serializers.CharField(), source="get_lineage", read_only=True) + + +class ObjectTagSerializer(ObjectTagMinimalSerializer): + """ + Serializer for the ObjectTag model. + """ + class Meta: + model = ObjectTag + fields = ObjectTagMinimalSerializer.Meta.fields + [ + # The taxonomy name "name", - "value", "taxonomy_id", # If the Tag or Taxonomy has been deleted, this ObjectTag shouldn't be shown to users. "is_deleted", ] +class ObjectTagsByTaxonomySerializer(serializers.ModelSerializer): + """ + Serialize a group of ObjectTags, grouped by taxonomy + """ + def to_representation(self, instance: list[ObjectTag]) -> dict: + """ + Convert this list of ObjectTags to the serialized dictionary, grouped by Taxonomy + """ + by_object: dict[str, dict[str, Any]] = {} + for obj_tag in instance: + if obj_tag.object_id not in by_object: + by_object[obj_tag.object_id] = { + "taxonomies": [] + } + taxonomies = by_object[obj_tag.object_id]["taxonomies"] + tax_entry = next((t for t in taxonomies if t["taxonomy_id"] == obj_tag.taxonomy_id), None) + if tax_entry is None: + tax_entry = { + "name": obj_tag.name, + "taxonomy_id": obj_tag.taxonomy_id, + "editable": (not obj_tag.taxonomy.cast().system_defined) if obj_tag.taxonomy else False, + "tags": [] + } + taxonomies.append(tax_entry) + tax_entry["tags"].append(ObjectTagMinimalSerializer(obj_tag).data) + return by_object + + class ObjectTagUpdateBodySerializer(serializers.Serializer): # pylint: disable=abstract-method """ Serializer of the body for the ObjectTag UPDATE view diff --git a/openedx_tagging/core/tagging/rest_api/v1/urls.py b/openedx_tagging/core/tagging/rest_api/v1/urls.py index 72ff87d0d..b7841b574 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/urls.py +++ b/openedx_tagging/core/tagging/rest_api/v1/urls.py @@ -10,6 +10,7 @@ router = DefaultRouter() router.register("taxonomies", views.TaxonomyView, basename="taxonomy") router.register("object_tags", views.ObjectTagView, basename="object_tag") +router.register("object_tag_counts", views.ObjectTagCountsView, basename="object_tag_counts") urlpatterns = [ path("", include(router.urls)), diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 951c8e6f2..d2566ca2c 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -3,6 +3,8 @@ """ from __future__ import annotations +from typing import Any + from django.db import models from django.http import Http404, HttpResponse from rest_framework import mixins, status @@ -26,12 +28,13 @@ from ...data import TagDataQuerySet from ...import_export.api import export_tags from ...import_export.parsers import ParserFormat -from ...models import Taxonomy +from ...models import ObjectTag, Taxonomy from ...rules import ObjectTagPermissionItem from ..paginators import TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination from .permissions import ObjectTagObjectPermissions, TagObjectPermissions, TaxonomyObjectPermissions from .serializers import ( ObjectTagListQueryParamsSerializer, + ObjectTagsByTaxonomySerializer, ObjectTagSerializer, ObjectTagUpdateBodySerializer, ObjectTagUpdateQueryParamsSerializer, @@ -265,10 +268,6 @@ class ObjectTagView( * 400 - Invalid query parameter * 403 - Permission denied - **Create Query Returns** - * 403 - Permission denied - * 405 - Method not allowed - **Update Parameters** * object_id (required): - The Object ID to add ObjectTags for. @@ -306,20 +305,20 @@ def get_queryset(self) -> models.QuerySet: taxonomy = taxonomy.cast() taxonomy_id = taxonomy.id - perm = "oel_tagging.view_objecttag" - perm_obj = ObjectTagPermissionItem( - taxonomy=taxonomy, - object_id=object_id, - ) - - if not self.request.user.has_perm( - perm, - # The obj arg expects a model, but we are passing an object - perm_obj, # type: ignore[arg-type] - ): - raise PermissionDenied( - "You do not have permission to view object tags for this taxonomy or object_id." - ) + if object_id.endswith("*") or "," in object_id: # pylint: disable=no-else-raise + raise ValidationError("Retrieving tags from multiple objects is not yet supported.") + # Note: This API is actually designed so that in the future it can be extended to return tags for multiple + # objects, e.g. if object_id.endswith("*") then it results in a object_id__startswith query. However, for + # now we have no use case for that so we retrieve tags for one object at a time. + else: + if not self.request.user.has_perm( + "oel_tagging.view_objecttag", + # The obj arg expects a model, but we are passing an object + ObjectTagPermissionItem(taxonomy=taxonomy, object_id=object_id), # type: ignore[arg-type] + ): + raise PermissionDenied( + "You do not have permission to view object tags for this taxonomy or object_id." + ) return get_object_tags(object_id, taxonomy_id) @@ -335,8 +334,13 @@ def retrieve(self, request, *args, **kwargs) -> Response: behavior we want. """ object_tags = self.filter_queryset(self.get_queryset()) - serializer = ObjectTagSerializer(object_tags, many=True) - return Response(serializer.data) + serializer = ObjectTagsByTaxonomySerializer(list(object_tags)) + response_data = serializer.data + if self.kwargs["object_id"] not in response_data: + # For consistency, the key with the object_id should always be present in the response, even if there + # are no tags at all applied to this object. + response_data[self.kwargs["object_id"]] = {"taxonomies": []} + return Response(response_data) def update(self, request, *args, **kwargs) -> Response: """ @@ -405,6 +409,52 @@ def update(self, request, *args, **kwargs) -> Response: return self.retrieve(request, object_id) +@view_auth_classes +class ObjectTagCountsView( + mixins.RetrieveModelMixin, + GenericViewSet, +): + """ + View to retrieve the count of ObjectTags for all matching object IDs. + + This API does NOT bother doing any permission checks as the "# of tags" is not considered sensitive information. + + **Retrieve Parameters** + * object_id_pattern (required): - The Object ID to retrieve ObjectTags for. Can contain '*' at the end + for wildcard matching, or use ',' to separate multiple object IDs. + + **Retrieve Example Requests** + GET api/tagging/v1/object_tag_counts/:object_id_pattern + + **Retrieve Query Returns** + * 200 - Success + """ + + serializer_class = ObjectTagSerializer + lookup_field = "object_id_pattern" + + def retrieve(self, request, *args, **kwargs) -> Response: + """ + Retrieve the counts of object tags that belong to a given object_id pattern + + Note: We override `retrieve` here instead of `list` because we are + passing in the Object ID (object_id) in the path (as opposed to passing + it in as a query_param) to retrieve the ObjectTag counts. + """ + # This API does NOT bother doing any permission checks as the # of tags is not considered sensitive information. + object_id_pattern = self.kwargs["object_id_pattern"] + qs: Any = ObjectTag.objects + if object_id_pattern.endswith("*"): + qs = qs.filter(object_id__startswith=object_id_pattern[0:len(object_id_pattern) - 1]) + elif "*" in object_id_pattern: + raise ValidationError("Wildcard matches are only supported if the * is at the end.") + else: + qs = qs.filter(object_id__in=object_id_pattern.split(",")) + + qs = qs.values("object_id").annotate(num_tags=models.Count("id")).order_by("object_id") + return Response({row["object_id"]: row["num_tags"] for row in qs}) + + @view_auth_classes class TaxonomyTagsView(ListAPIView, RetrieveUpdateDestroyAPIView): """ diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 909495a4e..a8c2e7569 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -85,17 +85,19 @@ def test_get_taxonomies(self) -> None: enabled = list(tagging_api.get_taxonomies()) assert enabled == [ tax1, + self.free_text_taxonomy, tax3, self.language_taxonomy, self.taxonomy, self.system_taxonomy, self.user_taxonomy, - ] + self.dummy_taxonomies + ] assert str(enabled[0]) == f" ({tax1.id}) Enabled" - assert str(enabled[1]) == " (5) Import Taxonomy Test" - assert str(enabled[2]) == " (-1) Languages" - assert str(enabled[3]) == " (1) Life on Earth" - assert str(enabled[4]) == " (4) System defined taxonomy" + assert str(enabled[1]) == f" ({self.free_text_taxonomy.id}) Free Text" + assert str(enabled[2]) == " (5) Import Taxonomy Test" + assert str(enabled[3]) == " (-1) Languages" + assert str(enabled[4]) == " (1) Life on Earth" + assert str(enabled[5]) == " (4) System defined taxonomy" with self.assertNumQueries(1): disabled = list(tagging_api.get_taxonomies(enabled=False)) @@ -107,12 +109,13 @@ def test_get_taxonomies(self) -> None: assert both == [ tax2, tax1, + self.free_text_taxonomy, tax3, self.language_taxonomy, self.taxonomy, self.system_taxonomy, self.user_taxonomy, - ] + self.dummy_taxonomies + ] def test_get_tags(self) -> None: assert pretty_format_tags(tagging_api.get_tags(self.taxonomy), parent=False) == [ @@ -263,20 +266,20 @@ def test_resync_object_tags(self) -> None: # At first, none of these will be deleted: assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + ("bar", False), + ("foo", False), (self.archaea.value, False), (self.bacteria.value, False), - ("foo", False), - ("bar", False), ] # Delete "bacteria" from the taxonomy: - self.bacteria.delete() # TODO: add an API method for this + tagging_api.delete_tags_from_taxonomy(self.taxonomy, [self.bacteria.value], with_subtags=True) assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + ("bar", False), + ("foo", False), (self.archaea.value, False), (self.bacteria.value, True), # <--- deleted! But the value is preserved. - ("foo", False), - ("bar", False), ] # Re-syncing the tags at this point does nothing: @@ -291,10 +294,10 @@ def test_resync_object_tags(self) -> None: # Now the tag is not deleted: assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + ("bar", False), + ("foo", False), (self.archaea.value, False), (self.bacteria.value, False), # <--- not deleted - ("foo", False), - ("bar", False), ] # Re-syncing the tags now does nothing: @@ -311,12 +314,12 @@ def test_tag_object(self): self.chordata, ], [ - self.chordata, self.archaebacteria, + self.chordata, ], [ - self.archaebacteria, self.archaea, + self.archaebacteria, ], ] @@ -520,8 +523,9 @@ def test_tag_object_limit(self) -> None: """ Test that the tagging limit is enforced. """ + dummy_taxonomies = self.create_100_taxonomies() # The user can add up to 100 tags to a object - for taxonomy in self.dummy_taxonomies: + for taxonomy in dummy_taxonomies: tagging_api.tag_object( taxonomy, ["Dummy Tag"], @@ -539,7 +543,7 @@ def test_tag_object_limit(self) -> None: assert "Cannot add more than 100 tags to" in str(exc.exception) # Updating existing tags should work - for taxonomy in self.dummy_taxonomies: + for taxonomy in dummy_taxonomies: tagging_api.tag_object( taxonomy, ["New Dummy Tag"], @@ -547,7 +551,7 @@ def test_tag_object_limit(self) -> None: ) # Updating existing tags adding a new one should fail - for taxonomy in self.dummy_taxonomies: + for taxonomy in dummy_taxonomies: with self.assertRaises(ValueError) as exc: tagging_api.tag_object( taxonomy, @@ -558,15 +562,16 @@ def test_tag_object_limit(self) -> None: assert "Cannot add more than 100 tags to" in str(exc.exception) def test_get_object_tags(self) -> None: - # Alpha tag has no taxonomy + # Alpha tag has no taxonomy (as if the taxonomy had been deleted) alpha = ObjectTag(object_id="abc") alpha.name = self.taxonomy.name - alpha.value = self.mammalia.value + alpha.value = "alpha" alpha.save() # Beta tag has a closed taxonomy beta = ObjectTag.objects.create( object_id="abc", taxonomy=self.taxonomy, + tag=self.taxonomy.tag_set.get(value="Protista"), ) # Fetch all the tags for a given object ID diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 8c006a14c..88bfeaad2 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -33,12 +33,14 @@ class TestTagTaxonomyMixin: def setUp(self): super().setUp() + # Core pre-defined taxonomies for testing: self.taxonomy = Taxonomy.objects.get(name="Life on Earth") - self.system_taxonomy = Taxonomy.objects.get( - name="System defined taxonomy" - ) + self.system_taxonomy = Taxonomy.objects.get(name="System defined taxonomy") self.language_taxonomy = LanguageTaxonomy.objects.get(name="Languages") self.user_taxonomy = Taxonomy.objects.get(name="User Authors").cast() + self.free_text_taxonomy = Taxonomy.objects.create(name="Free Text", allow_free_text=True) + + # References to some tags: self.archaea = get_tag("Archaea") self.archaebacteria = get_tag("Archaebacteria") self.bacteria = get_tag("Bacteria") @@ -73,7 +75,41 @@ def setUp(self): get_tag("System Tag 4"), ] - self.dummy_taxonomies = [] + def create_sort_test_taxonomy(self) -> Taxonomy: + """ + Helper method to create a taxonomy that's difficult to sort correctly in tree order. + """ + # pylint: disable=unused-variable + taxonomy = api.create_taxonomy("Sort Test") + + root1 = Tag.objects.create(taxonomy=taxonomy, value="1") + child1_1 = Tag.objects.create(taxonomy=taxonomy, value="11", parent=root1) + child1_2 = Tag.objects.create(taxonomy=taxonomy, value="2", parent=root1) + child1_3 = Tag.objects.create(taxonomy=taxonomy, value="1 A", parent=root1) + child1_4 = Tag.objects.create(taxonomy=taxonomy, value="11111", parent=root1) + grandchild1_4_1 = Tag.objects.create(taxonomy=taxonomy, value="1111-grandchild", parent=child1_4) + + root2 = Tag.objects.create(taxonomy=taxonomy, value="111") + child2_1 = Tag.objects.create(taxonomy=taxonomy, value="11111111", parent=root2) + child2_2 = Tag.objects.create(taxonomy=taxonomy, value="123", parent=root2) + + root1 = Tag.objects.create(taxonomy=taxonomy, value="ALPHABET") + child1_1 = Tag.objects.create(taxonomy=taxonomy, value="Android", parent=root1) + child1_2 = Tag.objects.create(taxonomy=taxonomy, value="abacus", parent=root1) + child1_2 = Tag.objects.create(taxonomy=taxonomy, value="azure", parent=root1) + child1_3 = Tag.objects.create(taxonomy=taxonomy, value="aardvark", parent=root1) + child1_4 = Tag.objects.create(taxonomy=taxonomy, value="ANVIL", parent=root1) + + root2 = Tag.objects.create(taxonomy=taxonomy, value="abstract") + child2_1 = Tag.objects.create(taxonomy=taxonomy, value="Andes", parent=root2) + child2_2 = Tag.objects.create(taxonomy=taxonomy, value="azores islands", parent=root2) + return taxonomy + + def create_100_taxonomies(self): + """ + Helper method to create 100 taxonomies and to apply a tag from each to an object + """ + dummy_taxonomies = [] for i in range(100): taxonomy = Taxonomy.objects.create( name=f"ZZ Dummy Taxonomy {i:03}", @@ -86,7 +122,8 @@ def setUp(self): _name=taxonomy.name, _value="Dummy Tag", ) - self.dummy_taxonomies.append(taxonomy) + dummy_taxonomies.append(taxonomy) + return dummy_taxonomies class TaxonomyTestSubclassA(Taxonomy): @@ -203,13 +240,34 @@ def test_unique_tags(self): ("eubacteria", ["Bacteria", "Eubacteria"]), # Third level tags return three levels ("chordata", ["Eukaryota", "Animalia", "Chordata"]), - # Lineage beyond TAXONOMY_MAX_DEPTH won't trace back to the root - ("mammalia", ["Animalia", "Chordata", "Mammalia"]), + # Even fourth level tags work + ("mammalia", ["Eukaryota", "Animalia", "Chordata", "Mammalia"]), ) @ddt.unpack def test_get_lineage(self, tag_attr, lineage): assert getattr(self, tag_attr).get_lineage() == lineage + def test_trailing_whitespace(self): + """ + Test that tags automatically strip out trailing/leading whitespace + """ + t = self.taxonomy.add_tag(" white space ") + assert t.value == "white space" + # And via the API: + t2 = api.add_tag_to_taxonomy(self.taxonomy, "\t value\n") + assert t2.value == "value" + + def test_no_tab(self): + """ + Test that tags cannot contain a TAB character, which we use as a field + separator in the database when computing lineage. + """ + with pytest.raises(ValidationError): + self.taxonomy.add_tag("has\ttab") + # And via the API: + with pytest.raises(ValidationError): + api.add_tag_to_taxonomy(self.taxonomy, "first\tsecond") + @ddt.ddt class TestFilteredTagsClosedTaxonomy(TestTagTaxonomyMixin, TestCase): @@ -444,22 +502,13 @@ def test_usage_count(self) -> None: "Bacteria (None) (used: 3, children: 2)", ] - def test_pathological_tree_sort(self) -> None: + def test_tree_sort(self) -> None: """ - Check for bugs in how tree sorting happens, if the tag names are very - similar. + Verify that taxonomies can be sorted correctly in tree orer (case insensitive). + + The taxonomy used contains values that are tricky to sort correctly unless the tree sort algorithm is correct. """ - # pylint: disable=unused-variable - taxonomy = api.create_taxonomy("Sort Test") - root1 = Tag.objects.create(taxonomy=taxonomy, value="1") - child1_1 = Tag.objects.create(taxonomy=taxonomy, value="11", parent=root1) - child1_2 = Tag.objects.create(taxonomy=taxonomy, value="2", parent=root1) - child1_3 = Tag.objects.create(taxonomy=taxonomy, value="1 A", parent=root1) - child1_4 = Tag.objects.create(taxonomy=taxonomy, value="11111", parent=root1) - grandchild1_4_1 = Tag.objects.create(taxonomy=taxonomy, value="1111-grandchild", parent=child1_4) - root2 = Tag.objects.create(taxonomy=taxonomy, value="111") - child2_1 = Tag.objects.create(taxonomy=taxonomy, value="11111111", parent=root2) - child2_2 = Tag.objects.create(taxonomy=taxonomy, value="123", parent=root2) + taxonomy = self.create_sort_test_taxonomy() result = pretty_format_tags(taxonomy.get_filtered_tags()) assert result == [ "1 (None) (children: 4)", @@ -471,27 +520,6 @@ def test_pathological_tree_sort(self) -> None: "111 (None) (children: 2)", " 11111111 (111) (children: 0)", " 123 (111) (children: 0)", - ] - - def test_case_insensitive_sort(self) -> None: - """ - Make sure the sorting is case-insensitive - """ - # pylint: disable=unused-variable - taxonomy = api.create_taxonomy("Sort Test") - root1 = Tag.objects.create(taxonomy=taxonomy, value="ALPHABET") - child1_1 = Tag.objects.create(taxonomy=taxonomy, value="Android", parent=root1) - child1_2 = Tag.objects.create(taxonomy=taxonomy, value="abacus", parent=root1) - child1_2 = Tag.objects.create(taxonomy=taxonomy, value="azure", parent=root1) - child1_3 = Tag.objects.create(taxonomy=taxonomy, value="aardvark", parent=root1) - child1_4 = Tag.objects.create(taxonomy=taxonomy, value="ANVIL", parent=root1) - - root2 = Tag.objects.create(taxonomy=taxonomy, value="abstract") - child2_1 = Tag.objects.create(taxonomy=taxonomy, value="Andes", parent=root2) - child2_2 = Tag.objects.create(taxonomy=taxonomy, value="azores islands", parent=root2) - - result = pretty_format_tags(taxonomy.get_filtered_tags()) - assert result == [ "abstract (None) (children: 2)", " Andes (abstract) (children: 0)", " azores islands (abstract) (children: 0)", @@ -503,16 +531,6 @@ def test_case_insensitive_sort(self) -> None: " azure (ALPHABET) (children: 0)", ] - # And it's case insensitive when getting only a single level: - result = pretty_format_tags(taxonomy.get_filtered_tags(parent_tag_value="ALPHABET", depth=1)) - assert result == [ - " aardvark (ALPHABET) (children: 0)", - " abacus (ALPHABET) (children: 0)", - " Android (ALPHABET) (children: 0)", - " ANVIL (ALPHABET) (children: 0)", - " azure (ALPHABET) (children: 0)", - ] - class TestFilteredTagsFreeTextTaxonomy(TestCase): """ @@ -669,16 +687,13 @@ def test_object_tag_lineage(self): assert object_tag.get_lineage() == ["Another tag"] def test_validate_value_free_text(self): - open_taxonomy = Taxonomy.objects.create( - name="Freetext Life", - allow_free_text=True, - ) + assert self.free_text_taxonomy.allow_free_text # An empty string or other non-string is not valid in a free-text taxonomy - assert open_taxonomy.validate_value("") is False - assert open_taxonomy.validate_value(None) is False - assert open_taxonomy.validate_value(True) is False + assert self.free_text_taxonomy.validate_value("") is False + assert self.free_text_taxonomy.validate_value(None) is False + assert self.free_text_taxonomy.validate_value(True) is False # But any other string value is valid: - assert open_taxonomy.validate_value("Any text we want") is True + assert self.free_text_taxonomy.validate_value("Any text we want") is True def test_validate_value_closed(self): """ @@ -730,43 +745,53 @@ def test_tag_case(self) -> None: tag=self.chordata, ).save() + def test_invalid_id(self): + """ + Test attempting to create object tags with invalid characters in the object ID + """ + args = {"tags": ["test"], "taxonomy": self.free_text_taxonomy} + with pytest.raises(ValidationError): + api.tag_object(object_id="wildcard*", **args) + with pytest.raises(ValidationError): + api.tag_object(object_id="one,two,three", **args) + api.tag_object(object_id="valid", **args) + def test_is_deleted(self): self.taxonomy.allow_multiple = True self.taxonomy.save() - open_taxonomy = Taxonomy.objects.create(name="Freetext Life", allow_free_text=True, allow_multiple=True) object_id = "obj1" # Create some tags: api.tag_object(self.taxonomy, [self.archaea.value, self.bacteria.value], object_id) # Regular tags - api.tag_object(open_taxonomy, ["foo", "bar", "tribble"], object_id) # Free text tags + api.tag_object(self.free_text_taxonomy, ["foo", "bar", "tribble"], object_id) # Free text tags # At first, none of these will be deleted: assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ - (self.archaea.value, False), - (self.bacteria.value, False), - ("foo", False), ("bar", False), + ("foo", False), ("tribble", False), + (self.archaea.value, False), + (self.bacteria.value, False), ] # Delete "bacteria" from the taxonomy: - self.bacteria.delete() # TODO: add an API method for this + api.delete_tags_from_taxonomy(self.taxonomy, ["Bacteria"], with_subtags=True) assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ - (self.archaea.value, False), - (self.bacteria.value, True), # <--- deleted! But the value is preserved. - ("foo", False), ("bar", False), + ("foo", False), ("tribble", False), + (self.archaea.value, False), + (self.bacteria.value, True), # <--- deleted! But the value is preserved. ] # Then delete the whole free text taxonomy - open_taxonomy.delete() + self.free_text_taxonomy.delete() assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ - (self.archaea.value, False), - (self.bacteria.value, True), # <--- deleted! But the value is preserved. - ("foo", True), # <--- Deleted, but the value is preserved ("bar", True), # <--- Deleted, but the value is preserved + ("foo", True), # <--- Deleted, but the value is preserved ("tribble", True), # <--- Deleted, but the value is preserved + (self.archaea.value, False), + (self.bacteria.value, True), # <--- deleted! But the value is preserved. ] diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 6a3125464..bdb214aed 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -12,6 +12,7 @@ from rest_framework import status from rest_framework.test import APITestCase +from openedx_tagging.core.tagging import api from openedx_tagging.core.tagging.import_export import api as import_export_api from openedx_tagging.core.tagging.import_export.parsers import ParserFormat from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy @@ -19,6 +20,7 @@ from openedx_tagging.core.tagging.rest_api.paginators import TagsPagination from openedx_tagging.core.tagging.rules import can_change_object_tag_objectid, can_view_object_tag_objectid +from .test_models import TestTagTaxonomyMixin from .utils import pretty_format_tags User = get_user_model() @@ -30,6 +32,7 @@ OBJECT_TAGS_RETRIEVE_URL = "/tagging/rest_api/v1/object_tags/{object_id}/" +OBJECT_TAG_COUNTS_URL = "/tagging/rest_api/v1/object_tag_counts/{object_id_pattern}/" OBJECT_TAGS_UPDATE_URL = "/tagging/rest_api/v1/object_tags/{object_id}/?taxonomy={taxonomy_id}" LANGUAGE_TAXONOMY_ID = -1 @@ -100,9 +103,9 @@ class TestTaxonomyViewSet(TestTaxonomyViewMixin): ) @ddt.unpack def test_list_taxonomy_queryparams(self, enabled, expected_status: int, expected_count: int | None): - Taxonomy.objects.create(name="Taxonomy enabled 1", enabled=True).save() - Taxonomy.objects.create(name="Taxonomy enabled 2", enabled=True).save() - Taxonomy.objects.create(name="Taxonomy disabled", enabled=False).save() + api.create_taxonomy(name="Taxonomy enabled 1", enabled=True) + api.create_taxonomy(name="Taxonomy enabled 2", enabled=True) + api.create_taxonomy(name="Taxonomy disabled", enabled=False) url = TAXONOMY_LIST_URL @@ -136,11 +139,11 @@ def test_list_taxonomy(self, user_attr: str | None, expected_status: int): def test_list_taxonomy_pagination(self) -> None: url = TAXONOMY_LIST_URL - Taxonomy.objects.create(name="T1", enabled=True).save() - Taxonomy.objects.create(name="T2", enabled=True).save() - Taxonomy.objects.create(name="T3", enabled=False).save() - Taxonomy.objects.create(name="T4", enabled=False).save() - Taxonomy.objects.create(name="T5", enabled=False).save() + api.create_taxonomy(name="T1", enabled=True) + api.create_taxonomy(name="T2", enabled=True) + api.create_taxonomy(name="T3", enabled=False) + api.create_taxonomy(name="T4", enabled=False) + api.create_taxonomy(name="T5", enabled=False) self.client.force_authenticate(user=self.staff) @@ -177,7 +180,7 @@ def test_list_invalid_page(self) -> None: @ddt.unpack def test_detail_taxonomy(self, user_attr: str | None, taxonomy_data: dict[str, bool], expected_status: int): create_data = {"name": "taxonomy detail test", **taxonomy_data} - taxonomy = Taxonomy.objects.create(**create_data) + taxonomy = api.create_taxonomy(**create_data) # type: ignore[arg-type] url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) if user_attr: @@ -265,12 +268,11 @@ def test_create_taxonomy_system_defined(self, create_data): ) @ddt.unpack def test_update_taxonomy(self, user_attr, expected_status): - taxonomy = Taxonomy.objects.create( + taxonomy = api.create_taxonomy( name="test update taxonomy", description="taxonomy description", enabled=True, ) - taxonomy.save() url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) @@ -303,10 +305,10 @@ def test_update_taxonomy_system_defined(self, system_defined, expected_status): """ Test that we can't update system_defined field """ - taxonomy = Taxonomy.objects.create(name="test system taxonomy") - if system_defined: - taxonomy.taxonomy_class = SystemDefinedTaxonomy - taxonomy.save() + taxonomy = api.create_taxonomy( + name="test system taxonomy", + taxonomy_class=SystemDefinedTaxonomy if system_defined else None, + ) url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) self.client.force_authenticate(user=self.staff) @@ -327,8 +329,7 @@ def test_update_taxonomy_404(self): ) @ddt.unpack def test_patch_taxonomy(self, user_attr, expected_status): - taxonomy = Taxonomy.objects.create(name="test patch taxonomy", enabled=False) - taxonomy.save() + taxonomy = api.create_taxonomy(name="test patch taxonomy", enabled=False) url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) @@ -360,10 +361,10 @@ def test_patch_taxonomy_system_defined(self, system_defined, expected_status): """ Test that we can't patch system_defined field """ - taxonomy = Taxonomy.objects.create(name="test system taxonomy") - if system_defined: - taxonomy.taxonomy_class = SystemDefinedTaxonomy - taxonomy.save() + taxonomy = api.create_taxonomy( + name="test system taxonomy", + taxonomy_class=SystemDefinedTaxonomy if system_defined else None, + ) url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) self.client.force_authenticate(user=self.staff) @@ -384,8 +385,7 @@ def test_patch_taxonomy_404(self): ) @ddt.unpack def test_delete_taxonomy(self, user_attr, expected_status): - taxonomy = Taxonomy.objects.create(name="test delete taxonomy") - taxonomy.save() + taxonomy = api.create_taxonomy(name="test delete taxonomy") url = TAXONOMY_DETAIL_URL.format(pk=taxonomy.pk) @@ -417,8 +417,7 @@ def test_export_taxonomy(self, output_format, content_type): """ Tests if a user can export a taxonomy """ - taxonomy = Taxonomy.objects.create(name="T1", enabled=True) - taxonomy.save() + taxonomy = api.create_taxonomy(name="T1") for i in range(20): # Valid ObjectTags Tag.objects.create(taxonomy=taxonomy, value=f"Tag {i}").save() @@ -445,11 +444,9 @@ def test_export_taxonomy_download(self, output_format, content_type): """ Tests if a user can export a taxonomy with download option """ - taxonomy = Taxonomy.objects.create(name="T1", enabled=True) - taxonomy.save() + taxonomy = api.create_taxonomy(name="T1") for i in range(20): - # Valid ObjectTags - Tag.objects.create(taxonomy=taxonomy, value=f"Tag {i}").save() + api.add_tag_to_taxonomy(taxonomy=taxonomy, tag=f"Tag {i}") url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) @@ -469,8 +466,7 @@ def test_export_taxonomy_invalid_param_output_format(self): """ Tests if a user can export a taxonomy using an invalid output_format param """ - taxonomy = Taxonomy.objects.create(name="T1", enabled=True) - taxonomy.save() + taxonomy = api.create_taxonomy(name="T1") url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) @@ -482,8 +478,7 @@ def test_export_taxonomy_invalid_param_download(self): """ Tests if a user can export a taxonomy using an invalid output_format param """ - taxonomy = Taxonomy.objects.create(name="T1", enabled=True) - taxonomy.save() + taxonomy = api.create_taxonomy(name="T1") url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) @@ -496,8 +491,7 @@ def test_export_taxonomy_unauthorized(self): Tests if a user can export a taxonomy that he doesn't have authorization """ # Only staff can view a disabled taxonomy - taxonomy = Taxonomy.objects.create(name="T1", enabled=False) - taxonomy.save() + taxonomy = api.create_taxonomy(name="T1", enabled=False) url = TAXONOMY_EXPORT_URL.format(pk=taxonomy.pk) @@ -509,12 +503,13 @@ def test_export_taxonomy_unauthorized(self): @ddt.ddt -class TestObjectTagViewSet(APITestCase): +class TestObjectTagViewSet(TestTagTaxonomyMixin, APITestCase): """ Testing various cases for the ObjectTagView. """ def setUp(self): + super().setUp() def _change_object_permission(user, object_id: str) -> bool: """ @@ -534,88 +529,34 @@ def _view_object_permission(user, object_id: str) -> bool: return can_view_object_tag_objectid(user, object_id) - super().setUp() - - self.user = User.objects.create( - username="user", - email="user@example.com", - ) - - self.staff = User.objects.create( - username="staff", - email="staff@example.com", - is_staff=True, - ) - - # System-defined language taxonomy with valid ObjectTag - self.system_taxonomy = SystemDefinedTaxonomy.objects.create(name="System Taxonomy") - self.tag1 = Tag.objects.create(taxonomy=self.system_taxonomy, value="Tag 1") - ObjectTag.objects.create(object_id="abc", taxonomy=self.system_taxonomy, tag=self.tag1) - - # Language system-defined language taxonomy - self.language_taxonomy = Taxonomy.objects.get(pk=LANGUAGE_TAXONOMY_ID) - - # Closed Taxonomies created by taxonomy admins, each with 20 ObjectTags - self.enabled_taxonomy = Taxonomy.objects.create(name="Enabled Taxonomy", allow_multiple=False) - self.disabled_taxonomy = Taxonomy.objects.create(name="Disabled Taxonomy", enabled=False, allow_multiple=False) - self.multiple_taxonomy = Taxonomy.objects.create(name="Multiple Taxonomy", allow_multiple=True) - for i in range(20): - # Valid ObjectTags - tag_enabled = Tag.objects.create(taxonomy=self.enabled_taxonomy, value=f"Tag {i}") - tag_disabled = Tag.objects.create(taxonomy=self.disabled_taxonomy, value=f"Tag {i}") - tag_multiple = Tag.objects.create(taxonomy=self.multiple_taxonomy, value=f"Tag {i}") - ObjectTag.objects.create( - object_id="abc", taxonomy=self.enabled_taxonomy, tag=tag_enabled, _value=tag_enabled.value - ) - ObjectTag.objects.create( - object_id="abc", taxonomy=self.disabled_taxonomy, tag=tag_disabled, _value=tag_disabled.value - ) - ObjectTag.objects.create( - object_id="abc", taxonomy=self.multiple_taxonomy, tag=tag_multiple, _value=tag_multiple.value - ) - - # Free-Text Taxonomies created by taxonomy admins, each linked - # to 10 ObjectTags - self.open_taxonomy_enabled = Taxonomy.objects.create( - name="Enabled Free-Text Taxonomy", allow_free_text=True, allow_multiple=False, - ) - self.open_taxonomy_disabled = Taxonomy.objects.create( - name="Disabled Free-Text Taxonomy", allow_free_text=True, enabled=False, allow_multiple=False, - ) - for i in range(10): - ObjectTag.objects.create(object_id="abc", taxonomy=self.open_taxonomy_enabled, _value=f"Free Text {i}") - ObjectTag.objects.create(object_id="abc", taxonomy=self.open_taxonomy_disabled, _value=f"Free Text {i}") - - self.dummy_taxonomies = [] - for i in range(100): - taxonomy = Taxonomy.objects.create( - name=f"Dummy Taxonomy {i}", - allow_free_text=True, - allow_multiple=True - ) - ObjectTag.objects.create( - object_id="limit_tag_count", - taxonomy=taxonomy, - _name=taxonomy.name, - _value="Dummy Tag" - ) - self.dummy_taxonomies.append(taxonomy) - # Override the object permission for the test rules.set_perm("oel_tagging.change_objecttag_objectid", _change_object_permission) rules.set_perm("oel_tagging.view_objecttag_objectid", _view_object_permission) + # Create a staff user: + self.staff = User.objects.create(username="staff", email="staff@example.com", is_staff=True) + + # For this test, allow multiple "Life on Earth" tags: + self.taxonomy.allow_multiple = True + self.taxonomy.save() + @ddt.data( - (None, status.HTTP_401_UNAUTHORIZED, None), - ("user", status.HTTP_200_OK, 81), - ("staff", status.HTTP_200_OK, 81), + (None, status.HTTP_401_UNAUTHORIZED), + ("user_1", status.HTTP_200_OK), + ("staff", status.HTTP_200_OK), ) @ddt.unpack - def test_retrieve_object_tags(self, user_attr, expected_status, expected_count): + def test_retrieve_object_tags(self, user_attr, expected_status): """ Test retrieving object tags """ - url = OBJECT_TAGS_RETRIEVE_URL.format(object_id="abc") + object_id = "problem15" + + # Apply the object tags that we're about to retrieve: + api.tag_object(object_id=object_id, taxonomy=self.taxonomy, tags=["Mammalia", "Fungi"]) + api.tag_object(object_id=object_id, taxonomy=self.user_taxonomy, tags=[self.user_1.username]) + + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) if user_attr: user = getattr(self, user_attr) @@ -625,7 +566,126 @@ def test_retrieve_object_tags(self, user_attr, expected_status, expected_count): assert response.status_code == expected_status if status.is_success(expected_status): - assert len(response.data) == expected_count + # Check the response, first converting from OrderedDict to regular dicts for simplicity. + assert response.data == { + # In the future, this API may allow retrieving tags for multiple objects at once, so it's grouped by + # object ID. + "problem15": { + "taxonomies": [ + { + "name": "Life on Earth", + "taxonomy_id": 1, + "editable": True, + "tags": [ + # Note: based on tree order (Animalia before Fungi), this tag comes first even though it + # starts with "M" and Fungi starts with "F" + { + "value": "Mammalia", + "lineage": ["Eukaryota", "Animalia", "Chordata", "Mammalia"], + }, + { + "value": "Fungi", + "lineage": ["Eukaryota", "Fungi"], + }, + ] + }, + { + "name": "User Authors", + "taxonomy_id": 3, + "editable": False, + "tags": [ + { + "value": "test_user_1", + "lineage": ["test_user_1"], + }, + ], + } + ], + }, + } + + def prepare_for_sort_test(self) -> tuple[str, list[dict]]: + """ + Tag an object with tags from the "sort test" taxonomy + """ + object_id = "problem7" + # Some selected tags to use, from the taxonomy create by self.create_sort_test_taxonomy() + sort_test_tags = [ + "ANVIL", + "Android", + "azores islands", + "abstract", + "11111111", + "111", + "123", + "1 A", + "1111-grandchild", + ] + + # Apply the object tags: + taxonomy = self.create_sort_test_taxonomy() + api.tag_object(object_id=object_id, taxonomy=taxonomy, tags=sort_test_tags) + + # The result we expect to see when retrieving the object tags, after applying the list above. + # Note: the full taxonomy looks like the following, so this is the order we + # expect, although not all of these tags were included. + # 1 + # 1 A + # 11 + # 11111 + # 1111-grandchild + # 2 + # 111 + # 11111111 + # 123 + # abstract + # Andes + # azores islands + # ALPHABET + # aardvark + # abacus + # Android + # ANVIL + # azure + sort_test_applied_result = [ + {"value": "1 A", "lineage": ["1", "1 A"]}, + {"value": "1111-grandchild", "lineage": ["1", "11111", "1111-grandchild"]}, + {"value": "111", "lineage": ["111"]}, + {"value": "11111111", "lineage": ["111", "11111111"]}, + {"value": "123", "lineage": ["111", "123"]}, + {"value": "abstract", "lineage": ["abstract"]}, + {"value": "azores islands", "lineage": ["abstract", "azores islands"]}, + {"value": "Android", "lineage": ["ALPHABET", "Android"]}, + {"value": "ANVIL", "lineage": ["ALPHABET", "ANVIL"]}, + ] + return object_id, sort_test_applied_result + + def test_retrieve_object_tags_sorted(self): + """ + Test the sort order of the object tags retrieved from the get object + tags API. + """ + object_id, sort_test_applied_result = self.prepare_for_sort_test() + + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) + self.client.force_authenticate(user=self.user_1) + response = self.client.get(url) + assert response.status_code == 200 + assert response.data[object_id]["taxonomies"][0]["name"] == "Sort Test" + assert response.data[object_id]["taxonomies"][0]["tags"] == sort_test_applied_result + + def test_retrieve_object_tags_query_count(self): + """ + Test how many queries are used when retrieving object tags + """ + object_id, sort_test_applied_result = self.prepare_for_sort_test() + + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) + self.client.force_authenticate(user=self.user_1) + with self.assertNumQueries(1): + response = self.client.get(url) + assert response.status_code == 200 + assert response.data[object_id]["taxonomies"][0]["tags"] == sort_test_applied_result def test_retrieve_object_tags_unauthorized(self): """ @@ -637,34 +697,56 @@ def test_retrieve_object_tags_unauthorized(self): assert response.status_code == status.HTTP_403_FORBIDDEN @ddt.data( - (None, "abc", status.HTTP_401_UNAUTHORIZED, None), - ("user", "abc", status.HTTP_200_OK, 20), - ("staff", "abc", status.HTTP_200_OK, 20), + (None, status.HTTP_401_UNAUTHORIZED), + ("user_1", status.HTTP_200_OK), + ("staff", status.HTTP_200_OK), ) @ddt.unpack def test_retrieve_object_tags_taxonomy_queryparam( - self, user_attr, object_id, expected_status, expected_count + self, user_attr, expected_status, ): """ Test retrieving object tags for specific taxonomies provided """ + object_id = "html7" + + # Apply the object tags that we're about to retrieve: + api.tag_object(object_id=object_id, taxonomy=self.taxonomy, tags=["Mammalia", "Fungi"]) + api.tag_object(object_id=object_id, taxonomy=self.user_taxonomy, tags=[self.user_1.username]) + url = OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id) if user_attr: user = getattr(self, user_attr) self.client.force_authenticate(user=user) - response = self.client.get(url, {"taxonomy": self.enabled_taxonomy.pk}) + response = self.client.get(url, {"taxonomy": self.user_taxonomy.pk}) assert response.status_code == expected_status if status.is_success(expected_status): - assert len(response.data) == expected_count - for object_tag in response.data: - assert object_tag.get("is_deleted") is False - assert object_tag.get("taxonomy_id") == self.enabled_taxonomy.pk + assert response.data == { + # In the future, this API may allow retrieving tags for multiple objects at once, so it's grouped by + # object ID. + object_id: { + "taxonomies": [ + # The "Life on Earth" tags are excluded here... + { + "name": "User Authors", + "taxonomy_id": 3, + "editable": False, + "tags": [ + { + "value": "test_user_1", + "lineage": ["test_user_1"], + }, + ], + } + ], + }, + } @ddt.data( (None, "abc", status.HTTP_401_UNAUTHORIZED), - ("user", "abc", status.HTTP_400_BAD_REQUEST), + ("user_1", "abc", status.HTTP_400_BAD_REQUEST), ("staff", "abc", status.HTTP_400_BAD_REQUEST), ) @ddt.unpack @@ -686,9 +768,9 @@ def test_retrieve_object_tags_invalid_taxonomy_queryparam(self, user_attr, objec (None, "POST", status.HTTP_401_UNAUTHORIZED), (None, "PATCH", status.HTTP_401_UNAUTHORIZED), (None, "DELETE", status.HTTP_401_UNAUTHORIZED), - ("user", "POST", status.HTTP_405_METHOD_NOT_ALLOWED), - ("user", "PATCH", status.HTTP_405_METHOD_NOT_ALLOWED), - ("user", "DELETE", status.HTTP_405_METHOD_NOT_ALLOWED), + ("user_1", "POST", status.HTTP_405_METHOD_NOT_ALLOWED), + ("user_1", "PATCH", status.HTTP_405_METHOD_NOT_ALLOWED), + ("user_1", "DELETE", status.HTTP_405_METHOD_NOT_ALLOWED), ("staff", "POST", status.HTTP_405_METHOD_NOT_ALLOWED), ("staff", "PATCH", status.HTTP_405_METHOD_NOT_ALLOWED), ("staff", "DELETE", status.HTTP_405_METHOD_NOT_ALLOWED), @@ -724,148 +806,102 @@ def test_object_tags_remaining_http_methods( @ddt.data( # Users and staff can add tags - (None, "language_taxonomy", ["Portuguese"], status.HTTP_401_UNAUTHORIZED), - ("user", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), - ("staff", "language_taxonomy", ["Portuguese"], status.HTTP_200_OK), - # Users and staff can clear add tags - (None, "enabled_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), - ("user", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), - ("staff", "enabled_taxonomy", ["Tag 1"], status.HTTP_200_OK), + (None, "language_taxonomy", {}, ["Portuguese"], status.HTTP_401_UNAUTHORIZED), + ("user_1", "language_taxonomy", {}, ["Portuguese"], status.HTTP_200_OK), + ("staff", "language_taxonomy", {}, ["Portuguese"], status.HTTP_200_OK), + # user_1s and staff can clear add tags + (None, "taxonomy", {}, ["Fungi"], status.HTTP_401_UNAUTHORIZED), + ("user_1", "taxonomy", {}, ["Fungi"], status.HTTP_200_OK), + ("staff", "taxonomy", {}, ["Fungi"], status.HTTP_200_OK), # Nobody can add tag using a disabled taxonomy - (None, "disabled_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), - ("user", "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", ["Tag 1"], status.HTTP_403_FORBIDDEN), - # Users and staff can add a single tag using a allow_multiple=True taxonomy - (None, "multiple_taxonomy", ["Tag 1"], status.HTTP_401_UNAUTHORIZED), - ("user", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), - ("staff", "multiple_taxonomy", ["Tag 1"], status.HTTP_200_OK), - # Users and staff can add tags using an open taxonomy - (None, "open_taxonomy_enabled", ["tag1"], status.HTTP_401_UNAUTHORIZED), - ("user", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), - ("staff", "open_taxonomy_enabled", ["tag1"], status.HTTP_200_OK), + (None, "taxonomy", {"enabled": False}, ["Fungi"], status.HTTP_401_UNAUTHORIZED), + ("user_1", "taxonomy", {"enabled": False}, ["Fungi"], status.HTTP_403_FORBIDDEN), + ("staff", "taxonomy", {"enabled": False}, ["Fungi"], status.HTTP_403_FORBIDDEN), + # If allow_multiple=True, multiple tags can be added, but not if it's false: + ("user_1", "taxonomy", {"allow_multiple": True}, ["Mammalia", "Fungi"], status.HTTP_200_OK), + ("user_1", "taxonomy", {"allow_multiple": False}, ["Mammalia", "Fungi"], status.HTTP_400_BAD_REQUEST), + # user_1s and staff can add tags using an open taxonomy + (None, "free_text_taxonomy", {}, ["tag1"], status.HTTP_401_UNAUTHORIZED), + ("user_1", "free_text_taxonomy", {}, ["tag1", "tag2"], status.HTTP_200_OK), + ("staff", "free_text_taxonomy", {}, ["tag1", "tag4"], status.HTTP_200_OK), # Nobody can add tags using a disabled open taxonomy - (None, "open_taxonomy_disabled", ["tag1"], status.HTTP_401_UNAUTHORIZED), - ("user", "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), - ("staff", "open_taxonomy_disabled", ["tag1"], status.HTTP_403_FORBIDDEN), + (None, "free_text_taxonomy", {"enabled": False}, ["tag1"], status.HTTP_401_UNAUTHORIZED), + ("user_1", "free_text_taxonomy", {"enabled": False}, ["tag1"], status.HTTP_403_FORBIDDEN), + ("staff", "free_text_taxonomy", {"enabled": False}, ["tag1"], status.HTTP_403_FORBIDDEN), + # Can't add invalid/nonexistent tags using a closed taxonomy + (None, "language_taxonomy", {}, ["Invalid"], status.HTTP_401_UNAUTHORIZED), + ("user_1", "language_taxonomy", {}, ["Invalid"], status.HTTP_400_BAD_REQUEST), + ("staff", "language_taxonomy", {}, ["Invalid"], status.HTTP_400_BAD_REQUEST), + ("staff", "taxonomy", {}, ["Invalid"], status.HTTP_400_BAD_REQUEST), ) @ddt.unpack - def test_tag_object(self, user_attr, taxonomy_attr, tag_values, expected_status): + def test_tag_object(self, user_attr, taxonomy_attr, taxonomy_flags, tag_values, expected_status): if user_attr: user = getattr(self, user_attr) self.client.force_authenticate(user=user) taxonomy = getattr(self, taxonomy_attr) + if taxonomy_flags: + for (k, v) in taxonomy_flags.items(): + setattr(taxonomy, k, v) + taxonomy.save() - url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + object_id = "abc" - response = self.client.put(url, {"tags": tag_values}, format="json") - assert response.status_code == expected_status - if status.is_success(expected_status): - assert len(response.data) == len(tag_values) - assert set(t["value"] for t in response.data) == set(tag_values) - - @ddt.data( - # Can't add invalid tags using a closed taxonomy - (None, "language_taxonomy", ["Invalid"], status.HTTP_401_UNAUTHORIZED), - ("user", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), - ("staff", "language_taxonomy", ["Invalid"], status.HTTP_400_BAD_REQUEST), - (None, "enabled_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), - ("user", "enabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), - ("staff", "enabled_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), - (None, "multiple_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), - ("user", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), - ("staff", "multiple_taxonomy", ["invalid"], status.HTTP_400_BAD_REQUEST), - # Nobody can edit object tags using a disabled taxonomy. - (None, "disabled_taxonomy", ["invalid"], status.HTTP_401_UNAUTHORIZED), - ("user", "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", ["invalid"], status.HTTP_403_FORBIDDEN), - ) - @ddt.unpack - def test_tag_object_invalid(self, user_attr, taxonomy_attr, tag_values, expected_status): - if user_attr: - user = getattr(self, user_attr) - self.client.force_authenticate(user=user) - - taxonomy = getattr(self, taxonomy_attr) - - url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=taxonomy.pk) response = self.client.put(url, {"tags": tag_values}, format="json") assert response.status_code == expected_status - assert not status.is_success(expected_status) # No success cases here + if status.is_success(expected_status): + assert [t["value"] for t in response.data[object_id]["taxonomies"][0]["tags"]] == tag_values + # And retrieving the object tags again should return an identical response: + assert response.data == self.client.get(OBJECT_TAGS_RETRIEVE_URL.format(object_id=object_id)).data @ddt.data( # Users and staff can clear tags - (None, "enabled_taxonomy", [], status.HTTP_401_UNAUTHORIZED), - ("user", "enabled_taxonomy", [], status.HTTP_200_OK), - ("staff", "enabled_taxonomy", [], status.HTTP_200_OK), - # Users and staff can clear object tags using a allow_multiple=True taxonomy - (None, "multiple_taxonomy", [], status.HTTP_401_UNAUTHORIZED), - ("user", "multiple_taxonomy", [], status.HTTP_200_OK), - ("staff", "multiple_taxonomy", [], status.HTTP_200_OK), + (None, {}, status.HTTP_401_UNAUTHORIZED), + ("user_1", {}, status.HTTP_200_OK), + ("staff", {}, status.HTTP_200_OK), # Nobody can clear tags using a disabled taxonomy - (None, "disabled_taxonomy", [], status.HTTP_401_UNAUTHORIZED), - ("user", "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", [], status.HTTP_403_FORBIDDEN), - (None, "open_taxonomy_disabled", [], status.HTTP_401_UNAUTHORIZED), - ("user", "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), - ("staff", "open_taxonomy_disabled", [], status.HTTP_403_FORBIDDEN), + (None, {"enabled": False}, status.HTTP_401_UNAUTHORIZED), + ("user_1", {"enabled": False}, status.HTTP_403_FORBIDDEN), + ("staff", {"enabled": False}, status.HTTP_403_FORBIDDEN), + # ... and it doesn't matter if it's free text or closed: + ("staff", {"enabled": False, "allow_free_text": False}, status.HTTP_403_FORBIDDEN), ) @ddt.unpack - def test_tag_object_clear(self, user_attr, taxonomy_attr, tag_values, expected_status): - if user_attr: - user = getattr(self, user_attr) - self.client.force_authenticate(user=user) - - taxonomy = getattr(self, taxonomy_attr) + def test_tag_object_clear(self, user_attr, taxonomy_flags, expected_status): + """ + Test that authorized users can *remove* tags using this API + """ + object_id = "abc" - url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + # First create an object tag: + api.tag_object(object_id=object_id, taxonomy=self.taxonomy, tags=["Fungi"]) - response = self.client.put(url, {"tags": tag_values}, format="json") - assert response.status_code == expected_status - if status.is_success(expected_status): - assert len(response.data) == len(tag_values) - assert set(t["value"] for t in response.data) == set(tag_values) - - @ddt.data( - # Users and staff can add multiple tags using a allow_multiple=True taxonomy - (None, "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), - ("user", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), - ("staff", "multiple_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_200_OK), - (None, "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_401_UNAUTHORIZED), - ("user", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), - ("staff", "open_taxonomy_enabled", ["tag1", "tag2"], status.HTTP_400_BAD_REQUEST), - # Users and staff can't add multple tags using a allow_multiple=False taxonomy - (None, "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), - ("user", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), - ("staff", "enabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_400_BAD_REQUEST), - (None, "language_taxonomy", ["Portuguese", "English"], status.HTTP_401_UNAUTHORIZED), - ("user", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), - ("staff", "language_taxonomy", ["Portuguese", "English"], status.HTTP_400_BAD_REQUEST), - # Nobody can edit tags using a disabled taxonomy. - (None, "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_401_UNAUTHORIZED), - ("user", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), - ("staff", "disabled_taxonomy", ["Tag 1", "Tag 2"], status.HTTP_403_FORBIDDEN), - ) - @ddt.unpack - def test_tag_object_multiple(self, user_attr, taxonomy_attr, tag_values, expected_status): if user_attr: user = getattr(self, user_attr) self.client.force_authenticate(user=user) - taxonomy = getattr(self, taxonomy_attr) + if taxonomy_flags: + for (k, v) in taxonomy_flags.items(): + setattr(self.taxonomy, k, v) + self.taxonomy.save() - url = OBJECT_TAGS_UPDATE_URL.format(object_id="abc", taxonomy_id=taxonomy.pk) + url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=self.taxonomy.pk) - response = self.client.put(url, {"tags": tag_values}, format="json") + response = self.client.put(url, {"tags": []}, format="json") assert response.status_code == expected_status if status.is_success(expected_status): - assert len(response.data) == len(tag_values) - assert set(t["value"] for t in response.data) == set(tag_values) + # Now there are no tags applied: + assert response.data[object_id]["taxonomies"] == [] + else: + # Make sure the object tags are unchanged: + assert [ot.value for ot in api.get_object_tags(object_id=object_id)] == ["Fungi"] @ddt.data( (None, status.HTTP_401_UNAUTHORIZED), - ("user", status.HTTP_403_FORBIDDEN), + ("user_1", status.HTTP_403_FORBIDDEN), ("staff", status.HTTP_403_FORBIDDEN), ) @ddt.unpack @@ -874,7 +910,7 @@ def test_tag_object_without_permission(self, user_attr, expected_status): user = getattr(self, user_attr) self.client.force_authenticate(user=user) - url = OBJECT_TAGS_UPDATE_URL.format(object_id="view_only", taxonomy_id=self.enabled_taxonomy.pk) + url = OBJECT_TAGS_UPDATE_URL.format(object_id="view_only", taxonomy_id=self.taxonomy.pk) response = self.client.put(url, {"tags": ["Tag 1"]}, format="json") assert response.status_code == expected_status @@ -885,25 +921,82 @@ def test_tag_object_count_limit(self): Checks if the limit of 100 tags per object is enforced """ object_id = "limit_tag_count" - url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=self.enabled_taxonomy.pk) + dummy_taxonomies = self.create_100_taxonomies() + + url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=self.taxonomy.pk) self.client.force_authenticate(user=self.staff) response = self.client.put(url, {"tags": ["Tag 1"]}, format="json") # Can't add another tag because the object already has 100 tags assert response.status_code == status.HTTP_400_BAD_REQUEST # The user can edit the tags that are already on the object - for taxonomy in self.dummy_taxonomies: + for taxonomy in dummy_taxonomies: url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=taxonomy.pk) response = self.client.put(url, {"tags": ["New Tag"]}, format="json") assert response.status_code == status.HTTP_200_OK # Editing tags adding another one will fail - for taxonomy in self.dummy_taxonomies: + for taxonomy in dummy_taxonomies: url = OBJECT_TAGS_UPDATE_URL.format(object_id=object_id, taxonomy_id=taxonomy.pk) response = self.client.put(url, {"tags": ["New Tag 1", "New Tag 2"]}, format="json") assert response.status_code == status.HTTP_400_BAD_REQUEST +class TestObjectTagCountsViewSet(TestTagTaxonomyMixin, APITestCase): + """ + Testing various cases for counting how many tags are applied to several objects. + """ + + def test_get_counts(self): + """ + Test retrieving the counts of tags applied to various content objects. + + This API does NOT bother doing any permission checks as the "# of tags" is not considered sensitive information. + """ + # Course 2 + api.tag_object(object_id="course02-unit01-problem01", taxonomy=self.free_text_taxonomy, tags=["other"]) + # Course 7 Unit 1 + api.tag_object(object_id="course07-unit01-problem01", taxonomy=self.free_text_taxonomy, tags=["a", "b", "c"]) + api.tag_object(object_id="course07-unit01-problem02", taxonomy=self.free_text_taxonomy, tags=["a", "b"]) + # Course 7 Unit 2 + api.tag_object(object_id="course07-unit02-problem01", taxonomy=self.free_text_taxonomy, tags=["b"]) + api.tag_object(object_id="course07-unit02-problem02", taxonomy=self.free_text_taxonomy, tags=["c", "d"]) + api.tag_object(object_id="course07-unit02-problem03", taxonomy=self.free_text_taxonomy, tags=["N", "M", "x"]) + + def check(object_id_pattern: str): + result = self.client.get(OBJECT_TAG_COUNTS_URL.format(object_id_pattern=object_id_pattern)) + assert result.status_code == status.HTTP_200_OK + return result.data + + with self.assertNumQueries(1): + assert check(object_id_pattern="course02-*") == { + "course02-unit01-problem01": 1, + } + with self.assertNumQueries(1): + assert check(object_id_pattern="course07-unit01-*") == { + "course07-unit01-problem01": 3, + "course07-unit01-problem02": 2, + } + with self.assertNumQueries(1): + assert check(object_id_pattern="course07-unit*") == { + "course07-unit01-problem01": 3, + "course07-unit01-problem02": 2, + "course07-unit02-problem01": 1, + "course07-unit02-problem02": 2, + "course07-unit02-problem03": 3, + } + # Can also use a comma to separate explicit object IDs: + with self.assertNumQueries(1): + assert check(object_id_pattern="course07-unit01-problem01") == { + "course07-unit01-problem01": 3, + } + with self.assertNumQueries(1): + assert check(object_id_pattern="course07-unit01-problem01,course07-unit02-problem02") == { + "course07-unit01-problem01": 3, + "course07-unit02-problem02": 2, + } + + class TestTaxonomyTagsView(TestTaxonomyViewMixin): """ Tests the list/create/update/delete tags of taxonomy view From 2a376bebd328f0b8a670d88125fa476020806c1e Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 5 Nov 2023 19:21:24 -0500 Subject: [PATCH 075/282] chore: Updating Python Requirements --- requirements/base.txt | 12 ++++++------ requirements/ci.txt | 4 ++-- requirements/dev.txt | 22 +++++++++++----------- requirements/doc.txt | 18 +++++++++--------- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 18 +++++++++--------- requirements/test.txt | 14 +++++++------- 7 files changed, 45 insertions(+), 45 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index ecea8617b..d1aeb7422 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -24,7 +24,7 @@ cffi==1.16.0 # via # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via requests click==8.1.7 # via @@ -41,7 +41,7 @@ click-repl==0.3.0 # via celery cryptography==41.0.5 # via pyjwt -django==3.2.22 +django==3.2.23 # via # -c requirements/constraints.txt # -r requirements/base.in @@ -64,9 +64,9 @@ djangorestframework==3.14.0 # edx-drf-extensions drf-jwt==1.19.2 # via edx-drf-extensions -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via edx-drf-extensions -edx-drf-extensions==8.12.0 +edx-drf-extensions==8.13.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via edx-drf-extensions @@ -124,10 +124,10 @@ tzdata==2023.3 # celery urllib3==2.0.7 # via requests -vine==5.0.0 +vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.8 +wcwidth==0.2.9 # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index 436463d18..e358d6148 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -8,13 +8,13 @@ click==8.1.7 # via import-linter distlib==0.3.7 # via virtualenv -filelock==3.13.0 +filelock==3.13.1 # via # tox # virtualenv grimp==3.1 # via import-linter -import-linter==1.12.0 +import-linter==1.12.1 # via -r requirements/ci.in packaging==23.2 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 8540c2e3b..af7157e18 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -46,7 +46,7 @@ cffi==1.16.0 # pynacl chardet==5.2.0 # via diff-cover -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/quality.txt # requests @@ -107,7 +107,7 @@ distlib==0.3.7 # via # -r requirements/ci.txt # virtualenv -django==3.2.22 +django==3.2.23 # via # -c requirements/constraints.txt # -r requirements/quality.txt @@ -157,11 +157,11 @@ drf-jwt==1.19.2 # via # -r requirements/quality.txt # edx-drf-extensions -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/quality.txt # edx-drf-extensions -edx-drf-extensions==8.12.0 +edx-drf-extensions==8.13.0 # via -r requirements/quality.txt edx-i18n-tools==1.3.0 # via -r requirements/dev.in @@ -175,7 +175,7 @@ exceptiongroup==1.1.3 # via # -r requirements/quality.txt # pytest -filelock==3.13.0 +filelock==3.13.1 # via # -r requirements/ci.txt # tox @@ -189,7 +189,7 @@ idna==3.4 # via # -r requirements/quality.txt # requests -import-linter==1.12.0 +import-linter==1.12.1 # via # -r requirements/ci.txt # -r requirements/quality.txt @@ -382,7 +382,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/quality.txt -pytest-django==4.5.2 +pytest-django==4.6.0 # via -r requirements/quality.txt python-dateutil==2.8.2 # via @@ -476,7 +476,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tomlkit==0.12.1 +tomlkit==0.12.2 # via # -r requirements/quality.txt # pylint @@ -529,7 +529,7 @@ urllib3==2.0.7 # requests # twine # types-requests -vine==5.0.0 +vine==5.1.0 # via # -r requirements/quality.txt # amqp @@ -539,11 +539,11 @@ virtualenv==20.24.6 # via # -r requirements/ci.txt # tox -wcwidth==0.2.8 +wcwidth==0.2.9 # via # -r requirements/quality.txt # prompt-toolkit -wheel==0.41.2 +wheel==0.41.3 # via # -r requirements/pip-tools.txt # pip-tools diff --git a/requirements/doc.txt b/requirements/doc.txt index e5663a3c1..453b99580 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -45,7 +45,7 @@ cffi==1.16.0 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/test.txt # requests @@ -84,7 +84,7 @@ cryptography==41.0.5 # pyjwt ddt==1.6.0 # via -r requirements/test.txt -django==3.2.22 +django==3.2.23 # via # -c requirements/constraints.txt # -r requirements/test.txt @@ -137,11 +137,11 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.12.0 +edx-drf-extensions==8.13.0 # via -r requirements/test.txt edx-opaque-keys==2.5.1 # via @@ -161,7 +161,7 @@ idna==3.4 # requests imagesize==1.4.1 # via sphinx -import-linter==1.12.0 +import-linter==1.12.1 # via -r requirements/test.txt importlib-metadata==6.8.0 # via sphinx @@ -228,7 +228,7 @@ pycparser==2.21 # via # -r requirements/test.txt # cffi -pydata-sphinx-theme==0.14.2 +pydata-sphinx-theme==0.14.3 # via sphinx-book-theme pygments==2.16.1 # via @@ -258,7 +258,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.5.2 +pytest-django==4.6.0 # via -r requirements/test.txt python-dateutil==2.8.2 # via @@ -385,13 +385,13 @@ urllib3==2.0.7 # -r requirements/test.txt # requests # types-requests -vine==5.0.0 +vine==5.1.0 # via # -r requirements/test.txt # amqp # celery # kombu -wcwidth==0.2.8 +wcwidth==0.2.9 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 50d35f22e..ea347319e 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -21,7 +21,7 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.41.2 +wheel==0.41.3 # via pip-tools zipp==3.17.0 # via importlib-metadata diff --git a/requirements/quality.txt b/requirements/quality.txt index 5bc7af81b..7fbf7b103 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -39,7 +39,7 @@ cffi==1.16.0 # -r requirements/test.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/test.txt # requests @@ -87,7 +87,7 @@ ddt==1.6.0 # via -r requirements/test.txt dill==0.3.7 # via pylint -django==3.2.22 +django==3.2.23 # via # -c requirements/constraints.txt # -r requirements/test.txt @@ -132,11 +132,11 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.12.0 +edx-drf-extensions==8.13.0 # via -r requirements/test.txt edx-lint==5.3.6 # via -r requirements/quality.in @@ -156,7 +156,7 @@ idna==3.4 # via # -r requirements/test.txt # requests -import-linter==1.12.0 +import-linter==1.12.1 # via -r requirements/test.txt importlib-metadata==6.8.0 # via @@ -289,7 +289,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.5.2 +pytest-django==4.6.0 # via -r requirements/test.txt python-dateutil==2.8.2 # via @@ -362,7 +362,7 @@ tomli==2.0.1 # mypy # pylint # pytest -tomlkit==0.12.1 +tomlkit==0.12.2 # via pylint twine==4.0.2 # via -r requirements/quality.in @@ -405,13 +405,13 @@ urllib3==2.0.7 # requests # twine # types-requests -vine==5.0.0 +vine==5.1.0 # via # -r requirements/test.txt # amqp # celery # kombu -wcwidth==0.2.8 +wcwidth==0.2.9 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/test.txt b/requirements/test.txt index 05af47ff2..2b89b9143 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -35,7 +35,7 @@ cffi==1.16.0 # -r requirements/base.txt # cryptography # pynacl -charset-normalizer==3.3.1 +charset-normalizer==3.3.2 # via # -r requirements/base.txt # requests @@ -113,11 +113,11 @@ drf-jwt==1.19.2 # via # -r requirements/base.txt # edx-drf-extensions -edx-django-utils==5.7.0 +edx-django-utils==5.8.0 # via # -r requirements/base.txt # edx-drf-extensions -edx-drf-extensions==8.12.0 +edx-drf-extensions==8.13.0 # via -r requirements/base.txt edx-opaque-keys==2.5.1 # via @@ -131,7 +131,7 @@ idna==3.4 # via # -r requirements/base.txt # requests -import-linter==1.12.0 +import-linter==1.12.1 # via -r requirements/test.in iniconfig==2.0.0 # via pytest @@ -198,7 +198,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.in -pytest-django==4.5.2 +pytest-django==4.6.0 # via -r requirements/test.in python-dateutil==2.8.2 # via @@ -278,13 +278,13 @@ urllib3==2.0.7 # -r requirements/base.txt # requests # types-requests -vine==5.0.0 +vine==5.1.0 # via # -r requirements/base.txt # amqp # celery # kombu -wcwidth==0.2.8 +wcwidth==0.2.9 # via # -r requirements/base.txt # prompt-toolkit From 531838b8db4a686115a013f7b79ba6c8c8b4e24b Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 9 Nov 2023 18:07:12 -0800 Subject: [PATCH 076/282] Minor fixes for tagging django admin [FC-0036] (#114) * feat: improve django admin for tagging/taxonomy models * fix: some existing instances had wrong class for Language taxonomy * fix: allows taxonomy.pk to be a negative number when parsing REST API URLs --- openedx_tagging/core/tagging/admin.py | 38 ++++++++++++++++++- .../tagging/migrations/0014_minor_fixes.py | 36 ++++++++++++++++++ openedx_tagging/core/tagging/models/base.py | 1 + .../core/tagging/rest_api/v1/views.py | 3 +- .../core/tagging/test_views.py | 25 ++++++++++++ 5 files changed, 100 insertions(+), 3 deletions(-) create mode 100644 openedx_tagging/core/tagging/migrations/0014_minor_fixes.py diff --git a/openedx_tagging/core/tagging/admin.py b/openedx_tagging/core/tagging/admin.py index 5be444cfc..4016df6ef 100644 --- a/openedx_tagging/core/tagging/admin.py +++ b/openedx_tagging/core/tagging/admin.py @@ -1,10 +1,44 @@ """ Tagging app admin """ +from __future__ import annotations + from django.contrib import admin from .models import ObjectTag, Tag, Taxonomy admin.site.register(Taxonomy) -admin.site.register(Tag) -admin.site.register(ObjectTag) + + +@admin.register(Tag) +class TagAdmin(admin.ModelAdmin): + """ + Admin definition for Tag model + """ + autocomplete_fields = ["parent"] + search_fields = ["value", "external_id"] + list_display = ["__str__", "taxonomy", "external_id"] + list_filter = ["taxonomy"] + + def has_add_permission(self, request): + """ + Don't create Tags using the django admin. Use the API or UI. + """ + return False + + +@admin.register(ObjectTag) +class ObjectTagAdmin(admin.ModelAdmin): + """ + Admin definition for ObjectTag model + """ + fields = ["object_id", "taxonomy", "tag", "_value"] + autocomplete_fields = ["tag"] + list_display = ["object_id", "name", "value"] + readonly_fields = ["object_id"] + + def has_add_permission(self, request): + """ + Don't create ObjectTags using the django admin. Use the API or UI. + """ + return False diff --git a/openedx_tagging/core/tagging/migrations/0014_minor_fixes.py b/openedx_tagging/core/tagging/migrations/0014_minor_fixes.py new file mode 100644 index 000000000..e09177899 --- /dev/null +++ b/openedx_tagging/core/tagging/migrations/0014_minor_fixes.py @@ -0,0 +1,36 @@ +# Generated by Django 3.2.22 on 2023-11-08 22:37 + +from django.db import migrations, models + +import openedx_learning.lib.fields + + +def fix_language_taxonomy(apps, schema_editor): + """ + Ensure that the language taxonomy has the correct taxonomy_class. Some + previous versions of this app's migration history allowed it to be created + without the right taxonomy_class. + """ + Taxonomy = apps.get_model("oel_tagging", "Taxonomy") + correct_class = "openedx_tagging.core.tagging.models.system_defined.LanguageTaxonomy" + lang_taxonomy = Taxonomy.objects.get(pk=-1) + if lang_taxonomy._taxonomy_class != correct_class: + lang_taxonomy._taxonomy_class = correct_class + lang_taxonomy.save(update_fields=["_taxonomy_class"]) + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_tagging', '0013_tag_parent_blank'), + ] + + operations = [ + # Allow taxonomy_class to be blank: + migrations.AlterField( + model_name='taxonomy', + name='_taxonomy_class', + field=models.CharField(blank=True, help_text='Taxonomy subclass used to instantiate this instance; must be a fully-qualified module and class name. If the module/class cannot be imported, an error is logged and the base Taxonomy class is used instead.', max_length=255, null=True), + ), + migrations.RunPython(fix_language_taxonomy, None), + ] diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index 86f948770..f79bb2522 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -227,6 +227,7 @@ class Taxonomy(models.Model): "Taxonomy subclass used to instantiate this instance; must be a fully-qualified module and class name." " If the module/class cannot be imported, an error is logged and the base Taxonomy class is used instead." ), + blank=True, ) class Meta: diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index d2566ca2c..e2145200f 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -170,7 +170,8 @@ class TaxonomyView(ModelViewSet): """ - lookup_value_regex = r"\d+" + # System taxonomies use negative numbers for their primary keys + lookup_value_regex = r'-?\d+' serializer_class = TaxonomySerializer permission_classes = [TaxonomyObjectPermissions] diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index bdb214aed..1a1c69ada 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -169,6 +169,24 @@ def test_list_invalid_page(self) -> None: assert response.status_code == status.HTTP_404_NOT_FOUND + def test_language_taxonomy(self): + """ + Test the "Language" taxonomy that's included. + """ + self.client.force_authenticate(user=self.user) + response = self.client.get(TAXONOMY_LIST_URL) + assert response.status_code == status.HTTP_200_OK + taxonomy_list = response.data["results"] + assert len(taxonomy_list) == 1 + check_taxonomy( + taxonomy_list[0], + taxonomy_id=LANGUAGE_TAXONOMY_ID, + name="Languages", + description="Languages that are enabled on this system.", + allow_multiple=False, # We may change this in the future to allow multiple language tags + system_defined=True, + ) + @ddt.data( (None, {"enabled": True}, status.HTTP_401_UNAUTHORIZED), (None, {"enabled": False}, status.HTTP_401_UNAUTHORIZED), @@ -193,6 +211,13 @@ def test_detail_taxonomy(self, user_attr: str | None, taxonomy_data: dict[str, b if status.is_success(expected_status): check_taxonomy(response.data, taxonomy.pk, **create_data) + def test_detail_system_taxonomy(self): + url = TAXONOMY_DETAIL_URL.format(pk=LANGUAGE_TAXONOMY_ID) + self.client.force_authenticate(user=self.user) + + response = self.client.get(url) + assert response.status_code == status.HTTP_200_OK + def test_detail_taxonomy_404(self) -> None: url = TAXONOMY_DETAIL_URL.format(pk=123123) From 579790d01dcefa2579cad5b8a85633523a4386be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Thu, 9 Nov 2023 23:09:18 -0300 Subject: [PATCH 077/282] feat: add import taxonomy endpoint (#112) Adds endpoints to the taxonomy view to create-and-import tags (POST) or import-and-update tags on an existing taxonomy (PUT). --- openedx_learning/__init__.py | 2 +- .../core/tagging/rest_api/v1/serializers.py | 29 ++ .../core/tagging/rest_api/v1/views.py | 83 +++- openedx_tagging/core/tagging/rules.py | 1 - .../core/tagging/test_views.py | 393 ++++++++++++++++++ 5 files changed, 501 insertions(+), 7 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index 711ab06c2..a5b838c53 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.3.2" +__version__ = "0.3.3" diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 3281af3f0..62d4ef3ed 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -9,6 +9,7 @@ from rest_framework.reverse import reverse from openedx_tagging.core.tagging.data import TagData +from openedx_tagging.core.tagging.import_export.parsers import ParserFormat from openedx_tagging.core.tagging.models import ObjectTag, Tag, Taxonomy @@ -216,3 +217,31 @@ class TaxonomyTagDeleteBodySerializer(serializers.Serializer): # pylint: disabl child=serializers.CharField(), required=True ) with_subtags = serializers.BooleanField(required=False) + + +class TaxonomyImportBodySerializer(serializers.Serializer): # pylint: disable=abstract-method + """ + Serializer of the body for the Taxonomy Import request + """ + file = serializers.FileField(required=True) + + def validate(self, attrs): + """ + Validates the file extension and add parser_format to the data + """ + filename = attrs["file"].name + ext = filename.split('.')[-1] + parser_format = getattr(ParserFormat, ext.upper(), None) + if not parser_format: + raise serializers.ValidationError({"file": f'File type not supported: {ext.lower()}'}, 'file') + + attrs['parser_format'] = parser_format + return attrs + + +class TaxonomyImportNewBodySerializer(TaxonomyImportBodySerializer): # pylint: disable=abstract-method + """ + Serializer of the body for the Taxonomy Create and Import request + """ + taxonomy_name = serializers.CharField(required=True) + taxonomy_description = serializers.CharField(default="") diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index e2145200f..170a80c58 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -11,6 +11,7 @@ from rest_framework.decorators import action from rest_framework.exceptions import MethodNotAllowed, PermissionDenied, ValidationError from rest_framework.generics import ListAPIView, RetrieveUpdateDestroyAPIView +from rest_framework.request import Request from rest_framework.response import Response from rest_framework.viewsets import GenericViewSet, ModelViewSet @@ -26,7 +27,7 @@ update_tag_in_taxonomy, ) from ...data import TagDataQuerySet -from ...import_export.api import export_tags +from ...import_export.api import export_tags, get_last_import_log, import_tags from ...import_export.parsers import ParserFormat from ...models import ObjectTag, Taxonomy from ...rules import ObjectTagPermissionItem @@ -40,6 +41,8 @@ ObjectTagUpdateQueryParamsSerializer, TagDataSerializer, TaxonomyExportQueryParamsSerializer, + TaxonomyImportBodySerializer, + TaxonomyImportNewBodySerializer, TaxonomyListQueryParamsSerializer, TaxonomySerializer, TaxonomyTagCreateBodySerializer, @@ -52,7 +55,7 @@ @view_auth_classes class TaxonomyView(ModelViewSet): """ - View to list, create, retrieve, update, delete or export Taxonomies. + View to list, create, retrieve, update, delete, export or import Taxonomies. **List Query Parameters** * enabled (optional) - Filter by enabled status. Valid values: true, @@ -167,7 +170,29 @@ class TaxonomyView(ModelViewSet): * 400 - Invalid query parameter * 403 - Permission denied + **Import/Create Taxonomy Example Requests** + POST /tagging/rest_api/v1/taxonomy/import/ + { + "taxonomy_name": "Taxonomy Name", + "taxonomy_description": "This is a description", + "file": , + } + + **Import/Create Taxonomy Query Returns** + * 200 - Success + * 400 - Bad request + * 403 - Permission denied + + **Import/Update Taxonomy Example Requests** + PUT /tagging/rest_api/v1/taxonomy/:pk/tags/import/ + { + "file": , + } + **Import/Update Taxonomy Query Returns** + * 200 - Success + * 400 - Bad request + * 403 - Permission denied """ # System taxonomies use negative numbers for their primary keys @@ -216,9 +241,6 @@ def export(self, request, **_kwargs) -> HttpResponse: Export a taxonomy. """ taxonomy = self.get_object() - perm = "oel_tagging.export_taxonomy" - if not request.user.has_perm(perm, taxonomy): - raise PermissionDenied("You do not have permission to export this taxonomy.") query_params = TaxonomyExportQueryParamsSerializer( data=request.query_params.dict() ) @@ -243,6 +265,57 @@ def export(self, request, **_kwargs) -> HttpResponse: return HttpResponse(tags, content_type=content_type) + @action(detail=False, url_path="import", methods=["post"]) + def create_import(self, request: Request, **_kwargs) -> Response: + """ + Creates a new taxonomy and imports the tags from the uploaded file. + """ + body = TaxonomyImportNewBodySerializer(data=request.data) + body.is_valid(raise_exception=True) + + taxonomy_name = body.validated_data["taxonomy_name"] + taxonomy_description = body.validated_data["taxonomy_description"] + file = body.validated_data["file"].file + parser_format = body.validated_data["parser_format"] + + taxonomy = create_taxonomy(taxonomy_name, taxonomy_description) + try: + import_success = import_tags(taxonomy, file, parser_format) + + if import_success: + serializer = self.get_serializer(taxonomy) + return Response(serializer.data, status=status.HTTP_201_CREATED) + else: + import_error = get_last_import_log(taxonomy) + taxonomy.delete() + return Response(import_error, status=status.HTTP_400_BAD_REQUEST) + except ValueError as e: + return Response(str(e), status=status.HTTP_400_BAD_REQUEST) + + @action(detail=True, url_path="tags/import", methods=["put"]) + def update_import(self, request: Request, **_kwargs) -> Response: + """ + Imports tags from the uploaded file to an already created taxonomy. + """ + body = TaxonomyImportBodySerializer(data=request.data) + body.is_valid(raise_exception=True) + + file = body.validated_data["file"].file + parser_format = body.validated_data["parser_format"] + + taxonomy = self.get_object() + try: + import_success = import_tags(taxonomy, file, parser_format) + + if import_success: + serializer = self.get_serializer(taxonomy) + return Response(serializer.data) + else: + import_error = get_last_import_log(taxonomy) + return Response(import_error, status=status.HTTP_400_BAD_REQUEST) + except ValueError as e: + return Response(str(e), status=status.HTTP_400_BAD_REQUEST) + @view_auth_classes class ObjectTagView( diff --git a/openedx_tagging/core/tagging/rules.py b/openedx_tagging/core/tagging/rules.py index 97ca56de2..761392448 100644 --- a/openedx_tagging/core/tagging/rules.py +++ b/openedx_tagging/core/tagging/rules.py @@ -171,7 +171,6 @@ def can_change_object_tag( rules.add_perm("oel_tagging.change_taxonomy", can_change_taxonomy) rules.add_perm("oel_tagging.delete_taxonomy", can_change_taxonomy) rules.add_perm("oel_tagging.view_taxonomy", can_view_taxonomy) -rules.add_perm("oel_tagging.export_taxonomy", can_view_taxonomy) # Tag rules.add_perm("oel_tagging.add_tag", can_change_tag) diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 1a1c69ada..c11de1a2e 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -3,12 +3,14 @@ """ from __future__ import annotations +import json from urllib.parse import parse_qs, quote, quote_plus, urlparse import ddt # type: ignore[import] # typing support in rules depends on https://github.com/dfunckt/django-rules/pull/177 import rules # type: ignore[import] from django.contrib.auth import get_user_model +from django.core.files.uploadedfile import SimpleUploadedFile from rest_framework import status from rest_framework.test import APITestCase @@ -29,6 +31,8 @@ TAXONOMY_DETAIL_URL = "/tagging/rest_api/v1/taxonomies/{pk}/" TAXONOMY_EXPORT_URL = "/tagging/rest_api/v1/taxonomies/{pk}/export/" TAXONOMY_TAGS_URL = "/tagging/rest_api/v1/taxonomies/{pk}/tags/" +TAXONOMY_TAGS_IMPORT_URL = "/tagging/rest_api/v1/taxonomies/{pk}/tags/import/" +TAXONOMY_CREATE_IMPORT_URL = "/tagging/rest_api/v1/taxonomies/import/" OBJECT_TAGS_RETRIEVE_URL = "/tagging/rest_api/v1/object_tags/{object_id}/" @@ -1974,3 +1978,392 @@ def test_delete_tag_in_taxonomy_without_subtags(self): # Check that Tag no longer exists with self.assertRaises(Tag.DoesNotExist): existing_tag.refresh_from_db() + + +class ImportTaxonomyMixin(TestTaxonomyViewMixin): + """ + Mixin to test importing taxonomies. + """ + def _get_file(self, tags: list, file_format: str) -> SimpleUploadedFile: + """ + Returns a file for the given format. + """ + if file_format == "csv": + csv_data = "id,value" + for tag in tags: + csv_data += f"\n{tag['id']},{tag['value']}" + return SimpleUploadedFile("taxonomy.csv", csv_data.encode(), content_type="text/csv") + else: # json + json_data = {"tags": tags} + return SimpleUploadedFile("taxonomy.json", json.dumps(json_data).encode(), content_type="application/json") + + +@ddt.ddt +class TestCreateImportView(ImportTaxonomyMixin, APITestCase): + """ + Tests the create/import taxonomy action. + """ + @ddt.data( + "csv", + "json", + ) + def test_import(self, file_format: str) -> None: + """ + Tests importing a valid taxonomy file. + """ + url = TAXONOMY_CREATE_IMPORT_URL + new_tags = [ + {"id": "tag_1", "value": "Tag 1"}, + {"id": "tag_2", "value": "Tag 2"}, + {"id": "tag_3", "value": "Tag 3"}, + {"id": "tag_4", "value": "Tag 4"}, + ] + file = self._get_file(new_tags, file_format) + + self.client.force_authenticate(user=self.staff) + response = self.client.post( + url, + { + "taxonomy_name": "Imported Taxonomy name", + "taxonomy_description": "Imported Taxonomy description", + "file": file, + }, + format="multipart" + ) + assert response.status_code == status.HTTP_201_CREATED + + # Check if the taxonomy was created + taxonomy = response.data + assert taxonomy["name"] == "Imported Taxonomy name" + assert taxonomy["description"] == "Imported Taxonomy description" + + # Check if the tags were created + url = TAXONOMY_TAGS_URL.format(pk=taxonomy["id"]) + response = self.client.get(url) + tags = response.data["results"] + assert len(tags) == len(new_tags) + for i, tag in enumerate(tags): + assert tag["value"] == new_tags[i]["value"] + + def test_import_no_file(self) -> None: + """ + Tests importing a taxonomy without a file. + """ + url = TAXONOMY_CREATE_IMPORT_URL + self.client.force_authenticate(user=self.staff) + response = self.client.post( + url, + { + "taxonomy_name": "Imported Taxonomy name", + "taxonomy_description": "Imported Taxonomy description", + }, + format="multipart" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["file"][0] == "No file was submitted." + + # Check if the taxonomy was not created + assert not Taxonomy.objects.filter(name="Imported Taxonomy name").exists() + + @ddt.data( + "csv", + "json", + ) + def test_import_no_name(self, file_format) -> None: + """ + Tests importing a taxonomy without specifing a name. + """ + url = TAXONOMY_CREATE_IMPORT_URL + file = SimpleUploadedFile(f"taxonomy.{file_format}", b"invalid file content") + self.client.force_authenticate(user=self.staff) + response = self.client.post( + url, + { + "taxonomy_description": "Imported Taxonomy description", + "file": file, + }, + format="multipart" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["taxonomy_name"][0] == "This field is required." + + # Check if the taxonomy was not created + assert not Taxonomy.objects.filter(name="Imported Taxonomy name").exists() + + def test_import_invalid_format(self) -> None: + """ + Tests importing a taxonomy with an invalid file format. + """ + url = TAXONOMY_CREATE_IMPORT_URL + file = SimpleUploadedFile("taxonomy.invalid", b"invalid file content") + self.client.force_authenticate(user=self.staff) + response = self.client.post( + url, + { + "taxonomy_name": "Imported Taxonomy name", + "taxonomy_description": "Imported Taxonomy description", + "file": file, + }, + format="multipart" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["file"][0] == "File type not supported: invalid" + + # Check if the taxonomy was not created + assert not Taxonomy.objects.filter(name="Imported Taxonomy name").exists() + + @ddt.data( + "csv", + "json", + ) + def test_import_invalid_content(self, file_format) -> None: + """ + Tests importing a taxonomy with an invalid file content. + """ + url = TAXONOMY_CREATE_IMPORT_URL + file = SimpleUploadedFile(f"taxonomy.{file_format}", b"invalid file content") + self.client.force_authenticate(user=self.staff) + response = self.client.post( + url, + { + "taxonomy_name": "Imported Taxonomy name", + "taxonomy_description": "Imported Taxonomy description", + "file": file, + }, + format="multipart" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert f"Invalid '.{file_format}' format:" in response.data + + # Check if the taxonomy was not created + assert not Taxonomy.objects.filter(name="Imported Taxonomy name").exists() + + def test_import_no_perm(self) -> None: + """ + Tests importing a taxonomy using a user without permission. + """ + url = TAXONOMY_CREATE_IMPORT_URL + new_tags = [ + {"id": "tag_1", "value": "Tag 1"}, + {"id": "tag_2", "value": "Tag 2"}, + {"id": "tag_3", "value": "Tag 3"}, + {"id": "tag_4", "value": "Tag 4"}, + ] + file = self._get_file(new_tags, "json") + + self.client.force_authenticate(user=self.user) + response = self.client.post( + url, + { + "taxonomy_name": "Imported Taxonomy name", + "taxonomy_description": "Imported Taxonomy description", + "file": file, + }, + format="multipart" + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + + # Check if the taxonomy was not created + assert not Taxonomy.objects.filter(name="Imported Taxonomy name").exists() + + +@ddt.ddt +class TestImportTagsView(ImportTaxonomyMixin, APITestCase): + """ + Tests the taxonomy import tags action. + """ + def setUp(self): + ImportTaxonomyMixin.setUp(self) + + self.taxonomy = Taxonomy.objects.create( + name="Test import taxonomy", + ) + tag_1 = Tag.objects.create( + taxonomy=self.taxonomy, + external_id="old_tag_1", + value="Old tag 1", + ) + tag_2 = Tag.objects.create( + taxonomy=self.taxonomy, + external_id="old_tag_2", + value="Old tag 2", + ) + self.old_tags = [tag_1, tag_2] + + @ddt.data( + "csv", + "json", + ) + def test_import(self, file_format: str) -> None: + """ + Tests importing a valid taxonomy file. + """ + url = TAXONOMY_TAGS_IMPORT_URL.format(pk=self.taxonomy.id) + new_tags = [ + {"id": "tag_1", "value": "Tag 1"}, + {"id": "tag_2", "value": "Tag 2"}, + {"id": "tag_3", "value": "Tag 3"}, + {"id": "tag_4", "value": "Tag 4"}, + ] + file = self._get_file(new_tags, file_format) + + self.client.force_authenticate(user=self.staff) + response = self.client.put( + url, + {"file": file}, + format="multipart" + ) + assert response.status_code == status.HTTP_200_OK + + # Check if the tags were created + url = TAXONOMY_TAGS_URL.format(pk=self.taxonomy.id) + response = self.client.get(url) + tags = response.data["results"] + all_tags = [{"value": tag.value} for tag in self.old_tags] + new_tags + assert len(tags) == len(all_tags) + for i, tag in enumerate(tags): + assert tag["value"] == all_tags[i]["value"] + + def test_import_no_file(self) -> None: + """ + Tests importing a taxonomy without a file. + """ + url = TAXONOMY_TAGS_IMPORT_URL.format(pk=self.taxonomy.id) + self.client.force_authenticate(user=self.staff) + response = self.client.put( + url, + {}, + format="multipart" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["file"][0] == "No file was submitted." + + # Check if the taxonomy was not changed + url = TAXONOMY_TAGS_URL.format(pk=self.taxonomy.id) + response = self.client.get(url) + tags = response.data["results"] + assert len(tags) == len(self.old_tags) + for i, tag in enumerate(tags): + assert tag["value"] == self.old_tags[i].value + + def test_import_invalid_format(self) -> None: + """ + Tests importing a taxonomy with an invalid file format. + """ + url = TAXONOMY_TAGS_IMPORT_URL.format(pk=self.taxonomy.id) + file = SimpleUploadedFile("taxonomy.invalid", b"invalid file content") + self.client.force_authenticate(user=self.staff) + response = self.client.put( + url, + {"file": file}, + format="multipart" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data["file"][0] == "File type not supported: invalid" + + # Check if the taxonomy was not changed + url = TAXONOMY_TAGS_URL.format(pk=self.taxonomy.id) + response = self.client.get(url) + tags = response.data["results"] + assert len(tags) == len(self.old_tags) + for i, tag in enumerate(tags): + assert tag["value"] == self.old_tags[i].value + + @ddt.data( + "csv", + "json", + ) + def test_import_invalid_content(self, file_format) -> None: + """ + Tests importing a taxonomy with an invalid file content. + """ + url = TAXONOMY_TAGS_IMPORT_URL.format(pk=self.taxonomy.id) + file = SimpleUploadedFile(f"taxonomy.{file_format}", b"invalid file content") + self.client.force_authenticate(user=self.staff) + response = self.client.put( + url, + { + "taxonomy_name": "Imported Taxonomy name", + "taxonomy_description": "Imported Taxonomy description", + "file": file, + }, + format="multipart" + ) + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert f"Invalid '.{file_format}' format:" in response.data + + # Check if the taxonomy was not changed + url = TAXONOMY_TAGS_URL.format(pk=self.taxonomy.id) + response = self.client.get(url) + tags = response.data["results"] + assert len(tags) == len(self.old_tags) + for i, tag in enumerate(tags): + assert tag["value"] == self.old_tags[i].value + + @ddt.data( + "csv", + "json", + ) + def test_import_free_text(self, file_format) -> None: + """ + Tests that importing tags into a free text taxonomy is not allowed. + """ + self.taxonomy.allow_free_text = True + self.taxonomy.save() + url = TAXONOMY_TAGS_IMPORT_URL.format(pk=self.taxonomy.id) + new_tags = [ + {"id": "tag_1", "value": "Tag 1"}, + {"id": "tag_2", "value": "Tag 2"}, + {"id": "tag_3", "value": "Tag 3"}, + {"id": "tag_4", "value": "Tag 4"}, + ] + file = self._get_file(new_tags, file_format) + + self.client.force_authenticate(user=self.staff) + response = self.client.put( + url, + {"file": file}, + format="multipart" + ) + + assert response.status_code == status.HTTP_400_BAD_REQUEST + assert response.data == f"Invalid taxonomy ({self.taxonomy.id}): You cannot import a free-form taxonomy." + + # Check if the taxonomy was no tags, since it is free text + url = TAXONOMY_TAGS_URL.format(pk=self.taxonomy.id) + response = self.client.get(url) + tags = response.data["results"] + assert len(tags) == 0 + + def test_import_no_perm(self) -> None: + """ + Tests importing a taxonomy using a user without permission. + """ + url = TAXONOMY_TAGS_IMPORT_URL.format(pk=self.taxonomy.id) + new_tags = [ + {"id": "tag_1", "value": "Tag 1"}, + {"id": "tag_2", "value": "Tag 2"}, + {"id": "tag_3", "value": "Tag 3"}, + {"id": "tag_4", "value": "Tag 4"}, + ] + file = self._get_file(new_tags, "json") + + self.client.force_authenticate(user=self.user) + response = self.client.put( + url, + { + "taxonomy_name": "Imported Taxonomy name", + "taxonomy_description": "Imported Taxonomy description", + "file": file, + }, + format="multipart" + ) + assert response.status_code == status.HTTP_403_FORBIDDEN + + # Check if the taxonomy was not changed + url = TAXONOMY_TAGS_URL.format(pk=self.taxonomy.id) + response = self.client.get(url) + tags = response.data["results"] + assert len(tags) == len(self.old_tags) + for i, tag in enumerate(tags): + assert tag["value"] == self.old_tags[i].value From e9a2ff5588297cf0d1498a6293bce198e718ca93 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 9 Nov 2023 12:29:00 -0800 Subject: [PATCH 078/282] feat: Make get_object_tag_counts available as a python API --- openedx_tagging/core/tagging/api.py | 20 +++++++++++++++++++ .../core/tagging/rest_api/v1/views.py | 19 ++++++------------ .../openedx_tagging/core/tagging/test_api.py | 18 +++++++++++++++++ .../core/tagging/test_views.py | 8 ++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index 8c5ed7648..c9dc1b58c 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -12,6 +12,8 @@ """ from __future__ import annotations +from typing import Any + from django.db import models, transaction from django.db.models import F, QuerySet, Value from django.db.models.functions import Coalesce, Concat, Lower @@ -185,6 +187,24 @@ def get_object_tags( return tags +def get_object_tag_counts(object_id_pattern: str) -> dict[str, int]: + """ + Given an object ID, a "starts with" glob pattern like + "course-v1:foo+bar+baz@*", or a list of "comma,separated,IDs", return a + dict of matching object IDs and how many tags each object has. + """ + qs: Any = ObjectTag.objects + if object_id_pattern.endswith("*"): + qs = qs.filter(object_id__startswith=object_id_pattern[0:len(object_id_pattern) - 1]) + elif "*" in object_id_pattern: + raise ValueError("Wildcard matches are only supported if the * is at the end.") + else: + qs = qs.filter(object_id__in=object_id_pattern.split(",")) + + qs = qs.values("object_id").annotate(num_tags=models.Count("id")).order_by("object_id") + return {row["object_id"]: row["num_tags"] for row in qs} + + def delete_object_tags(object_id: str): """ Delete all ObjectTag entries for a given object. diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index 170a80c58..a531e5bc0 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -3,8 +3,6 @@ """ from __future__ import annotations -from typing import Any - from django.db import models from django.http import Http404, HttpResponse from rest_framework import mixins, status @@ -20,6 +18,7 @@ add_tag_to_taxonomy, create_taxonomy, delete_tags_from_taxonomy, + get_object_tag_counts, get_object_tags, get_taxonomies, get_taxonomy, @@ -29,7 +28,7 @@ from ...data import TagDataQuerySet from ...import_export.api import export_tags, get_last_import_log, import_tags from ...import_export.parsers import ParserFormat -from ...models import ObjectTag, Taxonomy +from ...models import Taxonomy from ...rules import ObjectTagPermissionItem from ..paginators import TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination from .permissions import ObjectTagObjectPermissions, TagObjectPermissions, TaxonomyObjectPermissions @@ -517,16 +516,10 @@ def retrieve(self, request, *args, **kwargs) -> Response: """ # This API does NOT bother doing any permission checks as the # of tags is not considered sensitive information. object_id_pattern = self.kwargs["object_id_pattern"] - qs: Any = ObjectTag.objects - if object_id_pattern.endswith("*"): - qs = qs.filter(object_id__startswith=object_id_pattern[0:len(object_id_pattern) - 1]) - elif "*" in object_id_pattern: - raise ValidationError("Wildcard matches are only supported if the * is at the end.") - else: - qs = qs.filter(object_id__in=object_id_pattern.split(",")) - - qs = qs.values("object_id").annotate(num_tags=models.Count("id")).order_by("object_id") - return Response({row["object_id"]: row["num_tags"] for row in qs}) + try: + return Response(get_object_tag_counts(object_id_pattern)) + except ValueError as err: + raise ValidationError(err.args[0]) from err @view_auth_classes diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index a8c2e7569..665a6b8e6 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -673,3 +673,21 @@ def test_autocomplete_tags_closed_omit_object(self) -> None: # "Bacteria (children: 2)", # does not contain "cha" but a child does # " Archaebacteria (children: 0)", ] + + def test_get_object_tag_counts(self) -> None: + """ + Smoke test of get_object_tag_counts + """ + obj1 = "object_id1" + obj2 = "object_id2" + other = "other_object" + # Give each object 1-2 tags: + tagging_api.tag_object(object_id=obj1, taxonomy=self.taxonomy, tags=["DPANN"]) + tagging_api.tag_object(object_id=obj2, taxonomy=self.taxonomy, tags=["Chordata"]) + tagging_api.tag_object(object_id=obj2, taxonomy=self.free_text_taxonomy, tags=["has a notochord"]) + tagging_api.tag_object(object_id=other, taxonomy=self.free_text_taxonomy, tags=["other"]) + + assert tagging_api.get_object_tag_counts(obj1) == {obj1: 1} + assert tagging_api.get_object_tag_counts(obj2) == {obj2: 2} + assert tagging_api.get_object_tag_counts(f"{obj1},{obj2}") == {obj1: 1, obj2: 2} + assert tagging_api.get_object_tag_counts("object_*") == {obj1: 1, obj2: 2} diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index c11de1a2e..a6886ae12 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -1025,6 +1025,14 @@ def check(object_id_pattern: str): "course07-unit02-problem02": 2, } + def test_get_counts_invalid_spec(self): + """ + Test handling of an invalid object tag pattern + """ + result = self.client.get(OBJECT_TAG_COUNTS_URL.format(object_id_pattern="abc*def")) + assert result.status_code == status.HTTP_400_BAD_REQUEST + assert "Wildcard matches are only supported if the * is at the end." in str(result.content) + class TestTaxonomyTagsView(TestTaxonomyViewMixin): """ From b42dfabd00389133d7e03447ccb3970071156f74 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 9 Nov 2023 14:04:05 -0800 Subject: [PATCH 079/282] feat: APIs should not return deleted/disabled tags by default --- openedx_tagging/core/tagging/api.py | 20 +++- .../openedx_tagging/core/tagging/test_api.py | 112 ++++++++++++------ .../core/tagging/test_models.py | 6 +- .../core/tagging/test_views.py | 3 + 4 files changed, 101 insertions(+), 40 deletions(-) diff --git a/openedx_tagging/core/tagging/api.py b/openedx_tagging/core/tagging/api.py index c9dc1b58c..64d5c6a83 100644 --- a/openedx_tagging/core/tagging/api.py +++ b/openedx_tagging/core/tagging/api.py @@ -159,6 +159,7 @@ def resync_object_tags(object_tags: QuerySet | None = None) -> int: def get_object_tags( object_id: str, taxonomy_id: int | None = None, + include_deleted: bool = False, object_tag_class: type[ObjectTag] = ObjectTag ) -> QuerySet[ObjectTag]: """ @@ -167,8 +168,16 @@ def get_object_tags( Pass taxonomy_id to limit the returned object_tags to a specific taxonomy. """ filters = {"taxonomy_id": taxonomy_id} if taxonomy_id else {} + base_qs = ( + object_tag_class.objects + .filter(object_id=object_id, **filters) + .exclude(taxonomy__enabled=False) # Exclude if the whole taxonomy is disabled + ) + if not include_deleted: + base_qs = base_qs.exclude(taxonomy_id=None) # Exclude if the whole taxonomy was deleted + base_qs = base_qs.exclude(tag_id=None, taxonomy__allow_free_text=False) # Exclude if just the tag is deleted tags = ( - object_tag_class.objects.filter(object_id=object_id, **filters) + base_qs # Preload related objects, including data for the "get_lineage" method on ObjectTag/Tag: .select_related("taxonomy", "tag", "tag__parent", "tag__parent__parent") # Sort the tags within each taxonomy in "tree order". See Taxonomy._get_filtered_tags_deep for details on this: @@ -192,7 +201,11 @@ def get_object_tag_counts(object_id_pattern: str) -> dict[str, int]: Given an object ID, a "starts with" glob pattern like "course-v1:foo+bar+baz@*", or a list of "comma,separated,IDs", return a dict of matching object IDs and how many tags each object has. + + Deleted tags and disabled taxonomies are excluded from the counts, even if + ObjectTag data about them is present. """ + # Note: in the future we may add an option to exclude system taxonomies from the count. qs: Any = ObjectTag.objects if object_id_pattern.endswith("*"): qs = qs.filter(object_id__startswith=object_id_pattern[0:len(object_id_pattern) - 1]) @@ -200,7 +213,10 @@ def get_object_tag_counts(object_id_pattern: str) -> dict[str, int]: raise ValueError("Wildcard matches are only supported if the * is at the end.") else: qs = qs.filter(object_id__in=object_id_pattern.split(",")) - + # Don't include deleted tags or disabled taxonomies: + qs = qs.exclude(taxonomy_id=None) # The whole taxonomy was deleted + qs = qs.exclude(taxonomy__enabled=False) # The whole taxonomy is disabled + qs = qs.exclude(tag_id=None, taxonomy__allow_free_text=False) # The taxonomy exists but the tag is deleted qs = qs.values("object_id").annotate(num_tags=models.Count("id")).order_by("object_id") return {row["object_id"]: row["num_tags"] for row in qs} diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 665a6b8e6..4173c5a1a 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -265,7 +265,7 @@ def test_resync_object_tags(self) -> None: tagging_api.tag_object(open_taxonomy, ["foo", "bar"], object_id) # Free text tags # At first, none of these will be deleted: - assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id, include_deleted=True)] == [ ("bar", False), ("foo", False), (self.archaea.value, False), @@ -275,7 +275,7 @@ def test_resync_object_tags(self) -> None: # Delete "bacteria" from the taxonomy: tagging_api.delete_tags_from_taxonomy(self.taxonomy, [self.bacteria.value], with_subtags=True) - assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id, include_deleted=True)] == [ ("bar", False), ("foo", False), (self.archaea.value, False), @@ -293,7 +293,7 @@ def test_resync_object_tags(self) -> None: assert changed == 1 # Now the tag is not deleted: - assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id)] == [ + assert [(t.value, t.is_deleted) for t in tagging_api.get_object_tags(object_id, include_deleted=True)] == [ ("bar", False), ("foo", False), (self.archaea.value, False), @@ -342,19 +342,21 @@ def test_tag_object(self): assert object_tag.name == self.taxonomy.name assert object_tag.object_id == "biology101" - def test_tag_object_free_text(self): - self.taxonomy.allow_free_text = True + def test_tag_object_free_text(self) -> None: + """ + Test tagging an object using a free text taxonomy + """ tagging_api.tag_object( - self.taxonomy, - ["Eukaryota Xenomorph"], - "biology101", + object_id="biology101", + taxonomy=self.free_text_taxonomy, + tags=["Eukaryota Xenomorph"], ) object_tags = tagging_api.get_object_tags("biology101") assert len(object_tags) == 1 object_tag = object_tags[0] object_tag.full_clean() # Should not raise any ValidationErrors - assert object_tag.taxonomy == self.taxonomy - assert object_tag.name == self.taxonomy.name + assert object_tag.taxonomy == self.free_text_taxonomy + assert object_tag.name == self.free_text_taxonomy.name assert object_tag._value == "Eukaryota Xenomorph" # pylint: disable=protected-access assert object_tag.get_lineage() == ["Eukaryota Xenomorph"] assert object_tag.object_id == "biology101" @@ -561,32 +563,42 @@ def test_tag_object_limit(self) -> None: assert exc.exception assert "Cannot add more than 100 tags to" in str(exc.exception) - def test_get_object_tags(self) -> None: - # Alpha tag has no taxonomy (as if the taxonomy had been deleted) - alpha = ObjectTag(object_id="abc") - alpha.name = self.taxonomy.name - alpha.value = "alpha" - alpha.save() - # Beta tag has a closed taxonomy - beta = ObjectTag.objects.create( - object_id="abc", - taxonomy=self.taxonomy, - tag=self.taxonomy.tag_set.get(value="Protista"), - ) - - # Fetch all the tags for a given object ID - assert list( - tagging_api.get_object_tags(object_id="abc") - ) == [ - alpha, - beta, + def test_get_object_tags_deleted_disabled(self) -> None: + """ + Test that get_object_tags doesn't return tags from disabled taxonomies + or tags that have been deleted or taxonomies that have been deleted. + """ + obj_id = "object_id1" + self.taxonomy.allow_multiple = True + self.taxonomy.save() + disabled_taxonomy = tagging_api.create_taxonomy("Disabled Taxonomy", allow_free_text=True) + tagging_api.tag_object(object_id=obj_id, taxonomy=self.taxonomy, tags=["DPANN", "Chordata"]) + tagging_api.tag_object(object_id=obj_id, taxonomy=self.language_taxonomy, tags=["English"]) + tagging_api.tag_object(object_id=obj_id, taxonomy=self.free_text_taxonomy, tags=["has a notochord"]) + tagging_api.tag_object(object_id=obj_id, taxonomy=disabled_taxonomy, tags=["disabled tag"]) + + def get_object_tags(): + return [f"{ot.name}: {'>'.join(ot.get_lineage())}" for ot in tagging_api.get_object_tags(obj_id)] + + # Before deleting/disabling: + assert get_object_tags() == [ + "Disabled Taxonomy: disabled tag", + "Free Text: has a notochord", + "Languages: English", + "Life on Earth: Archaea>DPANN", + "Life on Earth: Eukaryota>Animalia>Chordata", ] - # Fetch all the tags for a given object ID + taxonomy - assert list( - tagging_api.get_object_tags(object_id="abc", taxonomy_id=self.taxonomy.pk) - ) == [ - beta, + # Now delete and disable things: + disabled_taxonomy.enabled = False + disabled_taxonomy.save() + self.free_text_taxonomy.delete() + tagging_api.delete_tags_from_taxonomy(self.taxonomy, ["DPANN"], with_subtags=False) + + # Now retrieve the tags again: + assert get_object_tags() == [ + "Languages: English", + "Life on Earth: Eukaryota>Animalia>Chordata", ] @ddt.data( @@ -676,7 +688,7 @@ def test_autocomplete_tags_closed_omit_object(self) -> None: def test_get_object_tag_counts(self) -> None: """ - Smoke test of get_object_tag_counts + Basic test of get_object_tag_counts """ obj1 = "object_id1" obj2 = "object_id2" @@ -691,3 +703,33 @@ def test_get_object_tag_counts(self) -> None: assert tagging_api.get_object_tag_counts(obj2) == {obj2: 2} assert tagging_api.get_object_tag_counts(f"{obj1},{obj2}") == {obj1: 1, obj2: 2} assert tagging_api.get_object_tag_counts("object_*") == {obj1: 1, obj2: 2} + + def test_get_object_tag_counts_deleted_disabled(self) -> None: + """ + Test that get_object_tag_counts doesn't "count" disabled taxonomies or + deleted tags. + """ + obj1 = "object_id1" + obj2 = "object_id2" + # Give each object 2 tags: + tagging_api.tag_object(object_id=obj1, taxonomy=self.taxonomy, tags=["DPANN"]) + tagging_api.tag_object(object_id=obj1, taxonomy=self.language_taxonomy, tags=["English"]) + tagging_api.tag_object(object_id=obj2, taxonomy=self.taxonomy, tags=["Chordata"]) + tagging_api.tag_object(object_id=obj2, taxonomy=self.free_text_taxonomy, tags=["has a notochord"]) + + # Before we delete tags / disable taxonomies, the counts are two each: + assert tagging_api.get_object_tag_counts("object_*") == {obj1: 2, obj2: 2} + # Delete the "DPANN" tag from self.taxonomy: + tagging_api.delete_tags_from_taxonomy(self.taxonomy, tags=["DPANN"], with_subtags=False) + assert tagging_api.get_object_tag_counts("object_*") == {obj1: 1, obj2: 2} + # Disable the free text taxonomy: + self.free_text_taxonomy.enabled = False + self.free_text_taxonomy.save() + assert tagging_api.get_object_tag_counts("object_*") == {obj1: 1, obj2: 1} + + # But, by the way, if we re-enable the taxonomy and restore the tag, the counts return: + self.free_text_taxonomy.enabled = True + self.free_text_taxonomy.save() + assert tagging_api.get_object_tag_counts("object_*") == {obj1: 1, obj2: 2} + tagging_api.add_tag_to_taxonomy(self.taxonomy, "DPANN", parent_tag_value="Archaea") + assert tagging_api.get_object_tag_counts("object_*") == {obj1: 2, obj2: 2} diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 88bfeaad2..367b735b3 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -766,7 +766,7 @@ def test_is_deleted(self): api.tag_object(self.free_text_taxonomy, ["foo", "bar", "tribble"], object_id) # Free text tags # At first, none of these will be deleted: - assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ + assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id, include_deleted=True)] == [ ("bar", False), ("foo", False), ("tribble", False), @@ -777,7 +777,7 @@ def test_is_deleted(self): # Delete "bacteria" from the taxonomy: api.delete_tags_from_taxonomy(self.taxonomy, ["Bacteria"], with_subtags=True) - assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ + assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id, include_deleted=True)] == [ ("bar", False), ("foo", False), ("tribble", False), @@ -788,7 +788,7 @@ def test_is_deleted(self): # Then delete the whole free text taxonomy self.free_text_taxonomy.delete() - assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id)] == [ + assert [(t.value, t.is_deleted) for t in api.get_object_tags(object_id, include_deleted=True)] == [ ("bar", True), # <--- Deleted, but the value is preserved ("foo", True), # <--- Deleted, but the value is preserved ("tribble", True), # <--- Deleted, but the value is preserved diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index a6886ae12..2dd3593d3 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -926,6 +926,9 @@ def test_tag_object_clear(self, user_attr, taxonomy_flags, expected_status): assert response.data[object_id]["taxonomies"] == [] else: # Make sure the object tags are unchanged: + if not self.taxonomy.enabled: # The taxonomy has to be enabled for us to see the tags though: + self.taxonomy.enabled = True + self.taxonomy.save() assert [ot.value for ot in api.get_object_tags(object_id=object_id)] == ["Fungi"] @ddt.data( From 9c19cf3816dd7f3adfa5e2bca05ec89647ce675a Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Mon, 13 Nov 2023 12:27:25 -0800 Subject: [PATCH 080/282] chore: Version bump --- openedx_learning/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index a5b838c53..ade40e2ca 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.3.3" +__version__ = "0.3.4" From bd7d20750e33f4d032dc54e8c057c2e674d8e38f Mon Sep 17 00:00:00 2001 From: UsamaSadiq Date: Wed, 15 Nov 2023 16:53:16 +0500 Subject: [PATCH 081/282] feat: add tox 4.0 support --- CHANGELOG.rst | 5 +++-- requirements/dev.in | 1 - requirements/dev.txt | 3 --- tox.ini | 4 ++-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 9c2000774..be5c0b25f 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -14,10 +14,11 @@ Change Log Unreleased ~~~~~~~~~~ +* Removed usage of ``tox-battery`` and added support for ``tox 4.0`` * Switch from ``edx-sphinx-theme`` to ``sphinx-book-theme`` since the former is deprecated. See https://github.com/openedx/edx-sphinx-theme/issues/184 for - more details. - + more details. + [0.1.0] - 2021-08-08 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/requirements/dev.in b/requirements/dev.in index 2c9492480..e2af39318 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -7,5 +7,4 @@ diff-cover # Changeset diff test coverage edx-i18n-tools # For i18n_tool dummy -tox-battery # Makes tox aware of requirements file changes django-debug-toolbar # Debugging DB queries primarily diff --git a/requirements/dev.txt b/requirements/dev.txt index af7157e18..f505facfb 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -484,9 +484,6 @@ tox==3.28.0 # via # -c requirements/constraints.txt # -r requirements/ci.txt - # tox-battery -tox-battery==0.6.2 - # via -r requirements/dev.in twine==4.0.2 # via -r requirements/quality.txt types-pytz==2023.3.1.1 diff --git a/tox.ini b/tox.ini index 284b213fc..4f57a8574 100644 --- a/tox.ini +++ b/tox.ini @@ -52,7 +52,7 @@ setenv = PYTHONPATH = {toxinidir} # Adding the option here instead of as a default in the docs Makefile because that Makefile is generated by sphinx. # SPHINXOPTS = -W -whitelist_externals = +allowlist_externals = make rm deps = @@ -66,7 +66,7 @@ commands = python setup.py check --restructuredtext --strict [testenv:quality] -whitelist_externals = +allowlist_externals = make rm touch From 3e82051b4a4ecd9c395aec06acf5240683b4cce1 Mon Sep 17 00:00:00 2001 From: farhan Date: Fri, 17 Nov 2023 16:44:43 +0500 Subject: [PATCH 082/282] fix: Enable common constraints --- requirements/constraints.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 6014c0923..99f31ef08 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -9,7 +9,7 @@ # linking to it here is good. # Common constraints for edx repos -# -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt +-c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # tox>4.0.0 isn't yet compatible with many tox plugins, causing CI failures in almost all repos. # Details can be found in this discussion: https://github.com/tox-dev/tox/discussions/1810 From ab5df6a698d45342bb5b662b21d7a7ec747a2220 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Mon, 20 Nov 2023 00:09:31 -0500 Subject: [PATCH 083/282] chore: Updating Python Requirements --- requirements/base.txt | 23 +++++++++++---------- requirements/ci.txt | 4 +++- requirements/dev.txt | 42 ++++++++++++++++++++------------------- requirements/doc.txt | 31 +++++++++++++++-------------- requirements/quality.txt | 43 +++++++++++++++++++++------------------- requirements/test.txt | 29 ++++++++++++++------------- 6 files changed, 91 insertions(+), 81 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index d1aeb7422..a3ddc94c7 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -4,7 +4,7 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via kombu asgiref==3.7.2 # via django @@ -14,11 +14,11 @@ backports-zoneinfo[tzdata]==0.2.1 # via # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via celery -celery==5.3.4 +celery==5.3.5 # via -r requirements/base.in -certifi==2023.7.22 +certifi==2023.11.17 # via requests cffi==1.16.0 # via @@ -43,6 +43,7 @@ cryptography==41.0.5 # via pyjwt django==3.2.23 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -c requirements/constraints.txt # -r requirements/base.in # django-crum @@ -66,19 +67,19 @@ drf-jwt==1.19.2 # via edx-drf-extensions edx-django-utils==5.8.0 # via edx-drf-extensions -edx-drf-extensions==8.13.0 +edx-drf-extensions==8.13.1 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via edx-drf-extensions idna==3.4 # via requests -kombu==5.3.2 +kombu==5.3.4 # via celery -newrelic==9.1.1 +newrelic==9.2.0 # via edx-django-utils -pbr==5.11.1 +pbr==6.0.0 # via stevedore -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via click-repl psutil==5.9.6 # via edx-django-utils @@ -122,12 +123,12 @@ tzdata==2023.3 # via # backports-zoneinfo # celery -urllib3==2.0.7 +urllib3==2.1.0 # via requests vine==5.1.0 # via # amqp # celery # kombu -wcwidth==0.2.9 +wcwidth==0.2.10 # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index e358d6148..d2e436159 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -19,7 +19,9 @@ import-linter==1.12.1 packaging==23.2 # via tox platformdirs==3.11.0 - # via virtualenv + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # virtualenv pluggy==1.3.0 # via tox py==1.11.0 diff --git a/requirements/dev.txt b/requirements/dev.txt index f505facfb..79528123b 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,7 +4,7 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via # -r requirements/quality.txt # kombu @@ -25,7 +25,7 @@ backports-zoneinfo[tzdata]==0.2.1 # backports-zoneinfo # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via # -r requirements/quality.txt # celery @@ -33,9 +33,9 @@ build==1.0.3 # via # -r requirements/pip-tools.txt # pip-tools -celery==5.3.4 +celery==5.3.5 # via -r requirements/quality.txt -certifi==2023.7.22 +certifi==2023.11.17 # via # -r requirements/quality.txt # requests @@ -95,9 +95,9 @@ cryptography==41.0.5 # -r requirements/quality.txt # pyjwt # secretstorage -ddt==1.6.0 +ddt==1.7.0 # via -r requirements/quality.txt -diff-cover==8.0.0 +diff-cover==8.0.1 # via -r requirements/dev.in dill==0.3.7 # via @@ -109,6 +109,7 @@ distlib==0.3.7 # virtualenv django==3.2.23 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -c requirements/constraints.txt # -r requirements/quality.txt # django-crum @@ -161,7 +162,7 @@ edx-django-utils==5.8.0 # via # -r requirements/quality.txt # edx-drf-extensions -edx-drf-extensions==8.13.0 +edx-drf-extensions==8.13.1 # via -r requirements/quality.txt edx-i18n-tools==1.3.0 # via -r requirements/dev.in @@ -200,7 +201,7 @@ importlib-metadata==6.8.0 # build # keyring # twine -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via # -r requirements/quality.txt # keyring @@ -226,11 +227,11 @@ jinja2==3.1.2 # -r requirements/quality.txt # code-annotations # diff-cover -keyring==24.2.0 +keyring==24.3.0 # via # -r requirements/quality.txt # twine -kombu==5.3.2 +kombu==5.3.4 # via # -r requirements/quality.txt # celery @@ -258,7 +259,7 @@ more-itertools==10.1.0 # via # -r requirements/quality.txt # jaraco-classes -mypy==1.6.1 +mypy==1.7.0 # via # -r requirements/quality.txt # djangorestframework-stubs @@ -268,7 +269,7 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/quality.txt -newrelic==9.1.1 +newrelic==9.2.0 # via # -r requirements/quality.txt # edx-django-utils @@ -286,7 +287,7 @@ packaging==23.2 # tox path==16.7.1 # via edx-i18n-tools -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/quality.txt # stevedore @@ -298,6 +299,7 @@ pkginfo==1.9.6 # twine platformdirs==3.11.0 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -r requirements/ci.txt # -r requirements/quality.txt # pylint @@ -311,7 +313,7 @@ pluggy==1.3.0 # tox polib==1.2.0 # via edx-i18n-tools -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via # -r requirements/quality.txt # click-repl @@ -331,7 +333,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.16.1 +pygments==2.17.1 # via # -r requirements/quality.txt # diff-cover @@ -382,7 +384,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/quality.txt -pytest-django==4.6.0 +pytest-django==4.7.0 # via -r requirements/quality.txt python-dateutil==2.8.2 # via @@ -421,7 +423,7 @@ rfc3986==2.0.0 # via # -r requirements/quality.txt # twine -rich==13.6.0 +rich==13.7.0 # via # -r requirements/quality.txt # twine @@ -476,7 +478,7 @@ tomli==2.0.1 # pyproject-hooks # pytest # tox -tomlkit==0.12.2 +tomlkit==0.12.3 # via # -r requirements/quality.txt # pylint @@ -520,7 +522,7 @@ tzdata==2023.3 # -r requirements/quality.txt # backports-zoneinfo # celery -urllib3==2.0.7 +urllib3==2.1.0 # via # -r requirements/quality.txt # requests @@ -536,7 +538,7 @@ virtualenv==20.24.6 # via # -r requirements/ci.txt # tox -wcwidth==0.2.9 +wcwidth==0.2.10 # via # -r requirements/quality.txt # prompt-toolkit diff --git a/requirements/doc.txt b/requirements/doc.txt index 453b99580..a8851c10f 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -8,7 +8,7 @@ accessible-pygments==0.0.4 # via pydata-sphinx-theme alabaster==0.7.13 # via sphinx -amqp==5.1.1 +amqp==5.2.0 # via # -r requirements/test.txt # kombu @@ -30,13 +30,13 @@ backports-zoneinfo[tzdata]==0.2.1 # kombu beautifulsoup4==4.12.2 # via pydata-sphinx-theme -billiard==4.1.0 +billiard==4.2.0 # via # -r requirements/test.txt # celery -celery==5.3.4 +celery==5.3.5 # via -r requirements/test.txt -certifi==2023.7.22 +certifi==2023.11.17 # via # -r requirements/test.txt # requests @@ -82,10 +82,11 @@ cryptography==41.0.5 # via # -r requirements/test.txt # pyjwt -ddt==1.6.0 +ddt==1.7.0 # via -r requirements/test.txt django==3.2.23 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -c requirements/constraints.txt # -r requirements/test.txt # django-crum @@ -141,7 +142,7 @@ edx-django-utils==5.8.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.13.0 +edx-drf-extensions==8.13.1 # via -r requirements/test.txt edx-opaque-keys==2.5.1 # via @@ -174,7 +175,7 @@ jinja2==3.1.2 # -r requirements/test.txt # code-annotations # sphinx -kombu==5.3.2 +kombu==5.3.4 # via # -r requirements/test.txt # celery @@ -184,7 +185,7 @@ markupsafe==2.1.3 # jinja2 mock==5.1.0 # via -r requirements/test.txt -mypy==1.6.1 +mypy==1.7.0 # via # -r requirements/test.txt # djangorestframework-stubs @@ -194,7 +195,7 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==9.1.1 +newrelic==9.2.0 # via # -r requirements/test.txt # edx-django-utils @@ -206,7 +207,7 @@ packaging==23.2 # pydata-sphinx-theme # pytest # sphinx -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/test.txt # stevedore @@ -216,7 +217,7 @@ pluggy==1.3.0 # pytest pprintpp==0.4.0 # via sphinxcontrib-django -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via # -r requirements/test.txt # click-repl @@ -230,7 +231,7 @@ pycparser==2.21 # cffi pydata-sphinx-theme==0.14.3 # via sphinx-book-theme -pygments==2.16.1 +pygments==2.17.1 # via # accessible-pygments # doc8 @@ -258,7 +259,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.6.0 +pytest-django==4.7.0 # via -r requirements/test.txt python-dateutil==2.8.2 # via @@ -380,7 +381,7 @@ tzdata==2023.3 # -r requirements/test.txt # backports-zoneinfo # celery -urllib3==2.0.7 +urllib3==2.1.0 # via # -r requirements/test.txt # requests @@ -391,7 +392,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.9 +wcwidth==0.2.10 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/quality.txt b/requirements/quality.txt index 7fbf7b103..f566fd6cc 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -4,7 +4,7 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via # -r requirements/test.txt # kombu @@ -24,13 +24,13 @@ backports-zoneinfo[tzdata]==0.2.1 # backports-zoneinfo # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via # -r requirements/test.txt # celery -celery==5.3.4 +celery==5.3.5 # via -r requirements/test.txt -certifi==2023.7.22 +certifi==2023.11.17 # via # -r requirements/test.txt # requests @@ -83,12 +83,13 @@ cryptography==41.0.5 # -r requirements/test.txt # pyjwt # secretstorage -ddt==1.6.0 +ddt==1.7.0 # via -r requirements/test.txt dill==0.3.7 # via pylint django==3.2.23 # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -c requirements/constraints.txt # -r requirements/test.txt # django-crum @@ -136,7 +137,7 @@ edx-django-utils==5.8.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.13.0 +edx-drf-extensions==8.13.1 # via -r requirements/test.txt edx-lint==5.3.6 # via -r requirements/quality.in @@ -162,7 +163,7 @@ importlib-metadata==6.8.0 # via # keyring # twine -importlib-resources==6.1.0 +importlib-resources==6.1.1 # via keyring iniconfig==2.0.0 # via @@ -182,9 +183,9 @@ jinja2==3.1.2 # via # -r requirements/test.txt # code-annotations -keyring==24.2.0 +keyring==24.3.0 # via twine -kombu==5.3.2 +kombu==5.3.4 # via # -r requirements/test.txt # celery @@ -202,7 +203,7 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.1.0 # via jaraco-classes -mypy==1.6.1 +mypy==1.7.0 # via # -r requirements/test.txt # djangorestframework-stubs @@ -212,7 +213,7 @@ mypy-extensions==1.0.0 # mypy mysqlclient==2.2.0 # via -r requirements/test.txt -newrelic==9.1.1 +newrelic==9.2.0 # via # -r requirements/test.txt # edx-django-utils @@ -222,19 +223,21 @@ packaging==23.2 # via # -r requirements/test.txt # pytest -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/test.txt # stevedore pkginfo==1.9.6 # via twine platformdirs==3.11.0 - # via pylint + # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt + # pylint pluggy==1.3.0 # via # -r requirements/test.txt # pytest -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via # -r requirements/test.txt # click-repl @@ -250,7 +253,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.in -pygments==2.16.1 +pygments==2.17.1 # via # readme-renderer # rich @@ -289,7 +292,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.txt -pytest-django==4.6.0 +pytest-django==4.7.0 # via -r requirements/test.txt python-dateutil==2.8.2 # via @@ -321,7 +324,7 @@ requests-toolbelt==1.0.0 # via twine rfc3986==2.0.0 # via twine -rich==13.6.0 +rich==13.7.0 # via twine rules==3.3 # via -r requirements/test.txt @@ -362,7 +365,7 @@ tomli==2.0.1 # mypy # pylint # pytest -tomlkit==0.12.2 +tomlkit==0.12.3 # via pylint twine==4.0.2 # via -r requirements/quality.in @@ -399,7 +402,7 @@ tzdata==2023.3 # -r requirements/test.txt # backports-zoneinfo # celery -urllib3==2.0.7 +urllib3==2.1.0 # via # -r requirements/test.txt # requests @@ -411,7 +414,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.9 +wcwidth==0.2.10 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/test.txt b/requirements/test.txt index 2b89b9143..5f2f346a3 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,7 +4,7 @@ # # make upgrade # -amqp==5.1.1 +amqp==5.2.0 # via # -r requirements/base.txt # kombu @@ -20,13 +20,13 @@ backports-zoneinfo[tzdata]==0.2.1 # backports-zoneinfo # celery # kombu -billiard==4.1.0 +billiard==4.2.0 # via # -r requirements/base.txt # celery -celery==5.3.4 +celery==5.3.5 # via -r requirements/base.txt -certifi==2023.7.22 +certifi==2023.11.17 # via # -r requirements/base.txt # requests @@ -71,9 +71,10 @@ cryptography==41.0.5 # via # -r requirements/base.txt # pyjwt -ddt==1.6.0 +ddt==1.7.0 # via -r requirements/test.in # via + # -c https://raw.githubusercontent.com/edx/edx-lint/master/edx_lint/files/common_constraints.txt # -c requirements/constraints.txt # -r requirements/base.txt # django-crum @@ -117,7 +118,7 @@ edx-django-utils==5.8.0 # via # -r requirements/base.txt # edx-drf-extensions -edx-drf-extensions==8.13.0 +edx-drf-extensions==8.13.1 # via -r requirements/base.txt edx-opaque-keys==2.5.1 # via @@ -137,7 +138,7 @@ iniconfig==2.0.0 # via pytest jinja2==3.1.2 # via code-annotations -kombu==5.3.2 +kombu==5.3.4 # via # -r requirements/base.txt # celery @@ -145,7 +146,7 @@ markupsafe==2.1.3 # via jinja2 mock==5.1.0 # via -r requirements/test.in -mypy==1.6.1 +mypy==1.7.0 # via # -r requirements/test.in # djangorestframework-stubs @@ -153,19 +154,19 @@ mypy-extensions==1.0.0 # via mypy mysqlclient==2.2.0 # via -r requirements/test.in -newrelic==9.1.1 +newrelic==9.2.0 # via # -r requirements/base.txt # edx-django-utils packaging==23.2 # via pytest -pbr==5.11.1 +pbr==6.0.0 # via # -r requirements/base.txt # stevedore pluggy==1.3.0 # via pytest -prompt-toolkit==3.0.39 +prompt-toolkit==3.0.41 # via # -r requirements/base.txt # click-repl @@ -198,7 +199,7 @@ pytest==7.4.3 # pytest-django pytest-cov==4.1.0 # via -r requirements/test.in -pytest-django==4.6.0 +pytest-django==4.7.0 # via -r requirements/test.in python-dateutil==2.8.2 # via @@ -273,7 +274,7 @@ tzdata==2023.3 # -r requirements/base.txt # backports-zoneinfo # celery -urllib3==2.0.7 +urllib3==2.1.0 # via # -r requirements/base.txt # requests @@ -284,7 +285,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.9 +wcwidth==0.2.10 # via # -r requirements/base.txt # prompt-toolkit From 3ae1c01f158cdca36733da86152fef10f3238c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 20 Nov 2023 23:08:15 -0300 Subject: [PATCH 084/282] fix: overwrite previous tags on import (#121) --- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/rest_api/v1/views.py | 5 +++-- tests/openedx_tagging/core/tagging/test_views.py | 5 ++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index ade40e2ca..d59f3d1e7 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.3.4" +__version__ = "0.3.5" diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index a531e5bc0..aa77bf643 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -294,7 +294,8 @@ def create_import(self, request: Request, **_kwargs) -> Response: @action(detail=True, url_path="tags/import", methods=["put"]) def update_import(self, request: Request, **_kwargs) -> Response: """ - Imports tags from the uploaded file to an already created taxonomy. + Imports tags from the uploaded file to an already created taxonomy, + overwriting any existing tags. """ body = TaxonomyImportBodySerializer(data=request.data) body.is_valid(raise_exception=True) @@ -304,7 +305,7 @@ def update_import(self, request: Request, **_kwargs) -> Response: taxonomy = self.get_object() try: - import_success = import_tags(taxonomy, file, parser_format) + import_success = import_tags(taxonomy, file, parser_format, replace=True) if import_success: serializer = self.get_serializer(taxonomy) diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 2dd3593d3..882de3dfa 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -2230,10 +2230,9 @@ def test_import(self, file_format: str) -> None: url = TAXONOMY_TAGS_URL.format(pk=self.taxonomy.id) response = self.client.get(url) tags = response.data["results"] - all_tags = [{"value": tag.value} for tag in self.old_tags] + new_tags - assert len(tags) == len(all_tags) + assert len(tags) == len(new_tags) for i, tag in enumerate(tags): - assert tag["value"] == all_tags[i]["value"] + assert tag["value"] == new_tags[i]["value"] def test_import_no_file(self) -> None: """ From 30260cb732b39ff065a61e4e0f88d7382464aa43 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Tue, 21 Nov 2023 09:35:47 -0800 Subject: [PATCH 085/282] fix: type hints and error where TypeError during import broke imports (#122) --- openedx_tagging/core/tagging/import_export/api.py | 7 +++---- openedx_tagging/core/tagging/import_export/parsers.py | 11 ++++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/openedx_tagging/core/tagging/import_export/api.py b/openedx_tagging/core/tagging/import_export/api.py index 6afa9096a..bdc0992ac 100644 --- a/openedx_tagging/core/tagging/import_export/api.py +++ b/openedx_tagging/core/tagging/import_export/api.py @@ -44,19 +44,18 @@ """ from __future__ import annotations -from io import BytesIO +from typing import BinaryIO from django.utils.translation import gettext as _ from ..models import TagImportTask, TagImportTaskState, Taxonomy -from .exceptions import TagImportError from .import_plan import TagImportPlan, TagImportTask from .parsers import ParserFormat, get_parser def import_tags( taxonomy: Taxonomy, - file: BytesIO, + file: BinaryIO, parser_format: ParserFormat, replace=False, ) -> bool: @@ -116,7 +115,7 @@ def import_tags( tag_import_plan.execute(task) task.end_success() return True - except (TagImportError, ValueError) as exception: + except Exception as exception: # pylint: disable=broad-exception-caught # Log any exception task.log_exception(exception) return False diff --git a/openedx_tagging/core/tagging/import_export/parsers.py b/openedx_tagging/core/tagging/import_export/parsers.py index 90268ff6a..441190bc9 100644 --- a/openedx_tagging/core/tagging/import_export/parsers.py +++ b/openedx_tagging/core/tagging/import_export/parsers.py @@ -6,7 +6,8 @@ import csv import json from enum import Enum -from io import BytesIO, StringIO, TextIOWrapper +from io import StringIO, TextIOWrapper +from typing import BinaryIO from django.utils.translation import gettext as _ @@ -51,7 +52,7 @@ class Parser: inital_row = 1 @classmethod - def parse_import(cls, file: BytesIO) -> tuple[list[TagItem], list[TagParserError]]: + def parse_import(cls, file: BinaryIO) -> tuple[list[TagItem], list[TagParserError]]: """ Parse tags in file an returns tags ready for use in TagImportPlan @@ -79,7 +80,7 @@ def export(cls, taxonomy: Taxonomy) -> str: return cls._export_data(tags, taxonomy) @classmethod - def _load_data(cls, file: BytesIO) -> tuple[list[dict], list[TagParserError]]: + def _load_data(cls, file: BinaryIO) -> tuple[list[dict], list[TagParserError]]: """ Each parser implements this function according to its format. This function reads the file and returns a list with the values of each tag. @@ -206,7 +207,7 @@ class JSONParser(Parser): inital_row = 0 @classmethod - def _load_data(cls, file: BytesIO) -> tuple[list[dict], list[TagParserError]]: + def _load_data(cls, file: BinaryIO) -> tuple[list[dict], list[TagParserError]]: """ Read a .json file and validates the root structure of the json """ @@ -259,7 +260,7 @@ class CSVParser(Parser): inital_row = 2 @classmethod - def _load_data(cls, file: BytesIO) -> tuple[list[dict], list[TagParserError]]: + def _load_data(cls, file: BinaryIO) -> tuple[list[dict], list[TagParserError]]: """ Read a .csv file and validates the header fields """ From b98f091fda1f0047bf620b028fc086c1fa44377e Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 23 Nov 2023 09:49:15 -0800 Subject: [PATCH 086/282] fix: Fix bugs in the "get tags" REST API, make it more consistent (#119) * fix: Fix bugs in the "get tags" REST API, make it more consistent * fix: shallow search was not returning parent tags with children that match * docs: update the ADR that describes the "get tags" endpoint * fix: don't throw an error when retrieving tags below a leaf tag * feat: return the number of matching children only with ?search_term=... * chore: Version bump --- .../0014-single-taxonomy-view-api.rst | 115 +++------- openedx_learning/__init__.py | 2 +- openedx_tagging/core/tagging/models/base.py | 4 +- .../core/tagging/rest_api/paginators.py | 9 +- .../core/tagging/rest_api/v1/serializers.py | 10 +- .../core/tagging/rest_api/v1/views.py | 76 ++++--- .../openedx_tagging/core/tagging/test_api.py | 34 +-- .../core/tagging/test_models.py | 26 ++- .../core/tagging/test_views.py | 214 ++++++++++++++---- 9 files changed, 305 insertions(+), 185 deletions(-) diff --git a/docs/decisions/0014-single-taxonomy-view-api.rst b/docs/decisions/0014-single-taxonomy-view-api.rst index ddd0de26b..c984aa0f6 100644 --- a/docs/decisions/0014-single-taxonomy-view-api.rst +++ b/docs/decisions/0014-single-taxonomy-view-api.rst @@ -4,10 +4,7 @@ Context -------- -This view returns tags of a closed taxonomy (for MVP has not been implemented yet -for open taxonomies). It is necessary to make a decision about what structure the tags are going -to have, how the pagination is going to work and how will the search for tags be implemented. -It was taken into account that taxonomies commonly have the following characteristics: +This view returns tags of a taxonomy (works with closed, open, and system-defined). It is necessary to make a decision about what structure the tags are going to have, how the pagination is going to work and how will the search for tags be implemented. It was taken into account that taxonomies commonly have the following characteristics: - It has few root tags. - It may a very large number of children for each tag. @@ -17,16 +14,18 @@ For the decisions, the following use cases were taken into account: - As a taxonomy administrator, I want to see all the tags available for use with a closed taxonomy, so that I can see the taxonomy's structure in the management interface. - - As a taxonomy administrator, I want to see the available tags as a lits of root tags - that can be expanded to show children tags. - - As a taxonomy administrator, I want to sort the list of root tags alphabetically: A-Z (default) and Z-A. - - As a taxonomy administrator, I want to expand all root tags to see all children tags. - - As a taxonomy administrator, I want to search for tags, so I can find root and children tags more easily. + + - As a taxonomy administrator, I want to see the available tags as a list of root tags + that can be expanded to show children tags. + - As a taxonomy administrator, I want to sort the list of root tags alphabetically: A-Z (default) and Z-A. + - As a taxonomy administrator, I want to expand all root tags to see all children tags. + - As a taxonomy administrator, I want to search for tags, so I can find root and children tags more easily. - As a course author, when I am editing the tags of a component, I want to see all the tags available from a particular taxonomy that I can use. - - As a course author, I want to see the available tags as a lits of root tags - that can be expanded to show children tags. - - As a course author, I want to search for tags, so I can find root and children tags more easily. + + - As a course author, I want to see the available tags as a list of root tags + that can be expanded to show children tags. + - As a course author, I want to search for tags, so I can find root and children tags more easily. Excluded use cases: @@ -41,107 +40,59 @@ Decision Views & Pagination ~~~~~~~~~~~~~~~~~~~ -Make one view: +We will have one REST API endpoint that can handle these use cases: -**get_matching_tags(parent_tag_id: str = None, search_term: str = None)** +**/tagging/rest_api/v1/taxonomies/:id/tags/?parent_tag=...** that can handle this cases: -- Get the root tags of the taxonomy. If ``parent_tag_id`` is ``None``. +- Get the root tags of the taxonomy. If ``parent_tag`` is omitted. - Get the children of a tag. Called each time the user expands a parent tag to see its children. - If ``parent_tag_id`` is not ``None``. + In this case, ``parent_tag`` is set to the value of the parent tag. In both cases the results are paginated. In addition to the common pagination metadata, it is necessary to return: - Total number of pages. -- Total number of root/children tags. +- Total number of tags in the result. - Range index of current page, Ex. Page 1: 1-12, Page 2: 13-24. - Total number of children of each tag. The pagination of root tags and child tags are independent. -In order to be able to fulfill the functionality of "Expand-all" in a scalable way, -the following has been agreed: -- Create a ``TAGS_THRESHOLD`` (default: 1000). -- If ``taxonomy.tags.count < TAGS_THRESHOLD``, then ``get_matching_tags()`` will return all tags on the taxonomy, - roots and children. -- Otherwise, ``get_matching_tags()`` will only return paginated root tags, and it will be necessary - to use ``get_matching_tags()`` to return paginated children. Also the "Expand-all" functionality will be disabled. +**Optional full-depth response** -For search you can see the next section (Search tags) +In order to be able to fulfill the functionality of "Expand-all" in a scalable way, and allow users to quickly browse taxonomies that have lots of small branches, the API will accept an optional parameter ``?full_depth_threshold``. If specified (e.g. ``?full_depth_threshold=1000``) and there are fewer results than this threshold, the full tree of tags will be returned a a single giant page, including all tags up to three levels deep. **Pros** -- It is the simplest way. +- This approach is simple and flexible. - Paging both root tags and children mitigates the huge number of tags that can be found in large taxonomies. Search tags ~~~~~~~~~~~~ -Support tag search on the backend. Return a subset of matching tags. -We will use the same view to perform a search with the same logic: - -**get_matching_tags(parent_tag_id: str = None, search_term: str = None)** - -We can use ``search_term`` to perform a search on all taxonomy tags or children tags depending of ``parent_tag_id``. -The result will be a pruned tree with the necessary tags to be able to reach the results from a root tag. -Ex. if in the result there may be a child tag of ``depth=2``, but the parents are not found in the result. -In this case, it is necessary to add the parent and the parent of the parent (root tag) to be able to show -the child tag that is in the result. - -For the search, ``SEARCH_TAGS_THRESHOLD`` will be used. (It is recommended that it be 20% of ``TAGS_THRESHOLD``). -It will work in the following way: - -- If ``search_result.count() < SEARCH_TAGS_THRESHOLD``, then it will return all tags on the result tree without pagination. -- Otherwise, it will return the roots of the result tree with pagination. Each root will have the entire pruned branch. +The same API endpoint will support an optional ``?search_term=...`` parameter to support searching/filtering tags by keyword. -It will work in the same way of ``TAGS_THRESHOLD`` (see Views & Pagination) - -**Pros** - -- It is the most scalable way. +The API endpoint will work exactly as described above (returning a single level of tags by default, paginated, optionally listing only the tags below a specific parent tag, optionally returning a deep tree if the results are small enough) - BUT only tags that match the keyword OR their ancestor tags will be returned. We return their ancestor tags (even if the ancestors themselves don't match the keyword) so that the tree of tags that do match can be displayed correctly. This also allows the UI to load the matching tags one layer at a time, paginated, if desired. Tag representation ~~~~~~~~~~~~~~~~~~~ -Return a list of root tags and within a link to obtain the children tags -or the complete list of children tags depending of ``TAGS_THRESHOLD`` or ``SEARCH_TAGS_THRESHOLD``. -The list of root tags will be ordered alphabetically. If it has child tags, they must also -be ordered alphabetically. - -**(taxonomy.tags.count < *_THRESHOLD)**:: - - { - "count": 100, - "tags": [ - { - "id": "tag_1", - "value": "Tag 1", - "taxonomy_id": "1", - "sub_tags": [ - { - "id": "tag_2", - "value": "Tag 2", - "taxonomy_id": "1", - "sub_tags": [ - (....) - ] - }, - (....) - ] - } +Return a list of root tags and within a link to obtain the children tags or the complete list of children tags depending on the requested ``?full_depth_threshold`` and the number of results. +The list of tags will be ordered in tree order (and alphabetically). If it has child tags, they must also be ordered alphabetically. - -**Otherwise**:: +**Example**:: { "count": 100, "tags": [ { - "id": "tag_1", "value": "Tag 1", - "taxonomy_id": "1", - "sub_tags_link": "http//api-call-to-get-children.com" + "depth": 0, + "external_id": None, + "child_count": 15, + "parent_value": None, + "sub_tags_url": "http//api-call-to-get-children.com" }, (....) ] @@ -155,6 +106,12 @@ be ordered alphabetically. - It is kept as a simple implementation. +Backend Python API +~~~~~~~~~~~~~~~~~~ + +On the backend, a very flexible API is available as ``Taxonomy.get_filtered_tags()`` which can cover all of the same use cases as the REST API endpoint (listing tags, shallow or deep, filtering on search term). However, the Python API returns a ``QuerySet`` of tag data dictionaries, rather than a JSON response. + + Rejected Options ----------------- @@ -199,4 +156,4 @@ can return all the tags in one page. So we can perform the tag search on the fro **Cons:** - It is not scalable. -- Sets limits of tags that can be created in the taxonomy. \ No newline at end of file +- Sets limits of tags that can be created in the taxonomy. diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index d59f3d1e7..d458a9e22 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.3.5" +__version__ = "0.3.6" diff --git a/openedx_tagging/core/tagging/models/base.py b/openedx_tagging/core/tagging/models/base.py index f79bb2522..981cffc36 100644 --- a/openedx_tagging/core/tagging/models/base.py +++ b/openedx_tagging/core/tagging/models/base.py @@ -493,13 +493,15 @@ def _get_filtered_tags_deep( if pk is not None: matching_ids.append(pk) qs = qs.filter(pk__in=matching_ids) + qs = qs.annotate(child_count=models.Count("children", filter=Q(children__pk__in=matching_ids))) elif excluded_values: raise NotImplementedError("Using excluded_values without search_term is not currently supported.") # We could implement this in the future but I'd prefer to get rid of the "excluded_values" API altogether. # It remains to be seen if it's useful to do that on the backend, or if we can do it better/simpler on the # frontend. + else: + qs = qs.annotate(child_count=models.Count("children")) - qs = qs.annotate(child_count=models.Count("children")) # Add the "depth" to each tag: qs = Tag.annotate_depth(qs) # Add the "lineage" as a field called "sort_key" to sort them in order correctly: diff --git a/openedx_tagging/core/tagging/rest_api/paginators.py b/openedx_tagging/core/tagging/rest_api/paginators.py index 8848e5374..2ebce12db 100644 --- a/openedx_tagging/core/tagging/rest_api/paginators.py +++ b/openedx_tagging/core/tagging/rest_api/paginators.py @@ -5,10 +5,7 @@ from edx_rest_framework_extensions.paginators import DefaultPagination # type: ignore[import] # From this point, the tags begin to be paginated -TAGS_THRESHOLD = 1000 - -# From this point, search tags begin to be paginated -SEARCH_TAGS_THRESHOLD = 200 +MAX_FULL_DEPTH_THRESHOLD = 10_000 class TagsPagination(DefaultPagination): @@ -29,5 +26,5 @@ class DisabledTagsPagination(DefaultPagination): It should be used if the number of tags within the taxonomy does not exceed `TAGS_THRESHOLD`. """ - page_size = TAGS_THRESHOLD - max_page_size = TAGS_THRESHOLD + 1 + page_size = MAX_FULL_DEPTH_THRESHOLD + max_page_size = MAX_FULL_DEPTH_THRESHOLD + 1 diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index 62d4ef3ed..ca95c91cb 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -4,6 +4,7 @@ from __future__ import annotations from typing import Any +from urllib.parse import urlencode from rest_framework import serializers from rest_framework.reverse import reverse @@ -162,12 +163,17 @@ def get_sub_tags_url(self, obj: TagData | Tag): child_count = obj.child_count if isinstance(obj, Tag) else obj["child_count"] if child_count > 0 and "taxonomy_id" in self.context: value = obj.value if isinstance(obj, Tag) else obj["value"] - query_params = f"?parent_tag={value}" request = self.context["request"] + query_params = request.query_params + new_query_params = {"parent_tag": value} + if "full_depth_threshold" in query_params: + new_query_params["full_depth_threshold"] = query_params["full_depth_threshold"] + if "search_term" in query_params: + new_query_params["search_term"] = query_params["search_term"] url_namespace = request.resolver_match.namespace # get the namespace, usually "oel_tagging" url = ( reverse(f"{url_namespace}:taxonomy-tags", args=[str(self.context["taxonomy_id"])]) - + query_params + + "?" + urlencode(new_query_params) ) return request.build_absolute_uri(url) return None diff --git a/openedx_tagging/core/tagging/rest_api/v1/views.py b/openedx_tagging/core/tagging/rest_api/v1/views.py index aa77bf643..591e95e15 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/views.py +++ b/openedx_tagging/core/tagging/rest_api/v1/views.py @@ -30,7 +30,7 @@ from ...import_export.parsers import ParserFormat from ...models import Taxonomy from ...rules import ObjectTagPermissionItem -from ..paginators import TAGS_THRESHOLD, DisabledTagsPagination, TagsPagination +from ..paginators import MAX_FULL_DEPTH_THRESHOLD, DisabledTagsPagination, TagsPagination from .permissions import ObjectTagObjectPermissions, TagObjectPermissions, TaxonomyObjectPermissions from .serializers import ( ObjectTagListQueryParamsSerializer, @@ -528,24 +528,28 @@ class TaxonomyTagsView(ListAPIView, RetrieveUpdateDestroyAPIView): """ View to list/create/update/delete tags of a taxonomy. - If you specify ?root_only or ?parent_tag_value=..., only one "level" of the - hierachy will be returned. Otherwise, several levels will be returned, in - tree order, up to the maximum supported depth. Additional levels/depth can - be retrieved by using ?parent_tag_value to load more data. + **List Request Details** - Note: If the taxonomy is particularly large (> 1,000 tags), ?root_only is - automatically set true by default and cannot be disabled. This way, users - can more easily select which tags they want to expand in the tree, and load - just that subset of the tree as needed. This may be changed in the future. + By default, only one "level" of the hierachy will be returned. But for more efficient handling of small taxonomies, + you can set ?full_depth_threshold=1000 which means that if there are fewer than 1,000 tags in the result set, they + will be returned in a single page of results that has all the tags in tree order up to the maximum supported depth. + + Additional levels/depth can be retrieved recursively by using the "sub_tags_url" field of any returned tag, or + using the ?parent_tag=value parameter manually. **List Query Parameters** * id (required) - The ID of the taxonomy to retrieve tags. + * search_term (optional) - Only return tags matching this term, plus their ancestors. Case insensitive. * parent_tag (optional) - Retrieve children of the tag with this value. - * root_only (optional) - If specified, only root tags are returned. - * include_counts (optional) - Include the count of how many times each - tag has been used. + * full_depth_threshold (optional) - If there are fewer than this many results, return the full (sub)tree of + results, up to maximum supported depth, in a single giant page of results. By default this is disabled + (equivalent to full_depth_threshold=0), which provides a more consistent and predictable response. + Using full_depth_threshold=1000 is recommended in general, but use lower values during development to ensure + compatibility with both large and small result sizes. + * include_counts (optional) - Include the count of how many times each tag has been used. * page (optional) - Page number (default: 1) - * page_size (optional) - Number of items per page (default: 10) + * page_size (optional) - Number of items per page (default: 30). Ignored when there are fewer tags than + specified by ?full_depth_threshold. **List Example Requests** GET api/tagging/v1/taxonomy/:id/tags - Get tags of taxonomy @@ -655,34 +659,42 @@ def get_queryset(self) -> TagDataQuerySet: taxonomy_id = int(self.kwargs.get("pk")) taxonomy = self.get_taxonomy(taxonomy_id) parent_tag_value = self.request.query_params.get("parent_tag", None) - root_only = "root_only" in self.request.query_params include_counts = "include_counts" in self.request.query_params search_term = self.request.query_params.get("search_term", None) - - if parent_tag_value: - # Fetching tags below a certain parent is always paginated and only returns the direct children - depth = 1 - if root_only: - raise ValidationError("?root_only and ?parent_tag cannot be used together") + try: + full_depth_threshold = int(self.request.query_params.get("full_depth_threshold", 0)) + except ValueError: + full_depth_threshold = -1 + if full_depth_threshold < 0 or full_depth_threshold > MAX_FULL_DEPTH_THRESHOLD: + raise ValidationError("Invalid full_depth_threshold") + + if full_depth_threshold or search_term: + # If full_depth_threshold is set, default to maximum depth and later prune to depth=1 if needed. + # We also need to do a deep search if there is a search term, to ensure we return parent items with children + # that match. + depth = None else: - if root_only: - depth = 1 # User Explicitly requested to load only the root tags for now - elif search_term: - depth = None # For search, default to maximum depth but use normal pagination - elif taxonomy.tag_set.count() > TAGS_THRESHOLD: - # This is a very large taxonomy. Only load the root tags at first, so users can choose what to load. - depth = 1 - else: - # We can load and display all the tags in the taxonomy at once: - self.pagination_class = DisabledTagsPagination - depth = None # Maximum depth + depth = 1 # Otherwise (default), load a single level of results only. - return taxonomy.get_filtered_tags( + results = taxonomy.get_filtered_tags( parent_tag_value=parent_tag_value, search_term=search_term, depth=depth, include_counts=include_counts, ) + if depth == 1: + # We're already returning just a single level. It will be paginated normally. + return results + elif full_depth_threshold and len(results) < full_depth_threshold: + # We can load and display all the tags in this (sub)tree at once: + self.pagination_class = DisabledTagsPagination + return results + else: + # We had to do a deep query, but we will only return one level of results. + # This is because the user did not request a deep response (via full_depth_threshold) or the result was too + # large (larger than the threshold). + # It will be paginated normally. + return results.filter(parent_value=parent_tag_value) def post(self, request, *args, **kwargs): """ diff --git a/tests/openedx_tagging/core/tagging/test_api.py b/tests/openedx_tagging/core/tagging/test_api.py index 4173c5a1a..5ad7e3fa4 100644 --- a/tests/openedx_tagging/core/tagging/test_api.py +++ b/tests/openedx_tagging/core/tagging/test_api.py @@ -193,11 +193,11 @@ def test_get_root_language_tags(self): def test_search_tags(self) -> None: result = tagging_api.search_tags(self.taxonomy, search_term='eU') assert pretty_format_tags(result, parent=False) == [ - 'Archaea (children: 3)', # Doesn't match 'eU' but is included because a child is included + 'Archaea (children: 1)', # Doesn't match 'eU' but is included because a child is included ' Euryarchaeida (children: 0)', - 'Bacteria (children: 2)', # Doesn't match 'eU' but is included because a child is included + 'Bacteria (children: 1)', # Doesn't match 'eU' but is included because a child is included ' Eubacteria (children: 0)', - 'Eukaryota (children: 5)', + 'Eukaryota (children: 0)', ] @override_settings(LANGUAGES=test_languages) @@ -603,30 +603,30 @@ def get_object_tags(): @ddt.data( ("ChA", [ - "Archaea (used: 1, children: 3)", + "Archaea (used: 1, children: 2)", " Euryarchaeida (used: 0, children: 0)", " Proteoarchaeota (used: 0, children: 0)", - "Bacteria (used: 0, children: 2)", # does not contain "cha" but a child does + "Bacteria (used: 0, children: 1)", # does not contain "cha" but a child does " Archaebacteria (used: 1, children: 0)", ]), ("ar", [ - "Archaea (used: 1, children: 3)", + "Archaea (used: 1, children: 2)", " Euryarchaeida (used: 0, children: 0)", " Proteoarchaeota (used: 0, children: 0)", - "Bacteria (used: 0, children: 2)", # does not contain "ar" but a child does + "Bacteria (used: 0, children: 1)", # does not contain "ar" but a child does " Archaebacteria (used: 1, children: 0)", - "Eukaryota (used: 0, children: 5)", - " Animalia (used: 1, children: 7)", # does not contain "ar" but a child does + "Eukaryota (used: 0, children: 1)", + " Animalia (used: 1, children: 2)", # does not contain "ar" but a child does " Arthropoda (used: 1, children: 0)", " Cnidaria (used: 0, children: 0)", ]), ("aE", [ - "Archaea (used: 1, children: 3)", + "Archaea (used: 1, children: 2)", " Euryarchaeida (used: 0, children: 0)", " Proteoarchaeota (used: 0, children: 0)", - "Bacteria (used: 0, children: 2)", # does not contain "ae" but a child does + "Bacteria (used: 0, children: 1)", # does not contain "ae" but a child does " Archaebacteria (used: 1, children: 0)", - "Eukaryota (used: 0, children: 5)", # does not contain "ae" but a child does + "Eukaryota (used: 0, children: 1)", # does not contain "ae" but a child does " Plantae (used: 1, children: 0)", ]), ("a", [ @@ -637,11 +637,11 @@ def get_object_tags(): "Bacteria (used: 0, children: 2)", " Archaebacteria (used: 1, children: 0)", " Eubacteria (used: 0, children: 0)", - "Eukaryota (used: 0, children: 5)", + "Eukaryota (used: 0, children: 4)", " Animalia (used: 1, children: 7)", " Arthropoda (used: 1, children: 0)", - " Chordata (used: 0, children: 1)", - " Cnidaria (used: 0, children: 0)", + " Chordata (used: 0, children: 0)", # <<< Chordata has a matching child but we only support searching + " Cnidaria (used: 0, children: 0)", # 3 levels deep at once for now. " Ctenophora (used: 0, children: 0)", " Gastrotrich (used: 1, children: 0)", " Placozoa (used: 1, children: 0)", @@ -678,11 +678,11 @@ def test_autocomplete_tags_closed_omit_object(self) -> None: tagging_api.tag_object(object_id=object_id, taxonomy=self.taxonomy, tags=["Archaebacteria"]) result = tagging_api.search_tags(self.taxonomy, "ChA", exclude_object_id=object_id) assert pretty_format_tags(result, parent=False) == [ - "Archaea (children: 3)", + "Archaea (children: 2)", " Euryarchaeida (children: 0)", " Proteoarchaeota (children: 0)", # These results are no longer included because of exclude_object_id: - # "Bacteria (children: 2)", # does not contain "cha" but a child does + # "Bacteria (children: 1)", # does not contain "cha" but a child does # " Archaebacteria (children: 0)", ] diff --git a/tests/openedx_tagging/core/tagging/test_models.py b/tests/openedx_tagging/core/tagging/test_models.py index 367b735b3..c730b16bc 100644 --- a/tests/openedx_tagging/core/tagging/test_models.py +++ b/tests/openedx_tagging/core/tagging/test_models.py @@ -417,10 +417,10 @@ def test_search(self) -> None: """ result = pretty_format_tags(self.taxonomy.get_filtered_tags(search_term="ARCH")) assert result == [ - "Archaea (None) (children: 3)", # Matches the value of this root tag, ARCHaea + "Archaea (None) (children: 2)", # Matches the value of this root tag, ARCHaea " Euryarchaeida (Archaea) (children: 0)", # Matches the value of this child tag " Proteoarchaeota (Archaea) (children: 0)", # Matches the value of this child tag - "Bacteria (None) (children: 2)", # Does not match this tag but matches a descendant: + "Bacteria (None) (children: 1)", # Does not match this tag but matches a descendant: " Archaebacteria (Bacteria) (children: 0)", # Matches the value of this child tag ] @@ -431,9 +431,25 @@ def test_search_2(self) -> None: """ result = pretty_format_tags(self.taxonomy.get_filtered_tags(search_term="chordata")) assert result == [ - "Eukaryota (None) (children: 5)", - " Animalia (Eukaryota) (children: 7)", - " Chordata (Animalia) (children: 1)", # this is the matching tag. + "Eukaryota (None) (children: 1)", # Has one child that matches + " Animalia (Eukaryota) (children: 1)", + " Chordata (Animalia) (children: 0)", # this is the matching tag. + ] + + def test_search_3(self) -> None: + """ + Another search test, that matches a tag deeper in the taxonomy to check + that the correct child_count is returned by the search. + """ + result = pretty_format_tags(self.taxonomy.get_filtered_tags(search_term="RO")) + assert result == [ + "Archaea (None) (children: 1)", + " Proteoarchaeota (Archaea) (children: 0)", + "Eukaryota (None) (children: 2)", # Note the "children: 2" is correct - 2 direct children are in the result + " Animalia (Eukaryota) (children: 2)", + " Arthropoda (Animalia) (children: 0)", # match + " Gastrotrich (Animalia) (children: 0)", # match + " Protista (Eukaryota) (children: 0)", # match ] def test_tags_deep(self) -> None: diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index 882de3dfa..f3f88208f 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -4,7 +4,7 @@ from __future__ import annotations import json -from urllib.parse import parse_qs, quote, quote_plus, urlparse +from urllib.parse import parse_qs, quote_plus, urlparse import ddt # type: ignore[import] # typing support in rules depends on https://github.com/dfunckt/django-rules/pull/177 @@ -1105,7 +1105,7 @@ def test_small_taxonomy_root(self): Test explicitly requesting only the root tags of a small taxonomy. """ self.client.force_authenticate(user=self.staff) - response = self.client.get(self.small_taxonomy_url + "?root_only&include_counts") + response = self.client.get(self.small_taxonomy_url + "?include_counts") assert response.status_code == status.HTTP_200_OK data = response.data @@ -1142,7 +1142,7 @@ def test_small_taxonomy(self): Test loading all the tags of a small taxonomy at once. """ self.client.force_authenticate(user=self.staff) - response = self.client.get(self.small_taxonomy_url) + response = self.client.get(self.small_taxonomy_url + "?full_depth_threshold=1000") assert response.status_code == status.HTTP_200_OK data = response.data @@ -1182,22 +1182,20 @@ def test_small_taxonomy_paged(self): Test loading only the first few of the tags of a small taxonomy. """ self.client.force_authenticate(user=self.staff) - response = self.client.get(self.small_taxonomy_url + "?page_size=5") + response = self.client.get(self.small_taxonomy_url + "?page_size=2") assert response.status_code == status.HTTP_200_OK data = response.data + # When pagination is active, we only load a single "layer" at a time: assert pretty_format_tags(data["results"]) == [ "Archaea (None) (children: 3)", - " DPANN (Archaea) (children: 0)", - " Euryarchaeida (Archaea) (children: 0)", - " Proteoarchaeota (Archaea) (children: 0)", "Bacteria (None) (children: 2)", ] # Checking pagination values assert data.get("next") is not None assert data.get("previous") is None - assert data.get("count") == 20 - assert data.get("num_pages") == 4 + assert data.get("count") == 3 + assert data.get("num_pages") == 2 assert data.get("current_page") == 1 # Get the next page: @@ -1205,11 +1203,7 @@ def test_small_taxonomy_paged(self): assert next_response.status_code == status.HTTP_200_OK next_data = next_response.data assert pretty_format_tags(next_data["results"]) == [ - " Archaebacteria (Bacteria) (children: 0)", - " Eubacteria (Bacteria) (children: 0)", "Eukaryota (None) (children: 5)", - " Animalia (Eukaryota) (children: 7)", - " Arthropoda (Animalia) (children: 0)", ] assert next_data.get("current_page") == 2 @@ -1218,18 +1212,18 @@ def test_small_search(self): Test performing a search """ search_term = 'eU' - url = f"{self.small_taxonomy_url}?search_term={search_term}" + url = f"{self.small_taxonomy_url}?search_term={search_term}&full_depth_threshold=100" self.client.force_authenticate(user=self.staff) response = self.client.get(url) assert response.status_code == status.HTTP_200_OK data = response.data assert pretty_format_tags(data["results"], parent=False) == [ - "Archaea (children: 3)", # No match in this tag, but a child matches so it's included + "Archaea (children: 1)", # No match in this tag, but a child matches so it's included " Euryarchaeida (children: 0)", - "Bacteria (children: 2)", # No match in this tag, but a child matches so it's included + "Bacteria (children: 1)", # No match in this tag, but a child matches so it's included " Eubacteria (children: 0)", - "Eukaryota (children: 5)", + "Eukaryota (children: 0)", ] # Checking pagination values @@ -1239,6 +1233,61 @@ def test_small_search(self): assert data.get("num_pages") == 1 assert data.get("current_page") == 1 + def test_small_search_shallow(self): + """ + Test performing a search without full_depth_threshold + """ + search_term = 'eU' + url = f"{self.small_taxonomy_url}?search_term={search_term}" + self.client.force_authenticate(user=self.staff) + response = self.client.get(url) + assert response.status_code == status.HTTP_200_OK + + data = response.data + assert pretty_format_tags(data["results"], parent=False) == [ + "Archaea (children: 1)", # No match in this tag, but a child matches so it's included + "Bacteria (children: 1)", # No match in this tag, but a child matches so it's included + "Eukaryota (children: 0)", + ] + + # Checking pagination values + assert data.get("next") is None + assert data.get("previous") is None + assert data.get("count") == 3 + assert data.get("num_pages") == 1 + assert data.get("current_page") == 1 + + # And we can load the sub_tags_url to "drill down" into the search: + sub_tags_response = self.client.get(data["results"][0]["sub_tags_url"]) + assert sub_tags_response.status_code == status.HTTP_200_OK + + assert pretty_format_tags(sub_tags_response.data["results"], parent=False) == [ + # This tag maches our search results and is a child of the previously returned Archaea tag: + " Euryarchaeida (children: 0)", + ] + + def test_empty_results(self): + """ + Test that various queries return an empty list + """ + self.client.force_authenticate(user=self.staff) + + def assert_empty(url): + response = self.client.get(url) + assert response.status_code == status.HTTP_200_OK + assert response.data["results"] == [] + assert response.data["count"] == 0 + + # Search terms that won't match any tags: + assert_empty(f"{self.small_taxonomy_url}?search_term=foobar") + assert_empty(f"{self.small_taxonomy_url}?search_term=foobar&full_depth_threshold=1000") + # Requesting children of leaf tags is always an empty result. + # Prior versions of the code would sometimes throw an exception when trying to handle these. + assert_empty(f"{self.small_taxonomy_url}?parent_tag=Fungi") + assert_empty(f"{self.small_taxonomy_url}?parent_tag=Fungi&full_depth_threshold=1000") + assert_empty(f"{self.small_taxonomy_url}?search_term=eu&parent_tag=Euryarchaeida") + assert_empty(f"{self.small_taxonomy_url}?search_term=eu&parent_tag=Euryarchaeida&full_depth_threshold=1000") + def test_large_taxonomy(self): """ Test listing the tags in a large taxonomy (~7,000 tags). @@ -1251,8 +1300,6 @@ def test_large_taxonomy(self): data = response.data results = data["results"] - # Even though we didn't specify root_only, only the root tags will have - # been returned, because of the taxonomy's size. assert pretty_format_tags(results) == [ "Tag 0 (None) (used: 0, children: 12)", "Tag 1099 (None) (used: 0, children: 12)", @@ -1276,7 +1323,7 @@ def test_large_taxonomy(self): assert results[0].get("sub_tags_url") == ( "http://testserver/tagging/" f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?parent_tag={quote(results[0]['value'])}" + f"/tags/?parent_tag={quote_plus(results[0]['value'])}" ) # Checking pagination values @@ -1320,44 +1367,91 @@ def test_large_search(self): Test searching in a large taxonomy """ self._build_large_taxonomy() - search_term = '11' - url = f"{self.large_taxonomy_url}?search_term={search_term}" self.client.force_authenticate(user=self.staff) - response = self.client.get(url) + + # Perform the search with full_depth_threshold=1000, which will give us the full tree of results, since + # there are less than 1000 matches: + search_term = '11' + response = self.client.get(f"{self.large_taxonomy_url}?search_term={search_term}&full_depth_threshold=1000") assert response.status_code == status.HTTP_200_OK data = response.data results = data["results"] - assert pretty_format_tags(results) == [ - "Tag 0 (None) (children: 12)", # First 2 results don't match but have children that match - " Tag 1 (Tag 0) (children: 12)", + assert pretty_format_tags(results)[:20] == [ + "Tag 0 (None) (children: 3)", # First 2 results don't match but have children that match + # Note the count here ---^ is not the total number of matching descendants, just the number of children + # once we filter the tree to include only matches and their ancestors. + " Tag 1 (Tag 0) (children: 1)", " Tag 11 (Tag 1) (children: 0)", - " Tag 105 (Tag 0) (children: 12)", # Non-match but children match + " Tag 105 (Tag 0) (children: 8)", # Non-match but children match " Tag 110 (Tag 105) (children: 0)", " Tag 111 (Tag 105) (children: 0)", " Tag 112 (Tag 105) (children: 0)", " Tag 113 (Tag 105) (children: 0)", " Tag 114 (Tag 105) (children: 0)", " Tag 115 (Tag 105) (children: 0)", - ] - assert data.get("count") == 362 - assert data.get("num_pages") == 37 - assert data.get("current_page") == 1 - # Get the next page: - next_response = self.client.get(response.data.get("next")) - assert next_response.status_code == status.HTTP_200_OK - assert pretty_format_tags(next_response.data["results"]) == [ " Tag 116 (Tag 105) (children: 0)", " Tag 117 (Tag 105) (children: 0)", - " Tag 118 (Tag 0) (children: 12)", + " Tag 118 (Tag 0) (children: 1)", " Tag 119 (Tag 118) (children: 0)", - "Tag 1099 (None) (children: 12)", + "Tag 1099 (None) (children: 9)", " Tag 1100 (Tag 1099) (children: 12)", " Tag 1101 (Tag 1100) (children: 0)", " Tag 1102 (Tag 1100) (children: 0)", " Tag 1103 (Tag 1100) (children: 0)", " Tag 1104 (Tag 1100) (children: 0)", ] + expected_num_results = 362 + assert data.get("count") == expected_num_results + assert len(results) == expected_num_results + assert data.get("num_pages") == 1 + assert data.get("current_page") == 1 + + # Now, perform the search with full_depth_threshold=100, which will give us paginated results, since there are + # more than 100 matches: + response2 = self.client.get(f"{self.large_taxonomy_url}?search_term={search_term}&full_depth_threshold=100") + assert response2.status_code == status.HTTP_200_OK + + data2 = response2.data + results2 = data2["results"] + assert pretty_format_tags(results2) == [ + # Notice that none of these root tags directly match the search query, but their children/grandchildren do + "Tag 0 (None) (children: 3)", + "Tag 1099 (None) (children: 9)", + "Tag 1256 (None) (children: 2)", + "Tag 1413 (None) (children: 1)", + "Tag 157 (None) (children: 2)", + "Tag 1570 (None) (children: 2)", + "Tag 1727 (None) (children: 1)", + "Tag 1884 (None) (children: 2)", + "Tag 2041 (None) (children: 1)", + "Tag 2198 (None) (children: 2)", + ] + assert data2.get("count") == 51 + assert data2.get("num_pages") == 6 + assert data2.get("current_page") == 1 + + # Now load the results that are in the subtree of the root tag 'Tag 0' + tag_0_subtags_url = results2[0]["sub_tags_url"] + assert "full_depth_threshold=100" in tag_0_subtags_url + response3 = self.client.get(tag_0_subtags_url) + data3 = response3.data + # Now the number of results is below our threshold (100), so the subtree gets returned as a single page: + assert pretty_format_tags(data3["results"]) == [ + " Tag 1 (Tag 0) (children: 1)", # Non-match but children match + " Tag 11 (Tag 1) (children: 0)", # Matches '11' + " Tag 105 (Tag 0) (children: 8)", # Non-match but children match + " Tag 110 (Tag 105) (children: 0)", # Matches '11' + " Tag 111 (Tag 105) (children: 0)", + " Tag 112 (Tag 105) (children: 0)", + " Tag 113 (Tag 105) (children: 0)", + " Tag 114 (Tag 105) (children: 0)", + " Tag 115 (Tag 105) (children: 0)", + " Tag 116 (Tag 105) (children: 0)", + " Tag 117 (Tag 105) (children: 0)", + " Tag 118 (Tag 0) (children: 1)", + " Tag 119 (Tag 118) (children: 0)", + ] def test_get_children(self): self._build_large_taxonomy() @@ -1385,7 +1479,7 @@ def test_get_children(self): assert results[0].get("sub_tags_url") == ( "http://testserver/tagging/" f"rest_api/v1/taxonomies/{self.large_taxonomy.id}" - f"/tags/?parent_tag={quote(tag.value)}" + f"/tags/?parent_tag={quote_plus(tag.value)}" ) # Checking pagination values @@ -1400,23 +1494,59 @@ def test_get_children(self): assert data.get("current_page") == 1 def test_get_leaves(self): - # Get tags depth=2 + """ + Test getting the tags at depth=2, using "full_depth_threshold=1000" to + load the whole subtree. + """ + # Get tags at depth=2 self.client.force_authenticate(user=self.staff) parent_tag = Tag.objects.get(value="Animalia") # Build url to get tags depth=2 - url = f"{self.small_taxonomy_url}?parent_tag={parent_tag.value}" + url = f"{self.small_taxonomy_url}?parent_tag={parent_tag.value}&full_depth_threshold=1000" response = self.client.get(url) results = response.data["results"] - # Even though we didn't specify root_only, only the root tags will have - # been returned, because of the taxonomy's size. + # Because the result is small, the result includes the complete tree below this one. assert pretty_format_tags(results) == [ " Arthropoda (Animalia) (children: 0)", " Chordata (Animalia) (children: 1)", + " Mammalia (Chordata) (children: 0)", + " Cnidaria (Animalia) (children: 0)", + " Ctenophora (Animalia) (children: 0)", + " Gastrotrich (Animalia) (children: 0)", + " Placozoa (Animalia) (children: 0)", + " Porifera (Animalia) (children: 0)", + ] + assert response.data.get("next") is None + + def test_get_leaves_paginated(self): + """ + Test getting depth=2 entries, disabling the feature to return the whole + subtree if the result is small enough. + """ + # Get tags at depth=2 + self.client.force_authenticate(user=self.staff) + parent_tag = Tag.objects.get(value="Animalia") + + # Build url to get tags depth=2 + url = f"{self.small_taxonomy_url}?parent_tag={parent_tag.value}&page_size=5" + response = self.client.get(url) + results = response.data["results"] + + # Because the result is small, the result includes the complete tree below this one. + assert pretty_format_tags(results) == [ + " Arthropoda (Animalia) (children: 0)", + " Chordata (Animalia) (children: 1)", # Note the child is not included " Cnidaria (Animalia) (children: 0)", " Ctenophora (Animalia) (children: 0)", " Gastrotrich (Animalia) (children: 0)", + ] + next_url = response.data.get("next") + assert next_url is not None + response2 = self.client.get(next_url) + results2 = response2.data["results"] + assert pretty_format_tags(results2) == [ " Placozoa (Animalia) (children: 0)", " Porifera (Animalia) (children: 0)", ] From 9fc1b888cac87ec98ba007a6035bef520c97b414 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 26 Nov 2023 19:21:04 -0500 Subject: [PATCH 087/282] chore: Updating Python Requirements --- requirements/base.txt | 6 +++--- requirements/ci.txt | 2 +- requirements/dev.txt | 16 ++++++++-------- requirements/doc.txt | 12 ++++++------ requirements/pip-tools.txt | 2 +- requirements/quality.txt | 12 ++++++------ requirements/test.txt | 10 +++++----- 7 files changed, 30 insertions(+), 30 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index a3ddc94c7..4681ee3ea 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -16,7 +16,7 @@ backports-zoneinfo[tzdata]==0.2.1 # kombu billiard==4.2.0 # via celery -celery==5.3.5 +celery==5.3.6 # via -r requirements/base.in certifi==2023.11.17 # via requests @@ -71,7 +71,7 @@ edx-drf-extensions==8.13.1 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via edx-drf-extensions -idna==3.4 +idna==3.6 # via requests kombu==5.3.4 # via celery @@ -130,5 +130,5 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.10 +wcwidth==0.2.12 # via prompt-toolkit diff --git a/requirements/ci.txt b/requirements/ci.txt index d2e436159..36407b101 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -40,5 +40,5 @@ typing-extensions==4.8.0 # via # grimp # import-linter -virtualenv==20.24.6 +virtualenv==20.24.7 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 79528123b..5d845e295 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -33,7 +33,7 @@ build==1.0.3 # via # -r requirements/pip-tools.txt # pip-tools -celery==5.3.5 +celery==5.3.6 # via -r requirements/quality.txt certifi==2023.11.17 # via @@ -172,7 +172,7 @@ edx-opaque-keys==2.5.1 # via # -r requirements/quality.txt # edx-drf-extensions -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # -r requirements/quality.txt # pytest @@ -186,7 +186,7 @@ grimp==3.1 # -r requirements/ci.txt # -r requirements/quality.txt # import-linter -idna==3.4 +idna==3.6 # via # -r requirements/quality.txt # requests @@ -259,7 +259,7 @@ more-itertools==10.1.0 # via # -r requirements/quality.txt # jaraco-classes -mypy==1.7.0 +mypy==1.7.1 # via # -r requirements/quality.txt # djangorestframework-stubs @@ -333,7 +333,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.txt -pygments==2.17.1 +pygments==2.17.2 # via # -r requirements/quality.txt # diff-cover @@ -534,15 +534,15 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.24.6 +virtualenv==20.24.7 # via # -r requirements/ci.txt # tox -wcwidth==0.2.10 +wcwidth==0.2.12 # via # -r requirements/quality.txt # prompt-toolkit -wheel==0.41.3 +wheel==0.42.0 # via # -r requirements/pip-tools.txt # pip-tools diff --git a/requirements/doc.txt b/requirements/doc.txt index a8851c10f..64408be21 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -34,7 +34,7 @@ billiard==4.2.0 # via # -r requirements/test.txt # celery -celery==5.3.5 +celery==5.3.6 # via -r requirements/test.txt certifi==2023.11.17 # via @@ -148,7 +148,7 @@ edx-opaque-keys==2.5.1 # via # -r requirements/test.txt # edx-drf-extensions -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest @@ -156,7 +156,7 @@ grimp==3.1 # via # -r requirements/test.txt # import-linter -idna==3.4 +idna==3.6 # via # -r requirements/test.txt # requests @@ -185,7 +185,7 @@ markupsafe==2.1.3 # jinja2 mock==5.1.0 # via -r requirements/test.txt -mypy==1.7.0 +mypy==1.7.1 # via # -r requirements/test.txt # djangorestframework-stubs @@ -231,7 +231,7 @@ pycparser==2.21 # cffi pydata-sphinx-theme==0.14.3 # via sphinx-book-theme -pygments==2.17.1 +pygments==2.17.2 # via # accessible-pygments # doc8 @@ -392,7 +392,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.10 +wcwidth==0.2.12 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index ea347319e..41203fd03 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -21,7 +21,7 @@ tomli==2.0.1 # build # pip-tools # pyproject-hooks -wheel==0.41.3 +wheel==0.42.0 # via pip-tools zipp==3.17.0 # via importlib-metadata diff --git a/requirements/quality.txt b/requirements/quality.txt index f566fd6cc..2da3f3ce8 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -28,7 +28,7 @@ billiard==4.2.0 # via # -r requirements/test.txt # celery -celery==5.3.5 +celery==5.3.6 # via -r requirements/test.txt certifi==2023.11.17 # via @@ -145,7 +145,7 @@ edx-opaque-keys==2.5.1 # via # -r requirements/test.txt # edx-drf-extensions -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # -r requirements/test.txt # pytest @@ -153,7 +153,7 @@ grimp==3.1 # via # -r requirements/test.txt # import-linter -idna==3.4 +idna==3.6 # via # -r requirements/test.txt # requests @@ -203,7 +203,7 @@ mock==5.1.0 # via -r requirements/test.txt more-itertools==10.1.0 # via jaraco-classes -mypy==1.7.0 +mypy==1.7.1 # via # -r requirements/test.txt # djangorestframework-stubs @@ -253,7 +253,7 @@ pycparser==2.21 # cffi pydocstyle==6.3.0 # via -r requirements/quality.in -pygments==2.17.1 +pygments==2.17.2 # via # readme-renderer # rich @@ -414,7 +414,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.10 +wcwidth==0.2.12 # via # -r requirements/test.txt # prompt-toolkit diff --git a/requirements/test.txt b/requirements/test.txt index 5f2f346a3..5b4b6e46d 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -24,7 +24,7 @@ billiard==4.2.0 # via # -r requirements/base.txt # celery -celery==5.3.5 +celery==5.3.6 # via -r requirements/base.txt certifi==2023.11.17 # via @@ -124,11 +124,11 @@ edx-opaque-keys==2.5.1 # via # -r requirements/base.txt # edx-drf-extensions -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via pytest grimp==3.1 # via import-linter -idna==3.4 +idna==3.6 # via # -r requirements/base.txt # requests @@ -146,7 +146,7 @@ markupsafe==2.1.3 # via jinja2 mock==5.1.0 # via -r requirements/test.in -mypy==1.7.0 +mypy==1.7.1 # via # -r requirements/test.in # djangorestframework-stubs @@ -285,7 +285,7 @@ vine==5.1.0 # amqp # celery # kombu -wcwidth==0.2.10 +wcwidth==0.2.12 # via # -r requirements/base.txt # prompt-toolkit From 06f4f5476f2dc70ee4664960d8f6988b3f83e505 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Chris=20Ch=C3=A1vez?= Date: Thu, 30 Nov 2023 17:10:25 -0500 Subject: [PATCH 088/282] feat: Add tags count to taxonomy serializer (#125) --- openedx_learning/__init__.py | 2 +- .../core/tagging/rest_api/v1/serializers.py | 6 +++ .../core/tagging/test_views.py | 43 +++++++++++++++++-- 3 files changed, 46 insertions(+), 5 deletions(-) diff --git a/openedx_learning/__init__.py b/openedx_learning/__init__.py index d458a9e22..73a0541a9 100644 --- a/openedx_learning/__init__.py +++ b/openedx_learning/__init__.py @@ -1,4 +1,4 @@ """ Open edX Learning ("Learning Core"). """ -__version__ = "0.3.6" +__version__ = "0.3.7" diff --git a/openedx_tagging/core/tagging/rest_api/v1/serializers.py b/openedx_tagging/core/tagging/rest_api/v1/serializers.py index ca95c91cb..211f3d534 100644 --- a/openedx_tagging/core/tagging/rest_api/v1/serializers.py +++ b/openedx_tagging/core/tagging/rest_api/v1/serializers.py @@ -34,6 +34,8 @@ class TaxonomySerializer(serializers.ModelSerializer): """ Serializer for the Taxonomy model. """ + tags_count = serializers.SerializerMethodField() + class Meta: model = Taxonomy fields = [ @@ -45,6 +47,7 @@ class Meta: "allow_free_text", "system_defined", "visible_to_authors", + "tags_count", ] def to_representation(self, instance): @@ -54,6 +57,9 @@ def to_representation(self, instance): instance = instance.cast() return super().to_representation(instance) + def get_tags_count(self, instance): + return instance.tag_set.count() + class ObjectTagListQueryParamsSerializer(serializers.Serializer): # pylint: disable=abstract-method """ diff --git a/tests/openedx_tagging/core/tagging/test_views.py b/tests/openedx_tagging/core/tagging/test_views.py index f3f88208f..00151f307 100644 --- a/tests/openedx_tagging/core/tagging/test_views.py +++ b/tests/openedx_tagging/core/tagging/test_views.py @@ -126,12 +126,20 @@ def test_list_taxonomy_queryparams(self, enabled, expected_status: int, expected assert len(response.data["results"]) == expected_count @ddt.data( - (None, status.HTTP_401_UNAUTHORIZED), - ("user", status.HTTP_200_OK), - ("staff", status.HTTP_200_OK), + (None, status.HTTP_401_UNAUTHORIZED, 0), + ("user", status.HTTP_200_OK, 10), + ("staff", status.HTTP_200_OK, 20), ) @ddt.unpack - def test_list_taxonomy(self, user_attr: str | None, expected_status: int): + def test_list_taxonomy(self, user_attr: str | None, expected_status: int, tags_count: int): + taxonomy = api.create_taxonomy(name="Taxonomy enabled 1", enabled=True) + for i in range(tags_count): + tag = Tag( + taxonomy=taxonomy, + value=f"Tag {i}", + ) + tag.save() + url = TAXONOMY_LIST_URL if user_attr: @@ -141,6 +149,33 @@ def test_list_taxonomy(self, user_attr: str | None, expected_status: int): response = self.client.get(url) assert response.status_code == expected_status + # Check results + if tags_count: + assert response.data["results"] == [ + { + "id": -1, + "name": "Languages", + "description": "Languages that are enabled on this system.", + "enabled": True, + "allow_multiple": False, + "allow_free_text": False, + "system_defined": True, + "visible_to_authors": True, + "tags_count": 0, + }, + { + "id": taxonomy.id, + "name": "Taxonomy enabled 1", + "description": "", + "enabled": True, + "allow_multiple": True, + "allow_free_text": False, + "system_defined": False, + "visible_to_authors": True, + "tags_count": tags_count, + }, + ] + def test_list_taxonomy_pagination(self) -> None: url = TAXONOMY_LIST_URL api.create_taxonomy(name="T1", enabled=True) From 57ecde0c7bc9321290672b12a15bb6a384206800 Mon Sep 17 00:00:00 2001 From: edX requirements bot Date: Sun, 3 Dec 2023 19:21:31 -0500 Subject: [PATCH 089/282] chore: Updating Python Requirements --- requirements/base.txt | 6 +++--- requirements/ci.txt | 2 +- requirements/dev.txt | 10 +++++----- requirements/doc.txt | 10 +++++----- requirements/pip-tools.txt | 2 +- requirements/quality.txt | 8 ++++---- requirements/test.txt | 6 +++--- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index 4681ee3ea..27a2c7fd4 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -39,7 +39,7 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -cryptography==41.0.5 +cryptography==41.0.7 # via pyjwt django==3.2.23 # via @@ -65,9 +65,9 @@ djangorestframework==3.14.0 # edx-drf-extensions drf-jwt==1.19.2 # via edx-drf-extensions -edx-django-utils==5.8.0 +edx-django-utils==5.9.0 # via edx-drf-extensions -edx-drf-extensions==8.13.1 +edx-drf-extensions==9.0.0 # via -r requirements/base.in edx-opaque-keys==2.5.1 # via edx-drf-extensions diff --git a/requirements/ci.txt b/requirements/ci.txt index 36407b101..ce1233e57 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -40,5 +40,5 @@ typing-extensions==4.8.0 # via # grimp # import-linter -virtualenv==20.24.7 +virtualenv==20.25.0 # via tox diff --git a/requirements/dev.txt b/requirements/dev.txt index 5d845e295..f53396380 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -90,7 +90,7 @@ coverage[toml]==7.3.2 # -r requirements/quality.txt # coverage # pytest-cov -cryptography==41.0.5 +cryptography==41.0.7 # via # -r requirements/quality.txt # pyjwt @@ -158,11 +158,11 @@ drf-jwt==1.19.2 # via # -r requirements/quality.txt # edx-drf-extensions -edx-django-utils==5.8.0 +edx-django-utils==5.9.0 # via # -r requirements/quality.txt # edx-drf-extensions -edx-drf-extensions==8.13.1 +edx-drf-extensions==9.0.0 # via -r requirements/quality.txt edx-i18n-tools==1.3.0 # via -r requirements/dev.in @@ -194,7 +194,7 @@ import-linter==1.12.1 # via # -r requirements/ci.txt # -r requirements/quality.txt -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via # -r requirements/pip-tools.txt # -r requirements/quality.txt @@ -534,7 +534,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.24.7 +virtualenv==20.25.0 # via # -r requirements/ci.txt # tox diff --git a/requirements/doc.txt b/requirements/doc.txt index 64408be21..6ebdf9d9f 100644 --- a/requirements/doc.txt +++ b/requirements/doc.txt @@ -78,7 +78,7 @@ coverage[toml]==7.3.2 # -r requirements/test.txt # coverage # pytest-cov -cryptography==41.0.5 +cryptography==41.0.7 # via # -r requirements/test.txt # pyjwt @@ -138,11 +138,11 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-django-utils==5.8.0 +edx-django-utils==5.9.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.13.1 +edx-drf-extensions==9.0.0 # via -r requirements/test.txt edx-opaque-keys==2.5.1 # via @@ -164,7 +164,7 @@ imagesize==1.4.1 # via sphinx import-linter==1.12.1 # via -r requirements/test.txt -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via sphinx iniconfig==2.0.0 # via @@ -229,7 +229,7 @@ pycparser==2.21 # via # -r requirements/test.txt # cffi -pydata-sphinx-theme==0.14.3 +pydata-sphinx-theme==0.14.4 # via sphinx-book-theme pygments==2.17.2 # via diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 41203fd03..93a9cee28 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -8,7 +8,7 @@ build==1.0.3 # via pip-tools click==8.1.7 # via pip-tools -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via build packaging==23.2 # via build diff --git a/requirements/quality.txt b/requirements/quality.txt index 2da3f3ce8..84696da4a 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -78,7 +78,7 @@ coverage[toml]==7.3.2 # -r requirements/test.txt # coverage # pytest-cov -cryptography==41.0.5 +cryptography==41.0.7 # via # -r requirements/test.txt # pyjwt @@ -133,11 +133,11 @@ drf-jwt==1.19.2 # via # -r requirements/test.txt # edx-drf-extensions -edx-django-utils==5.8.0 +edx-django-utils==5.9.0 # via # -r requirements/test.txt # edx-drf-extensions -edx-drf-extensions==8.13.1 +edx-drf-extensions==9.0.0 # via -r requirements/test.txt edx-lint==5.3.6 # via -r requirements/quality.in @@ -159,7 +159,7 @@ idna==3.6 # requests import-linter==1.12.1 # via -r requirements/test.txt -importlib-metadata==6.8.0 +importlib-metadata==7.0.0 # via # keyring # twine diff --git a/requirements/test.txt b/requirements/test.txt index 5b4b6e46d..0f8d02e2a 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -67,7 +67,7 @@ coverage[toml]==7.3.2 # via # -r requirements/test.in # pytest-cov -cryptography==41.0.5 +cryptography==41.0.7 # via # -r requirements/base.txt # pyjwt @@ -114,11 +114,11 @@ drf-jwt==1.19.2 # via # -r requirements/base.txt # edx-drf-extensions -edx-django-utils==5.8.0 +edx-django-utils==5.9.0 # via # -r requirements/base.txt # edx-drf-extensions -edx-drf-extensions==8.13.1 +edx-drf-extensions==9.0.0 # via -r requirements/base.txt edx-opaque-keys==2.5.1 # via From dc3060e23dcede7bd28bd4bf2c96a533524f47cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 4 Dec 2023 17:46:16 -0300 Subject: [PATCH 090/282] fix: taxonomy import: skip parent update from deleted tags (#126) --- .../core/tagging/import_export/actions.py | 10 +++-- .../core/tagging/import_export/import_plan.py | 18 ++++---- .../core/tagging/import_export/test_api.py | 41 ++++++++++++++++++- .../tagging/import_export/test_import_plan.py | 36 +++++----------- 4 files changed, 67 insertions(+), 38 deletions(-) diff --git a/openedx_tagging/core/tagging/import_export/actions.py b/openedx_tagging/core/tagging/import_export/actions.py index b968a6b0b..679f36ae2 100644 --- a/openedx_tagging/core/tagging/import_export/actions.py +++ b/openedx_tagging/core/tagging/import_export/actions.py @@ -373,8 +373,7 @@ class DeleteTag(ImportAction): """ def __str__(self) -> str: - taxonomy_tag = self._get_tag() - return str(_("Delete tag (external_id={external_id})").format(external_id=taxonomy_tag.external_id)) + return str(_("Delete tag (external_id={external_id})").format(external_id=self.tag.id)) name = "delete" @@ -397,8 +396,11 @@ def execute(self) -> None: """ Delete a tag """ - taxonomy_tag = self._get_tag() - taxonomy_tag.delete() + try: + taxonomy_tag = self._get_tag() + taxonomy_tag.delete() + except Tag.DoesNotExist: + pass # The tag may be already cascade deleted if the parent tag was deleted class WithoutChanges(ImportAction): diff --git a/openedx_tagging/core/tagging/import_export/import_plan.py b/openedx_tagging/core/tagging/import_export/import_plan.py index f58390df1..b5ce41305 100644 --- a/openedx_tagging/core/tagging/import_export/import_plan.py +++ b/openedx_tagging/core/tagging/import_export/import_plan.py @@ -93,14 +93,16 @@ def _build_delete_actions(self, tags: dict): # Verify if there is not a parent update before if not self._search_parent_update(child.external_id, tag.external_id): # Change parent to avoid delete childs - self._build_action( - UpdateParentTag, - TagItem( - id=child.external_id, - value=child.value, - parent_id=None, - ), - ) + if child.external_id not in tags: + # Only update parent if the child is not going to be deleted + self._build_action( + UpdateParentTag, + TagItem( + id=child.external_id, + value=child.value, + parent_id=None, + ), + ) # Delete action self._build_action( diff --git a/tests/openedx_tagging/core/tagging/import_export/test_api.py b/tests/openedx_tagging/core/tagging/import_export/test_api.py index 9721fec44..039933d44 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_api.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_api.py @@ -8,7 +8,7 @@ import openedx_tagging.core.tagging.import_export.api as import_export_api from openedx_tagging.core.tagging.import_export import ParserFormat -from openedx_tagging.core.tagging.models import LanguageTaxonomy, TagImportTask, TagImportTaskState, Taxonomy +from openedx_tagging.core.tagging.models import LanguageTaxonomy, Tag, TagImportTask, TagImportTaskState, Taxonomy from .mixins import TestImportExportMixin @@ -197,3 +197,42 @@ def test_import_with_export_output(self) -> None: if tag.parent: assert new_tag.parent assert tag.parent.external_id == new_tag.parent.external_id + + def test_import_removing_with_childs(self) -> None: + """ + Test import need to remove childs with parents that will also be removed + """ + new_taxonomy = Taxonomy(name="New taxonomy") + new_taxonomy.save() + level2 = Tag.objects.create( + id=1000, + external_id="tag_2", + value="Tag 2", + taxonomy=new_taxonomy, + ) + level1 = Tag.objects.create( + id=1001, + external_id="tag_1", + value="Tag 1", + taxonomy=new_taxonomy, + ) + level3 = Tag.objects.create( + id=1002, + external_id="tag_3", + value="Tag 3", + taxonomy=new_taxonomy, + ) + level2.parent = level1 + level2.save() + + level3.parent = level3 + level3.save() + + # Import with empty tags, to remove all tags + importFile = BytesIO(json.dumps({"tags": []}).encode()) + assert import_export_api.import_tags( + new_taxonomy, + importFile, + ParserFormat.JSON, + replace=True, + ) diff --git a/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py index 915ffb724..c8dbaa0c1 100644 --- a/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py +++ b/tests/openedx_tagging/core/tagging/import_export/test_import_plan.py @@ -60,23 +60,15 @@ def test_build_delete_actions(self) -> None: assert len(self.import_plan.errors) == 0 # Check actions in order - # #1 Update parent of 'tag_2' - assert self.import_plan.actions[0].name == 'update_parent' - assert self.import_plan.actions[0].tag.id == 'tag_2' - assert self.import_plan.actions[0].tag.parent_id is None - # #2 Delete 'tag_1' + # #1 Delete 'tag_1' + assert self.import_plan.actions[0].name == 'delete' + assert self.import_plan.actions[0].tag.id == 'tag_1' + # #2 Delete 'tag_2' assert self.import_plan.actions[1].name == 'delete' - assert self.import_plan.actions[1].tag.id == 'tag_1' - # #3 Delete 'tag_2' + assert self.import_plan.actions[1].tag.id == 'tag_2' + # #3 Delete 'tag_3' assert self.import_plan.actions[2].name == 'delete' - assert self.import_plan.actions[2].tag.id == 'tag_2' - # #4 Update parent of 'tag_4' - assert self.import_plan.actions[3].name == 'update_parent' - assert self.import_plan.actions[3].tag.id == 'tag_4' - assert self.import_plan.actions[3].tag.parent_id is None - # #5 Delete 'tag_3' - assert self.import_plan.actions[4].name == 'delete' - assert self.import_plan.actions[4].tag.id == 'tag_3' + assert self.import_plan.actions[2].tag.id == 'tag_3' @ddt.data( # Test valid actions @@ -193,10 +185,6 @@ def test_build_delete_actions(self) -> None: 'name': 'without_changes', 'id': 'tag_4', }, - { - 'name': 'update_parent', - 'id': 'tag_2', - }, { 'name': 'delete', 'id': 'tag_1', @@ -330,13 +318,11 @@ def test_generate_actions(self, tags, replace, expected_errors, expected_actions "Import plan for Import Taxonomy Test\n" "--------------------------------\n" "#1: No changes needed for tag (external_id=tag_4)\n" - "#2: Update the parent of tag (external_id=tag_2) from parent (external_id=tag_1) " - "to parent (external_id=None).\n" - "#3: Delete tag (external_id=tag_1)\n" - "#4: Delete tag (external_id=tag_2)\n" - "#5: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " + "#2: Delete tag (external_id=tag_1)\n" + "#3: Delete tag (external_id=tag_2)\n" + "#4: Update the parent of tag (external_id=tag_4) from parent (external_id=tag_3) " "to parent (external_id=None).\n" - "#6: Delete tag (external_id=tag_3)\n" + "#5: Delete tag (external_id=tag_3)\n" ), ) @ddt.unpack From 66f4fa222a7e5edd5078f401eaaa2ab0828588f3 Mon Sep 17 00:00:00 2001 From: David Ormsbee Date: Mon, 4 Dec 2023 17:38:13 -0500 Subject: [PATCH 091/282] docs: ADR for serving static assets (#110) --- docs/decisions/0015-serving-static-assets.rst | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 docs/decisions/0015-serving-static-assets.rst diff --git a/docs/decisions/0015-serving-static-assets.rst b/docs/decisions/0015-serving-static-assets.rst new file mode 100644 index 000000000..cd8a5845d --- /dev/null +++ b/docs/decisions/0015-serving-static-assets.rst @@ -0,0 +1,154 @@ +15. Serving Course Team Authored Static Assets +============================================== + +Context +-------- + +Both Studio and the LMS need to serve course team authored static assets as part of the authoring and learning experiences. "Static assets" in the edx-platform context presently refers to: image files, audio files, text document files like PDFs, older video transcript files, and even JavaScript and Python files. It does NOT typically include video files, which are treated separately because of their large file size and complex workflows (processing for multiple resolutions, using third-party dictation services, etc.) + +This ADR is the synthesis of various ideas that were discussed across a handful of pull requests and issues. These links are provided for extra context, but they are not required to understand this ADR: + +* `File uploads + Experimental Media Server #31 `_ +* `File Uploads + media_server app #33 `_ +* `Modeling Files and File Dependencies #70 `_ +* `Serving static assets (disorganized thoughts) #108 `_ + +Data Storage Implementation +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The underlying data models live in the openedx-learning repo. The most relevant models are: + +* `RawContent in contents/models.py `_ +* `Component and ComponentVersion in components/models.py `_ + +Key takeaways about how this data is stored: + +* Assets are associated and versioned with Components, where a Component is typically an XBlock. So you don't ask for "version 5 of /static/fig1.webp", you ask for "the /static/fig1.webp associated with version 5 of this block". +* This initial MVP would be to serve assets for v2 content libraries, where all static assets are associated with a particular component XBlock. Later on, we'll want to allow courses to port their existing files and uploads into this system in a backwards compatible way. We will probably do this by creating a non-XBlock, filesystem Component type that can treat the entire course's uploads as a Component. The specifics for how that is modeled on the backend are out of scope for this ADR, but this general approach is meant to work for both use cases. +* The actual raw asset data is stored in django-storages using its hash value as the file name. This makes it cheap to make many references to the same asset data under different names and versions, but it means that we cannot simply give direct links to the raw file data to the browser (see the next section for details). + +The Difficulty with Direct Links to Raw Data Files +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Since the raw data is stored as objects in an S3-like store, and the mapping of file names and versions to that raw data is stored in Django models, why not simply have a Django endpoint that redirects a request to the named asset to the hash-named raw data it corresponds to? + +**It will break relative links between assets.** + The raw data files exist in a flat space with hashes for names, meaning that any relative links between assets (e.g. a JavaScript file referencing an image) would break once a browser follows the redirect. + +**Setting Metadata: Content types, filenames, and caching.** + The assets won't generally "work" unless they're served with the correct Content-Type header. For users who want to download a file, it's quite inconvenient if the filename doesn't include the correct file extension (not to mention a friendly name instead of the hash). So we need to set the Content-Type and/or Content-Disposition: ; filename=... headers. + + Setting these values for each request has proved problematic because some (but not all) S3-compatible storage services (including S3 itself) only support setting those headers for each request if you issue a signed GET request, which then gets in the way of caching and introduces the probability of browsers caching expired links, leading to all kinds of annoying cache invalidation issues. + + Setting the filename value at upload time also doesn't work because the same data may be referenced under different filenames by different Components or even different versions of the same Component. + +Application Requirements +~~~~~~~~~~~~~~~~~~~~~~~~ + +**Relative links between assets must be preserved.** + Assets may reference each other in relative links, e.g. a JavaScript file that references images or other JavaScript files. That means that our solution cannot require querystring-based authorization tokens in the style of S3 signed URLs, since asset files would have no way to encode those into their relative links. + +**Multiple versions of the asset should be available at the same time.** + Our system should be able to serve at minimum the current draft and published versions of an asset. Ideally, it should be able to serve any version of an asset. This is a departure from the way Studio and the LMS currently handle files and uploads, since there is currently no versioning at all–assets exist in a flat namespace at the course level and are immediately published. + +Security Requirements +~~~~~~~~~~~~~~~~~~~~~ + +**Assets must enforce user+file read permissions at the Learning Context level.** + The MongoDB GridFS backed ContentStore currently supports course-level access checks that can be toggled on and off for individual assets. Uploaded assets are public by default, and can be downloaded by anyone who knows the URL, regardless of whether or not they are enrolled in the course. They can optionally be "locked", which will restrict downloads to students who are enrolled in the course. + +**Assets should enforce more granular permissions at the individual Component level.** + An important distinction between ContentStore and v2 Content Library assets is that the latter can be directly associated with a Component. As a long term goal, we should be able to make permissions check on per-Component basis. So if a student does not have permission to view a Component for whatever reason (wrong content group, exam hasn't started, etc.), then they should also not have permission to see static assets associated with that component. + + The further implication of this requirement is that *permissions checking must be extensible*. The openedx-learning repo will implement the details of how to serve an asset, but it will not have the necessary models and logic to determine whether it is allowed to. + +**Assets must be served from an entirely different domain than the LMS and Studio instances.** + To reduce our chance of maliciously uploaded JavaScript compromising LMS and Studio users, user-uploaded assets must live on an entirely different domain from LMS and Studio (i.e. not just another subdomain). So if our LMS is located at ``sandbox.openedx.org``, the files should be accessed at a URL like ``assets.sandbox.openedx.io``. + +Operational Requirements +~~~~~~~~~~~~~~~~~~~~~~~~ + +**The asset server must be capable of handling high levels of traffic.** + Django views are poor choice for streaming files at scale, especially when deploying using WSGI (as Open edX does), since it will tie down a worker process for the entire duration of the response. While a Django-based streaming response may sufficient for small-to-medium traffic sites, we should allow for a more scalable solution that fully takes advantage of modern CDN capabilities. + +**Serving assets should not *require* ASGI deployment.** + Deploying the LMS and Studio using ASGI would likely substantially improve the scalability of a Django-based streaming solution, but migrating and testing this new deployment type for the entire stack is a large task and is considered out of scope for this project. + +Decision +-------- + +URLs +~~~~ + +The format will be: ``https://{asset_server}/assets/apps/{app}/{learning_package_key}/{component_key}/{version}/{filepath}`` + +The assets will be served from a completely different domain from the LMS and Studio, and will not be a subdomain. + +A more concrete example: ``https://studio.assets.sandbox.openedx.io/apps/content_libraries/lib:Axim:200/xblock.v1:problem@826eb471-0db2-4943-b343-afa65a6fdeb5/v2/static/images/fig1.png`` + +The ``version`` can be: + +* ``draft`` indicating the latest draft version (viewed by authors in Studio). +* ``published`` indicating the latest published version (viewed by students in the LMS) +* ``v{num}`` meaning a specific version–e.g. ``v20`` for version 20. + +Asset Server Implementation +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There will be two asset server URLs–one corresponding to the LMS and one corresponding to Studio, each with their own subdomain. An example set of domains might be: + +* LMS: ``sandbox.openedx.org`` +* Studio: ``studio.sandbox.openedx.org`` +* LMS Assets: ``lms.assets.sandbox.openedx.io`` (note the ``.io`` top level domain) +* Studio Assets: ``studio.assets.sandbox.openedx.io`` + +The asset serving domains will be serviced by a Caddy instance that is configured as a reverse proxy to the LMS or Studio. Caddy will be configured to only proxy a specific set of paths that correspond to valid asset URLs. + +Django View Implemenation +~~~~~~~~~~~~~~~~~~~~~~~~~ + +The LMS and Studio will each have one or two apps that implement view endpoints by extending a view that will be provided by the Learning Core. These views will only respond to requests that come via the asset domains (i.e. they will not work if you request the same paths using the LMS or Studio domains). + +Django is poorly suited to serving large static assets, particularly when deployed using WSGI. Instead of streaming the actual file data, the Django views serving assets will make use of the ``X-Accel-Redirect`` header. This header is supported by both Caddy and Nginx, and will cause them to fetch the data from the specified URI to send to the user. This redirect happens internally in the proxy and does *not* change the browser address. For sites using an object store like S3, the Django view will generate and send a signed URL to the asset. For sites using file-based Django media storage, the view will send a URL that Caddy or Nginx knows how to load from the file system. + +The Django view will also be responsible for setting other important header information, such as size, content type, and caching information. + +Permissions +~~~~~~~~~~~ + +The Learning Core provided view will contain the logic for looking up and serving assets, but it will be the responsibility of an app in Studio or the LMS to extend it with permissions checking logic. This logic may vary from app to app. For instance, Studio would likely implement a simple permissions checking model that only examines the learning context and restricts access to course staff. LMS might eventually use a much more sophisticated model that looks at the individual Component that an asset belongs to. + +Cookie Authentication +~~~~~~~~~~~~~~~~~~~~~ + +Authentication will use a session cookie for each asset server domain. + +Assets that are publicly readable will not require authentication. + +Asset requests may return a 403 error if the user is logged in but not authorized to download the asset. They will return a 401 error for users that are not authenticated. + +There will be a new endpoint exposed in LMS/Studio that will force a redirect and login to the asset server. Pages that make use of assets will be expected to load that endpoint in their ```` before any page assets are loaded. The flow would go like this: + +#. There is a ``