diff --git a/accounts/tests/test_upload.py b/accounts/tests/test_upload.py index 47c94893f..edc890b55 100644 --- a/accounts/tests/test_upload.py +++ b/accounts/tests/test_upload.py @@ -29,9 +29,8 @@ from django.urls import reverse from sounds.forms import PackForm -from sounds.models import BulkUploadProgress, License, Pack, Sound +from sounds.models import BulkUploadProgress, License, Pack, PendingSound, PendingSoundAnalysis, Sound from tags.models import Tag -from utils.filesystem import File from utils.test_helpers import ( create_test_files, create_user_and_sounds, @@ -73,12 +72,29 @@ def test_select_uploaded_files_to_describe(self): user_upload_path = settings.UPLOADS_PATH + "/%i/" % user.id os.makedirs(user_upload_path, exist_ok=True) create_test_files(filenames, user_upload_path) + pending_objects = [] + for filename in filenames: + pending_objects.append( + PendingSound.objects.create( + user=user, + filename=filename, + path=os.path.join(user_upload_path, filename), + filesize=100, + ) + ) + for pending in pending_objects: + PendingSoundAnalysis.objects.create( + pending_sound=pending, + processing_state=PendingSoundAnalysis.ProcessingState.OK, + duration=1.0, + samplerate=44100.0, + ) # Check that files are displayed in the template resp = self.client.get(reverse("accounts-manage-sounds", args=["pending_description"])) self.assertEqual(resp.status_code, 200) self.assertListEqual( - sorted([os.path.basename(f.full_path) for f in resp.context["file_structure"].children]), sorted(filenames) + sorted([child.name for child in resp.context["file_structure"].children]), sorted(filenames) ) # Selecting one file redirects to /home/describe/sounds/ @@ -87,9 +103,7 @@ def test_select_uploaded_files_to_describe(self): reverse("accounts-manage-sounds", args=["pending_description"]), { "describe": "describe", - "sound-files": [ - f"file{idx}" for idx in sounds_to_describe_idx - ], # Note this is not the filename but the value of the "select" option + "sound-files": [f"file{pending_objects[idx].id}" for idx in sounds_to_describe_idx], }, ) session_key_prefix = resp.url.split("session=")[1] @@ -98,10 +112,8 @@ def test_select_uploaded_files_to_describe(self): self.client.session[f"{session_key_prefix}-len_original_describe_sounds"], len(sounds_to_describe_idx) ) self.assertListEqual( - sorted( - [os.path.basename(f.full_path) for f in self.client.session[f"{session_key_prefix}-describe_sounds"]] - ), - sorted([filenames[idx] for idx in sounds_to_describe_idx]), + sorted(self.client.session[f"{session_key_prefix}-describe_pending_ids"]), + sorted([pending_objects[idx].id for idx in sounds_to_describe_idx]), ) # Selecting multiple file redirects to /home/describe/license/ @@ -110,9 +122,7 @@ def test_select_uploaded_files_to_describe(self): reverse("accounts-manage-sounds", args=["pending_description"]), { "describe": "describe", - "sound-files": [ - f"file{idx}" for idx in sounds_to_describe_idx - ], # Note this is not the filename but the value of the "select" option + "sound-files": [f"file{pending_objects[idx].id}" for idx in sounds_to_describe_idx], }, ) session_key_prefix = resp.url.split("session=")[1] @@ -121,10 +131,8 @@ def test_select_uploaded_files_to_describe(self): self.client.session[f"{session_key_prefix}-len_original_describe_sounds"], len(sounds_to_describe_idx) ) self.assertListEqual( - sorted( - [os.path.basename(f.full_path) for f in self.client.session[f"{session_key_prefix}-describe_sounds"]] - ), - sorted([filenames[idx] for idx in sounds_to_describe_idx]), + sorted(self.client.session[f"{session_key_prefix}-describe_pending_ids"]), + sorted([pending_objects[idx].id for idx in sounds_to_describe_idx]), ) # Selecting files to delete, deletes the files @@ -133,13 +141,11 @@ def test_select_uploaded_files_to_describe(self): reverse("accounts-manage-sounds", args=["pending_description"]), { "delete_confirm": "delete_confirm", - "sound-files": [ - f"file{idx}" for idx in sounds_to_delete_idx - ], # Note this is not the filename but the value of the "select" option, + "sound-files": [f"file{pending_objects[idx].id}" for idx in sounds_to_delete_idx], }, ) self.assertRedirects(resp, reverse("accounts-manage-sounds", args=["pending_description"])) - self.assertEqual(len(os.listdir(user_upload_path)), len(filenames) - len(sounds_to_delete_idx)) + self.assertEqual(PendingSound.objects.filter(user=user).count(), len(filenames) - len(sounds_to_delete_idx)) @override_uploads_path_with_temp_directory def test_describe_selected_files(self): @@ -159,13 +165,29 @@ def test_describe_selected_files(self): # Set license and pack data in session session = self.client.session session_key_prefix = "304298eb" - session[f"{session_key_prefix}-describe_license"] = License.objects.all()[0] - session[f"{session_key_prefix}-describe_pack"] = False session[f"{session_key_prefix}-len_original_describe_sounds"] = 2 - session[f"{session_key_prefix}-describe_sounds"] = [ - File(1, filenames[0], user_upload_path + filenames[0], False), - File(2, filenames[1], user_upload_path + filenames[1], False), + pending_objects = [ + PendingSound.objects.create( + user=user, + filename=filename, + path=os.path.join(user_upload_path, filename), + filesize=100, + ) + for filename in filenames ] + for pending in pending_objects: + PendingSoundAnalysis.objects.create( + pending_sound=pending, + processing_state=PendingSoundAnalysis.ProcessingState.OK, + duration=1.0, + samplerate=44100.0, + ) + license_obj = License.objects.all()[0] + for pending in pending_objects: + pending.license = license_obj + pending.pack = None + pending.save() + session[f"{session_key_prefix}-describe_pending_ids"] = [pending_objects[0].id, pending_objects[1].id] session.save() # Post description information @@ -173,6 +195,7 @@ def test_describe_selected_files(self): f"/home/describe/sounds/?session={session_key_prefix}", { "0-audio_filename": filenames[0], + "0-pending_sound_id": pending_objects[0].id, "0-lat": "46.31658418182218", "0-lon": "3.515625", "0-zoom": "16", @@ -184,6 +207,7 @@ def test_describe_selected_files(self): "0-bst_category": "ss-n", "0-name": filenames[0], "1-audio_filename": filenames[1], + "1-pending_sound_id": pending_objects[1].id, "1-license": "3", "1-description": "another test description", "1-lat": "", diff --git a/accounts/views.py b/accounts/views.py index 4945080e4..8e1d2394f 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -94,12 +94,21 @@ from general.templatetags.util import license_with_version from messages.models import Message from sounds.forms import LicenseForm, PackForm -from sounds.models import BulkUploadProgress, Download, Pack, PackDownload, Sound, SoundLicenseHistory -from sounds.views import edit_and_describe_sounds_helper +from sounds.models import ( + BulkUploadProgress, + Download, + Pack, + PackDownload, + PendingSound, + PendingSoundAnalysis, + Sound, + SoundLicenseHistory, +) +from sounds.views import clear_session_describe_data, edit_and_describe_sounds_helper from tickets.models import Ticket, TicketComment, UserAnnotation from utils.cache import invalidate_user_template_caches from utils.dbtime import DBTime -from utils.filesystem import generate_tree, remove_directory_if_empty +from utils.filesystem import File, remove_directory_if_empty from utils.images import extract_square from utils.logging_filters import get_client_ip from utils.mail import send_mail_template, send_mail_template_to_support @@ -604,8 +613,11 @@ def process_filter_and_sort_options(request, sort_options, tab): sounds_processing_count = sounds_processing_base_qs.count() packs_base_qs = Pack.objects.filter(user=request.user).exclude(is_deleted=True) packs_count = packs_base_qs.count() - file_structure, files = generate_tree(request.user.profile.locations()["uploads_dir"]) - sounds_pending_description_count = len(files) + pending_sounds_qs = PendingSound.objects.filter(user=request.user, status=PendingSound.Status.UPLOADED).order_by( + "-created" + ) + sounds_pending_description_count = pending_sounds_qs.count() + pending_sounds = list(pending_sounds_qs) if tab == "pending_description" else [] tvars = { "tab": tab, @@ -620,7 +632,7 @@ def process_filter_and_sort_options(request, sort_options, tab): if tab == "pending_description": unclosed_bulkdescribe = BulkUploadProgress.objects.filter(user=request.user).exclude(progress_type="C") tvars.update({"unclosed_bulkdescribe": unclosed_bulkdescribe}) - tvars_or_redirect = sounds_pending_description_helper(request, file_structure, files) + tvars_or_redirect = sounds_pending_description_helper(request, pending_sounds) if isinstance(tvars_or_redirect, dict): tvars.update(tvars_or_redirect) else: @@ -802,11 +814,47 @@ def edit_sounds(request): ) # Note that the list of sounds to describe is stored in the session object -def sounds_pending_description_helper(request, file_structure, files): +def sounds_pending_description_helper(request, pending_sounds): + def build_file_structure(): + root = File(0, "", "", True) + files_lookup = {} + for index, pending in enumerate(pending_sounds, start=1): + file_obj = File(index, pending.filename, pending.path, False) + file_obj.id = f"file{pending.id}" + try: + processing_state = pending.analysis.processing_state # type: ignore[attr-defined] + except PendingSoundAnalysis.DoesNotExist: # type: ignore[attr-defined] + processing_state = PendingSoundAnalysis.ProcessingState.PENDING + except AttributeError: + processing_state = PendingSoundAnalysis.ProcessingState.PENDING + file_obj.processing_state = processing_state + file_obj.processing_state_label = PendingSoundAnalysis.ProcessingState(processing_state).label + file_obj.is_ready_for_actions = processing_state in [ + PendingSoundAnalysis.ProcessingState.OK, + PendingSoundAnalysis.ProcessingState.FAILED, + ] + root.children.append(file_obj) + files_lookup[file_obj.id] = pending + return root, files_lookup + + file_structure, files_lookup = build_file_structure() file_structure.name = "" + files_for_form = {file_id: file_id for file_id in files_lookup.keys()} + + unavailable_pending_ids = set() + for pending in pending_sounds: + try: + processing_state = pending.analysis.processing_state # type: ignore[attr-defined] + except (PendingSoundAnalysis.DoesNotExist, AttributeError): # type: ignore[attr-defined] + processing_state = PendingSoundAnalysis.ProcessingState.PENDING + if processing_state in [ + PendingSoundAnalysis.ProcessingState.PENDING, + PendingSoundAnalysis.ProcessingState.PROCESSING, + ]: + unavailable_pending_ids.add(pending.id) if request.method == "POST": - form = FileChoiceForm(files, request.POST, prefix="sound") + form = FileChoiceForm(files_for_form, request.POST, prefix="sound") csv_form = BulkDescribeForm(request.POST, request.FILES, prefix="bulk") if csv_form.is_valid(): directory = os.path.join(settings.CSV_PATH, str(request.user.id)) @@ -827,17 +875,25 @@ def sounds_pending_description_helper(request, file_structure, files): tasks.validate_bulk_describe_csv.delay(bulk_upload_progress_object_id=bulk.id) return HttpResponseRedirect(reverse("accounts-bulk-describe", args=[bulk.id])) elif form.is_valid(): - if "delete_confirm" in request.POST: - for f in form.cleaned_data["files"]: + selected_pending = [ + files_lookup[file_id] for file_id in form.cleaned_data["files"] if file_id in files_lookup + ] + if not selected_pending: + form.add_error(None, "No valid files selected.") + elif any(pending.id in unavailable_pending_ids for pending in selected_pending): + form.add_error(None, "Some selected files are still being processed. Please try again later.") + elif "delete_confirm" in request.POST: + for pending in selected_pending: try: - os.remove(files[f].full_path) - utils.sound_upload.clean_processing_before_describe_files(files[f].full_path) - remove_uploaded_file_from_mirror_locations(files[f].full_path) + os.remove(pending.path) + utils.sound_upload.clean_processing_before_describe_files(pending.path) + remove_uploaded_file_from_mirror_locations(pending.path) except OSError as e: if e.errno == errno.ENOENT: upload_logger.info("Failed to remove file %s", str(e)) else: raise + pending.delete() # Remove user uploads directory if there are no more files to describe user_uploads_dir = request.user.profile.locations()["uploads_dir"] @@ -848,28 +904,25 @@ def sounds_pending_description_helper(request, file_structure, files): session_key_prefix = str(uuid.uuid4())[ 0:8 ] # Use a new so we don't interfere with other active description/editing processes - request.session[f"{session_key_prefix}-describe_sounds"] = [ - files[x] for x in form.cleaned_data["files"] - ] - request.session[f"{session_key_prefix}-len_original_describe_sounds"] = len( - request.session[f"{session_key_prefix}-describe_sounds"] - ) + pending_ids = [pending.id for pending in selected_pending] + request.session[f"{session_key_prefix}-describe_pending_ids"] = pending_ids + request.session[f"{session_key_prefix}-len_original_describe_sounds"] = len(pending_ids) # If only one file is chosen, go straight to the last step of the describe process, otherwise go to license selection step - if len(request.session[f"{session_key_prefix}-describe_sounds"]) > 1: + if len(pending_ids) > 1: return HttpResponseRedirect(reverse("accounts-describe-license") + f"?session={session_key_prefix}") else: return HttpResponseRedirect(reverse("accounts-describe-sounds") + f"?session={session_key_prefix}") else: - form = FileChoiceForm(files) + form = FileChoiceForm(files_for_form) tvars = {"form": form, "file_structure": file_structure} return tvars else: csv_form = BulkDescribeForm(prefix="bulk") - form = FileChoiceForm(files, prefix="sound") + form = FileChoiceForm(files_for_form, prefix="sound") tvars = { "form": form, "file_structure": file_structure, - "n_files": len(files), + "n_files": len(pending_sounds), "csv_form": csv_form, "describe_enabled": settings.UPLOAD_AND_DESCRIPTION_ENABLED, } @@ -879,16 +932,23 @@ def sounds_pending_description_helper(request, file_structure, files): @login_required def describe_license(request): session_key_prefix = request.GET.get("session", "") + pending_ids = request.session.get(f"{session_key_prefix}-describe_pending_ids", []) + pending_qs = PendingSound.objects.filter(id__in=pending_ids, user=request.user) + if not pending_ids or not pending_qs.exists(): + clear_session_describe_data(request, session_key_prefix=session_key_prefix) + return HttpResponseRedirect(reverse("accounts-manage-sounds", args=["pending_description"])) if request.method == "POST": form = LicenseForm(request.POST, hide_old_license_versions=True) if form.is_valid(): - request.session[f"{session_key_prefix}-describe_license"] = form.cleaned_data["license"] + PendingSound.objects.filter(id__in=pending_ids, user=request.user).update( + license=form.cleaned_data["license"] + ) return HttpResponseRedirect(reverse("accounts-describe-pack") + f"?session={session_key_prefix}") else: form = LicenseForm(hide_old_license_versions=True) tvars = { "form": form, - "num_files": request.session.get(f"{session_key_prefix}-len_original_describe_sounds", 0), + "num_files": len(pending_ids), "session_key_prefix": session_key_prefix, } return render(request, "accounts/describe_license.html", tvars) @@ -898,23 +958,28 @@ def describe_license(request): def describe_pack(request): packs = Pack.objects.filter(user=request.user).exclude(is_deleted=True) session_key_prefix = request.GET.get("session", "") + pending_ids = request.session.get(f"{session_key_prefix}-describe_pending_ids", []) + pending_qs = PendingSound.objects.filter(id__in=pending_ids, user=request.user) + if not pending_ids or not pending_qs.exists(): + clear_session_describe_data(request, session_key_prefix=session_key_prefix) + return HttpResponseRedirect(reverse("accounts-manage-sounds", args=["pending_description"])) if request.method == "POST": form = PackForm(packs, request.POST, prefix="pack") if form.is_valid(): data = form.cleaned_data if data["new_pack"]: pack, created = Pack.objects.get_or_create(user=request.user, name=data["new_pack"]) - request.session[f"{session_key_prefix}-describe_pack"] = pack + PendingSound.objects.filter(id__in=pending_ids, user=request.user).update(pack=pack) elif data["pack"]: - request.session[f"{session_key_prefix}-describe_pack"] = data["pack"] + PendingSound.objects.filter(id__in=pending_ids, user=request.user).update(pack=data["pack"]) else: - request.session[f"{session_key_prefix}-describe_pack"] = False + PendingSound.objects.filter(id__in=pending_ids, user=request.user).update(pack=None) return HttpResponseRedirect(reverse("accounts-describe-sounds") + f"?session={session_key_prefix}") else: form = PackForm(packs, prefix="pack") tvars = { "form": form, - "num_files": request.session.get(f"{session_key_prefix}-len_original_describe_sounds", 0), + "num_files": len(pending_ids), "session_key_prefix": session_key_prefix, } return render(request, "accounts/describe_pack.html", tvars) @@ -1412,18 +1477,19 @@ def handle_uploaded_file(user_id, f): # Move or copy the uploaded file from the temporary folder created by Django to the /uploads path dest_directory = os.path.join(settings.UPLOADS_PATH, str(user_id)) os.makedirs(dest_directory, exist_ok=True) - dest_path = os.path.join(dest_directory, os.path.basename(f.name)).encode("utf-8") - upload_logger.info( - "handling file upload from user %s, filename %s and saving to %s", user_id, f.name, dest_path.decode("utf-8") - ) + unique_prefix = uuid.uuid4().hex[:8] + stored_filename = f"{unique_prefix}_{os.path.basename(f.name)}" + dest_path = os.path.join(dest_directory, stored_filename) + dest_path_bytes = dest_path.encode("utf-8") + upload_logger.info("handling file upload from user %s, filename %s and saving to %s", user_id, f.name, dest_path) starttime = time.monotonic() if settings.MOVE_TMP_UPLOAD_FILES_INSTEAD_OF_COPYING and isinstance(f, TemporaryUploadedFile): # Big files (bigger than ~2MB, this is configured by Django and can be customized) will be delivered via a # TemporaryUploadedFile which has already been streamed in disk, so we only need to move the already existing # file instead of copying it try: - os.rename(f.temporary_file_path(), dest_path) - os.chmod(dest_path, 0o644) # Set appropriate permissions so that file can be downloaded from nginx + os.rename(f.temporary_file_path(), dest_path_bytes) + os.chmod(dest_path_bytes, 0o644) # Set appropriate permissions so that file can be downloaded from nginx except Exception as e: upload_logger.warning("file upload (%s) - failed moving TemporaryUploadedFile error: %s", user_id, str(e)) return False @@ -1431,21 +1497,35 @@ def handle_uploaded_file(user_id, f): # Small files will be of type InMemoryUploadedFile instead of TemporaryUploadedFile and will be initially # stored in memory, so we need to copy them to the destination try: - destination = open(dest_path, "wb") + destination = open(dest_path_bytes, "wb") for chunk in f.chunks(): destination.write(chunk) except Exception as e: upload_logger.warning("file upload (%s) - failed writing uploaded file error: %s", user_id, str(e)) return False + try: + pending_sound = PendingSound.objects.create( + user_id=user_id, + filename=f.name, + path=dest_path, + filesize=f.size, + ) + PendingSoundAnalysis.objects.create(pending_sound=pending_sound) + except Exception as e: + upload_logger.warning("file upload (%s) - failed creating PendingSound error: %s", user_id, str(e)) + try: + os.remove(dest_path_bytes) + except OSError: + pass + return False + # trigger processing of uploaded files before description - tasks.process_before_description.delay(audio_file_path=dest_path.decode()) - upload_logger.info( - "file upload (%s) - sent uploaded file %s to processing before description", user_id, dest_path.decode("utf-8") - ) + tasks.process_before_description.delay(pending_sound_id=pending_sound.id) + upload_logger.info("file upload (%s) - sent uploaded file %s to processing before description", user_id, dest_path) # NOTE: if we enable mirror locations for uploads and the copying below causes problems, we could do it async - copy_uploaded_file_to_mirror_locations(dest_path) + copy_uploaded_file_to_mirror_locations(dest_path_bytes) upload_logger.info( "file upload (%s) - handling file upload done, took %.2f seconds", user_id, time.monotonic() - starttime ) diff --git a/docker-compose.yml b/docker-compose.yml index db6c947f5..2c38efc2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: - ./freesound-data/db_dev_dump:/freesound-data/db_dev_dump ports: - "${FS_BIND_HOST:-127.0.0.1}:${LOCAL_PORT_PREFIX}5432:5432" + command: postgres -c shared_buffers=2048MB -c work_mem=256MB -c maintenance_work_mem=512MB environment: - POSTGRES_USER=freesound - POSTGRES_DB=freesound diff --git a/freesound/static/bw-frontend/src/components/addSoundsModal.js b/freesound/static/bw-frontend/src/components/addSoundsModal.js index 81ea97eaf..3148ec0ed 100644 --- a/freesound/static/bw-frontend/src/components/addSoundsModal.js +++ b/freesound/static/bw-frontend/src/components/addSoundsModal.js @@ -3,7 +3,7 @@ import {initializeObjectSelector, updateObjectSelectorDataProperties} from '../c import {serializedIdListToIntList, combineIdsLists} from "../utils/data" const handleAddSoundsModal = (modalId, modalUrl, selectedSoundsDestinationElement, onSoundsSelectedCallback) => { - handleGenericModal(modalUrl, (modalContainer) => { + handleGenericModal(modalUrl, (modalContainer) => { const inputElement = modalContainer.getElementsByTagName('input')[0]; inputElement.addEventListener("keypress", function(event) { if (event.key === "Enter") { @@ -44,11 +44,22 @@ const prepareAddSoundsModalAndFields = (container) => { const addSoundsButtons = [...container.querySelectorAll(`[data-toggle^="add-sounds-modal"]`)]; addSoundsButtons.forEach(addSoundsButton => { const removeSoundsButton = addSoundsButton.nextElementSibling; - removeSoundsButton.disabled = true; + if (removeSoundsButton) { + removeSoundsButton.disabled = true; + } - const selectedSoundsDestinationElement = addSoundsButton.parentNode.parentNode.querySelector('.bw-object-selector-container[data-type="sounds"]'); + const selectedSoundsDestinationElement = addSoundsButton.parentNode?.parentNode?.querySelector('.bw-object-selector-container[data-type="sounds"]'); + if (!selectedSoundsDestinationElement) { + addSoundsButton.disabled = true; + if (removeSoundsButton) { + removeSoundsButton.disabled = true; + } + return; + } initializeObjectSelector(selectedSoundsDestinationElement, (element) => { - removeSoundsButton.disabled = element.dataset.selectedIds == "" + if (removeSoundsButton) { + removeSoundsButton.disabled = element.dataset.selectedIds == ""; + } }); const soundsInput = selectedSoundsDestinationElement.parentNode.parentNode.getElementsByTagName('input')[0]; @@ -62,7 +73,7 @@ const prepareAddSoundsModalAndFields = (container) => { const soundsLabel = selectedSoundsDestinationElement.parentNode.parentNode.getElementsByTagName('label')[0]; const itemCountElementInLabel = soundsLabel === (null || undefined) ? null : soundsLabel.querySelector('#element-count'); - + const maxSounds = selectedSoundsDestinationElement.dataset.maxElements; const maxSoundsHelpText = selectedSoundsDestinationElement.parentNode.parentNode.getElementsByClassName('helptext')[0] if(maxSounds !== "None"){ @@ -72,7 +83,8 @@ const prepareAddSoundsModalAndFields = (container) => { } } - removeSoundsButton.addEventListener('click', (evt) => { + if (removeSoundsButton){ + removeSoundsButton.addEventListener('click', (evt) => { evt.preventDefault(); const soundCheckboxes = selectedSoundsDestinationElement.querySelectorAll('input.bw-checkbox'); soundCheckboxes.forEach(checkbox => { @@ -91,6 +103,7 @@ const prepareAddSoundsModalAndFields = (container) => { itemCountElementInLabel.innerHTML = selectedSoundsDestinationElement.children.length} removeSoundsButton.disabled = true; }); + } addSoundsButton.addEventListener('click', (evt) => { evt.preventDefault(); @@ -107,11 +120,13 @@ const prepareAddSoundsModalAndFields = (container) => { if (itemCountElementInLabel){ itemCountElementInLabel.innerHTML = selectedSoundsDestinationElement.children.length} initializeObjectSelector(selectedSoundsDestinationElement, (element) => { - removeSoundsButton.disabled = element.dataset.selectedIds == "" + if (removeSoundsButton){ + removeSoundsButton.disabled = element.dataset.selectedIds == "" + } }); }); - - }); + + }); }); } diff --git a/freesound/static/bw-frontend/src/pages/manageSounds.js b/freesound/static/bw-frontend/src/pages/manageSounds.js index 646fa8859..efd9f0535 100644 --- a/freesound/static/bw-frontend/src/pages/manageSounds.js +++ b/freesound/static/bw-frontend/src/pages/manageSounds.js @@ -159,8 +159,10 @@ if (selectAllButton !== null) { selectAllButton.addEventListener('click', evt => { const describeFileCheckboxes = describeFileCheckboxesWrapper.querySelectorAll('input'); describeFileCheckboxes.forEach(checkboxElement => { - checkboxElement.checked = true; - onCheckboxChanged(checkboxElement, describeFileCheckboxes); + if (!checkboxElement.disabled) { + checkboxElement.checked = true; + onCheckboxChanged(checkboxElement, describeFileCheckboxes); + } }); }) } @@ -171,7 +173,9 @@ if (selectNoneButton !== null) { const describeFileCheckboxes = describeFileCheckboxesWrapper.querySelectorAll('input'); describeFileCheckboxes.forEach(checkboxElement => { checkboxElement.checked = false; - onCheckboxChanged(checkboxElement, describeFileCheckboxes); + if (!checkboxElement.disabled) { + onCheckboxChanged(checkboxElement, describeFileCheckboxes); + } }); }) } \ No newline at end of file diff --git a/general/tasks.py b/general/tasks.py index 0edd9beef..2f3395b6c 100644 --- a/general/tasks.py +++ b/general/tasks.py @@ -579,24 +579,56 @@ def process_sound(sound_id, skip_previews=False, skip_displays=False): @shared_task(name=PROCESS_BEFORE_DESCRIPTION_TASK_NAME, queue=settings.CELERY_ASYNC_TASKS_QUEUE_NAME) -def process_before_description(audio_file_path): +def process_before_description(pending_sound_id): """Processes an uploaded sound file before the sound is described and saves generated previews and wave/spectral images in a specific directory so these can be served in the sound players used in the description phase. Args: - audio_file_path (str): path to the uploaded file + pending_sound_id (int): ID of PendingSound entry """ + PendingSound = apps.get_model("sounds", "PendingSound") + PendingSoundAnalysis = apps.get_model("sounds", "PendingSoundAnalysis") + + pending_sound = None + analysis = None + try: + pending_sound = PendingSound.objects.select_related("analysis").get(id=pending_sound_id) + except PendingSound.DoesNotExist: + workers_logger.warning( + "PendingSound %s does not exist, skipping processing-before-describe task", + pending_sound_id, + ) + return + + audio_file_path = pending_sound.path + try: + analysis = pending_sound.analysis + except PendingSound.analysis.RelatedObjectDoesNotExist: # type: ignore[attr-defined] + analysis = PendingSoundAnalysis.objects.create(pending_sound=pending_sound) set_timeout_alarm(settings.WORKER_TIMEOUT, f"Processing-before-describe of sound {audio_file_path} timed out") workers_logger.info( "Starting processing-before-describe of sound (%s)" % json.dumps({"task_name": PROCESS_BEFORE_DESCRIPTION_TASK_NAME, "audio_file_path": audio_file_path}) ) start_time = time.monotonic() + if analysis: + analysis.processing_state = PendingSoundAnalysis.ProcessingState.PROCESSING + analysis.save(update_fields=["processing_state"]) try: check_if_free_space() result = FreesoundAudioProcessorBeforeDescription(audio_file_path=audio_file_path).process() if result: + if analysis: + analysis.duration = result.get("duration") + analysis.samplerate = result.get("samplerate") + analysis.channels = result.get("channels") + analysis.bitdepth = result.get("bitdepth") + analysis.md5 = result.get("md5") + analysis.processing_state = PendingSoundAnalysis.ProcessingState.OK + analysis.save( + update_fields=["duration", "samplerate", "channels", "bitdepth", "md5", "processing_state"] + ) workers_logger.info( "Finished processing-before-describe of sound (%s)" % json.dumps( @@ -609,6 +641,9 @@ def process_before_description(audio_file_path): ) ) else: + if analysis: + analysis.processing_state = PendingSoundAnalysis.ProcessingState.FAILED + analysis.save(update_fields=["processing_state"]) workers_logger.info( "Finished processing-before-describe of sound (%s)" % json.dumps( @@ -622,6 +657,9 @@ def process_before_description(audio_file_path): ) except WorkerException as e: + if analysis: + analysis.processing_state = PendingSoundAnalysis.ProcessingState.FAILED + analysis.save(update_fields=["processing_state"]) workers_logger.info( "WorkerException while processing-before-describe sound (%s)" % json.dumps( @@ -636,6 +674,9 @@ def process_before_description(audio_file_path): sentry_sdk.capture_exception(e) except Exception as e: + if analysis: + analysis.processing_state = PendingSoundAnalysis.ProcessingState.FAILED + analysis.save(update_fields=["processing_state"]) workers_logger.info( "Unexpected error while processing-before-describe sound (%s)" % json.dumps( diff --git a/sounds/migrations/0060_pendingsound_alter_sound_bst_category_and_more.py b/sounds/migrations/0060_pendingsound_alter_sound_bst_category_and_more.py new file mode 100644 index 000000000..03a0d95a9 --- /dev/null +++ b/sounds/migrations/0060_pendingsound_alter_sound_bst_category_and_more.py @@ -0,0 +1,53 @@ +# Generated by Django 4.2.19 on 2025-11-27 11:47 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('sounds', '0059_alter_soundsimilarityvector'), + ] + + operations = [ + migrations.CreateModel( + name='PendingSound', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('filename', models.CharField(max_length=512)), + ('path', models.CharField(max_length=1024)), + ('filesize', models.PositiveIntegerField()), + ('created', models.DateTimeField(auto_now_add=True)), + ('status', models.CharField(choices=[('UP', 'Uploaded'), ('DE', 'Described'), ('DL', 'Deleted')], default='UP', max_length=2)), + ('license', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='sounds.license')), + ('pack', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to='sounds.pack')), + ], + ), + migrations.CreateModel( + name='PendingSoundAnalysis', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', models.DateTimeField(auto_now_add=True)), + ('processing_state', models.CharField(choices=[('PE', 'Pending'), ('PR', 'Processing'), ('OK', 'OK'), ('FA', 'Failed')], default='PE', max_length=2)), + ('duration', models.FloatField(blank=True, null=True)), + ('samplerate', models.FloatField(blank=True, null=True)), + ('channels', models.IntegerField(blank=True, null=True)), + ('bitdepth', models.IntegerField(blank=True, null=True)), + ('md5', models.CharField(blank=True, db_index=True, max_length=32, null=True)), + ('pending_sound', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='analysis', to='sounds.pendingsound')), + ], + ), + migrations.AddField( + model_name='pendingsound', + name='sound', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='pending_sound', to='sounds.sound'), + ), + migrations.AddField( + model_name='pendingsound', + name='user', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='pending_sounds', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/sounds/models.py b/sounds/models.py index 18ed2ce75..64d0567e3 100644 --- a/sounds/models.py +++ b/sounds/models.py @@ -38,7 +38,7 @@ from django.contrib.sites.models import Site from django.core.exceptions import ObjectDoesNotExist from django.db import models -from django.db.models import Avg, Exists, F, OuterRef, Prefetch, Sum +from django.db.models import Avg, Exists, F, OuterRef, Prefetch, Sum, TextChoices from django.db.models.functions import Greatest, JSONObject from django.db.models.signals import post_delete, post_save, pre_delete from django.dispatch import receiver @@ -77,6 +77,52 @@ sounds_logger = logging.getLogger("sounds") +class PendingSound(models.Model): + """Tracks uploaded audio files awaiting description.""" + + class Status(TextChoices): + UPLOADED = "UP", "Uploaded" + DESCRIBED = "DE", "Described" + DELETED = "DL", "Deleted" + + user = models.ForeignKey(User, related_name="pending_sounds", on_delete=models.CASCADE) + filename = models.CharField(max_length=512) + path = models.CharField(max_length=1024) + filesize = models.PositiveIntegerField() + created = models.DateTimeField(auto_now_add=True) + license = models.ForeignKey("License", null=True, blank=True, on_delete=models.SET_NULL) + pack = models.ForeignKey("Pack", null=True, blank=True, on_delete=models.SET_NULL) + sound = models.OneToOneField( + "Sound", null=True, blank=True, on_delete=models.SET_NULL, related_name="pending_sound" + ) + status = models.CharField(max_length=2, choices=Status.choices, default=Status.UPLOADED) + + def __str__(self): + return f"{self.filename} uploaded by {self.user} at {self.created} (Status={self.status})" + + +class PendingSoundAnalysis(models.Model): + """Stores processing results for PendingSound instances.""" + + class ProcessingState(TextChoices): + PENDING = "PE", "Pending" + PROCESSING = "PR", "Processing" + OK = "OK", "OK" + FAILED = "FA", "Failed" + + pending_sound = models.OneToOneField(PendingSound, related_name="analysis", on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + processing_state = models.CharField(max_length=2, choices=ProcessingState.choices, default=ProcessingState.PENDING) + duration = models.FloatField(null=True, blank=True) + samplerate = models.FloatField(null=True, blank=True) + channels = models.IntegerField(null=True, blank=True) + bitdepth = models.IntegerField(null=True, blank=True) + md5 = models.CharField(max_length=32, null=True, blank=True, db_index=True) + + def __str__(self): + return f"Analysis for pending sound {self.pending_sound_id}" + + class License(models.Model): """A creative commons license model""" diff --git a/sounds/views.py b/sounds/views.py index 6c2fa01e0..0f4174eed 100644 --- a/sounds/views.py +++ b/sounds/views.py @@ -55,6 +55,7 @@ Pack, PackDownload, PackDownloadSound, + PendingSound, RemixGroup, Sound, SoundOfTheDay, @@ -438,7 +439,7 @@ def clear_session_edit_data(request, session_key_prefix=""): def clear_session_describe_data(request, session_key_prefix=""): # Clear pre-existing edit/describe sound related data in the session - for key in ["describe_sounds", "describe_license", "describe_pack", "len_original_describe_sounds"]: + for key in ["describe_pending_ids", "len_original_describe_sounds"]: request.session.pop(f"{session_key_prefix}-{key}", None) @@ -454,9 +455,16 @@ def create_sounds(request, forms): sounds_to_process = [] dirty_packs = [] for form in forms: - file_full_path = form.file_full_path - if not file_full_path.endswith(form.audio_filename_from_form): - continue # Double check that we are not writing to the wrong file + pending_sound = getattr(form, "pending_sound", None) + if pending_sound: + posted_pending_id = getattr(form, "pending_sound_id_from_form", None) + if str(pending_sound.id) != str(posted_pending_id): + continue + file_full_path = pending_sound.path + else: + file_full_path = form.file_full_path + if not file_full_path.endswith(form.audio_filename_from_form): + continue # Double check that we are not writing to the right file sound_fields = { "name": form.cleaned_data["name"], "bst_category": form.cleaned_data["bst_category"], @@ -491,6 +499,10 @@ def create_sounds(request, forms): if sound_sources: sound.set_sources(sound_sources) sounds_to_process.append(sound) + if pending_sound: + pending_sound.sound = sound + pending_sound.status = PendingSound.Status.DESCRIBED + pending_sound.save(update_fields=["sound", "status"]) if user.profile.is_whitelisted: messages.add_message( request, @@ -598,10 +610,23 @@ def update_edited_sound(sound, data): for pack_to_process in packs_to_process: pack_to_process.process() - files = request.session.get( - f"{session_key_prefix}-describe_sounds", None - ) # List of File objects of sounds to describe + pending_ids = request.session.get(f"{session_key_prefix}-describe_pending_ids", None) sounds = request.session.get(f"{session_key_prefix}-edit_sounds", None) # List of Sound objects to edit + if describing: + if pending_ids is None: + return HttpResponseRedirect(reverse("accounts-manage-sounds", args=["pending_description"])) + pending_lookup = { + ps.id: ps + for ps in PendingSound.objects.filter(id__in=pending_ids, user=request.user).select_related("analysis") + } + files = [pending_lookup.get(pending_id) for pending_id in pending_ids if pending_lookup.get(pending_id)] + if not files: + clear_session_describe_data(request, session_key_prefix=session_key_prefix) + return HttpResponseRedirect(reverse("accounts-manage-sounds", args=["pending_description"])) + if len(files) != len(pending_ids): + request.session[f"{session_key_prefix}-describe_pending_ids"] = [ps.id for ps in files] + else: + files = None if (describing and files is None) or (not describing and sounds is None): # Expecting either a list of sounds or audio files to describe, got none. Redirect to main manage sounds page. return HttpResponseRedirect(reverse("accounts-manage-sounds", args=["published"])) @@ -620,17 +645,11 @@ def update_edited_sound(sound, data): (len_original_describe_edit_sounds - len(all_remaining_sounds_to_edit_or_describe)) / forms_per_round + 1 ) files_data_for_players = [] # Used when describing sounds (not when editing) to be able to show sound players - preselected_license = request.session.get( - f"{session_key_prefix}-describe_license", False - ) # Pre-selected from the license selection page when describing multiple sounds - preselected_pack = request.session.get( - f"{session_key_prefix}-describe_pack", False - ) # Pre-selected from the pack selection page when describing multiple sounds for count, element in enumerate(sounds_to_edit_or_describe): prefix = str(count) if describing: - audio_file_path = element.full_path + audio_file_path = element.path duration = get_duration_from_processing_before_describe_files(audio_file_path) if duration > 0.0: processing_before_describe_base_url = get_processing_before_describe_sound_base_url(audio_file_path) @@ -653,13 +672,16 @@ def update_edited_sound(sound, data): form = SoundEditAndDescribeForm( request.POST, prefix=prefix, - file_full_path=element.full_path if describing else None, + file_full_path=element.path if describing else None, explicit_disable=element.is_explicit if not describing else False, hide_old_license_versions="3.0" not in element.license.deed_url if not describing else True, user_packs=Pack.objects.filter(user=request.user if describing else element.user).exclude( is_deleted=True ), ) + if describing: + form.pending_sound = element + form.pending_sound_id = element.id forms.append(form) if form.is_valid(): if not describing: @@ -673,16 +695,15 @@ def update_edited_sound(sound, data): update_edited_sound(element, form.cleaned_data) else: # Don't do anything here except storing the audio filename from the POST data which will be later used - # as a double check to make sure we don't write the description to the wrong file - audio_filename_from_form = request.POST.get(f"{prefix}-audio_filename", None) - form.audio_filename_from_form = audio_filename_from_form + form.pending_sound_id_from_form = request.POST.get(f"{prefix}-pending_sound_id", None) + form.audio_filename_from_form = request.POST.get(f"{prefix}-audio_filename", None) else: all_forms_validated_ok = False form.sound_sources_ids = list( form.cleaned_data["sources"] ) # Add sources ids to list so sources sound selector can be initialized if describing: - form.audio_filename = element.name + form.audio_filename = element.filename else: form.sound_id = element.id else: @@ -702,11 +723,11 @@ def update_edited_sound(sound, data): ) else: sound_sources_ids = [] - initial = dict(name=os.path.splitext(element.name)[0]) - if preselected_license: - initial["license"] = preselected_license - if preselected_pack: - initial["pack"] = preselected_pack.id + initial = dict(name=os.path.splitext(element.filename)[0]) + if element.license: + initial["license"] = element.license + if element.pack: + initial["pack"] = element.pack.id form = SoundEditAndDescribeForm( prefix=prefix, explicit_disable=element.is_explicit if not describing else False, @@ -716,9 +737,12 @@ def update_edited_sound(sound, data): is_deleted=True ), ) + if describing: + form.pending_sound = element + form.pending_sound_id = element.id form.sound_sources_ids = sound_sources_ids if describing: - form.audio_filename = element.name + form.audio_filename = element.filename else: form.sound_id = element.id forms.append(form) @@ -746,7 +770,10 @@ def update_edited_sound(sound, data): create_sounds(request, forms) # Remove sounds successfully described from session data - request.session[f"{session_key_prefix}-describe_sounds"] = files[forms_per_round:] + remaining_pending_ids = request.session.get(f"{session_key_prefix}-describe_pending_ids", [])[ + forms_per_round: + ] + request.session[f"{session_key_prefix}-describe_pending_ids"] = remaining_pending_ids # If no more sounds to describe, redirect to manage sound page, otherwise redirect to same page to proceed with second round messages.add_message( @@ -754,7 +781,7 @@ def update_edited_sound(sound, data): messages.INFO, f"Successfully finished sound description round {current_round} of {num_rounds}!", ) - if not request.session[f"{session_key_prefix}-describe_sounds"]: + if not remaining_pending_ids: clear_session_describe_data(request, session_key_prefix=session_key_prefix) return HttpResponseRedirect(reverse("accounts-manage-sounds", args=["processing"])) else: diff --git a/templates/accounts/recursive_file.html b/templates/accounts/recursive_file.html index 03b112055..73bc2b182 100644 --- a/templates/accounts/recursive_file.html +++ b/templates/accounts/recursive_file.html @@ -4,7 +4,16 @@
{% for child in file.children %}
- +
{% endfor %}
diff --git a/templates/sounds/edit_and_describe.html b/templates/sounds/edit_and_describe.html index f0b11e155..ab354de2f 100644 --- a/templates/sounds/edit_and_describe.html +++ b/templates/sounds/edit_and_describe.html @@ -10,9 +10,9 @@ {% block page-content %}
-

{% if describing and total_sounds_to_describe > 1 %}This is the last step of the description process. {% endif %}{% if num_rounds > 1%}If you've selected more than {{ sounds_per_round }} sounds, +

{% if describing and total_sounds_to_describe > 1 %}This is the last step of the description process. {% endif %}{% if num_rounds > 1%}If you've selected more than {{ sounds_per_round }} sounds, you will be asked to {%if describe %}describe{% else %}edit{% endif %} them in rounds of {{ sounds_per_round }} sounds until all are done. {% endif %} - {% if num_forms > 1 %}For each sound y{% else %}Y{% endif %}ou'll have to at least specify a name, some tags, a description and a license. Additionally, you can also provide a pack, geolocation data (only recommended for + {% if num_forms > 1 %}For each sound y{% else %}Y{% endif %}ou'll have to at least specify a name, some tags, a description and a license. Additionally, you can also provide a pack, geolocation data (only recommended for field recordings), and a list of sound sources, that is to say, a list of other Freesound sounds that were used to create this one. {% if num_forms > 1%}Click on the sound filenames below to display the form for {% if describing %}describing{% else %}editing{% endif %} each sound.{% endif %}

@@ -55,14 +55,15 @@

{% if describing %}{{ form.audio_filename }}{% else %}{{ form.name.value }}{ {% if describing %} + {% else %} {% endif %} - +
{{ form.non_field_errors }}
- +
Basic information
@@ -108,7 +109,7 @@
Geolocation
Sound sources
{% include "molecules/sources_form_field.html" %}
- +
{% if not forloop.last %}
@@ -117,7 +118,7 @@

Sound sources
{% endfor %} {% if num_forms == 1 %} - + {% endif %}
@@ -133,13 +134,13 @@

Sounds to {% if describing %}describe{% else %

{% endfor %} {% if num_forms > 1 %} - + {% endif %}
{% if num_forms > 1 %} - + {% endif %}
diff --git a/utils/audioprocessing/freesound_audio_processing.py b/utils/audioprocessing/freesound_audio_processing.py index ed4456aab..29123ea07 100644 --- a/utils/audioprocessing/freesound_audio_processing.py +++ b/utils/audioprocessing/freesound_audio_processing.py @@ -31,6 +31,7 @@ import utils.audioprocessing.processing as audioprocessing from utils.audioprocessing.processing import AudioProcessingException +from utils.filesystem import md5file from utils.mirror_files import copy_displays_to_mirror_locations, copy_previews_to_mirror_locations from utils.sound_upload import get_processing_before_describe_sound_folder @@ -533,4 +534,5 @@ def process(self): info["audio_file_path"] = self.audio_file_path with open(self.output_info_file, "w") as f: json.dump(info, f) - return True + info["md5"] = md5file(self.audio_file_path) + return info diff --git a/utils/sound_upload.py b/utils/sound_upload.py index 3bf03a1e7..809b91c07 100644 --- a/utils/sound_upload.py +++ b/utils/sound_upload.py @@ -84,24 +84,6 @@ def clean_processing_before_describe_files(audio_file_path): remove_directory(directory_path, ignore_errors=True) -def get_duration_from_processing_before_describe_files(audio_file_path): - info_file_path = os.path.join(get_processing_before_describe_sound_folder(audio_file_path), "info.json") - try: - with open(info_file_path) as f: - return float(json.load(f)["duration"]) - except Exception: - return 0.0 - - -def get_samplerate_from_processing_before_describe_files(audio_file_path): - info_file_path = os.path.join(get_processing_before_describe_sound_folder(audio_file_path), "info.json") - try: - with open(info_file_path) as f: - return float(json.load(f)["samplerate"]) - except Exception: - return 44100.0 - - def get_processing_before_describe_sound_folder(audio_file_path): """ Get the path to the folder where the sound files generated during processing-before-describe @@ -117,6 +99,33 @@ def get_processing_before_describe_sound_base_url(audio_file_path): return settings.PROCESSING_BEFORE_DESCRIPTION_URL + "/".join(path.split("/")[-2:]) + "/" +def _get_latest_analysis_for_path(audio_file_path): + PendingSoundAnalysis = apps.get_model("sounds", "PendingSoundAnalysis") + try: + return ( + PendingSoundAnalysis.objects.select_related("pending_sound") + .filter(pending_sound__path=audio_file_path) + .order_by("-created") + .first() + ) + except PendingSoundAnalysis.DoesNotExist: + return None + + +def get_duration_from_processing_before_describe_files(audio_file_path): + analysis = _get_latest_analysis_for_path(audio_file_path) + if analysis and analysis.duration is not None: + return float(analysis.duration) + return 0.0 + + +def get_samplerate_from_processing_before_describe_files(audio_file_path): + analysis = _get_latest_analysis_for_path(audio_file_path) + if analysis and analysis.samplerate is not None: + return float(analysis.samplerate) + return 44100.0 + + def create_sound(user, sound_fields, apiv2_client=None, bulk_upload_progress=None, process=True, remove_exists=False): """ This function is used to create sound objects uploaded via the sound describe form, the API or the bulk describe diff --git a/utils/tests/test_processing.py b/utils/tests/test_processing.py index b5f4eafe6..d8aaa2618 100644 --- a/utils/tests/test_processing.py +++ b/utils/tests/test_processing.py @@ -312,7 +312,7 @@ def test_process_before_description(self): uploaded_file_path = os.path.join(settings.UPLOADS_PATH, str(self.user.id), filename) create_test_files(paths=[uploaded_file_path], make_valid_wav_files=True, duration=2) result = FreesoundAudioProcessorBeforeDescription(audio_file_path=uploaded_file_path).process() - self.assertTrue(result) + self.assertIsInstance(result, dict) self.assertListEqual( sorted(os.listdir(get_processing_before_describe_sound_folder(uploaded_file_path))), sorted(["wave.png", "spectral.png", "preview.ogg", "preview.mp3", "info.json"]),