diff --git a/annotationweb/migrations/0002_auto_20191118_1104.py b/annotationweb/migrations/0002_auto_20191118_1104.py deleted file mode 100644 index 7217c22..0000000 --- a/annotationweb/migrations/0002_auto_20191118_1104.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 2.2.1 on 2019-11-18 10:04 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('annotationweb', '0001_initial'), - ] - - operations = [ - migrations.AddField( - model_name='task', - name='post_processing_method', - field=models.CharField(default='', help_text='Name of post processing method to use', max_length=255), - ), - migrations.AlterField( - model_name='label', - name='color_blue', - field=models.PositiveSmallIntegerField(default=0), - ), - migrations.AlterField( - model_name='label', - name='color_green', - field=models.PositiveSmallIntegerField(default=0), - ), - migrations.AlterField( - model_name='label', - name='color_red', - field=models.PositiveSmallIntegerField(default=255), - ), - ] diff --git a/annotationweb/migrations/0003_keyframeannotation_frame_metadata.py b/annotationweb/migrations/0003_keyframeannotation_frame_metadata.py deleted file mode 100644 index d97100c..0000000 --- a/annotationweb/migrations/0003_keyframeannotation_frame_metadata.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.1 on 2019-11-19 13:47 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('annotationweb', '0002_auto_20191118_1104'), - ] - - operations = [ - migrations.AddField( - model_name='keyframeannotation', - name='frame_metadata', - field=models.CharField(default='', help_text='A text field for storing arbitrary metadata on the current frame', max_length=512), - ), - ] diff --git a/annotationweb/migrations/0004_imageannotation_finished.py b/annotationweb/migrations/0004_imageannotation_finished.py deleted file mode 100644 index 97de500..0000000 --- a/annotationweb/migrations/0004_imageannotation_finished.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.1 on 2020-11-16 14:28 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('annotationweb', '0003_keyframeannotation_frame_metadata'), - ] - - operations = [ - migrations.AddField( - model_name='imageannotation', - name='finished', - field=models.BooleanField(default=True), - ), - ] diff --git a/annotationweb/migrations/0005_auto_20201117_1447.py b/annotationweb/migrations/0005_auto_20201117_1447.py deleted file mode 100644 index 4f71cd0..0000000 --- a/annotationweb/migrations/0005_auto_20201117_1447.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.1 on 2020-11-17 13:47 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('annotationweb', '0004_imageannotation_finished'), - ] - - operations = [ - migrations.AlterField( - model_name='task', - name='post_processing_method', - field=models.CharField(blank=True, default='', help_text='Name of post processing method to use', max_length=255), - ), - ] diff --git a/annotationweb/models.py b/annotationweb/models.py index 698722c..9e8eadc 100644 --- a/annotationweb/models.py +++ b/annotationweb/models.py @@ -45,6 +45,7 @@ class Task(models.Model): CARDIAC_PLAX_SEGMENTATION = 'cardiac_plax_segmentation' CARDIAC_ALAX_SEGMENTATION = 'cardiac_alax_segmentation' SPLINE_SEGMENTATION = 'spline_segmentation' + SPLINE_LINE_POINT = 'spline_line_point' TASK_TYPES = ( (CLASSIFICATION, 'Classification'), (BOUNDING_BOX, 'Bounding box'), @@ -52,7 +53,8 @@ class Task(models.Model): (CARDIAC_SEGMENTATION, 'Cardiac apical segmentation'), (CARDIAC_PLAX_SEGMENTATION, 'Cardiac PLAX segmentation'), (CARDIAC_ALAX_SEGMENTATION, 'Cardiac ALAX segmentation'), - (SPLINE_SEGMENTATION, 'Spline segmentation') + (SPLINE_SEGMENTATION, 'Spline segmentation'), + (SPLINE_LINE_POINT, 'Splines, lines & point segmentation') ) name = models.CharField(max_length=200) diff --git a/annotationweb/settings.py b/annotationweb/settings.py index fc590d7..871ee9c 100644 --- a/annotationweb/settings.py +++ b/annotationweb/settings.py @@ -24,7 +24,7 @@ # python manage.py shell -c 'from django.core.management import utils; print(utils.get_random_secret_key())' # You can safely change this at any time, but note that logged in users will be logged out. # See more info here: https://medium.com/@bayraktar.eralp/changing-rotating-django-secret-key-without-logging-users-out-804a29d3ea65 -#SECRET_KEY = 'insert secret key her!' +#SECRET_KEY = 'insert secret key here!' # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True @@ -48,6 +48,7 @@ 'user', 'cardiac', 'spline_segmentation', + 'spline_line_point', 'cardiac_parasternal_long_axis', 'cardiac_apical_long_axis', 'django_otp', diff --git a/annotationweb/urls.py b/annotationweb/urls.py index ab79955..cec7470 100644 --- a/annotationweb/urls.py +++ b/annotationweb/urls.py @@ -40,7 +40,8 @@ path('cardiac/', include('cardiac.urls')), path('cardiac-plax/', include('cardiac_parasternal_long_axis.urls')), path('cardiac-alax/', include('cardiac_apical_long_axis.urls')), - path('spline-segmentation/', include('spline_segmentation.urls')) + path('spline-segmentation/', include('spline_segmentation.urls')), + path('spline-line-point/', include('spline_line_point.urls')) ] # This is for making statics in a development environment diff --git a/annotationweb/views.py b/annotationweb/views.py index 467ecbb..7f31433 100644 --- a/annotationweb/views.py +++ b/annotationweb/views.py @@ -446,6 +446,8 @@ def task_description(request, task_id): url = reverse('cardiac_parasternal_long_axis:segment_image', args=[task_id]) elif task.type == task.CARDIAC_ALAX_SEGMENTATION: url = reverse('cardiac_apical_long_axis:segment_image', args=[task_id]) + elif task.type == task.SPLINE_LINE_POINT: + url = reverse('spline_line_point:segment_image', args=[task_id]) else: raise NotImplementedError() @@ -601,6 +603,8 @@ def get_redirection(task): return 'cardiac_apical_long_axis:segment_image' elif task.type == Task.SPLINE_SEGMENTATION: return 'spline_segmentation:segment_image' + elif task.type == Task.SPLINE_LINE_POINT: + return 'spline_line_point:segment_image' # @register.simple_tag diff --git a/exporters/spline_line_point_exporter.py b/exporters/spline_line_point_exporter.py new file mode 100644 index 0000000..2500413 --- /dev/null +++ b/exporters/spline_line_point_exporter.py @@ -0,0 +1,185 @@ +from math import sqrt, floor, ceil +from common.exporter import Exporter +from common.metaimage import MetaImage +from common.utility import create_folder, copy_image +from annotationweb.models import ImageAnnotation, Dataset, Task, Label, Subject, KeyFrameAnnotation, ImageMetadata +from spline_segmentation.models import ControlPoint +from django import forms +import os +from os.path import join +from shutil import rmtree, copyfile +import numpy as np +import json +from scipy.ndimage.morphology import binary_fill_holes +import PIL + + +class SplineLinePointExporterForm(forms.Form): + path = forms.CharField(label='Storage path', max_length=1000) + delete_existing_data = forms.BooleanField(label='Delete any existing data at storage path', initial=False, required=False) + + def __init__(self, task, data=None): + super().__init__(data) + self.fields['subjects'] = forms.ModelMultipleChoiceField( + queryset=Subject.objects.filter(dataset__task=task)) + + +class SplineLinePointExporter(Exporter): + """ + asdads + """ + + task_type = Task.SPLINE_LINE_POINT + name = 'Splines, lines & point segmentation exporter' + + def get_form(self, data=None): + return SplineLinePointExporterForm(self.task, data=data) + + def export(self, form): + delete_existing_data = form.cleaned_data['delete_existing_data'] + # Create dir, delete old if it exists + path = form.cleaned_data['path'] + if delete_existing_data: + # Delete path + try: + os.stat(path) + rmtree(path) + except: + # Folder does not exist + pass + + # Create folder if it doesn't exist + create_folder(path) + + self.add_subjects_to_path(path, form.cleaned_data['subjects']) + + return True, path + + def add_subjects_to_path(self, path, data): + + # For each subject + for subject in data: + subject_path = join(path, subject.dataset.name, subject.name) + frames = KeyFrameAnnotation.objects.filter(image_annotation__task=self.task, image_annotation__image__subject=subject) + for frame in frames: + # Check if image was rejected + if frame.image_annotation.rejected: + continue + # Get image sequence + image_sequence = frame.image_annotation.image + + # Copy image frames + sequence_id = os.path.basename(os.path.dirname(image_sequence.format)) + subject_subfolder = join(subject_path, str(sequence_id)) + create_folder(subject_subfolder) + + target_name = os.path.basename(image_sequence.format).replace('#',str(frame.frame_nr)) + target_gt_name = os.path.splitext(target_name)[0]+"_gt.mhd" + + filename = image_sequence.format.replace('#', str(frame.frame_nr)) + new_filename = join(subject_subfolder, target_name) + copy_image(filename, new_filename) + + # Get control points to create segmentation + if new_filename.endswith('.mhd'): + image_mhd = MetaImage(filename=new_filename) + image_size = image_mhd.get_size() + spacing = image_mhd.get_spacing() + else: + image_pil = PIL.Image.open(new_filename) + image_size = image_pil.size + spacing = [1, 1] + self.save_segmentation(frame, image_size, join(subject_subfolder, target_gt_name), spacing) + + return True, path + + def get_object_segmentation(self, image_size, frame): + segmentation = np.zeros(image_size, dtype=np.uint8) + tension = 0.5 + + labels = Label.objects.filter(task=frame.image_annotation.task).order_by('id') + counter = 1 + for label in labels: + objects = ControlPoint.objects.filter(label=label, image=frame).only('object').distinct() + for object in objects: + previous_x = None + previous_y = None + control_points = ControlPoint.objects.filter(label=label, image=frame, object=object.object).order_by('index') + max_index = len(control_points) + + if max_index == 1: + x = int(round(control_points[0].x)) + y = int(round(control_points[0].y)) + segmentation[y, x] = counter + else: + for i in range(max_index): + if i == 0: + first = max_index-1 + else: + first = i-1 + a = control_points[first] + b = control_points[i] + c = control_points[(i+1) % max_index] + d = control_points[(i+2) % max_index] + length = sqrt((b.x - c.x)*(b.x - c.x) + (b.y - c.y)*(b.y - c.y)) + # Not a very elegant solution ... could try to estimate the spline length instead + # or draw straight lines between consecutive points instead + step_size = min(0.01, 1.0 / (length*2)) + for t in np.arange(0, 1, step_size): + x = (2 * t * t * t - 3 * t * t + 1) * b.x + \ + (1 - tension) * (t * t * t - 2.0 * t * t + t) * (c.x - a.x) + \ + (-2 * t * t * t + 3 * t * t) * c.x + \ + (1 - tension) * (t * t * t - t * t) * (d.x - b.x) + y = (2 * t * t * t - 3 * t * t + 1) * b.y + \ + (1 - tension) * (t * t * t - 2.0 * t * t + t) * (c.y - a.y) + \ + (-2 * t * t * t + 3 * t * t) * c.y + \ + (1 - tension) * (t * t * t - t * t) * (d.y - b.y) + + # Round and snap to borders + x = int(round(x)) + x = min(image_size[1]-1, max(0, x)) + y = int(round(y)) + y = min(image_size[0]-1, max(0, y)) + + if previous_x is not None and (abs(previous_x - x) > 1 or abs(previous_y - y) > 1): + # Draw a straight line between the points + end_pos = np.array([x,y]) + start_pos = np.array([previous_x,previous_y]) + direction = end_pos - start_pos + segment_length = np.linalg.norm(end_pos - start_pos) + direction = direction / segment_length # Normalize + for i in np.arange(0.0, np.ceil(segment_length), 0.5): + current = start_pos + direction * (float(i)/np.ceil(segment_length)) + current = np.round(current).astype(np.int32) + current[0] = min(image_size[1]-1, max(0, current[0])) + current[1] = min(image_size[0]-1, max(0, current[1])) + segmentation[current[1], current[0]] = counter + + previous_x = x + previous_y = y + + segmentation[y, x] = counter + + # Fill the hole + segmentation[binary_fill_holes(segmentation == counter)] = counter + + counter += 1 + + return segmentation + + def save_segmentation(self, frame, image_size, filename, spacing): + image_size = [image_size[1], image_size[0]] + + # Create compounded segmentation object + segmentation = self.get_object_segmentation(image_size, frame) + + segmentation_mhd = MetaImage(data=segmentation) + segmentation_mhd.set_attribute('ImageQuality', frame.image_annotation.image_quality) + segmentation_mhd.set_attribute('View', frame.image_annotation.comments) + segmentation_mhd.set_attribute('FrameType', frame.frame_metadata) + segmentation_mhd.set_spacing(spacing) + metadata = ImageMetadata.objects.filter(image=frame.image_annotation.image) + for item in metadata: + segmentation_mhd.set_attribute(item.name, item.value) + segmentation_mhd.write(filename) + diff --git a/importers/image_sequence_importer.py b/importers/image_sequence_importer.py index 10d80a9..e1f26dd 100644 --- a/importers/image_sequence_importer.py +++ b/importers/image_sequence_importer.py @@ -17,15 +17,13 @@ def __init__(self, data=None): class ImageSequenceImporter(Importer): """ Data should be sorted in the following way in the root folder: - Subject 1/ - Sequence 1/ - _0.mhd or .png - _1.mhd or .png - ... - Sequence 2/ - ... - Subject 2/ + Sequence 1/ + _0.mhd or .png + _1.mhd or .png ... + Sequence 2/ + ... + Allow both .mhd and .png sequences from the same root folder. Assumes that a single sequence does not mix .mhd and .png @@ -43,93 +41,97 @@ def import_data(self, form): raise Exception('Dataset must be given to importer') path = form.cleaned_data['path'] - # Go through each subfolder and create a subject for each - for file in os.listdir(path): - subject_dir = join(path, file) - if not os.path.isdir(subject_dir): + # Go through each subfolder and create a subject for each. EDIT: Assume path points to one subject + #for file in os.listdir(path): + #subject_dir = join(path, file) + subject_dir = path # NEW + file = subject_dir.split('/')[-1] # NEW + print('Filename:', file) + + if not os.path.isdir(subject_dir): + raise Exception('Provided path is not a directory') + + try: + # Check if subject exists in this dataset first + subject = Subject.objects.get(name=file, dataset=self.dataset) + except Subject.DoesNotExist: + # Create new subject + subject = Subject() + subject.name = file + subject.dataset = self.dataset + subject.save() + + for file2 in os.listdir(subject_dir): + image_sequence_dir = join(subject_dir, file2) + if not os.path.isdir(image_sequence_dir): continue - try: - # Check if subject exists in this dataset first - subject = Subject.objects.get(name=file, dataset=self.dataset) - except Subject.DoesNotExist: - # Create new subject - subject = Subject() - subject.name = file - subject.dataset = self.dataset - subject.save() - - for file2 in os.listdir(subject_dir): - image_sequence_dir = join(subject_dir, file2) - if not os.path.isdir(image_sequence_dir): - continue + # Count nr of frames + # Handle only monotype sequence: .mhd or .png + frames = [] + extension = None + name = '' + for file3 in os.listdir(image_sequence_dir): + if file3[-4:] == '.mhd': + image_filename = join(image_sequence_dir, file3) + frames.append(image_filename) + name = file3[:file3.rfind('_')] + if extension is None or extension == '.mhd': + extension = '.mhd' + else: + raise Exception('Found both mhd and png images in the same folder.') + elif file3[-4:] == '.png': + image_filename = join(image_sequence_dir, file3) + frames.append(image_filename) + name = file3[:file3.rfind('_')] + if extension is None or extension == '.png': + extension = '.png' + else: + raise Exception('Found both mhd and png images in the same folder.') - # Count nr of frames - # Handle only monotype sequence: .mhd or .png - frames = [] - extension = None - name = '' - for file3 in os.listdir(image_sequence_dir): - if file3[-4:] == '.mhd': - image_filename = join(image_sequence_dir, file3) - frames.append(image_filename) - name = file3[:file3.rfind('_')] - if extension is None or extension == '.mhd': - extension = '.mhd' - else: - raise Exception('Found both mhd and png images in the same folder.') - elif file3[-4:] == '.png': - image_filename = join(image_sequence_dir, file3) - frames.append(image_filename) - name = file3[:file3.rfind('_')] - if extension is None or extension == '.png': - extension = '.png' - else: - raise Exception('Found both mhd and png images in the same folder.') - - if len(frames) == 0: - continue + if len(frames) == 0: + continue - filename_format = join(image_sequence_dir, name + '_#') - filename_format += extension - try: - # Check to see if sequence exist - image_sequence = ImageSequence.objects.get(format=filename_format, subject=subject) - # Check to see that nr of sequences is correct - if image_sequence.nr_of_frames < len(frames): - # Delete this sequnce, and redo it - image_sequence.delete() - # Create new - image_sequence = ImageSequence() - image_sequence.format = filename_format - image_sequence.subject = subject - image_sequence.nr_of_frames = len(frames) - image_sequence.save() - else: - # Skip importing data, as this has already have been done - continue - except ImageSequence.DoesNotExist: - # Create new image sequence + filename_format = join(image_sequence_dir, name + '_#') + filename_format += extension + try: + # Check to see if sequence exist + image_sequence = ImageSequence.objects.get(format=filename_format, subject=subject) + # Check to see that nr of sequences is correct + if image_sequence.nr_of_frames < len(frames): + # Delete this sequnce, and redo it + image_sequence.delete() + # Create new image_sequence = ImageSequence() image_sequence.format = filename_format image_sequence.subject = subject image_sequence.nr_of_frames = len(frames) image_sequence.save() - - # Check if metadata.txt exists, and if so parse it and add - metadata_filename = join(image_sequence_dir, 'metadata.txt') - if os.path.exists(metadata_filename): - with open(metadata_filename, 'r') as f: - for line in f: - parts = line.split(':') - if len(parts) != 2: - raise Exception('Excepted 2 parts when spliting metadata in file ' + metadata_filename) - - # Save to DB - metadata = ImageMetadata() - metadata.image = image_sequence - metadata.name = parts[0].strip() - metadata.value = parts[1].strip() - metadata.save() + else: + # Skip importing data, as this has already have been done + continue + except ImageSequence.DoesNotExist: + # Create new image sequence + image_sequence = ImageSequence() + image_sequence.format = filename_format + image_sequence.subject = subject + image_sequence.nr_of_frames = len(frames) + image_sequence.save() + + # Check if metadata.txt exists, and if so parse it and add + metadata_filename = join(image_sequence_dir, 'metadata.txt') + if os.path.exists(metadata_filename): + with open(metadata_filename, 'r') as f: + for line in f: + parts = line.split(':') + if len(parts) != 2: + raise Exception('Excepted 2 parts when spliting metadata in file ' + metadata_filename) + + # Save to DB + metadata = ImageMetadata() + metadata.image = image_sequence + metadata.name = parts[0].strip() + metadata.value = parts[1].strip() + metadata.save() return True, path diff --git a/spline_line_point/__init__.py b/spline_line_point/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spline_line_point/admin.py b/spline_line_point/admin.py new file mode 100644 index 0000000..925adee --- /dev/null +++ b/spline_line_point/admin.py @@ -0,0 +1,4 @@ +from django.contrib import admin + + + diff --git a/spline_line_point/apps.py b/spline_line_point/apps.py new file mode 100644 index 0000000..b2f98b7 --- /dev/null +++ b/spline_line_point/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class SplineLinePoint(AppConfig): + name = 'spline_line_point' diff --git a/spline_line_point/migrations/__init__.py b/spline_line_point/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spline_line_point/models.py b/spline_line_point/models.py new file mode 100644 index 0000000..e69de29 diff --git a/spline_line_point/static/spline_line_point/segmentation.js b/spline_line_point/static/spline_line_point/segmentation.js new file mode 100644 index 0000000..5fcd263 --- /dev/null +++ b/spline_line_point/static/spline_line_point/segmentation.js @@ -0,0 +1,489 @@ +var g_backgroundImage; +var g_currentColor = null; +var g_controlPoints = {}; // control point dictionary [key_frame_nr] contains a dictionary/map of objects with a dictionary with label: label data, control_points: list +var g_currentObject = 0; // Which index in g_controlPoints[key_frame_nr] list we are in +var g_move = false; +var g_pointToMove = -1; +var g_labelToMove = -1; +var g_moveDistanceThreshold = 8; +var g_drawLine = false; +var g_currentLabel = -1; +var g_targetFrameTypes = {}; + +function getMaxObjectID() { + var max = -1; + for(var key in g_controlPoints[g_currentFrameNr]) { + key = parseInt(key); // dictionary keys are strings + if(key > max) + max = key; + } + return max; +} + +function setupSegmentation() { + + // Define event callbacks + $('#canvas').mousedown(function(e) { + if(e.ctrlKey && g_controlPoints[g_currentFrameNr][g_currentObject].control_points.length >= 1) { // Create new object if ctrl key is held down + g_currentObject = getMaxObjectID()+1; + data = {label: g_labelButtons[g_currentLabel], control_points: []}; + g_controlPoints[g_currentFrameNr][g_currentObject] = data; + } + + var scale = g_canvasWidth / $('#canvas').width(); + var mouseX = (e.pageX - this.offsetLeft)*scale; + var mouseY = (e.pageY - this.offsetTop)*scale; + var point = getClosestPoint(mouseX, mouseY); + if(point !== false) { + // Activate object of this point + g_currentObject = point.object; + $('.labelButton').removeClass('activeLabel'); + $('#labelButton' + point.label_idx).addClass('activeLabel'); + g_currentLabel = getLabelIdxWithId(point.label_idx); + // Move point + g_move = true; + g_pointToMove = point.index; + g_labelToMove = point.label_idx; + } else { + var section = isPointOnSpline(mouseX, mouseY); + if(section >= 0) { + // Insert point + insertControlPoint(mouseX, mouseY, g_labelButtons[g_currentLabel].id, section); + } else { + addControlPointsForNewFrame(g_currentFrameNr); + // Add point at end + addControlPoint(mouseX, mouseY, g_currentFrameNr, g_currentObject, g_labelButtons[g_currentLabel].id, g_shiftKeyPressed); + } + } + redrawSequence(); + }); + + $('#canvas').mousemove(function(e) { + var scale = g_canvasWidth / $('#canvas').width(); + var mouseX = (e.pageX - this.offsetLeft)*scale; + var mouseY = (e.pageY - this.offsetTop)*scale; + var cursor = 'default'; + + + if(g_move) { + cursor = 'move'; + setControlPoint(g_pointToMove, g_currentObject, mouseX, mouseY); + redrawSequence(); + } else { + if(!e.ctrlKey && + g_currentFrameNr in g_controlPoints && + g_currentObject in g_controlPoints[g_currentFrameNr] && + g_controlPoints[g_currentFrameNr][g_currentObject].control_points.length > 0 && + isPointOnSpline(mouseX, mouseY) < 0) { + // If mouse is not close to spline, draw dotted drawing line + var line = { + x0: getControlPoint(-1, g_currentObject).x, + y0: getControlPoint(-1, g_currentObject).y, + x1: mouseX, + y1: mouseY + }; + g_drawLine = line; + } else { + cursor = 'pointer'; + g_drawLine = false; + } + redrawSequence(); + } + $('#canvas').css({'cursor' : cursor}); + e.preventDefault(); + }); + + $('#canvas').mouseup(function(e) { + g_move = false; + }); + + $('#canvas').mouseleave(function(e) { + g_drawLine = false; + redrawSequence(); + }); + + $('#canvas').dblclick(function(e) { + if(g_move) + return; + var scale = g_canvasWidth / $('#canvas').width(); + var mouseX = (e.pageX - this.offsetLeft)*scale; + var mouseY = (e.pageY - this.offsetTop)*scale; + var point = getClosestPoint(mouseX, mouseY); + if(point !== false) { + g_controlPoints[g_currentFrameNr][g_currentObject].control_points.splice(point.index, 1); + //if(g_controlPoints[g_currentFrameNr[g_currentObject]].control_points.length == 0) { + // g_controlPoints[g_currentFrameNr].splice(g_currentObject, 1); // remove this object + //} + } + redrawSequence(); + }); + + + $("#clearButton").click(function() { + g_annotationHasChanged = true; + g_controlPoints[g_currentFrameNr][g_currentObject].control_points = []; + redrawSequence(); + }); + + $('#removeFrameButton').click(function() { + // Remove splines in this frame + if(g_currentFrameNr in g_controlPoints){ + delete g_controlPoints[g_currentFrameNr]; + } + redrawSequence(); + }); + + /* + $(document).keydown(function(event) { // Had to remove this to make saving work + if(event.ctrlKey) { + g_drawLine = false; + redrawSequence(); + } + }); + */ + + $("#addNormalFrameButton").click(function() { + setPlayButton(false); + if(g_targetFrames.includes(g_currentFrameNr)) // Already exists + return; + addKeyFrame(g_currentFrameNr, '#555555'); + g_targetFrameTypes[g_currentFrameNr] = 'Normal'; + g_currentTargetFrameIndex = g_targetFrames.length-1; + }); + + + $("#addEDFrameButton").click(function() { + setPlayButton(false); + if(g_targetFrames.includes(g_currentFrameNr)) // Already exists + return; + addKeyFrame(g_currentFrameNr, '#CC3434'); + g_targetFrameTypes[g_currentFrameNr] = 'ED'; + g_currentTargetFrameIndex = g_targetFrames.length-1; + }); + + + $("#addESFrameButton").click(function() { + setPlayButton(false); + if(g_targetFrames.includes(g_currentFrameNr)) // Already exists + return; + addKeyFrame(g_currentFrameNr, '#0077b3'); + g_targetFrameTypes[g_currentFrameNr] = 'ES'; + g_currentTargetFrameIndex = g_targetFrames.length-1; + }); + + $('#copyAnnotation').click(function() { + // Verify that we are on a target frame; + if(g_targetFrames.indexOf(g_currentFrameNr) < 0 || g_targetFrames.length === 1) + return; + + // Find previous frame + var frame_index = g_targetFrames.findIndex(index => index === g_currentFrameNr); + var copy_index = frame_index - 1; + if(copy_index < 0) + return; + + // Copy and potentially replace previous segmentation + g_controlPoints[g_currentFrameNr] = JSON.parse(JSON.stringify(g_controlPoints[g_targetFrames[copy_index]])); // Hack for doing deep copy + redrawSequence(); + }); + + // Set first label active + changeLabel(g_labelButtons[0].id) + + redraw(); + + +} + +function addControlPointsForNewFrame(frameNr) { + if(frameNr in g_controlPoints) // Already exists + return; + g_controlPoints[frameNr] = {}; +} + +function loadSegmentationTask(image_sequence_id) { + + g_backgroundImage = new Image(); + g_backgroundImage.src = '/show_frame/' + image_sequence_id + '/' + 0 + '/' + g_taskID + '/'; + g_backgroundImage.onload = function() { + g_canvasWidth = this.width; + g_canvasHeight = this.height; + setupSegmentation(); + }; +} + +function getClosestPoint(x, y) { + var minDistance = g_canvasWidth*g_canvasHeight; + var minPoint = -1; + var minLabel = -1; + var minObject = -1; + + for(var j in g_controlPoints[g_currentFrameNr]) { + for(var i = 0; i < g_controlPoints[g_currentFrameNr][j].control_points.length; i++) { + var point = getControlPoint(i, j); + var distance = Math.sqrt((point.x - x) * (point.x - x) + (point.y - y) * (point.y - y)); + if(distance < minDistance) { + minPoint = i; + minDistance = distance; + minLabel = g_controlPoints[g_currentFrameNr][j].label.id; + minObject = j; + } + } + } + + if(minDistance < g_moveDistanceThreshold) { + return {index: minPoint, label_idx: minLabel, object: minObject}; + } else { + return false; + } +} + +function createControlPoint(x, y, label, uncertain) { + // Find label index + var labelIndex = 0; + for(var i = 0; i < g_labelButtons.length; i++) { + if(g_labelButtons[i].id == label) { + labelIndex = i; + break; + } + } + + var controlPoint = { + x: x, + y: y, + label_id: label, // actual DB id + label: labelIndex, // index: only used for color + uncertain: uncertain // whether the user is uncertain about this point or not + }; + return controlPoint; +} + +function insertControlPoint(x, y, label, index) { + var controlPoint = createControlPoint(x, y, label, g_shiftKeyPressed); + g_controlPoints[g_currentFrameNr][g_currentObject].control_points.splice(index+1, 0, controlPoint); +} + +function addControlPoint(x, y, target_frame, object, label, uncertain) { + var controlPoint = createControlPoint(x, y, label, uncertain); + if(!(object in g_controlPoints[target_frame])) // add object if it doesn't exist + g_controlPoints[target_frame][object] = {label: g_labelButtons[getLabelIdxWithId(label)], control_points: []}; + g_controlPoints[target_frame][object].control_points.push(controlPoint); +} + +function getControlPoint(index, object) { + if(index === -1) { + index = g_controlPoints[g_currentFrameNr][object].control_points.length-1; + } + return g_controlPoints[g_currentFrameNr][object].control_points[index]; +} + +function setControlPoint(index, object, x, y) { + g_controlPoints[g_currentFrameNr][object].control_points[index].x = x; + g_controlPoints[g_currentFrameNr][object].control_points[index].y = y; +} + +function isPointOnSpline(pointX, pointY) { + for(var object in g_controlPoints[g_currentFrameNr]) { + for (var i = 0; i < g_controlPoints[g_currentFrameNr][object].control_points.length; ++i) { + var a = getControlPoint(max(0, i - 1), object); + var b = getControlPoint(i, object); + var c = getControlPoint(min(g_controlPoints[g_currentFrameNr][object].control_points.length - 1, i + 1), object); + var d = getControlPoint(min(g_controlPoints[g_currentFrameNr][object].control_points.length - 1, i + 2), object); + + var step = 0.1; + var tension = 0.5; + for (var t = 0.0; t < 1; t += step) { + var x = + (2 * t * t * t - 3 * t * t + 1) * b.x + + (1 - tension) * (t * t * t - 2.0 * t * t + t) * (c.x - a.x) + + (-2 * t * t * t + 3 * t * t) * c.x + + (1 - tension) * (t * t * t - t * t) * (d.x - b.x); + var y = + (2 * t * t * t - 3 * t * t + 1) * b.y + + (1 - tension) * (t * t * t - 2.0 * t * t + t) * (c.y - a.y) + + (-2 * t * t * t + 3 * t * t) * c.y + + (1 - tension) * (t * t * t - t * t) * (d.y - b.y); + if (Math.sqrt((pointX - x) * (pointX - x) + (pointY - y) * (pointY - y)) < g_moveDistanceThreshold) { + return i; + } + } + } + } + return -1; +} + +function redraw(){ + var index = g_currentFrameNr - g_startFrame; + g_context.drawImage(g_sequence[index], 0, 0, g_canvasWidth, g_canvasHeight); + + if(!(g_currentFrameNr in g_controlPoints)) + return; + + var controlPointSize = 6; + var landmarkPointSize = controlPointSize*1.5 + g_context.lineWidth = 2; + + // Draw controlPoint + for(var j in g_controlPoints[g_currentFrameNr]) { + var label = g_controlPoints[g_currentFrameNr][j].label; + for(var i = 0; i < g_controlPoints[g_currentFrameNr][j].control_points.length; i++) { + g_context.beginPath(); + g_context.strokeStyle = colorToHexString(label.red, label.green, label.blue); + + // Draw line as spline + + var maxIndex = g_controlPoints[g_currentFrameNr][j].control_points.length; + var first; + if (i === 0) { + first = maxIndex - 1; + } else { + first = i - 1; + } + + var a = getControlPoint(first, j); + var b = getControlPoint(i, j); + var c = getControlPoint((i + 1) % maxIndex, j); + var d = getControlPoint((i + 2) % maxIndex, j); + + var step = 0.1; + var tension = 0.5; + + if (b.uncertain || c.uncertain) { + // Draw uncertain segments with dashed line + g_context.setLineDash([1, 5]); // dashes are 5px and spaces are 5px + } else { + g_context.setLineDash([]); // reset + } + + var pointList = [a, b, c, d]; + drawSpline(pointList, step, tension) + + // Draw control point + if (g_controlPoints[g_currentFrameNr][j].control_points.length == 1) { + g_context.fillStyle = g_context.strokeStyle; + g_context.fillRect(b.x - landmarkPointSize / 2, b.y - landmarkPointSize / 2, landmarkPointSize, landmarkPointSize); + } else { + g_context.fillStyle = colorToHexString(255, 255, 0); + g_context.fillRect(b.x - controlPointSize / 2, b.y - controlPointSize / 2, controlPointSize, controlPointSize); + } + + + } + + /* + // Draw spline between end points + if(g_controlPoints[labelIndex].length > 4) { + var a = getControlPoint(g_controlPoints[labelIndex].length-2, labelIndex); + var b = getControlPoint(g_controlPoints[labelIndex].length-1, labelIndex); + var c = getControlPoint(0, labelIndex); + var d = getControlPoint(1, labelIndex); + + var pointList = [a,b,c,d]; + drawSpline(pointList, step, tension); + }*/ + } + + if(g_drawLine !== false) { + g_context.beginPath(); + g_context.setLineDash([5, 5]); // dashes are 5px and spaces are 5px + var label = g_labelButtons[g_currentLabel]; + g_context.strokeStyle = colorToHexString(label.red, label.green, label.blue); + g_context.moveTo(g_drawLine.x0, g_drawLine.y0); + g_context.lineTo(g_drawLine.x1, g_drawLine.y1); + g_context.stroke(); + g_context.setLineDash([]); // Clear + } +} + +// Override redraw sequence in sequence.js +function redrawSequence() { + var index = g_currentFrameNr - g_startFrame; + g_context.drawImage(g_sequence[index], 0, 0, g_canvasWidth, g_canvasHeight); + redraw(); +} + +function getLabelIdxWithId(id) { + for(var i = 0; i < g_labelButtons.length; i++) { + if(g_labelButtons[i].id == id) { + return i; + } + } +} + +// Override of annotationweb.js +function changeLabel(label_id) { + console.log('changing label to: ', label_id) + $('.labelButton').removeClass('activeLabel'); + $('#labelButton' + label_id).addClass('activeLabel'); + g_currentLabel = getLabelIdxWithId(label_id); + + // changeLabel is called when we press the label button (may or may not have an object), + // and when we press a control point (already have object) + if(g_currentFrameNr in g_controlPoints) { + var found = false; + for(var j in g_controlPoints[g_currentFrameNr]) { + if(g_controlPoints[g_currentFrameNr][j].label.id == label_id) { + g_currentObject = j; + found = true; + break; + } + } + if(!found) { + // Create new object id + g_currentObject = getMaxObjectID()+1; + g_controlPoints[g_currentFrameNr][g_currentObject] = {label: g_labelButtons[getLabelIdxWithId(label)], control_points: []}; + } + } +} + +function drawSpline(pointList, step_size, tension){ + var a = pointList[0]; + var b = pointList[1]; + var c = pointList[2]; + var d = pointList[3]; + + prev_x = -1; + prev_y = -1; + + for(var t = 0.0; t < 1; t += step_size) { + var x = (2 * t * t * t - 3 * t * t + 1) * b.x + + (1 - tension) * (t * t * t - 2.0 * t * t + t) * (c.x - a.x) + + (-2 * t * t * t + 3 * t * t) * c.x + + (1 - tension) * (t * t * t - t * t) * (d.x - b.x); + var y = (2 * t * t * t - 3 * t * t + 1) * b.y + + (1 - tension) * (t * t * t - 2.0 * t * t + t) * (c.y - a.y) + + (-2 * t * t * t + 3 * t * t) * c.y + + (1 - tension) * (t * t * t - t * t) * (d.y - b.y); + + // Draw line + if(prev_x >= 0) { + g_context.moveTo(prev_x, prev_y); + g_context.lineTo(x, y); + g_context.stroke(); + } + prev_x = x; + prev_y = y; + } +} + + +function sendDataForSave() { + return $.ajax({ + type: "POST", + url: "/spline-line-point/save/", + data: { + control_points: JSON.stringify(g_controlPoints), + target_frames: JSON.stringify(g_targetFrames), + target_frame_types: JSON.stringify(g_targetFrameTypes), + n_labels: g_labelButtons.length, + width: g_canvasWidth, + height: g_canvasHeight, + image_id: g_imageID, + task_id: g_taskID, + rejected: g_rejected ? 'true':'false', + comments: $('#comments').val(), + quality: $('input[name=quality]:checked').val(), + }, + dataType: "json" // Need this do get result back as JSON + }); +} diff --git a/spline_line_point/templates/spline_line_point/do_task.html b/spline_line_point/templates/spline_line_point/do_task.html new file mode 100644 index 0000000..985d3c8 --- /dev/null +++ b/spline_line_point/templates/spline_line_point/do_task.html @@ -0,0 +1,182 @@ + +{% extends 'annotationweb/two_column_layout.html' %} + +{% block javascript %} + +initializeAnnotation({{ task.id }}, {{ image.id }}); + +{% if image_sequence %} +loadSequence( + {{ image_sequence.id }}, + {{ image_sequence.start_frame_nr }}, + {{ image_sequence.nr_of_frames }}, + {% if task.show_entire_sequence %}true{% else %}false{% endif %}, + {% if task.user_frame_selection %}true{% else %}false{% endif %}, + {% if task.annotate_single_frame %}true{% else %}false{% endif %}, + [ {% for frame_nr in frames %}{{ frame_nr }},{% endfor %}], + {{ task.frames_before }}, + {{ task.frames_after }}, + {% if task.auto_play %}true{% else %}false{% endif %} +); +{% endif %} + +{# Add label buttons #} +{% for label in labels %} + addLabelButton({{ label.id }}, {{ label.color_red }}, {{ label.color_green }}, {{ label.color_blue }}, {% if label.parent %}{{ label.parent.id }}{% else %}0{% endif %}); +{% endfor %} + +{% for frame in target_frames %} + addControlPointsForNewFrame({{ frame.frame_nr }}); + g_targetFrameTypes['{{ frame.frame_nr }}'] = '{{ frame.frame_metadata }}'; + // A hack for updating markers + g_progressbar.on('markercomplete', function() { + var marker_index = {{ frame.frame_nr }}; + var type = g_targetFrameTypes['{{ frame.frame_nr }}']; + var color = '#555555'; + if(type == 'ED') { + color = '#CC3434'; + } else if(type == 'ES') { + color = '#0077b3'; + } + console.log('Updating marker ' + marker_index + ' with ' + color); + $('#sliderMarker' + marker_index).css('background-color', color); + }); +{% endfor %} + +{% block task_javascript %} +{% endblock %} + +{% if return_url %} +setReturnURL('{{ return_url|safe }}'); +{% endif %} + +{% endblock javascript %} + +{% block content_left %} + +{% for message in messages %} +{{ message }} +{% endfor %} +
+ +

{{ task.name }}

+ +{{ task.number_of_annotated_images }} of {{ task.total_number_of_images}} videos/images have been labeled ({{ task.percentage_finished }}%) + +{% if image_sequence %} +

Sequence

+
Loading...
+
Drag the slider to view the other frames in the sequence. Current frame
+
+
+ + {% if task.user_frame_selection %} + + + + + {% endif %} + + +
+{% endif %} +

Actions

+
+ {% if previous_image_id %} + + {% endif %} + {% if next_image_id %} + + {% endif %} + + + + +
+
+ You have done changes to the annotation.
+ Do you wish to save the changes before going to the next/previous image? +
+

Annotation

+
+
+ Image quality: + {% for quality_id, quality_name in image_quality_choices %} + {{ quality_name }} + {% endfor %} +
+
+ +
+{% block label_buttons %} +
+ {% for label in toplabels %} + + {% endfor %} +
+{% for sublabel in sublabels %} +
+
+
+ {% for label in sublabel.labels %} + + {% endfor %} +
+
+{% endfor %} +{% endblock label_buttons %} +
+
+
+{% block task_instructions %} +{% endblock %} +
+ +
+

Comments

+ +
+ +{% endblock content_left %} + +{% block content_right %} + +Failed to show images. Canvas probably not supported in the browser. + +{% block task_content %} +{% endblock %} + +
+ +
+ Dataset: {{ image_sequence.subject.dataset.name }} + Subject: {{ image_sequence.subject.name }}
+ Filename: {{ image_sequence.format }}
+
+ + + +{% if image.metadata_set.count > 0 %} +

Image metadata

+{% for metadata in image.metadata_set.all %} +{{ metadata.name }}: {{ metadata.value }}
+{% endfor %} +{% endif %} + +{% if task.description|length > 0 %} +

Task description

+ +{{ task.description|safe }} +{% endif %} + +{% endblock content_right %} diff --git a/spline_line_point/templates/spline_line_point/segment_image.html b/spline_line_point/templates/spline_line_point/segment_image.html new file mode 100644 index 0000000..5d53fe8 --- /dev/null +++ b/spline_line_point/templates/spline_line_point/segment_image.html @@ -0,0 +1,35 @@ +{% extends 'spline_line_point/do_task.html' %} + +{% block task_javascript %} + {% if image_sequence %} + loadSegmentationTask({{ image_sequence.id }}); + {% endif %} + + {% for frame in target_frames %} + addControlPointsForNewFrame({{ frame.frame_nr }}); + {% endfor %} + + {% for control_point in control_points %} + addControlPoint({{ control_point.x }}, {{ control_point.y }}, {{ control_point.image.frame_nr }}, {{ control_point.object }}, {{ control_point.label_id }}, {% if control_point.uncertain %}true{% else %}false{% endif %}); + {% endfor %} +{% endblock task_javascript %} + +{% block task_instructions %} + + + +{% endblock task_instructions %} \ No newline at end of file diff --git a/spline_line_point/tests.py b/spline_line_point/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/spline_line_point/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/spline_line_point/urls.py b/spline_line_point/urls.py new file mode 100644 index 0000000..74fbde2 --- /dev/null +++ b/spline_line_point/urls.py @@ -0,0 +1,10 @@ +from django.urls import path +from . import views + +app_name = 'spline_line_point' +urlpatterns = [ + path('segment-image//', views.segment_next_image, name='segment_image'), + path('segment-image///', views.segment_image, name='segment_image'), + path('show///', views.show_segmentation, name='show_segmentation'), + path('save/', views.save_segmentation, name='save'), +] diff --git a/spline_line_point/views.py b/spline_line_point/views.py new file mode 100644 index 0000000..72bca15 --- /dev/null +++ b/spline_line_point/views.py @@ -0,0 +1,89 @@ +from django.shortcuts import render, redirect +from django.contrib import messages +from django.http import JsonResponse, HttpResponseRedirect +from annotationweb.models import Task, KeyFrameAnnotation +import common.task +import json +from spline_segmentation.models import * +from django.db import transaction + + +def segment_next_image(request, task_id): + return segment_image(request, task_id, None) + + +def segment_image(request, task_id, image_id): + try: + context = common.task.setup_task_context(request, task_id, Task.SPLINE_LINE_POINT, image_id) + context['javascript_files'] = ['spline_line_point/segmentation.js'] + + # Check if image is already segmented, if so get data and pass to template + try: + annotations = KeyFrameAnnotation.objects.filter(image_annotation__task_id=task_id, image_annotation__image_id=image_id) + control_points = ControlPoint.objects.filter(image__in=annotations).order_by('index') + context['control_points'] = control_points + context['target_frames'] = annotations + except KeyFrameAnnotation.DoesNotExist: + pass + + return render(request, 'spline_line_point/segment_image.html', context) + except common.task.NoMoreImages: + messages.info(request, 'This task is finished, no more images to segment.') + return redirect('index') + except RuntimeError as e: + messages.error(request, str(e)) + return HttpResponseRedirect(request.META.get('HTTP_REFERER')) + + +def save_segmentation(request): + error_messages = '' + + control_points = json.loads(request.POST['control_points']) + target_frame_types = json.loads(request.POST['target_frame_types']) + n_labels = int(request.POST['n_labels']) + + try: + # Use atomic transaction here so if something crashes the annotations are restored.. + with transaction.atomic(): + annotations = common.task.save_annotation(request) + + # Save segmentation + # Save control points + for annotation in annotations: + frame_nr = str(annotation.frame_nr) + + # Set frame metadata + annotation.frame_metadata = target_frame_types[frame_nr] + annotation.save() + + for object in control_points[frame_nr]: + nr_of_control_points = len(control_points[frame_nr][object]['control_points']) + if nr_of_control_points < 1: + continue + for point in range(nr_of_control_points): + control_point = ControlPoint() + control_point.image = annotation + control_point.x = float(control_points[frame_nr][object]['control_points'][point]['x']) + control_point.y = float(control_points[frame_nr][object]['control_points'][point]['y']) + control_point.index = point + control_point.object = int(object) + control_point.label = Label.objects.get(id=int(control_points[frame_nr][object]['label']['id'])) + control_point.uncertain = bool(control_points[frame_nr][object]['control_points'][point]['uncertain']) + control_point.save() + + response = { + 'success': 'true', + 'message': 'Annotation saved', + } + except Exception as e: + response = { + 'success': 'false', + 'message': str(e), + } + raise e + + return JsonResponse(response) + + +def show_segmentation(request, task_id, image_id): + pass