Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion backend/settings.gradle
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
rootProject.name = 'backend'
include 'smartjam-api'
include 'smartjam-analyzer'
include 'smartjam-analyzer'
include 'smartjam-common'
3 changes: 3 additions & 0 deletions backend/smartjam-analyzer/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,7 @@ dependencies {
implementation 'be.tarsos.dsp:core:2.5'
implementation 'be.tarsos.dsp:jvm:2.5'


implementation project(':smartjam-common')

}
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;

import com.smartjam.smartjamanalyzer.api.kafka.dto.S3EventDto;
import com.smartjam.common.dto.s3.S3EventDto;
import com.smartjam.smartjamanalyzer.application.AudioAnalysisUseCase;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
Expand Down Expand Up @@ -32,9 +32,7 @@ public class S3StorageListener {
topics = "s3-events",
groupId = "smartjam-analyzer-group",
concurrency = "3",
properties = {
"spring.json.value.default.type=com.smartjam.smartjamanalyzer.api" + ".kafka.dto" + ".S3EventDto"
})
properties = {"spring.json.value.default.type=com.smartjam.common.dto.s3.S3EventDto"})
public void onFileUploaded(S3EventDto event, Acknowledgment ack) {
if (event == null || event.records() == null || event.records().isEmpty()) {
if (ack != null) ack.acknowledge();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@

import java.util.List;

import com.smartjam.common.dto.FeedbackEvent;

/**
* Encapsulates the results of a performance evaluation, including overall scores, time-aligned feedback events, and
* internal evaluation artifacts (like warping paths).
Expand Down Expand Up @@ -41,12 +43,4 @@ private static double[][] copyMatrix(double[][] src) {
public boolean isPassed() {
return totalScore > 80;
}

public record FeedbackEvent(
double teacherStartTime,
double teacherEndTime,
double studentStartTime,
double studentEndTime,
String message,
double severity) {}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import java.util.Collections;
import java.util.List;

import com.smartjam.common.dto.FeedbackEvent;
import com.smartjam.common.model.FeedbackType;
import com.smartjam.smartjamanalyzer.domain.model.AnalysisResult;
import com.smartjam.smartjamanalyzer.domain.model.FeatureSequence;
import com.smartjam.smartjamanalyzer.domain.port.PerformanceEvaluator;
Expand Down Expand Up @@ -37,20 +39,17 @@ public AnalysisResult evaluate(FeatureSequence reference, FeatureSequence studen
double[][] dtwMatrix = computeCostMatrix(reference.frames(), student.frames());
List<int[]> path = findWarpingPath(dtwMatrix);

List<AnalysisResult.FeedbackEvent> feedbacks = new ArrayList<>();
List<FeedbackEvent> feedbacks = new ArrayList<>();
PathMetrics metrics = analyzePath(path, reference, student, feedbacks);

return buildFinalResult(metrics, dtwMatrix, path, feedbacks);
}

private PathMetrics analyzePath(
List<int[]> path,
FeatureSequence reference,
FeatureSequence student,
List<AnalysisResult.FeedbackEvent> feedbacks) {
List<int[]> path, FeatureSequence reference, FeatureSequence student, List<FeedbackEvent> feedbacks) {

ErrorState pitchES = new ErrorState("Wrong note", reference.frameRate(), student.frameRate());
ErrorState rhythmES = new ErrorState("Wrong rhythm", reference.frameRate(), student.frameRate());
ErrorState pitchES = new ErrorState(FeedbackType.WRONG_NOTE, reference.frameRate(), student.frameRate());
ErrorState rhythmES = new ErrorState(FeedbackType.WRONG_RHYTHM, reference.frameRate(), student.frameRate());

double totalPitchDist = 0;
double totalRhythmDrift = 0;
Expand Down Expand Up @@ -86,7 +85,7 @@ private PathMetrics analyzePath(

private void processErrorState(
ErrorState state,
List<AnalysisResult.FeedbackEvent> feedbacks,
List<FeedbackEvent> feedbacks,
double tRef,
double tStud,
double severity,
Expand All @@ -97,15 +96,15 @@ private void processErrorState(
}
}

private void flushIfActive(ErrorState state, List<AnalysisResult.FeedbackEvent> feedbacks) {
private void flushIfActive(ErrorState state, List<FeedbackEvent> feedbacks) {
if (state.isActive()) {
AnalysisResult.FeedbackEvent ev = state.flush();
FeedbackEvent ev = state.flush();
if (ev != null) feedbacks.add(ev);
}
}

private AnalysisResult buildFinalResult(
PathMetrics metrics, double[][] matrix, List<int[]> path, List<AnalysisResult.FeedbackEvent> feedbacks) {
PathMetrics metrics, double[][] matrix, List<int[]> path, List<FeedbackEvent> feedbacks) {

double avgPitchError = metrics.totalPitchDist / path.size();
double pScore = Math.max(0.0, 100.0 * (1.0 - Math.pow(avgPitchError / 0.5, 2)));
Expand Down Expand Up @@ -188,7 +187,7 @@ List<int[]> findWarpingPath(double[][] dtw) {
private record PathMetrics(double totalPitchDist, double totalRhythmDrift) {}

private static class ErrorState {
private final String message;
private final FeedbackType type;
private final double refFrameDuration;
private final double studFrameDuration;
private double rStart = -1;
Expand All @@ -198,8 +197,8 @@ private static class ErrorState {
private double maxSev = 0;
private int graceCounter = 0;

ErrorState(String message, float refFrameRate, float studFrameRate) {
this.message = message;
ErrorState(FeedbackType type, float refFrameRate, float studFrameRate) {
this.type = type;
this.refFrameDuration = 1.0 / refFrameRate;
this.studFrameDuration = 1.0 / studFrameRate;
}
Expand Down Expand Up @@ -230,7 +229,7 @@ boolean isActive() {
return rStart >= 0;
}

AnalysisResult.FeedbackEvent flush() {
FeedbackEvent flush() {
double refDuration = (rEnd - rStart) + refFrameDuration;
double studDuration = (sEnd - sStart) + studFrameDuration;

Expand All @@ -239,8 +238,8 @@ AnalysisResult.FeedbackEvent flush() {
return null;
}

AnalysisResult.FeedbackEvent event = new AnalysisResult.FeedbackEvent(
rStart, rEnd + refFrameDuration, sStart, sEnd + studFrameDuration, message, maxSev);
FeedbackEvent event =
new FeedbackEvent(rStart, rEnd + refFrameDuration, sStart, sEnd + studFrameDuration, type, maxSev);
reset();
return event;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
import java.util.Collections;
import java.util.List;

import com.smartjam.common.dto.s3.S3EventDto;
import com.smartjam.smartjamanalyzer.api.kafka.S3StorageListener;
import com.smartjam.smartjamanalyzer.api.kafka.dto.S3EventDto;
import com.smartjam.smartjamanalyzer.application.AudioAnalysisUseCase;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ void shouldPerformFullAnalysisCycle() throws Exception {
String feedbackReport = result.feedback().stream()
.map(e -> String.format(
"-> [%s] с %5.2fs до %5.2fs (Ученик: с %5.2fs до %5.2fs) | Тяжесть: %.2f",
e.message(),
e.type(),
e.teacherStartTime(),
e.teacherEndTime(),
e.studentStartTime(),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
import java.util.Collections;
import java.util.List;

import com.smartjam.common.dto.FeedbackEvent;
import com.smartjam.common.model.FeedbackType;
import com.smartjam.smartjamanalyzer.domain.model.AnalysisResult;
import com.smartjam.smartjamanalyzer.domain.model.FeatureSequence;
import org.junit.jupiter.api.DisplayName;
Expand Down Expand Up @@ -103,7 +105,7 @@ void shouldPenalizeEarlyTermination() {

assertTrue(result.totalScore() < 50.0, "Score должен быть низким при обрыве записи");
boolean hasRhythmError =
result.feedback().stream().anyMatch(e -> e.message().equals("Wrong rhythm"));
result.feedback().stream().anyMatch(e -> e.type().equals(FeedbackType.WRONG_RHYTHM));
assertTrue(hasRhythmError);
}

Expand All @@ -128,8 +130,8 @@ void shouldMergeFrequentShortErrorsRealistic() {

AnalysisResult result = evaluator.evaluate(new FeatureSequence(ref, 20f), new FeatureSequence(stud, 20f));

List<AnalysisResult.FeedbackEvent> pitchEvents = result.feedback().stream()
.filter(e -> e.message().equals("Wrong note"))
List<FeedbackEvent> pitchEvents = result.feedback().stream()
.filter(e -> e.type().equals(FeedbackType.WRONG_NOTE))
.toList();
assertTrue(pitchEvents.size() < 5, "Ошибки должны склеиваться, лог не должен спамить");
}
Expand All @@ -149,7 +151,7 @@ void shouldBeSensitiveToSmallPitchShiftsRealistic() {
result.pitchScore() > 20.0 && result.pitchScore() < 80.0,
"Оценка должна быть промежуточной при реалистичных признаках");
boolean hasPitchError =
result.feedback().stream().anyMatch(e -> e.message().equals("Wrong note"));
result.feedback().stream().anyMatch(e -> e.type().equals(FeedbackType.WRONG_NOTE));
assertTrue(hasPitchError);
}

Expand All @@ -166,7 +168,7 @@ void severityShouldNotBeMaxForSmallError() {
AnalysisResult result = evaluator.evaluate(new FeatureSequence(ref, 20f), new FeatureSequence(stud, 20f));

result.feedback().stream()
.filter(e -> e.message().equals("Wrong note"))
.filter(e -> e.type().equals(FeedbackType.WRONG_NOTE))
.forEach(e -> assertTrue(e.severity() < 0.9, "Severity должна быть <0.9"));
}

Expand Down Expand Up @@ -206,11 +208,11 @@ void shortGapsShouldMergeRealistic() {

AnalysisResult result = evaluator.evaluate(refSeq, studSeq);

List<AnalysisResult.FeedbackEvent> pitchEvents = result.feedback().stream()
.filter(e -> e.message().equals("Wrong note"))
List<FeedbackEvent> pitchEvents = result.feedback().stream()
.filter(e -> e.type().equals(FeedbackType.WRONG_NOTE))
.toList();
assertEquals(1, pitchEvents.size(), "Ошибки должны склеиться в одно событие");
AnalysisResult.FeedbackEvent event = pitchEvents.getFirst();
FeedbackEvent event = pitchEvents.getFirst();
assertTrue(event.studentEndTime() - event.studentStartTime() > 2.0);
}

Expand Down Expand Up @@ -242,9 +244,9 @@ void longGapShouldSplitRealistic() {

AnalysisResult result = evaluator.evaluate(refSeq, studSeq);

List<AnalysisResult.FeedbackEvent> pitchEvents;
List<FeedbackEvent> pitchEvents;
pitchEvents = result.feedback().stream()
.filter(e -> e.message().equals("Wrong note"))
.filter(e -> e.type().equals(FeedbackType.WRONG_NOTE))
.toList();
assertEquals(2, pitchEvents.size(), "Длинный перерыв должен разделить ошибки");
}
Expand Down
3 changes: 3 additions & 0 deletions backend/smartjam-api/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -9,4 +9,7 @@ dependencies {


implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:3.0.2'


implementation project(':smartjam-common')
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package com.smartjam.smartjamapi.infrastructure.openapi;

import io.swagger.v3.oas.models.Components;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.info.Contact;
import io.swagger.v3.oas.models.info.Info;
import io.swagger.v3.oas.models.security.SecurityRequirement;
import io.swagger.v3.oas.models.security.SecurityScheme;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

/**
* Configuration for OpenAPI documentation using SpringDoc. Defines metadata, security schemes, and global requirements
* for the API.
*/
@Configuration
public class OpenApiConfig {

private static final String SECURITY_SCHEME_NAME = "bearerAuth";
/**
* Customizes the OpenAPI definition.
*
* @return a fully configured OpenAPI object.
*/
@Bean
public OpenAPI customOpenAPI() {
return new OpenAPI()
.info(createApiInfo())
.addSecurityItem(new SecurityRequirement().addList(SECURITY_SCHEME_NAME))
.components(new Components().addSecuritySchemes(SECURITY_SCHEME_NAME, createSecurityScheme()));
}

private Info createApiInfo() {
return new Info()
.title("SmartJam API")
.version("1.0")
.description("""
Interactive Music Learning and Performance Analysis System.

Development Team:
- Sanjar Satlykov ([satlykovs@gmail.com](mailto:satlykovs@gmail.com))
- Anton Podrezov ([toni.podrezov@gmail.com](mailto:toni.podrezov@gmail.com))
- Serj Baskov ([baskovs450@gmail.com](mailto:baskovs450@gmail.com))

Supervised by:
- Andrey Sheremeev ([sheremeev.andrey@gmail.com](mailto:sheremeev.andrey@gmail.com))
""")
.contact(new Contact()
.name("SmartJam Team")
.email("satlykovs@gmail.com")
.url("https://github.com/Satlykovs/SmartJam"));
}

private SecurityScheme createSecurityScheme() {
return new SecurityScheme()
.name(SECURITY_SCHEME_NAME)
.type(SecurityScheme.Type.HTTP)
.scheme("bearer")
.bearerFormat("JWT")
.description("Provide your JWT token obtained from the login endpoint.");
}
}
Binary file modified backend/smartjam-api/src/main/resources/application.yaml
Binary file not shown.
20 changes: 20 additions & 0 deletions backend/smartjam-common/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@

plugins {
id 'java-library'
}

bootJar {
enabled = false
}

jar {
enabled = true
}

dependencies {
api 'com.fasterxml.jackson.core:jackson-annotations'
api 'com.fasterxml.jackson.core:jackson-databind'
}

group = 'com.smartjam'
version = '0.0.1-SNAPSHOT'
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
package com.smartjam.common.dto;

import com.smartjam.common.model.FeedbackType;

/**
* A shared contract for a single feedback occurrence. Used by Analyzer to produce results and by API/Mobile to display
* them.
*/
public record FeedbackEvent(
double teacherStartTime,
double teacherEndTime,
double studentStartTime,
double studentEndTime,
FeedbackType type,
double severity) {}
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package com.smartjam.smartjamanalyzer.api.kafka.dto;
package com.smartjam.common.dto.s3;

import java.util.List;

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package com.smartjam.common.model;

/**
* Represents the lifecycle stages of an audio processing task. Used to track the state from the moment a database
* record is created until the final analysis result is stored.
*/
public enum AudioProcessingStatus {
AWAITING_UPLOAD,
UPLOADED,
ANALYZING,
COMPLETED,
FAILED
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package com.smartjam.common.model;

/**
* Categories of musical discrepancies identified during the comparison of a student's performance against the teacher's
* reference.
*/
public enum FeedbackType {
WRONG_NOTE,
WRONG_RHYTHM
}
Loading