Skip to content

text_complexite_microservice added #84

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gwyndol1ne wants to merge 2 commits into
base: master
Choose a base branch
from
Open

text_complexite_microservice added #84

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a6273b5
text_complexite_microservice added
Oct 11, 2024
a1c5f7c
fix
Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions src/main/webapp/src/components/TextComplexity/TextComplexityPage.css
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
.metrics-result {
padding: 20px;
max-width: 800px;
margin: 0 auto;
background-color: #f9f9f9;
border-radius: 8px;
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.1);
}

.metrics-list {
list-style-type: none;
padding: 0;
display: flex;
flex-direction: column;
gap: 10px;
}

.metric-item {
padding: 10px;
border: 1px solid #ddd;
border-radius: 4px;
cursor: pointer;
background: #ffffff;
display: flex;
justify-content: space-between;
align-items: center;
}

.metric-item:hover {
background-color: #f0f0f0;
}

.metric-name {
font-weight: bold;
}

.metric-value {
color: #333;
}

.metric-details {
margin-top: 20px;
background: #ffffff;
border: 1px solid #ddd;
border-radius: 8px;
padding: 15px;
box-shadow: 0 2px 5px rgba(0, 0, 0, 0.15);
}

.close-button {
background: #ff6b6b;
color: #ffffff;
border: none;
border-radius: 4px;
padding: 5px 10px;
cursor: pointer;
}

.close-button:hover {
background: #ff4a4a;
}
51 changes: 51 additions & 0 deletions src/main/webapp/src/components/TextComplexity/TextComplexityPage.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
import React, { useState } from 'react';
import './MetricsResult.css';

type Metric = {
name: string;
value: number | string;
description: string;
};

interface MetricsResultProps {
data: Metric[];
}

const MetricsResult: React.FC<MetricsResultProps> = ({ data }) => {
const [selectedMetric, setSelectedMetric] = useState<Metric | null>(null);

const handleMetricClick = (metric: Metric) => {
setSelectedMetric(metric);
};

const handleClosePanel = () => {
setSelectedMetric(null);
};

return (
<div className="metrics-result">
<h2>Результаты анализа предложений</h2>
<ul className="metrics-list">
{data.map((metric, index) => (
<li
key={index}
className="metric-item"
onClick={() => handleMetricClick(metric)}
>
<span className="metric-name">{metric.name}:</span>
<span className="metric-value">{metric.value}</span>
</li>
))}
</ul>

{selectedMetric && (
<div className="metric-details">
<button className="close-button" onClick={handleClosePanel}>Закрыть</button>
<h3>{selectedMetric.name}</h3>
</div>
)}
</div>
);
};

export default MetricsResult;
69 changes: 69 additions & 0 deletions src/main/webapp/src/pages/guide.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import React, { useState } from 'react';
import { Card, Text, Button, Title } from '@mantine/core';
import firstStepGif from "../resources/firstStep.gif"
import secondStepGif from "../resources/secondStep.gif"
import thirdStepGif from "../resources/thirdStep.gif"
import fourthStepGif from "../resources/fourthStep.gif"
import fifthStepGif from "../resources/fifthStep.gif"
import sixthStepGif from "../resources/sixthStep.gif"
import '../css/guide.css';
const GuidePage: React.FC = () => {
const [selectedCard, setSelectedCard] = useState<number | null>(null);
const arrowDownCard = 2;
const arrowRightCards = [0,1];
const arrowLeftCards = [5,4];
const steps = [
{ text: 'Для начала вам следует выбрать набор правил. Если вы хотите провести проверку со всеми правилами, тогда игнорируйте этот пункт. Вам следует перейти в пункт "Наборы правил и там выбрать один из существующих наборов или создать свой."', gif: firstStepGif },
{ text: 'После выбора набора правил, вам следует перейти на страницу для загрузки файлов. Здесь надо выбрать файл и начать загрузку. После обработки нажмите на количество ошибок и сможете перейти на обзор файла.', gif: secondStepGif },
{ text: 'Вам откроется страница с тремя полями, в первом идет список ошибок, нажав на которые их можно просмотреть. Также тут возможно фильтровать ошибки по названиям или страницам.', gif: thirdStepGif },
{ text: 'Наконец, если среди нарушений вы нашли ошибочное, например, вам кажется, что тут все правильно, а отчет говорит обратное, то стоит отправить нам отчет, нажав на соответствующую кнопку. ', gif: sixthStepGif },
{ text: 'По центру будет поле со страницей, на которой подчеркнута ошибка. Если ошибка занимает больше одной строки, то будут подчеркнуты несколько строк.', gif: fifthStepGif },
{ text: 'Слева будет поле, на котором будут все найденные типы ошибок. Если навести на тип ошибки, то можно увидеть описание данной ошибки. Также рядом написан тип ошибки: "предупреждение" или "ошибка". Предупреждения не так принципиальны, как ошибки.', gif: fourthStepGif },
];

const handleCardClick = (index: number) => {
if (selectedCard === index) {
setSelectedCard(null);
} else {
setSelectedCard(index);
}
};
return (
<div className='guide'>
<Title className="name">
Приложение{' '}<br />
<Text component="span" color="violet">
MAP
</Text>{' '}<br />-
это инструмент<br /> для проверки студенческих <br />работ
</Title>
<div>
<Text className='description'>Ниже приведен демо файл для тестового прогона программы. Также снизу есть пошаговая инструкция.</Text>
</div>
<Button component="a" href={require('../resources/Demo.pdf')} download>Скачать файл</Button>
<div style={{ display: 'flex', flexWrap: 'wrap' }}>
{steps.map((step, index) => (
<Card
className='stepCard'
key={index}
shadow="sm"
style={{
transform: selectedCard === index ? 'rotateY(180deg)' : 'rotateY(0deg)'
}}
onClick={() => handleCardClick(index)}
>
<div style={{ transform: selectedCard === index ? 'rotateY(180deg)' : 'rotateY(0deg)'}}>
<Text>{step.text}</Text>
{index === arrowDownCard && selectedCard !== index && <div className="arrow down">➜➤</div>}
{arrowLeftCards.includes(index) && selectedCard !== index && <div className="arrow left">➜➤</div>}
{selectedCard !== index && arrowRightCards.includes(index) && <div className="arrow right">➜➤</div>}
{selectedCard === index && <img src={step.gif} alt={`Step ${index + 1}`} style={{ width: '100%', height: '70%' }} />}
</div>
</Card>
))}
</div>
</div>
);
};

export default GuidePage;
6 changes: 6 additions & 0 deletions text_complexity_microservice/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
FROM python:3.10-slim-buster
WORKDIR /code
COPY ./requirements.txt /code/requirements.txt
RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
COPY ./src/app /code/src/app
CMD ["uvicorn", "src.app.api:app", "--host", "0.0.0.0", "--port", "8000"]
6 changes: 6 additions & 0 deletions text_complexity_microservice/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
import uvicorn

from src.app.api import app

if __name__ == '__main__':
uvicorn.run(app, host="localhost", port=8000)
9 changes: 9 additions & 0 deletions text_complexity_microservice/requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
fastapi==0.85.1
uvicorn==0.19.0
orjson==3.8.1
spacy==3.4.2
spacy_conll==3.2.0
pydantic==1.10.2
textcomplexity==0.11.0

ru_core_news_sm @ https://github.com/explosion/spacy-models/releases/download/ru_core_news_sm-3.4.0/ru_core_news_sm-3.4.0-py3-none-any.whl
Empty file added text_complexity_microservice/src/__init__.py
View file
Open in desktop
Empty file.
Empty file added text_complexity_microservice/src/app/__init__.py
View file
Open in desktop
Empty file.
48 changes: 48 additions & 0 deletions text_complexity_microservice/src/app/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
from fastapi import FastAPI, Query
from fastapi.responses import ORJSONResponse

from src.app import measure_calculator
from src.app.conll_converter import ConllConverter
from src.app.measure import SentenceMeasures
from src.app.models.request import SentenceMeasuresRequest, MeasureName
from src.app.models.response import SentenceMeasuresResponse, MeasureResult

app = FastAPI(title="text-complexity-service", default_response_class=ORJSONResponse)

conll_converters = {}


@app.on_event("startup")
async def startup_event():
for lang, conv in ConllConverter.all_language_converters().items():
conll_converters[lang] = conv


@app.get("/", include_in_schema=False)
async def openapi():
return app.openapi()


@app.post("/sentence_measures", response_model=list[SentenceMeasuresResponse])
async def sentence_measures(body: list[SentenceMeasuresRequest], measure: set[MeasureName] = Query(default=None)):
response = []
if not measure:
measure = [m for m in MeasureName]
measure = [SentenceMeasures[m] for m in measure]

for item in body:
if not item.sentence.strip():
response.append(SentenceMeasuresResponse(id=item.id, error="Empty sentence"))
continue

conll_sentences = conll_converters[item.language].text2conll_str(item.sentence).split("\n\n")
if len(conll_sentences) != 1:
response.append(SentenceMeasuresResponse(id=item.id, error="Required one sentence"))
continue

conll_sentence = conll_sentences[0]
calculated_measures = [(m.name, measure_calculator.calculate_for_sentence(conll_sentence, m))
for m in measure]
calculated_measures = [MeasureResult(name=res[0], value=res[1]) for res in calculated_measures]
response.append(SentenceMeasuresResponse(id=item.id, measures=calculated_measures))
return response
28 changes: 28 additions & 0 deletions text_complexity_microservice/src/app/conll_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
from spacy_conll import init_parser


class ConllConverter:
_spacy_models = {
"ru": "ru_core_news_sm",
}

def __init__(self, lang: str):
if lang not in ConllConverter.supported_languages():
raise ValueError("Unsupported language")

self.nlp = init_parser(ConllConverter._spacy_models[lang],
"spacy",
conversion_maps={"deprel": {"ROOT": "root"}})

def text2conll_str(self, text: str) -> str:
doc = self.nlp(text)
return doc._.conll_str

@staticmethod
def supported_languages() -> list[str]:
return list(ConllConverter._spacy_models.keys())

@staticmethod
def all_language_converters() -> dict[str, 'ConllConverter']:
languages = ConllConverter.supported_languages()
return dict([(lang, ConllConverter(lang)) for lang in languages])
58 changes: 58 additions & 0 deletions text_complexity_microservice/src/app/measure.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
from enum import Enum
from typing import Callable, NamedTuple

from textcomplexity import surface, dependency, sentence


class MeasureType(Enum):
text = 0
sentence = 1
punctuation = 2
graph = 3


class Measure(NamedTuple):
text_name: str
type: MeasureType
func: Callable


class SentenceMeasures(Measure, Enum):
type_token_ratio = "type-token ratio", MeasureType.text, surface.type_token_ratio
guiraud_r = "Guiraud's R", MeasureType.text, surface.guiraud_r
herdan_c = "Herdan's C", MeasureType.text, surface.herdan_c
dugast_k = "Dugast's k", MeasureType.text, surface.dugast_k
maas_a2 = "Maas' a²", MeasureType.text, surface.maas_a2
dugast_u = "Dugast's U", MeasureType.text, surface.dugast_u
tuldava_ln = "Tuldava's LN", MeasureType.text, surface.tuldava_ln
brunet_w = "Brunet's W", MeasureType.text, surface.brunet_w
cttr = "CTTR", MeasureType.text, surface.cttr
summer_s = "Summer's S", MeasureType.text, surface.summer_s
sichel_s = "Sichel's S", MeasureType.text, surface.sichel_s
michea_m = "Michéa's M", MeasureType.text, surface.michea_m
honore_h = "Honoré's H", MeasureType.text, surface.honore_h
entropy = "entropy", MeasureType.text, surface.entropy
evenness = "evenness", MeasureType.text, surface.evenness
jarvis_evenness = "Jarvis's evenness", MeasureType.text, surface.jarvis_evenness
yule_k = "Yule's K", MeasureType.text, surface.yule_k
simpson_d = "Simpson's D", MeasureType.text, surface.simpson_d
herdan_vm = "Herdan's Vm", MeasureType.text, surface.herdan_vm
hdd = "HD-D", MeasureType.text, surface.hdd
average_token_length = "average token length", MeasureType.text, surface.average_token_length
orlov_z = "Orlov's Z", MeasureType.text, surface.orlov_z
mtld = "MTLD", MeasureType.text, surface.mtld

sentence_length_tokens = "sentence length (tokens)", MeasureType.sentence, sentence._sentence_length_tokens
sentence_length_characters = ("sentence length (characters)", MeasureType.sentence,
sentence._sentence_length_characters)

sentence_length_words = "sentence length (words)", MeasureType.punctuation, sentence._sentence_length_words
punctuation_per_sentence = "punctuation per sentence", MeasureType.punctuation, sentence._punctuation_per_sentence

average_dependency_distance = ("average dependency distance", MeasureType.graph,
dependency._average_dependency_distance)
closeness_centrality = "closeness centrality", MeasureType.graph, dependency._closeness_centrality
outdegree_centralization = "outdegree centralization", MeasureType.graph, dependency._outdegree_centralization
closeness_centralization = "closeness centralization", MeasureType.graph, dependency._closeness_centralization
longest_shortest_path = "longest shortest path", MeasureType.graph, dependency._longest_shortest_path
dependents_per_word = "dependents per word", MeasureType.graph, dependency._dependents_per_word
36 changes: 36 additions & 0 deletions text_complexity_microservice/src/app/measure_calculator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import itertools
import math

from textcomplexity.utils import conllu
from textcomplexity.utils.text import Text

from src.app.measure import MeasureType, Measure


def calculate_for_sentence(conll_sentence: str, measure: Measure, punct_tags: list[str] = None) -> float:
if not punct_tags:
punct_tags = ["PUNCT"]

conll_sentence = conll_sentence.split("\n")
sentences, graphs = zip(*conllu.read_conllu_sentences(conll_sentence, ignore_case=True))
if len(sentences) > 1:
raise ValueError()
tokens = list(itertools.chain.from_iterable(sentences))
graph = graphs[0]

try:
match measure.type:
case MeasureType.text:
tokens = [t for t in tokens if t.pos not in punct_tags]
text = Text.from_tokens(tokens)
return measure.func(text)
case MeasureType.sentence:
return measure.func(tokens)
case MeasureType.punctuation:
return measure.func(tokens, punct_tags)
case MeasureType.graph:
return measure.func(graph)
case _:
raise TypeError()
except (ZeroDivisionError, ValueError):
return math.nan
Empty file added text_complexity_microservice/src/app/models/__init__.py
View file
Open in desktop
Empty file.
Loading