Skip to content

ratingSyncJob 메모리 절감 #136

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
asp345 wants to merge 1 commit into
base: develop
Choose a base branch
from
Draft

ratingSyncJob 메모리 절감 #136

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 25 additions & 12 deletions batch/src/main/kotlin/com/wafflestudio/snuttev/sync/SnuttRatingSyncJobConfig.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package com.wafflestudio.snuttev.sync

import com.wafflestudio.snuttev.core.domain.lecture.model.SnuttLectureIdMap
import com.wafflestudio.snuttev.core.domain.lecture.repository.LectureRepository
import jakarta.persistence.EntityManagerFactory
import org.springframework.batch.core.job.Job
Expand Down Expand Up @@ -31,7 +30,7 @@ class SnuttRatingSyncJobConfig(
companion object {
const val RATING_SYNC_JOB_NAME = "RATING_SYNC_JOB"
private const val CUSTOM_READER_JOB_STEP = RATING_SYNC_JOB_NAME + "_STEP"
private const val CHUNK_SIZE = 1000000
private const val CHUNK_SIZE = 1000
}

@Bean
Expand All @@ -42,7 +41,7 @@ class SnuttRatingSyncJobConfig(

private fun customReaderStep(jobRepository: JobRepository): Step =
StepBuilder(CUSTOM_READER_JOB_STEP, jobRepository)
.chunk<SnuttLectureIdMap, SnuttLectureIdMap>(
.chunk<SnuttLectureRatingSyncTarget, SnuttLectureRatingSyncTarget>(
CHUNK_SIZE,
JpaTransactionManager().apply {
this.entityManagerFactory = this@SnuttRatingSyncJobConfig.entityManagerFactory
Expand All @@ -51,24 +50,33 @@ class SnuttRatingSyncJobConfig(
.writer(writer())
.build()

private fun reader(): JpaPagingItemReader<SnuttLectureIdMap> =
JpaPagingItemReaderBuilder<SnuttLectureIdMap>()
.name("snuttLectureIdMapReader")
private fun reader(): JpaPagingItemReader<SnuttLectureRatingSyncTarget> =
JpaPagingItemReaderBuilder<SnuttLectureRatingSyncTarget>()
.name("snuttLectureRatingSyncReader")
.entityManagerFactory(entityManagerFactory)
.queryString("SELECT s FROM SnuttLectureIdMap s JOIN FETCH s.semesterLecture")
.pageSize(CHUNK_SIZE)
.queryString(
"""
SELECT new com.wafflestudio.snuttev.sync.SnuttLectureRatingSyncTarget(s.snuttId, sl.lecture.id)
FROM SnuttLectureIdMap s
JOIN s.semesterLecture sl
""".trimIndent(),
).pageSize(CHUNK_SIZE)
.build()

private fun writer(): ItemWriter<SnuttLectureIdMap> =
private fun writer(): ItemWriter<SnuttLectureRatingSyncTarget> =
ItemWriter { items ->
val lectureIdtoLectureRatingMap =
val lectureIdToLectureRatingMap =
lectureRepository
.findAllRatingsByLectureIds(
items.mapNotNull { it.semesterLecture.lecture.id },
items
.asSequence()
.map { it.lectureId }
.distinct()
.toList(),
).associateBy { it.id }
val bulkOps = mongoTemplate.bulkOps(BulkOperations.BulkMode.UNORDERED, "lectures")
items.forEach {
val evInfo = lectureIdtoLectureRatingMap[it.semesterLecture.lecture.id]
val evInfo = lectureIdToLectureRatingMap[it.lectureId]
bulkOps.updateOne(
Query(Criteria.where("_id").`is`(it.snuttId)),
Update()
Expand All @@ -80,3 +88,8 @@ class SnuttRatingSyncJobConfig(
bulkOps.execute()
}
}

data class SnuttLectureRatingSyncTarget(
val snuttId: String,
val lectureId: Long,
)