Skip to content

Commit f52fa2c

Browse files
mangsuyomangsuyo
authored
Merge pull request #366 from buddy-ya/test/insert-data#365
feat: 대규모 데이터 삽입을 위한 Spring Batch Job 구현
2 parents 42f7dac + 053fc41 commit f52fa2c

21 files changed

Lines changed: 748 additions & 3 deletions

‎be-submodule‎

‎build.gradle‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ dependencies {
2828
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
2929
implementation 'org.springframework.boot:spring-boot-starter-security'
3030
implementation 'org.springframework.boot:spring-boot-starter-web'
31+
implementation 'org.springframework.boot:spring-boot-starter-batch'
3132
// 채팅 웹 소켓
3233
implementation 'org.springframework.boot:spring-boot-starter-websocket'
3334
implementation group: 'io.jsonwebtoken', name: 'jjwt-api', version: '0.11.5'
@@ -41,7 +42,7 @@ dependencies {
4142
//Flyway
4243
implementation 'org.flywaydb:flyway-core'
4344
implementation 'org.flywaydb:flyway-mysql'
44-
45+
4546
implementation 'com.fasterxml.jackson.core:jackson-databind'
4647
testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
4748
compileOnly 'org.projectlombok:lombok'
@@ -69,4 +70,4 @@ tasks.register('copySecret', Copy) {
6970
from './be-submodule' // 서브 모듈 디렉토리 경로
7071
include "*.properties" // 설정 파일 복사
7172
into 'src/main/resources' // 붙여넣을 위치
72-
}
73+
}

‎src/main/java/com/team/buddyya/job/SeedDataJobConfig.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package com.team.buddyya.job;
2+
3+
import com.team.buddyya.job.feed.FeedInsertJobConfig;
4+
import com.team.buddyya.job.feed.bookmark.BookmarkInsertJobConfig;
5+
import com.team.buddyya.job.feed.comment.CommentInsertJobConfig;
6+
import com.team.buddyya.job.feed.image.FeedImageInsertJobConfig;
7+
import com.team.buddyya.job.feed.like.FeedLikeInsertJobConfig;
8+
import com.team.buddyya.job.student.StudentInsertJobConfig;
9+
import lombok.RequiredArgsConstructor;
10+
import org.springframework.batch.core.Job;
11+
import org.springframework.batch.core.Step;
12+
import org.springframework.batch.core.job.builder.JobBuilder;
13+
import org.springframework.batch.core.repository.JobRepository;
14+
import org.springframework.context.annotation.Bean;
15+
import org.springframework.context.annotation.Configuration;
16+
import org.springframework.context.annotation.Import;
17+
import org.springframework.context.annotation.Profile;
18+
19+
@Profile("local")
20+
@Import({StudentInsertJobConfig.class, FeedInsertJobConfig.class, CommentInsertJobConfig.class,
21+
FeedImageInsertJobConfig.class, FeedLikeInsertJobConfig.class, BookmarkInsertJobConfig.class})
22+
@Configuration
23+
@RequiredArgsConstructor
24+
public class SeedDataJobConfig {
25+
26+
public static final int STUDENT_COUNT = 100_000;
27+
public static final int FEED_COUNT = 1_000_000;
28+
public static final int COMMENT_COUNT = 100_000;
29+
public static final int LIKE_COUNT = 500_000;
30+
public static final int BOOKMARK_COUNT = 500_000;
31+
32+
private final JobRepository jobRepository;
33+
private final Step studentInsertStep;
34+
private final Step feedInsertStep;
35+
private final Step commentInsertStep;
36+
private final Step feedImageInsertStep;
37+
private final Step feedLikeInsertStep;
38+
private final Step bookmarkInsertStep;
39+
40+
@Bean
41+
public Job seedDataJob() {
42+
return new JobBuilder("seedDataJob", jobRepository)
43+
.start(studentInsertStep)
44+
.next(feedInsertStep)
45+
.next(commentInsertStep)
46+
.next(feedImageInsertStep)
47+
.next(feedLikeInsertStep)
48+
.next(bookmarkInsertStep)
49+
.build();
50+
}
51+
}

‎src/main/java/com/team/buddyya/job/feed/FeedInsertJobConfig.java‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
package com.team.buddyya.job.feed;
2+
3+
import static com.team.buddyya.job.SeedDataJobConfig.FEED_COUNT;
4+
5+
import javax.sql.DataSource;
6+
import lombok.RequiredArgsConstructor;
7+
import org.springframework.batch.core.Step;
8+
import org.springframework.batch.core.repository.JobRepository;
9+
import org.springframework.batch.core.step.builder.StepBuilder;
10+
import org.springframework.batch.item.ItemReader;
11+
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
12+
import org.springframework.batch.item.database.JdbcBatchItemWriter;
13+
import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
14+
import org.springframework.context.annotation.Bean;
15+
import org.springframework.context.annotation.Configuration;
16+
import org.springframework.jdbc.core.JdbcTemplate;
17+
import org.springframework.transaction.PlatformTransactionManager;
18+
19+
@Configuration
20+
@RequiredArgsConstructor
21+
public class FeedInsertJobConfig {
22+
23+
private final JobRepository jobRepository;
24+
private final PlatformTransactionManager platformTransactionManager;
25+
private final DataSource dataSource;
26+
private static final int CHUNK_SIZE = 1000;
27+
28+
@Bean
29+
public Step feedInsertStep(ItemReader<FeedJobDTO> feedItemReader, JdbcBatchItemWriter<FeedJobDTO> feedItemWriter) {
30+
return new StepBuilder("feedInsertStep", jobRepository)
31+
.<FeedJobDTO, FeedJobDTO>chunk(CHUNK_SIZE, platformTransactionManager)
32+
.reader(feedItemReader)
33+
.writer(feedItemWriter)
34+
.build();
35+
}
36+
37+
@Bean
38+
public ItemReader<FeedJobDTO> feedItemReader() {
39+
return new FeedItemReader(new JdbcTemplate(dataSource), FEED_COUNT);
40+
}
41+
42+
@Bean
43+
public JdbcBatchItemWriter<FeedJobDTO> feedItemWriter() {
44+
String sql = "INSERT INTO feed (title, content, is_profile_visible, student_id, category_id, university_id, like_count, comment_count, view_count, pinned, created_date, updated_date) VALUES (:title, :content, :profileVisible, :studentId, :categoryId, :universityId, 0, 0, 0, false, NOW(), NOW())";
45+
return new JdbcBatchItemWriterBuilder<FeedJobDTO>()
46+
.dataSource(dataSource)
47+
.sql(sql)
48+
.itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
49+
.build();
50+
}
51+
}

‎src/main/java/com/team/buddyya/job/feed/FeedItemReader.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package com.team.buddyya.job.feed;
2+
3+
import java.util.concurrent.ThreadLocalRandom;
4+
import java.util.concurrent.atomic.AtomicInteger;
5+
import org.springframework.batch.item.ItemReader;
6+
import org.springframework.jdbc.core.JdbcTemplate;
7+
8+
public class FeedItemReader implements ItemReader<FeedJobDTO> {
9+
10+
private final JdbcTemplate jdbcTemplate;
11+
private final int totalCount;
12+
private final AtomicInteger counter = new AtomicInteger(0);
13+
14+
private Long minStudentId;
15+
private Long maxStudentId;
16+
private Long minUniversityId;
17+
private Long maxUniversityId;
18+
19+
public FeedItemReader(JdbcTemplate jdbcTemplate, int totalCount) {
20+
this.jdbcTemplate = jdbcTemplate;
21+
this.totalCount = totalCount;
22+
}
23+
24+
@Override
25+
public FeedJobDTO read() throws Exception {
26+
if (minStudentId == null) {
27+
// Student, University 테이블의 데이터 존재 여부 및 ID 범위 초기화
28+
if (jdbcTemplate.queryForObject("SELECT COUNT(1) FROM student", Long.class) == 0) {
29+
throw new IllegalStateException("Prerequisite data (Student) is missing.");
30+
}
31+
this.minStudentId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM student", Long.class);
32+
this.maxStudentId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM student", Long.class);
33+
if (jdbcTemplate.queryForObject("SELECT COUNT(1) FROM university", Long.class) == 0) {
34+
throw new IllegalStateException("Prerequisite data (University) is missing.");
35+
}
36+
this.minUniversityId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM university", Long.class);
37+
this.maxUniversityId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM university", Long.class);
38+
}
39+
40+
if (counter.get() >= totalCount) {
41+
return null;
42+
}
43+
44+
int currentCount = counter.incrementAndGet();
45+
long randomStudentId = ThreadLocalRandom.current().nextLong(minStudentId, maxStudentId + 1);
46+
long randomUniversityId = ThreadLocalRandom.current().nextLong(minUniversityId, maxUniversityId + 1);
47+
// Category는 데이터가 적다고 가정하고 SQL로 랜덤 조회, 많아진다면 동일하게 MIN/MAX 방식으로 변경
48+
Long randomCategoryId = jdbcTemplate.queryForObject("SELECT id FROM category ORDER BY RAND() LIMIT 1",
49+
Long.class);
50+
51+
return FeedJobDTO.builder()
52+
.title("피드 제목 " + currentCount)
53+
.content("피드 내용입니다. " + currentCount)
54+
.profileVisible(ThreadLocalRandom.current().nextBoolean())
55+
.studentId(randomStudentId)
56+
.categoryId(randomCategoryId)
57+
.universityId(randomUniversityId)
58+
.build();
59+
}
60+
}

‎src/main/java/com/team/buddyya/job/feed/FeedJobDTO.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package com.team.buddyya.job.feed;
2+
3+
import lombok.Builder;
4+
import lombok.Getter;
5+
6+
@Getter
7+
@Builder
8+
public class FeedJobDTO {
9+
private String title;
10+
private String content;
11+
private boolean profileVisible;
12+
private Long studentId;
13+
private Long categoryId;
14+
private Long universityId;
15+
}

‎src/main/java/com/team/buddyya/job/feed/bookmark/BookmarkInsertJobConfig.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
package com.team.buddyya.job.feed.bookmark;
2+
3+
import static com.team.buddyya.job.SeedDataJobConfig.BOOKMARK_COUNT;
4+
5+
import javax.sql.DataSource;
6+
import lombok.RequiredArgsConstructor;
7+
import org.springframework.batch.core.Step;
8+
import org.springframework.batch.core.repository.JobRepository;
9+
import org.springframework.batch.core.step.builder.StepBuilder;
10+
import org.springframework.batch.item.ItemReader;
11+
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
12+
import org.springframework.batch.item.database.JdbcBatchItemWriter;
13+
import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
14+
import org.springframework.context.annotation.Bean;
15+
import org.springframework.context.annotation.Configuration;
16+
import org.springframework.jdbc.core.JdbcTemplate;
17+
import org.springframework.transaction.PlatformTransactionManager;
18+
19+
@Configuration
20+
@RequiredArgsConstructor
21+
public class BookmarkInsertJobConfig {
22+
23+
private final JobRepository jobRepository;
24+
private final PlatformTransactionManager platformTransactionManager;
25+
private final DataSource dataSource;
26+
private static final int CHUNK_SIZE = 1000;
27+
28+
@Bean
29+
public Step bookmarkInsertStep(ItemReader<BookmarkJobDTO> bookmarkItemReader,
30+
JdbcBatchItemWriter<BookmarkJobDTO> bookmarkItemWriter) {
31+
return new StepBuilder("bookmarkInsertStep", jobRepository)
32+
.<BookmarkJobDTO, BookmarkJobDTO>chunk(CHUNK_SIZE, platformTransactionManager)
33+
.reader(bookmarkItemReader)
34+
.writer(bookmarkItemWriter)
35+
.build();
36+
}
37+
38+
@Bean
39+
public ItemReader<BookmarkJobDTO> bookmarkItemReader() {
40+
return new BookmarkItemReader(new JdbcTemplate(dataSource), BOOKMARK_COUNT);
41+
}
42+
43+
@Bean
44+
public JdbcBatchItemWriter<BookmarkJobDTO> bookmarkItemWriter() {
45+
String sql = "INSERT IGNORE INTO bookmark (feed_id, student_id, created_date) VALUES (:feedId, :studentId, NOW())";
46+
return new JdbcBatchItemWriterBuilder<BookmarkJobDTO>()
47+
.dataSource(dataSource)
48+
.sql(sql)
49+
.itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
50+
.assertUpdates(false)
51+
.build();
52+
}
53+
}

‎src/main/java/com/team/buddyya/job/feed/bookmark/BookmarkItemReader.java‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
package com.team.buddyya.job.feed.bookmark;
2+
3+
import java.util.concurrent.ThreadLocalRandom;
4+
import java.util.concurrent.atomic.AtomicInteger;
5+
import org.springframework.batch.item.ItemReader;
6+
import org.springframework.jdbc.core.JdbcTemplate;
7+
8+
public class BookmarkItemReader implements ItemReader<BookmarkJobDTO> {
9+
10+
private final JdbcTemplate jdbcTemplate;
11+
private final int totalCount;
12+
private final AtomicInteger counter = new AtomicInteger(0);
13+
private final ThreadLocalRandom random = ThreadLocalRandom.current();
14+
15+
private Long minFeedId;
16+
private Long maxFeedId;
17+
private Long minStudentId;
18+
private Long maxStudentId;
19+
20+
public BookmarkItemReader(JdbcTemplate jdbcTemplate, int totalCount) {
21+
this.jdbcTemplate = jdbcTemplate;
22+
this.totalCount = totalCount;
23+
}
24+
25+
@Override
26+
public BookmarkJobDTO read() {
27+
if (minFeedId == null) {
28+
this.minFeedId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM feed", Long.class);
29+
this.maxFeedId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM feed", Long.class);
30+
this.minStudentId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM student", Long.class);
31+
this.maxStudentId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM student", Long.class);
32+
}
33+
if (counter.getAndIncrement() >= totalCount) {
34+
return null;
35+
}
36+
long randomFeedId = random.nextLong(minFeedId, maxFeedId + 1);
37+
long randomStudentId = random.nextLong(minStudentId, maxStudentId + 1);
38+
return BookmarkJobDTO.builder()
39+
.feedId(randomFeedId)
40+
.studentId(randomStudentId)
41+
.build();
42+
}
43+
}

‎src/main/java/com/team/buddyya/job/feed/bookmark/BookmarkJobDTO.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package com.team.buddyya.job.feed.bookmark;
2+
3+
import lombok.Builder;
4+
import lombok.Getter;
5+
6+
@Getter
7+
@Builder
8+
public class BookmarkJobDTO {
9+
private Long feedId;
10+
private Long studentId;
11+
}

‎src/main/java/com/team/buddyya/job/feed/comment/CommentInsertJobConfig.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package com.team.buddyya.job.feed.comment;
2+
3+
import static com.team.buddyya.job.SeedDataJobConfig.COMMENT_COUNT;
4+
5+
import javax.sql.DataSource;
6+
import lombok.RequiredArgsConstructor;
7+
import org.springframework.batch.core.Step;
8+
import org.springframework.batch.core.repository.JobRepository;
9+
import org.springframework.batch.core.step.builder.StepBuilder;
10+
import org.springframework.batch.item.ItemReader;
11+
import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
12+
import org.springframework.batch.item.database.JdbcBatchItemWriter;
13+
import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
14+
import org.springframework.context.annotation.Bean;
15+
import org.springframework.context.annotation.Configuration;
16+
import org.springframework.jdbc.core.JdbcTemplate;
17+
import org.springframework.transaction.PlatformTransactionManager;
18+
19+
@Configuration
20+
@RequiredArgsConstructor
21+
public class CommentInsertJobConfig {
22+
23+
private final JobRepository jobRepository;
24+
private final PlatformTransactionManager platformTransactionManager;
25+
private final DataSource dataSource;
26+
private static final int CHUNK_SIZE = 1000;
27+
28+
@Bean
29+
public Step commentInsertStep(ItemReader<CommentJobDTO> commentItemReader,
30+
JdbcBatchItemWriter<CommentJobDTO> commentItemWriter) {
31+
return new StepBuilder("commentInsertStep", jobRepository)
32+
.<CommentJobDTO, CommentJobDTO>chunk(CHUNK_SIZE, platformTransactionManager)
33+
.reader(commentItemReader)
34+
.writer(commentItemWriter)
35+
.build();
36+
}
37+
38+
@Bean
39+
public ItemReader<CommentJobDTO> commentItemReader() {
40+
return new CommentItemReader(new JdbcTemplate(dataSource), COMMENT_COUNT);
41+
}
42+
43+
@Bean
44+
public JdbcBatchItemWriter<CommentJobDTO> commentItemWriter() {
45+
String sql = "INSERT INTO comment (student_id, feed_id, parent_id, content, like_count, deleted, created_date, updated_date) VALUES (:studentId, :feedId, :parentId, :content, 0, false, NOW(), NOW())";
46+
return new JdbcBatchItemWriterBuilder<CommentJobDTO>()
47+
.dataSource(dataSource)
48+
.sql(sql)
49+
.itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
50+
.build();
51+
}
52+
}

0 commit comments

Comments
 (0)