feat: 대규모 데이터 삽입을 위한 Spring Batch Job 구현

build.gradle

@@ -28,6 +28,7 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
     implementation 'org.springframework.boot:spring-boot-starter-security'
     implementation 'org.springframework.boot:spring-boot-starter-web'
+    implementation 'org.springframework.boot:spring-boot-starter-batch'
     // 채팅 웹 소켓
     implementation 'org.springframework.boot:spring-boot-starter-websocket'
     implementation group: 'io.jsonwebtoken', name: 'jjwt-api', version: '0.11.5'
@@ -41,7 +42,7 @@ dependencies {
     //Flyway
     implementation 'org.flywaydb:flyway-core'
     implementation 'org.flywaydb:flyway-mysql'
-
+    
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     testImplementation 'org.junit.jupiter:junit-jupiter:5.8.1'
     compileOnly 'org.projectlombok:lombok'
@@ -69,4 +70,4 @@ tasks.register('copySecret', Copy) {
     from './be-submodule' // 서브 모듈 디렉토리 경로
     include "*.properties"  // 설정 파일 복사
     into 'src/main/resources'  // 붙여넣을 위치
-}
+}

src/main/java/com/team/buddyya/job/SeedDataJobConfig.java

@@ -0,0 +1,51 @@
+package com.team.buddyya.job;
+
+import com.team.buddyya.job.feed.FeedInsertJobConfig;
+import com.team.buddyya.job.feed.bookmark.BookmarkInsertJobConfig;
+import com.team.buddyya.job.feed.comment.CommentInsertJobConfig;
+import com.team.buddyya.job.feed.image.FeedImageInsertJobConfig;
+import com.team.buddyya.job.feed.like.FeedLikeInsertJobConfig;
+import com.team.buddyya.job.student.StudentInsertJobConfig;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.job.builder.JobBuilder;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Import;
+import org.springframework.context.annotation.Profile;
+
+@Profile("local")
+@Import({StudentInsertJobConfig.class, FeedInsertJobConfig.class, CommentInsertJobConfig.class,
+        FeedImageInsertJobConfig.class, FeedLikeInsertJobConfig.class, BookmarkInsertJobConfig.class})
+@Configuration
+@RequiredArgsConstructor
+public class SeedDataJobConfig {
+
+    public static final int STUDENT_COUNT = 100_000;
+    public static final int FEED_COUNT = 1_000_000;
+    public static final int COMMENT_COUNT = 100_000;
+    public static final int LIKE_COUNT = 500_000;
+    public static final int BOOKMARK_COUNT = 500_000;
+
+    private final JobRepository jobRepository;
+    private final Step studentInsertStep;
+    private final Step feedInsertStep;
+    private final Step commentInsertStep;
+    private final Step feedImageInsertStep;
+    private final Step feedLikeInsertStep;
+    private final Step bookmarkInsertStep;
+
+    @Bean
+    public Job seedDataJob() {
+        return new JobBuilder("seedDataJob", jobRepository)
+                .start(studentInsertStep)
+                .next(feedInsertStep)
+                .next(commentInsertStep)
+                .next(feedImageInsertStep)
+                .next(feedLikeInsertStep)
+                .next(bookmarkInsertStep)
+                .build();
+    }
+}

src/main/java/com/team/buddyya/job/feed/FeedInsertJobConfig.java

@@ -0,0 +1,51 @@
+package com.team.buddyya.job.feed;
+
+import static com.team.buddyya.job.SeedDataJobConfig.FEED_COUNT;
+
+import javax.sql.DataSource;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
+import org.springframework.batch.item.database.JdbcBatchItemWriter;
+import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.transaction.PlatformTransactionManager;
+
+@Configuration
+@RequiredArgsConstructor
+public class FeedInsertJobConfig {
+
+    private final JobRepository jobRepository;
+    private final PlatformTransactionManager platformTransactionManager;
+    private final DataSource dataSource;
+    private static final int CHUNK_SIZE = 1000;
+
+    @Bean
+    public Step feedInsertStep(ItemReader<FeedJobDTO> feedItemReader, JdbcBatchItemWriter<FeedJobDTO> feedItemWriter) {
+        return new StepBuilder("feedInsertStep", jobRepository)
+                .<FeedJobDTO, FeedJobDTO>chunk(CHUNK_SIZE, platformTransactionManager)
+                .reader(feedItemReader)
+                .writer(feedItemWriter)
+                .build();
+    }
+
+    @Bean
+    public ItemReader<FeedJobDTO> feedItemReader() {
+        return new FeedItemReader(new JdbcTemplate(dataSource), FEED_COUNT);
+    }
+
+    @Bean
+    public JdbcBatchItemWriter<FeedJobDTO> feedItemWriter() {
+        String sql = "INSERT INTO feed (title, content, is_profile_visible, student_id, category_id, university_id, like_count, comment_count, view_count, pinned, created_date, updated_date) VALUES (:title, :content, :profileVisible, :studentId, :categoryId, :universityId, 0, 0, 0, false, NOW(), NOW())";
+        return new JdbcBatchItemWriterBuilder<FeedJobDTO>()
+                .dataSource(dataSource)
+                .sql(sql)
+                .itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
+                .build();
+    }
+}

src/main/java/com/team/buddyya/job/feed/FeedItemReader.java

@@ -0,0 +1,60 @@
+package com.team.buddyya.job.feed;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+public class FeedItemReader implements ItemReader<FeedJobDTO> {
+
+    private final JdbcTemplate jdbcTemplate;
+    private final int totalCount;
+    private final AtomicInteger counter = new AtomicInteger(0);
+
+    private Long minStudentId;
+    private Long maxStudentId;
+    private Long minUniversityId;
+    private Long maxUniversityId;
+
+    public FeedItemReader(JdbcTemplate jdbcTemplate, int totalCount) {
+        this.jdbcTemplate = jdbcTemplate;
+        this.totalCount = totalCount;
+    }
+
+    @Override
+    public FeedJobDTO read() throws Exception {
+        if (minStudentId == null) {
+            // Student, University 테이블의 데이터 존재 여부 및 ID 범위 초기화
+            if (jdbcTemplate.queryForObject("SELECT COUNT(1) FROM student", Long.class) == 0) {
+                throw new IllegalStateException("Prerequisite data (Student) is missing.");
+            }
+            this.minStudentId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM student", Long.class);
+            this.maxStudentId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM student", Long.class);
+            if (jdbcTemplate.queryForObject("SELECT COUNT(1) FROM university", Long.class) == 0) {
+                throw new IllegalStateException("Prerequisite data (University) is missing.");
+            }
+            this.minUniversityId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM university", Long.class);
+            this.maxUniversityId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM university", Long.class);
+        }
+
+        if (counter.get() >= totalCount) {
+            return null;
+        }
+
+        int currentCount = counter.incrementAndGet();
+        long randomStudentId = ThreadLocalRandom.current().nextLong(minStudentId, maxStudentId + 1);
+        long randomUniversityId = ThreadLocalRandom.current().nextLong(minUniversityId, maxUniversityId + 1);
+        // Category는 데이터가 적다고 가정하고 SQL로 랜덤 조회, 많아진다면 동일하게 MIN/MAX 방식으로 변경
+        Long randomCategoryId = jdbcTemplate.queryForObject("SELECT id FROM category ORDER BY RAND() LIMIT 1",
+                Long.class);
+
+        return FeedJobDTO.builder()
+                .title("피드 제목 " + currentCount)
+                .content("피드 내용입니다. " + currentCount)
+                .profileVisible(ThreadLocalRandom.current().nextBoolean())
+                .studentId(randomStudentId)
+                .categoryId(randomCategoryId)
+                .universityId(randomUniversityId)
+                .build();
+    }
+}

src/main/java/com/team/buddyya/job/feed/FeedJobDTO.java

@@ -0,0 +1,15 @@
+package com.team.buddyya.job.feed;
+
+import lombok.Builder;
+import lombok.Getter;
+
+@Getter
+@Builder
+public class FeedJobDTO {
+    private String title;
+    private String content;
+    private boolean profileVisible;
+    private Long studentId;
+    private Long categoryId;
+    private Long universityId;
+}

src/main/java/com/team/buddyya/job/feed/bookmark/BookmarkInsertJobConfig.java

@@ -0,0 +1,53 @@
+package com.team.buddyya.job.feed.bookmark;
+
+import static com.team.buddyya.job.SeedDataJobConfig.BOOKMARK_COUNT;
+
+import javax.sql.DataSource;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
+import org.springframework.batch.item.database.JdbcBatchItemWriter;
+import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.transaction.PlatformTransactionManager;
+
+@Configuration
+@RequiredArgsConstructor
+public class BookmarkInsertJobConfig {
+
+    private final JobRepository jobRepository;
+    private final PlatformTransactionManager platformTransactionManager;
+    private final DataSource dataSource;
+    private static final int CHUNK_SIZE = 1000;
+
+    @Bean
+    public Step bookmarkInsertStep(ItemReader<BookmarkJobDTO> bookmarkItemReader,
+                                   JdbcBatchItemWriter<BookmarkJobDTO> bookmarkItemWriter) {
+        return new StepBuilder("bookmarkInsertStep", jobRepository)
+                .<BookmarkJobDTO, BookmarkJobDTO>chunk(CHUNK_SIZE, platformTransactionManager)
+                .reader(bookmarkItemReader)
+                .writer(bookmarkItemWriter)
+                .build();
+    }
+
+    @Bean
+    public ItemReader<BookmarkJobDTO> bookmarkItemReader() {
+        return new BookmarkItemReader(new JdbcTemplate(dataSource), BOOKMARK_COUNT);
+    }
+
+    @Bean
+    public JdbcBatchItemWriter<BookmarkJobDTO> bookmarkItemWriter() {
+        String sql = "INSERT IGNORE INTO bookmark (feed_id, student_id, created_date) VALUES (:feedId, :studentId, NOW())";
+        return new JdbcBatchItemWriterBuilder<BookmarkJobDTO>()
+                .dataSource(dataSource)
+                .sql(sql)
+                .itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
+                .assertUpdates(false)
+                .build();
+    }
+}

src/main/java/com/team/buddyya/job/feed/bookmark/BookmarkItemReader.java

@@ -0,0 +1,43 @@
+package com.team.buddyya.job.feed.bookmark;
+
+import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.jdbc.core.JdbcTemplate;
+
+public class BookmarkItemReader implements ItemReader<BookmarkJobDTO> {
+
+    private final JdbcTemplate jdbcTemplate;
+    private final int totalCount;
+    private final AtomicInteger counter = new AtomicInteger(0);
+    private final ThreadLocalRandom random = ThreadLocalRandom.current();
+
+    private Long minFeedId;
+    private Long maxFeedId;
+    private Long minStudentId;
+    private Long maxStudentId;
+
+    public BookmarkItemReader(JdbcTemplate jdbcTemplate, int totalCount) {
+        this.jdbcTemplate = jdbcTemplate;
+        this.totalCount = totalCount;
+    }
+
+    @Override
+    public BookmarkJobDTO read() {
+        if (minFeedId == null) {
+            this.minFeedId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM feed", Long.class);
+            this.maxFeedId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM feed", Long.class);
+            this.minStudentId = jdbcTemplate.queryForObject("SELECT MIN(id) FROM student", Long.class);
+            this.maxStudentId = jdbcTemplate.queryForObject("SELECT MAX(id) FROM student", Long.class);
+        }
+        if (counter.getAndIncrement() >= totalCount) {
+            return null;
+        }
+        long randomFeedId = random.nextLong(minFeedId, maxFeedId + 1);
+        long randomStudentId = random.nextLong(minStudentId, maxStudentId + 1);
+        return BookmarkJobDTO.builder()
+                .feedId(randomFeedId)
+                .studentId(randomStudentId)
+                .build();
+    }
+}

src/main/java/com/team/buddyya/job/feed/bookmark/BookmarkJobDTO.java

@@ -0,0 +1,11 @@
+package com.team.buddyya.job.feed.bookmark;
+
+import lombok.Builder;
+import lombok.Getter;
+
+@Getter
+@Builder
+public class BookmarkJobDTO {
+    private Long feedId;
+    private Long studentId;
+}

src/main/java/com/team/buddyya/job/feed/comment/CommentInsertJobConfig.java

@@ -0,0 +1,52 @@
+package com.team.buddyya.job.feed.comment;
+
+import static com.team.buddyya.job.SeedDataJobConfig.COMMENT_COUNT;
+
+import javax.sql.DataSource;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.batch.item.database.BeanPropertyItemSqlParameterSourceProvider;
+import org.springframework.batch.item.database.JdbcBatchItemWriter;
+import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.transaction.PlatformTransactionManager;
+
+@Configuration
+@RequiredArgsConstructor
+public class CommentInsertJobConfig {
+
+    private final JobRepository jobRepository;
+    private final PlatformTransactionManager platformTransactionManager;
+    private final DataSource dataSource;
+    private static final int CHUNK_SIZE = 1000;
+
+    @Bean
+    public Step commentInsertStep(ItemReader<CommentJobDTO> commentItemReader,
+                                  JdbcBatchItemWriter<CommentJobDTO> commentItemWriter) {
+        return new StepBuilder("commentInsertStep", jobRepository)
+                .<CommentJobDTO, CommentJobDTO>chunk(CHUNK_SIZE, platformTransactionManager)
+                .reader(commentItemReader)
+                .writer(commentItemWriter)
+                .build();
+    }
+
+    @Bean
+    public ItemReader<CommentJobDTO> commentItemReader() {
+        return new CommentItemReader(new JdbcTemplate(dataSource), COMMENT_COUNT);
+    }
+
+    @Bean
+    public JdbcBatchItemWriter<CommentJobDTO> commentItemWriter() {
+        String sql = "INSERT INTO comment (student_id, feed_id, parent_id, content, like_count, deleted, created_date, updated_date) VALUES (:studentId, :feedId, :parentId, :content, 0, false, NOW(), NOW())";
+        return new JdbcBatchItemWriterBuilder<CommentJobDTO>()
+                .dataSource(dataSource)
+                .sql(sql)
+                .itemSqlParameterSourceProvider(new BeanPropertyItemSqlParameterSourceProvider<>())
+                .build();
+    }
+}