Skip to content

Commit 92cb431

Browse files
kimD0ngjunkimD0ngjun
authored
Merge pull request #10 from CarToi/dev
[main deploy] 데이터 수집 로직 완성
2 parents 22251b4 + 8c36909 commit 92cb431

File tree

84 files changed

+3380
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

84 files changed

+3380
-0
lines changed

‎.github/workflows/cicd.yml‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
name: cicd
2+
3+
on:
4+
push:
5+
branches: [ "main" ]
6+
7+
jobs:
8+
build:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- uses: actions/checkout@v3
12+
- name: Install JDK 21
13+
uses: actions/setup-java@v3
14+
with:
15+
java-version: '21'
16+
distribution: 'temurin'
17+
18+
- name: Build with Gradle And Test
19+
run: |
20+
mkdir -p ./src/main/resources
21+
echo ${{ secrets.APPLICATION_YML }} | base64 --decode > ./src/main/resources/application.yml
22+
23+
if [ -s ./src/main/resources/application.yml ]; then
24+
echo "application.yml Create Success"
25+
else
26+
echo "application.yml Create Fail"
27+
exit 1
28+
fi
29+
30+
chmod +x ./gradlew
31+
./gradlew clean build
32+
33+
- name: Login to DockerHub
34+
uses: docker/login-action@v1
35+
with:
36+
username: ${{ secrets.DOCKERHUB_USERNAME }}
37+
password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
38+
39+
- name: Build Docker
40+
run: docker build --platform linux/amd64 -t ${{ secrets.DOCKERHUB_USERNAME }}/saegil .
41+
42+
- name: Push Docker
43+
run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/saegil:latest
44+
45+
deploy:
46+
needs: build
47+
runs-on: ubuntu-latest
48+
steps:
49+
- name: Get Github Actions VM IP
50+
id: ip
51+
uses: haythem/public-ip@v1.3
52+
53+
- name: Print VM IP
54+
run: |
55+
echo ${{ steps.ip.outputs.ipv4 }}
56+
57+
- name: Configure AWS credentials
58+
uses: aws-actions/configure-aws-credentials@v4
59+
with:
60+
aws-access-key-id: ${{ secrets.AWS_IAM_ACCESS_KEY }}
61+
aws-secret-access-key: ${{ secrets.AWS_IAM_ACCESS_SECRET_KEY }}
62+
aws-region: 'ap-northeast-2'
63+
64+
- name: Add GitHub Actions IP
65+
run: |
66+
aws ec2 authorize-security-group-ingress \
67+
--group-id ${{ secrets.AWS_SECURITY_GROUP_ID }} \
68+
--protocol tcp \
69+
--port 22 \
70+
--cidr ${{ steps.ip.outputs.ipv4 }}/32
71+
72+
# 일단 기존에 동작하던 도커 컨테이너를 중단하고 삭제해야될듯
73+
- name: Stop Current Server
74+
uses: appleboy/ssh-action@master
75+
with:
76+
username: ubuntu
77+
host: ${{ secrets.EC2_ELASTIC_IP }}
78+
key: ${{ secrets.EC2_SSH_KEY }}
79+
script: |
80+
sudo docker stop ${{ secrets.EC2_WAS }}
81+
sudo docker rm ${{ secrets.EC2_WAS }}
82+
sudo docker image prune -a -f
83+
84+
- name: Docker Compose
85+
uses: appleboy/ssh-action@master
86+
with:
87+
username: ubuntu
88+
host: ${{ secrets.EC2_ELASTIC_IP }}
89+
key: ${{ secrets.EC2_SSH_KEY }}
90+
script: |
91+
sudo docker pull ${{ secrets.DOCKERHUB_USERNAME }}/saegil:latest
92+
sudo docker-compose -f docker-compose.yml up -d
93+
94+
# 얘는 새롭게 뜬 도커 컨테이너 헬스체크
95+
- name: Check Deploy Server URL
96+
uses: jtalk/url-health-check-action@v3
97+
with:
98+
url: https://${{ secrets.EC2_HTTPS_DOMAIN }}/env
99+
max-attempts: 5
100+
retry-delay: 10s
101+
102+
- name: Remove GitHub Actions IP
103+
run: |
104+
aws ec2 revoke-security-group-ingress \
105+
--group-id ${{ secrets.AWS_SECURITY_GROUP_ID }} \
106+
--protocol tcp \
107+
--port 22 \
108+
--cidr ${{ steps.ip.outputs.ipv4 }}/32
109+

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ build/
55
!**/src/main/**/build/
66
!**/src/test/**/build/
77
application.yml
8+
application-deploy.yml
89

910
### STS ###
1011
.apt_generated

‎Dockerfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
FROM openjdk:21
2+
ARG JAR_FILE=build/libs/*.jar
3+
4+
# 두 ARG는 ec2 도커 컴포즈에 세팅된 변수 값을 따라가게 됨
5+
ARG PROFILES
6+
ARG ENV
7+
8+
COPY ${JAR_FILE} app.jar
9+
ENTRYPOINT ["java", "-DSpring.profiles.active=${PROFILES}", "-DServer.env=${ENV}", "-jar", "app.jar"]

‎build.gradle‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,13 @@ plugins {
22
id 'java'
33
id 'org.springframework.boot' version '3.5.0'
44
id 'io.spring.dependency-management' version '1.1.7'
5+
6+
// jmh
7+
id "me.champeau.jmh" version "0.7.3"
8+
}
9+
10+
ext {
11+
springAiVersion = "1.0.0"
512
}
613

714
group = 'org.jun'
@@ -19,10 +26,58 @@ repositories {
1926

2027
dependencies {
2128
implementation 'org.springframework.boot:spring-boot-starter-web'
29+
30+
implementation 'org.projectlombok:lombok'
31+
annotationProcessor 'org.projectlombok:lombok'
32+
testCompileOnly 'org.projectlombok:lombok'
33+
testAnnotationProcessor 'org.projectlombok:lombok'
34+
2235
testImplementation 'org.springframework.boot:spring-boot-starter-test'
2336
testRuntimeOnly 'org.junit.platform:junit-platform-launcher'
37+
38+
// MySQL, H2 DB & JPA
39+
runtimeOnly 'com.mysql:mysql-connector-j'
40+
runtimeOnly 'com.h2database:h2'
41+
implementation 'org.springframework.boot:spring-boot-starter-data-jpa'
42+
43+
// selenium
44+
// implementation("org.seleniumhq.selenium:selenium-java:4.33.0")
45+
46+
// JSoup
47+
implementation 'org.jsoup:jsoup:1.20.1'
48+
49+
// jmh
50+
jmh 'org.openjdk.jmh:jmh-core:1.37'
51+
jmh 'org.openjdk.jmh:jmh-generator-annprocess:1.37'
52+
jmh 'org.openjdk.jmh:jmh-generator-bytecode:1.37'
53+
54+
// discord
55+
implementation 'net.dv8tion:JDA:5.6.1'
56+
}
57+
58+
test {
59+
systemProperty "spring.profiles.active", "test"
60+
}
61+
62+
dependencyManagement {
63+
imports {
64+
mavenBom "org.springframework.ai:spring-ai-bom:$springAiVersion"
65+
}
2466
}
2567

2668
tasks.named('test') {
2769
useJUnitPlatform()
2870
}
71+
72+
jmh {
73+
warmupIterations = 2 // 워밍업 반복 횟수
74+
iterations = 5 // 측정 반복 횟수
75+
fork = 1 // 포크 수
76+
failOnError = true
77+
resultFormat = 'JSON' // 결과 포맷
78+
resultsFile = project.file("${project.buildDir}/reports/jmh/results.json") // 결과 파일 위치
79+
timeOnIteration = '1s' // 각 반복당 실행 시간
80+
timeUnit = 'ms' // 시간 단위
81+
includes = ['.*Benchmark.*'] // 벤치마크 클래스 패턴
82+
profilers = ['gc']
83+
}

‎src/jmh/java/org/jun/saemangeum/benchmark/process/ContentCollectServiceBenchmark.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package org.jun.saemangeum.benchmark.process;
2+
3+
import org.jun.saemangeum.process.application.service.ContentCollectService;
4+
import org.openjdk.jmh.annotations.*;
5+
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
6+
import org.springframework.context.annotation.ComponentScan;
7+
import org.springframework.context.annotation.Configuration;
8+
import org.springframework.context.annotation.PropertySource;
9+
10+
import java.util.concurrent.TimeUnit;
11+
12+
@State(Scope.Benchmark)
13+
@BenchmarkMode({Mode.AverageTime, Mode.Throughput})
14+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
15+
@Warmup(iterations = 5, time = 5)
16+
@Measurement(iterations = 10, time = 5)
17+
@Fork(1)
18+
public class ContentCollectServiceBenchmark {
19+
20+
private ContentCollectService contentCollectService;
21+
22+
@Setup(Level.Trial)
23+
public void setUp() {
24+
var context = new AnnotationConfigApplicationContext(org.jun.saemangeum.benchmark.process.ContentCollectServiceBenchmark.TestConfig.class);
25+
contentCollectService = context.getBean(ContentCollectService.class);
26+
}
27+
28+
@Configuration
29+
@ComponentScan(basePackages = "org.jun.saemangeum")
30+
@PropertySource("classpath:application.yml")
31+
static class TestConfig {
32+
}
33+
34+
/**
35+
* 단일 스레드 내 블로킹 테스트
36+
*/
37+
@Benchmark
38+
public void benchmarkCollectAndSave() {
39+
contentCollectService.collectAndSave();
40+
}
41+
42+
/**
43+
* 단일 스레드 내 병렬 스트림 테스트
44+
*/
45+
@Benchmark
46+
public void benchmarkCollectAndSaveByParallelStream() {
47+
contentCollectService.collectAndSaveByParallelStream();
48+
}
49+
50+
/**
51+
* 가상 스레드 기반 프로세스 테스트
52+
*/
53+
@Benchmark
54+
public void benchmarkCollectAndSaveAsync() {
55+
contentCollectService.collectAndSaveAsync().join();
56+
}
57+
}

‎src/jmh/java/org/jun/saemangeum/benchmark/process/MockContentCollectServiceBenchmark.java‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package org.jun.saemangeum.benchmark.process;
2+
3+
import org.jun.saemangeum.global.domain.Content;
4+
import org.jun.saemangeum.global.repository.ContentRepository;
5+
import org.jun.saemangeum.process.application.collect.base.Refiner;
6+
import org.jun.saemangeum.process.application.service.ContentCollectService;
7+
import org.jun.saemangeum.global.service.ContentService;
8+
import org.mockito.Mockito;
9+
import org.openjdk.jmh.annotations.*;
10+
import org.springframework.context.annotation.ComponentScan;
11+
import org.springframework.context.annotation.Configuration;
12+
import org.springframework.core.task.TaskExecutor;
13+
import org.springframework.core.task.support.TaskExecutorAdapter;
14+
15+
import java.util.List;
16+
import java.util.concurrent.Executors;
17+
import java.util.concurrent.TimeUnit;
18+
import java.util.stream.Collectors;
19+
import java.util.stream.IntStream;
20+
21+
@State(Scope.Benchmark)
22+
@BenchmarkMode({Mode.AverageTime, Mode.Throughput})
23+
@OutputTimeUnit(TimeUnit.MILLISECONDS)
24+
@Warmup(iterations = 5, time = 1)
25+
@Measurement(iterations = 10, time = 1)
26+
@Fork(1)
27+
public class MockContentCollectServiceBenchmark {
28+
29+
private ContentCollectService contentCollectService;
30+
31+
@Configuration
32+
@ComponentScan(basePackages = "org.jun.saemangeum")
33+
static class TestConfig {
34+
}
35+
36+
/**
37+
* 외부 소스 의존성 모킹처리
38+
*/
39+
@Setup
40+
public void setup() {
41+
ContentRepository mockRepo = Mockito.mock(ContentRepository.class);
42+
// 실제 저장 없이 전달된 리스트 파라미터 그대로 반환하도록 모킹 처리
43+
Mockito.when(mockRepo.saveAll(Mockito.anyList()))
44+
.thenAnswer(invocation -> invocation.getArgument(0));
45+
46+
// Refiner마다 고유의 지연 시간 (ms)을 지정
47+
int[] contentCounts = {
48+
40, // 공공데이터 기반 축제 정보
49+
51, // 공공데이터 기반 공연행사 정보
50+
9, // 새만금개발청 방조제 관광지
51+
41, // 새만금개발청 인접도시 관광지
52+
6, // 새만금개발공사 군도 관광지
53+
10 // 군산시 문화관광 축제행사 정보
54+
};
55+
56+
int[] simulatedDelays = {
57+
300, // 축제 정보 API 지연
58+
350, // 공연행사 API 지연
59+
200, // 방조제 크롤링 지연
60+
800, // 인접도시 크롤링 지연
61+
600, // 군도 크롤링 지연
62+
150 // 군산시 페이지 크롤링 지연
63+
};
64+
65+
List<Refiner> mockRefiners = IntStream.range(0, contentCounts.length)
66+
.mapToObj(i -> {
67+
List<Content> mockContent = IntStream.range(0, contentCounts[i])
68+
.mapToObj(j -> Mockito.mock(Content.class))
69+
.collect(Collectors.toList());
70+
71+
Refiner refiner = Mockito.mock(Refiner.class);
72+
Mockito.when(refiner.refine()).thenAnswer(invocation -> {
73+
try {
74+
Thread.sleep(simulatedDelays[i]); // 지연 시뮬레이션
75+
} catch (InterruptedException e) {
76+
Thread.currentThread().interrupt(); // 인터럽트 복구
77+
}
78+
return mockContent;
79+
});
80+
return refiner;
81+
})
82+
.toList();
83+
84+
TaskExecutor virtualThreadExecutor =
85+
new TaskExecutorAdapter(Executors.newVirtualThreadPerTaskExecutor());
86+
contentCollectService = new ContentCollectService(
87+
mockRefiners, virtualThreadExecutor, new ContentService(mockRepo));
88+
}
89+
90+
/**
91+
* 단일 스레드 내 블로킹 테스트
92+
*/
93+
@Benchmark
94+
public void benchmarkCollectAndSave() {
95+
contentCollectService.collectAndSave();
96+
}
97+
98+
/**
99+
* 단일 스레드 내 병렬 스트림 테스트
100+
*/
101+
@Benchmark
102+
public void benchmarkCollectAndSaveByParallelStream() {
103+
contentCollectService.collectAndSaveByParallelStream();
104+
}
105+
106+
/**
107+
* 가상 스레드 기반 프로세스 테스트
108+
*/
109+
@Benchmark
110+
public void benchmarkCollectAndSaveAsync() {
111+
contentCollectService.collectAndSaveAsync().join();
112+
}
113+
}

‎src/main/java/org/jun/saemangeum/global/controller/DeployController.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
package org.jun.saemangeum.global.controller;
2+
3+
import lombok.RequiredArgsConstructor;
4+
import org.springframework.beans.factory.annotation.Value;
5+
import org.springframework.web.bind.annotation.GetMapping;
6+
import org.springframework.web.bind.annotation.RequestMapping;
7+
import org.springframework.web.bind.annotation.RestController;
8+
9+
@RestController
10+
@RequestMapping("/env")
11+
@RequiredArgsConstructor
12+
public class DeployController {
13+
14+
@Value("${server.env}")
15+
private String env;
16+
17+
@GetMapping
18+
public String env() {
19+
return env;
20+
}
21+
}

0 commit comments

Comments
 (0)