diff --git a/dodam-application/dodam-rest-api/build.gradle b/dodam-application/dodam-rest-api/build.gradle index 2e9c411c..0021f5a6 100644 --- a/dodam-application/dodam-rest-api/build.gradle +++ b/dodam-application/dodam-rest-api/build.gradle @@ -22,6 +22,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-security' implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.boot:spring-boot-starter-batch' + implementation 'org.jsoup:jsoup:1.15.3' implementation 'org.springframework:spring-tx:6.0.6' diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java index b211d867..d09a5fa8 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java @@ -3,8 +3,7 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@SpringBootApplication -public class DodamRestApiApplication { +@SpringBootApplication public class DodamRestApiApplication { public static void main(String[] args) { SpringApplication.run(DodamRestApiApplication.class, args); diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java index 8992efa2..38cf7716 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java @@ -4,7 +4,6 @@ import b1nd.dodam.domain.rds.bus.entity.Bus; import b1nd.dodam.domain.rds.bus.entity.BusApplication; import b1nd.dodam.domain.rds.bus.entity.BusTime; -import b1nd.dodam.domain.rds.bus.entity.BusTimeToBus; import b1nd.dodam.domain.rds.bus.enumeration.BusApplicationStatus; import b1nd.dodam.domain.rds.bus.exception.BusAlreadyAppliedException; import b1nd.dodam.domain.rds.bus.repository.BusApplicationRepository; @@ -22,7 +21,6 @@ import org.springframework.transaction.annotation.Transactional; import java.util.List; -import java.util.Optional; @Component @Transactional(readOnly = true) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java new file mode 100644 index 00000000..944d2f81 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java @@ -0,0 +1,40 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.domain.rds.notice.entity.Notice; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +@RequiredArgsConstructor +public class BatchConfig { + + private final JobRepository jobRepository; + private final PlatformTransactionManager platformTransactionManager; + private final NoticeItemWriter noticeItemWriter; + private final NoticeItemProcessor noticeItemProcessor; + private final NoticelerItemReader noticelerItemReader; + + @Bean + public Job noticeCrawlingJob() { + return new JobBuilder("noticeCrawlingJob", jobRepository) + .start(noticeCrawlingStep()) + .build(); + } + + @Bean + public Step noticeCrawlingStep() { + return new StepBuilder("noticeCrawlingStep", jobRepository) + .chunk(10, platformTransactionManager) + .reader(noticelerItemReader) + .processor(noticeItemProcessor) + .writer(noticeItemWriter) + .build(); + } +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java new file mode 100644 index 00000000..bd29a808 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java @@ -0,0 +1,30 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class BatchJobRunner { + + private final JobLauncher jobLauncher; + private final Job job; + + @Scheduled(cron = "0 0 12 * * ?") + private void executeBatchJob() { + try { + JobParameters jobParameter = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + jobLauncher.run(job, jobParameter); + } catch (Exception e) { + e.printStackTrace(); + } + } + +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java new file mode 100644 index 00000000..197d1ea0 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java @@ -0,0 +1,15 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.domain.rds.notice.entity.Notice; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.stereotype.Component; + +@Component +public class NoticeItemProcessor implements ItemProcessor { + + @Override + public Notice process(Notice item) { + return item; + } + +} \ No newline at end of file diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java new file mode 100644 index 00000000..8bfa0f86 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java @@ -0,0 +1,21 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.domain.rds.notice.entity.Notice; +import b1nd.dodam.domain.rds.notice.repository.NoticeRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class NoticeItemWriter implements ItemWriter { + + private final NoticeRepository noticeRepository; + + @Override + public void write(Chunk chunk) { + noticeRepository.saveAll(chunk); + } + +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java new file mode 100644 index 00000000..0dc70227 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java @@ -0,0 +1,110 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.client.core.WebClientSupport; +import b1nd.dodam.domain.rds.member.entity.Member; +import b1nd.dodam.domain.rds.member.repository.MemberRepository; +import b1nd.dodam.domain.rds.notice.entity.Notice; +import b1nd.dodam.domain.rds.notice.enumration.NoticeStatus; +import b1nd.dodam.domain.redis.notice.service.NoticeRedisService; +import b1nd.dodam.restapi.support.async.AsyncConfig; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.item.ItemReader; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; + +@Component +@StepScope +@RequiredArgsConstructor +public class NoticelerItemReader implements ItemReader { + + private final MemberRepository memberRepository; + private final NoticeRedisService noticeRedisService; + private final WebClientSupport webClient; + private final AsyncConfig asyncConfig; + private static final String URL = "https://dgsw.dge.hs.kr"; + private static final String ADDITIONAL_URL = "/dgswh/na/ntt/selectNttList.do?mi=10091723&bbsId=10091723"; + private static final String DETAIL_ADDITIONAL_URL = "/dgswh/na/ntt/selectNttInfo.do?mi=10091723&bbsId=10091723"; + private List notices; + private int nextIndex = 0; + private Member teacher; + private Map cookies; + private CompletableFuture> fetchNoticesFuture; + + @PostConstruct + public void init() { + this.teacher = memberRepository.getById("re"); + fetchNoticesFuture = fetchNoticesArticlesAsync(); + } + + @Override + public Notice read() { + try { + if (fetchNoticesFuture != null && fetchNoticesFuture.isDone()) { + if (notices == null) notices = fetchNoticesFuture.get(); + if (nextIndex < notices.size()) return notices.get(nextIndex++); + } + } catch (InterruptedException | ExecutionException e) { + } + return null; + } + + public CompletableFuture> fetchNoticesArticlesAsync() { + return CompletableFuture.supplyAsync(() -> { + try { + Connection.Response response = Jsoup + .connect(URL + ADDITIONAL_URL) + .method(Connection.Method.GET) + .execute(); + cookies = response.cookies(); + + return fetchDocument(URL + ADDITIONAL_URL, cookies) + .flatMapMany(doc -> Flux.fromIterable(doc.select("tr"))) + .flatMap(this::fetchNoticeDetailAsync) + .collectList() + .block(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, asyncConfig.batchTaskExecutor()); + } + + private Mono fetchNoticeDetailAsync(Element post) { + String nttSn = post.select("tr").select("td").select("a").attr("data-id"); + if (nttSn.isEmpty()) return Mono.empty(); + + String cacheKey = "notice:" + nttSn; + if (noticeRedisService.validateNotice(cacheKey)) { + return Mono.empty(); + } + + //TODO 홈페이지에서 파일도 크롤링 하기 + return fetchDocument(URL + DETAIL_ADDITIONAL_URL + "&nttSn=" + nttSn, cookies) + .map(doc -> { + Element detailElement = doc.select(".bbs_ViewA").first(); + String title = detailElement.select("h3").text().trim(); + String content = detailElement.select(".bbsV_cont").text().trim(); + noticeRedisService.setNotice(title, nttSn); + return new Notice(title, content, NoticeStatus.CREATED, teacher); + }); + } + + private Mono fetchDocument(String url, Map cookies) { + return webClient.batchGet(url, cookies) + .map(Jsoup::parse) + .onErrorReturn(new Document("")); + } + +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java index 1e0d91ef..eb0ffdae 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java @@ -4,6 +4,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import org.springframework.core.task.TaskExecutor; import org.springframework.scheduling.annotation.AsyncConfigurer; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -28,6 +29,17 @@ public Executor asyncExecutor() { return executor; } + @Bean + public TaskExecutor batchTaskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(5); + executor.setMaxPoolSize(10); + executor.setQueueCapacity(100); + executor.setThreadNamePrefix("Batch-"); + executor.initialize(); + return executor; + } + @Override public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { return new AsyncExceptionHandler(); diff --git a/dodam-application/dodam-rest-api/src/main/resources/application.yml b/dodam-application/dodam-rest-api/src/main/resources/application.yml index 45737469..197795e7 100644 --- a/dodam-application/dodam-rest-api/src/main/resources/application.yml +++ b/dodam-application/dodam-rest-api/src/main/resources/application.yml @@ -2,6 +2,9 @@ server: port: 8080 spring: + batch: + job: + enabled: false datasource: driver-class-name: com.mysql.cj.jdbc.Driver url: ${DB_URL} diff --git a/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java b/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java index 28bb4f5a..41f9a941 100644 --- a/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java +++ b/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java @@ -11,8 +11,10 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; +import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; @Component @RequiredArgsConstructor @@ -29,6 +31,16 @@ public Mono get(String url, Class responseDtoClass, String... headers) .bodyToMono(responseDtoClass); } + public Mono batchGet(String url, Map cookies) { + return webClient.get() + .uri(url) + .header(HttpHeaders.COOKIE, cookies.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining("; "))) + .retrieve() + .bodyToMono(String.class); + } + public void post(String url, V body, String... headers) { webClient.post() .uri(url) diff --git a/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java new file mode 100644 index 00000000..c850e95b --- /dev/null +++ b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java @@ -0,0 +1,23 @@ +package b1nd.dodam.domain.redis.notice.service; + +import lombok.RequiredArgsConstructor; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; + +@Service +@RequiredArgsConstructor +public class NoticeRedisService { + private final StringRedisTemplate redisTemplate; + + public void setNotice(String title, String nttSn){ + String cacheKey = "notice:" + nttSn; + redisTemplate.opsForValue().set(cacheKey, title, Duration.ofDays(1)); + } + + public boolean validateNotice(String cacheKey) { + return redisTemplate.hasKey(cacheKey); + } + +}