Refactor/#250

dodam-application/dodam-rest-api/build.gradle

@@ -22,6 +22,8 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-security'
     implementation 'org.springframework.boot:spring-boot-starter-actuator'
     implementation 'org.springframework.boot:spring-boot-starter-validation'
+    implementation 'org.springframework.boot:spring-boot-starter-batch'
+    implementation 'org.jsoup:jsoup:1.15.3'
 
     implementation 'org.springframework:spring-tx:6.0.6'
 

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java

@@ -1,9 +1,11 @@
 package b1nd.dodam;
 
+import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
 @SpringBootApplication
+@EnableBatchProcessing
 public class DodamRestApiApplication {
 
     public static void main(String[] args) {

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java

@@ -4,7 +4,6 @@
 import b1nd.dodam.domain.rds.bus.entity.Bus;
 import b1nd.dodam.domain.rds.bus.entity.BusApplication;
 import b1nd.dodam.domain.rds.bus.entity.BusTime;
-import b1nd.dodam.domain.rds.bus.entity.BusTimeToBus;
 import b1nd.dodam.domain.rds.bus.enumeration.BusApplicationStatus;
 import b1nd.dodam.domain.rds.bus.exception.BusAlreadyAppliedException;
 import b1nd.dodam.domain.rds.bus.repository.BusApplicationRepository;
@@ -22,7 +21,6 @@
 import org.springframework.transaction.annotation.Transactional;
 
 import java.util.List;
-import java.util.Optional;
 
 @Component
 @Transactional(readOnly = true)

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java

@@ -0,0 +1,40 @@
+package b1nd.dodam.restapi.notice.infrastructure.batch;
+
+import b1nd.dodam.domain.rds.notice.entity.Notice;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.Step;
+import org.springframework.batch.core.job.builder.JobBuilder;
+import org.springframework.batch.core.repository.JobRepository;
+import org.springframework.batch.core.step.builder.StepBuilder;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.transaction.PlatformTransactionManager;
+
+@Configuration
+@RequiredArgsConstructor
+public class BatchConfig {
+
+    private final JobRepository jobRepository;
+    private final PlatformTransactionManager platformTransactionManager;
+    private final NoticeItemWriter noticeItemWriter;
+    private final NoticeItemProcessor noticeItemProcessor;
+    private final NoticelerItemReader noticelerItemReader;
+
+    @Bean
+    public Job noticeCrawlingJob() {
+        return new JobBuilder("noticeCrawlingJob", jobRepository)
+                .start(noticeCrawlingStep())
+                .build();
+    }
+
+    @Bean
+    public Step noticeCrawlingStep() {
+        return new StepBuilder("noticeCrawlingStep", jobRepository)
+                .<Notice, Notice>chunk(10, platformTransactionManager)
+                .reader(noticelerItemReader)
+                .processor(noticeItemProcessor)
+                .writer(noticeItemWriter)
+                .build();
+    }
+}

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java

@@ -0,0 +1,30 @@
+package b1nd.dodam.restapi.notice.infrastructure.batch;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.core.Job;
+import org.springframework.batch.core.JobParameters;
+import org.springframework.batch.core.JobParametersBuilder;
+import org.springframework.batch.core.launch.JobLauncher;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class BatchJobRunner {
+
+    private final JobLauncher jobLauncher;
+    private final Job job;
+
+    @Scheduled(cron = "0 0 12 * * ?")
+    private void executeBatchJob() {
+        try {
+            JobParameters jobParameter = new JobParametersBuilder()
+                    .addLong("time", System.currentTimeMillis())
+                    .toJobParameters();
+            jobLauncher.run(job, jobParameter);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+
+}

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java

@@ -0,0 +1,15 @@
+package b1nd.dodam.restapi.notice.infrastructure.batch;
+
+import b1nd.dodam.domain.rds.notice.entity.Notice;
+import org.springframework.batch.item.ItemProcessor;
+import org.springframework.stereotype.Component;
+
+@Component
+public class NoticeItemProcessor implements ItemProcessor<Notice, Notice> {
+
+    @Override
+    public Notice process(Notice item) throws Exception {
+        return item;
+    }
+
+}

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java

@@ -0,0 +1,21 @@
+package b1nd.dodam.restapi.notice.infrastructure.batch;
+
+import b1nd.dodam.domain.rds.notice.entity.Notice;
+import b1nd.dodam.domain.rds.notice.repository.NoticeRepository;
+import lombok.RequiredArgsConstructor;
+import org.springframework.batch.item.Chunk;
+import org.springframework.batch.item.ItemWriter;
+import org.springframework.stereotype.Component;
+
+@Component
+@RequiredArgsConstructor
+public class NoticeItemWriter implements ItemWriter<Notice> {
+
+    private final NoticeRepository noticeRepository;
+
+    @Override
+    public void write(Chunk<? extends Notice> chunk) {
+        noticeRepository.saveAll(chunk);
+    }
+
+}

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java

@@ -0,0 +1,110 @@
+package b1nd.dodam.restapi.notice.infrastructure.batch;
+
+import b1nd.dodam.client.core.WebClientSupport;
+import b1nd.dodam.domain.rds.member.entity.Member;
+import b1nd.dodam.domain.rds.member.repository.MemberRepository;
+import b1nd.dodam.domain.rds.notice.entity.Notice;
+import b1nd.dodam.domain.rds.notice.enumration.NoticeStatus;
+import b1nd.dodam.domain.redis.notice.service.NoticeRedisService;
+import b1nd.dodam.restapi.support.async.AsyncConfig;
+import jakarta.annotation.PostConstruct;
+import lombok.RequiredArgsConstructor;
+import org.jsoup.Connection;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.springframework.batch.core.configuration.annotation.StepScope;
+import org.springframework.batch.item.ItemReader;
+import org.springframework.stereotype.Component;
+import reactor.core.publisher.Flux;
+import reactor.core.publisher.Mono;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+
+@Component
+@StepScope
+@RequiredArgsConstructor
+public class NoticelerItemReader implements ItemReader<Notice> {
+
+    private final MemberRepository memberRepository;
+    private final NoticeRedisService noticeRedisService;
+    private final WebClientSupport webClient;
+    private final AsyncConfig asyncConfig;
+    private static final String URL = "https://dgsw.dge.hs.kr";
+    private static final String ADDITIONAL_URL = "/dgswh/na/ntt/selectNttList.do?mi=10091723&bbsId=10091723";
+    private static final String DETAIL_ADDITIONAL_URL = "/dgswh/na/ntt/selectNttInfo.do?mi=10091723&bbsId=10091723";
+    private List<Notice> notices;
+    private int nextIndex = 0;
+    private Member teacher;
+    private Map<String, String> cookies;
+    private CompletableFuture<List<Notice>> fetchNoticesFuture;
+
+    @PostConstruct
+    public void init() {
+        this.teacher = memberRepository.getById("re");
+        fetchNoticesFuture = fetchNoticesArticlesAsync();
+    }
+
+    @Override
+    public Notice read() {
+        try {
+            if (fetchNoticesFuture != null && fetchNoticesFuture.isDone()) {
+                if (notices == null) notices = fetchNoticesFuture.get();
+                if (nextIndex < notices.size()) return notices.get(nextIndex++);
+            }
+        } catch (InterruptedException | ExecutionException e) {
+        }
+        return null;
+    }
+
+    public CompletableFuture<List<Notice>> fetchNoticesArticlesAsync() {
+        return CompletableFuture.supplyAsync(() -> {
+            try {
+                Connection.Response response = Jsoup
+                        .connect(URL + ADDITIONAL_URL)
+                        .method(Connection.Method.GET)
+                        .execute();
+                cookies = response.cookies();
+
+                return fetchDocument(URL + ADDITIONAL_URL, cookies)
+                        .flatMapMany(doc -> Flux.fromIterable(doc.select("tr")))
+                        .flatMap(this::fetchNoticeDetailAsync)
+                        .collectList()
+                        .block();
+            } catch (IOException e) {
+                throw new RuntimeException(e);
+            }
+        }, asyncConfig.batchTaskExecutor());
+    }
+
+    private Mono<Notice> fetchNoticeDetailAsync(Element post) {
+        String nttSn = post.select("tr").select("td").select("a").attr("data-id");
+        if (nttSn.isEmpty()) return Mono.empty();
+
+        String cacheKey = "notice:" + nttSn;
+        if (noticeRedisService.validateNotice(cacheKey)) {
+            return Mono.empty();
+        }
+
+        return fetchDocument(URL + DETAIL_ADDITIONAL_URL + "&nttSn=" + nttSn, cookies)
+                .map(doc -> {
+                    Element detailElement = doc.select(".bbs_ViewA").first();
+                    String title = detailElement.select("h3").text().trim();
+                    String content = detailElement.select(".bbsV_cont").text().trim();
+//                    String file = detailElement.select(".fname").text().trim();
+                    noticeRedisService.setNotice(title, nttSn);
+                    return new Notice(title, content, NoticeStatus.CREATED, teacher);
+                });
+    }
+
+    private Mono<Document> fetchDocument(String url, Map<String, String> cookies) {
+        return webClient.batchGet(url, cookies)
+                .map(Jsoup::parse)
+                .onErrorReturn(new Document(""));
+    }
+
+}

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java

@@ -38,6 +38,12 @@ public ResponseData<List<NoticeRes>> getByStatus(
         return noticeUseCase.getNotices(keyword, lastId, limit);
     }
 
+//    @GetMapping("/craw")
+//    public Response getCraw(
+//    ) throws Exception {
+//        return noticeUseCase.get();
+//    }
+
     @GetMapping("/division")
     public ResponseData<List<NoticeRes>> getById(
             @RequestParam(required = false) Long id,

dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java

@@ -4,6 +4,7 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Primary;
+import org.springframework.core.task.TaskExecutor;
 import org.springframework.scheduling.annotation.AsyncConfigurer;
 import org.springframework.scheduling.annotation.EnableAsync;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
@@ -28,6 +29,17 @@ public Executor asyncExecutor() {
         return executor;
     }
 
+    @Bean
+    public TaskExecutor batchTaskExecutor() {
+        ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor();
+        executor.setCorePoolSize(5);
+        executor.setMaxPoolSize(10);
+        executor.setQueueCapacity(100);
+        executor.setThreadNamePrefix("Batch-");
+        executor.initialize();
+        return executor;
+    }
+
     @Override
     public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() {
         return new AsyncExceptionHandler();

dodam-application/dodam-rest-api/src/main/resources/application.yml

@@ -2,6 +2,9 @@ server:
   port: 8080
 
 spring:
+  batch:
+    job:
+      enabled: false
   datasource:
     driver-class-name: com.mysql.cj.jdbc.Driver
     url: ${DB_URL}

dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java

@@ -11,8 +11,10 @@
 import org.springframework.web.reactive.function.client.WebClient;
 import reactor.core.publisher.Mono;
 
+import java.util.Map;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 @Component
 @RequiredArgsConstructor
@@ -29,6 +31,16 @@ public <T> Mono<T> get(String url, Class<T> responseDtoClass, String... headers)
                 .bodyToMono(responseDtoClass);
     }
 
+    public <T> Mono<String> batchGet(String url, Map<String, String> cookies) {
+        return webClient.get()
+                .uri(url)
+                .header(HttpHeaders.COOKIE, cookies.entrySet().stream()
+                        .map(entry -> entry.getKey() + "=" + entry.getValue())
+                        .collect(Collectors.joining("; ")))
+                .retrieve()
+                .bodyToMono(String.class);
+    }
+
     public <V> void post(String url, V body, String... headers) {
         webClient.post()
                 .uri(url)

dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java

@@ -0,0 +1,23 @@
+package b1nd.dodam.domain.redis.notice.service;
+
+import lombok.RequiredArgsConstructor;
+import org.springframework.data.redis.core.StringRedisTemplate;
+import org.springframework.stereotype.Service;
+
+import java.time.Duration;
+
+@Service
+@RequiredArgsConstructor
+public class NoticeRedisService {
+    private final StringRedisTemplate redisTemplate;
+
+    public void setNotice(String title, String nttSn){
+        String cacheKey = "notice:" + nttSn;
+        redisTemplate.opsForValue().set(cacheKey, title, Duration.ofDays(1));
+    }
+
+    public boolean validateNotice(String cacheKey) {
+        return redisTemplate.hasKey(cacheKey);
+    }
+
+}