From 5cfed5d355a4e54958acaef762a98bbb8a334e96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Tue, 18 Mar 2025 10:01:18 +0900 Subject: [PATCH 1/9] =?UTF-8?q?Feat:=20springBatch=EC=99=80=20Jsoup?= =?UTF-8?q?=EB=A5=BC=20=ED=86=B5=ED=95=B4=20=ED=95=99=EA=B5=90=20=ED=99=88?= =?UTF-8?q?=ED=8E=98=EC=9D=B4=EC=A7=80=20=ED=81=AC=EB=A1=A4=EB=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dodam-application/dodam-rest-api/build.gradle | 2 + .../b1nd/dodam/DodamRestApiApplication.java | 2 + .../notice/application/NoticeUseCase.java | 12 ++- .../infrastructure/batch/BatchConfig.java | 41 +++++++ .../infrastructure/batch/BatchJobRunner.java | 26 +++++ .../batch/NoticeItemProcessor.java | 15 +++ .../batch/NoticeItemWriter.java | 20 ++++ .../batch/NoticelerItemReader.java | 102 ++++++++++++++++++ .../notice/presentation/NoticeController.java | 6 ++ .../src/main/resources/application.yml | 3 + 10 files changed, 228 insertions(+), 1 deletion(-) create mode 100644 dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java create mode 100644 dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java create mode 100644 dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java create mode 100644 dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java create mode 100644 dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java diff --git a/dodam-application/dodam-rest-api/build.gradle b/dodam-application/dodam-rest-api/build.gradle index 2e9c411c..0021f5a6 100644 --- a/dodam-application/dodam-rest-api/build.gradle +++ b/dodam-application/dodam-rest-api/build.gradle @@ -22,6 +22,8 @@ dependencies { implementation 'org.springframework.boot:spring-boot-starter-security' implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'org.springframework.boot:spring-boot-starter-validation' + implementation 'org.springframework.boot:spring-boot-starter-batch' + implementation 'org.jsoup:jsoup:1.15.3' implementation 'org.springframework:spring-tx:6.0.6' diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java index b211d867..e29c9690 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java @@ -1,9 +1,11 @@ package b1nd.dodam; +import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; @SpringBootApplication +@EnableBatchProcessing public class DodamRestApiApplication { public static void main(String[] args) { diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java index 9a3e21b2..ee09067b 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java @@ -13,6 +13,8 @@ import b1nd.dodam.restapi.notice.application.data.req.GenerateNoticeReq; import b1nd.dodam.restapi.notice.application.data.req.ModifyNoticeReq; import b1nd.dodam.restapi.notice.application.data.res.NoticeRes; +import b1nd.dodam.restapi.notice.infrastructure.batch.BatchJobRunner; +import b1nd.dodam.restapi.notice.infrastructure.batch.NoticelerItemReader; import b1nd.dodam.restapi.support.data.Response; import b1nd.dodam.restapi.support.data.ResponseData; import lombok.RequiredArgsConstructor; @@ -26,13 +28,21 @@ @Component @RequiredArgsConstructor -@Transactional(rollbackFor = Exception.class) +//@Transactional(rollbackFor = Exception.class) public class NoticeUseCase { private final NoticeService noticeService; private final DivisionService divisionService; private final DivisionMemberService divisionMemberService; private final MemberAuthenticationHolder memberAuthenticationHolder; + private final BatchJobRunner batchJobRunner; + private final NoticelerItemReader noticelerItemReader; + + public Response get() throws Exception { +// batchJobRunner.run(); + noticelerItemReader.fetchNoticesArticles(); + return Response.ok("성공"); + } public ResponseData register(GenerateNoticeReq generateNoticeReq) { Member member = memberAuthenticationHolder.current(); diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java new file mode 100644 index 00000000..f1db3182 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java @@ -0,0 +1,41 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.domain.rds.notice.entity.Notice; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.Step; +import org.springframework.batch.core.job.builder.JobBuilder; +import org.springframework.batch.core.repository.JobRepository; +import org.springframework.batch.core.step.builder.StepBuilder; +import org.springframework.batch.repeat.RepeatStatus; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.transaction.PlatformTransactionManager; + +@Configuration +@RequiredArgsConstructor +public class BatchConfig { + + private final JobRepository jobRepository; + private final PlatformTransactionManager platformTransactionManager; + private final NoticeItemWriter noticeItemWriter; + private final NoticeItemProcessor noticeItemProcessor; + private final NoticelerItemReader noticelerItemReader; + + @Bean + public Job noticeCrawlingJob() { + return new JobBuilder("noticeCrawlingJob", jobRepository) + .start(noticeCrawlingStep()) + .build(); + } + + @Bean + public Step noticeCrawlingStep() { + return new StepBuilder("noticeCrawlingStep", jobRepository) + .chunk(10, platformTransactionManager) + .reader(noticelerItemReader) + .processor(noticeItemProcessor) + .writer(noticeItemWriter) + .build(); + } +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java new file mode 100644 index 00000000..d735d5cb --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java @@ -0,0 +1,26 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import lombok.RequiredArgsConstructor; +import org.springframework.batch.core.Job; +import org.springframework.batch.core.JobParameters; +import org.springframework.batch.core.JobParametersBuilder; +import org.springframework.batch.core.launch.JobLauncher; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class BatchJobRunner implements CommandLineRunner { + + private final JobLauncher jobLauncher; + private final Job job; + + @Override + public void run(String... args) throws Exception { + JobParameters jobParameter = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + + jobLauncher.run(job, jobParameter); + } +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java new file mode 100644 index 00000000..c3f342e8 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java @@ -0,0 +1,15 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.domain.rds.notice.entity.Notice; +import org.springframework.batch.item.ItemProcessor; +import org.springframework.stereotype.Component; + +@Component +public class NoticeItemProcessor implements ItemProcessor { + + @Override + public Notice process(Notice item) throws Exception { + // 여기서 필요한 추가적인 처리 로직을 넣을 수 있습니다. + return item; // 예시로 그대로 반환 + } +} \ No newline at end of file diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java new file mode 100644 index 00000000..6fe4e789 --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java @@ -0,0 +1,20 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.domain.rds.notice.entity.Notice; +import b1nd.dodam.domain.rds.notice.repository.NoticeRepository; +import lombok.RequiredArgsConstructor; +import org.springframework.batch.item.Chunk; +import org.springframework.batch.item.ItemWriter; +import org.springframework.stereotype.Component; + +@Component +@RequiredArgsConstructor +public class NoticeItemWriter implements ItemWriter { + + private final NoticeRepository noticeRepository; + + @Override + public void write(Chunk chunk) throws Exception { + noticeRepository.saveAll(chunk); + } +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java new file mode 100644 index 00000000..59d3d65c --- /dev/null +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java @@ -0,0 +1,102 @@ +package b1nd.dodam.restapi.notice.infrastructure.batch; + +import b1nd.dodam.domain.rds.member.entity.Member; +import b1nd.dodam.domain.rds.member.repository.MemberRepository; +import b1nd.dodam.domain.rds.notice.entity.Notice; +import b1nd.dodam.domain.rds.notice.enumration.NoticeStatus; +import jakarta.annotation.PostConstruct; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jsoup.Connection; +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; +import org.springframework.batch.core.configuration.annotation.StepScope; +import org.springframework.batch.item.ItemReader; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Slf4j +@Component +//@StepScope +@RequiredArgsConstructor +public class NoticelerItemReader implements ItemReader { + + private final MemberRepository memberRepository; + private List notices; + private int nextIndex = 0; + private final String url = "https://dgsw.dge.hs.kr"; + private final String additionalUrl = "/dgswh/na/ntt/selectNttList.do?mi=10091723&bbsId=10091723"; + private Member testeacher; + +// @PostConstruct +// public void init() { +// this.testeacher = memberRepository.getById(String.valueOf(testeacher)); +// this.notices = fetchNoticesArticles(); +// } + + @PostConstruct + public void init() throws IOException { + this.testeacher = memberRepository.getById("re"); + this.notices = fetchNoticesArticles(); + } + + @Override + public Notice read() throws Exception { + if (nextIndex < notices.size()) { + return notices.get(nextIndex++); + } else { + return null; + } + } + + public List fetchNoticesArticles() throws IOException { + List notices = new ArrayList<>(); + Connection.Response response = Jsoup + .connect(url) + .method(Connection.Method.GET) + .execute(); + + Map cookies = response.cookies(); + Document document = Jsoup.connect(url + additionalUrl) + .cookies(cookies) + .get(); + + log.info("cookies : {}", cookies); + + Elements posts = document.select("td.bbs_tit > a"); + + log.info("posts : {}", posts); + for (Element post : posts) { + String nttSn = post.attr("data-id"); + String bbsId = "10091704"; + String mi = "10091704"; + + // 공지사항의 상세 URL 생성 + String detailUrl = url + additionalUrl + "&nttSn=" + nttSn; + log.info("detailUrl: {}", detailUrl); + + // 공지사항 상세 페이지 크롤링 + Document detailDocument = Jsoup.connect(detailUrl) + .cookies(cookies) + .get(); + log.info("detailDocument: {}", detailDocument); + + // 제목, 내용, 파일 등 필요한 데이터 추출 + String title = detailDocument.select(".bbs_ViewA").text(); // 제목 + String content = detailDocument.select(".bbsV_cont").text(); // 내용 + String file = detailDocument.select(".fname").text(); // 첨부파일 정보 + + // Notice 객체 생성 후 리스트에 추가 + notices.add(new Notice(title, content, NoticeStatus.CREATED, testeacher)); + log.info("Title: " + title); + log.info("Content: " + content); + } + return notices; + } +} \ No newline at end of file diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java index 484cb9d8..c28e0274 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java @@ -38,6 +38,12 @@ public ResponseData> getByStatus( return noticeUseCase.getNotices(keyword, lastId, limit); } + @GetMapping("/craw") + public Response getCraw( + ) throws Exception { + return noticeUseCase.get(); + } + @GetMapping("/division") public ResponseData> getById( @RequestParam(required = false) Long id, diff --git a/dodam-application/dodam-rest-api/src/main/resources/application.yml b/dodam-application/dodam-rest-api/src/main/resources/application.yml index 45737469..197795e7 100644 --- a/dodam-application/dodam-rest-api/src/main/resources/application.yml +++ b/dodam-application/dodam-rest-api/src/main/resources/application.yml @@ -2,6 +2,9 @@ server: port: 8080 spring: + batch: + job: + enabled: false datasource: driver-class-name: com.mysql.cj.jdbc.Driver url: ${DB_URL} From 4a6b5d054b0aec9b50f3e17f59e92107bce50446 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Tue, 25 Mar 2025 12:33:41 +0900 Subject: [PATCH 2/9] =?UTF-8?q?chore:=20=ED=95=99=EA=B5=90=20=ED=99=88?= =?UTF-8?q?=ED=8E=98=EC=9D=B4=EC=A7=80=20=ED=81=AC=EB=A1=A4=EB=A7=81=20?= =?UTF-8?q?=EC=88=98=EC=A0=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../application/BusApplicationUseCase.java | 2 - .../batch/NoticeItemProcessor.java | 3 +- .../batch/NoticelerItemReader.java | 48 +++++++++++-------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java index 8992efa2..38cf7716 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/bus/application/BusApplicationUseCase.java @@ -4,7 +4,6 @@ import b1nd.dodam.domain.rds.bus.entity.Bus; import b1nd.dodam.domain.rds.bus.entity.BusApplication; import b1nd.dodam.domain.rds.bus.entity.BusTime; -import b1nd.dodam.domain.rds.bus.entity.BusTimeToBus; import b1nd.dodam.domain.rds.bus.enumeration.BusApplicationStatus; import b1nd.dodam.domain.rds.bus.exception.BusAlreadyAppliedException; import b1nd.dodam.domain.rds.bus.repository.BusApplicationRepository; @@ -22,7 +21,6 @@ import org.springframework.transaction.annotation.Transactional; import java.util.List; -import java.util.Optional; @Component @Transactional(readOnly = true) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java index c3f342e8..56e13119 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java @@ -9,7 +9,6 @@ public class NoticeItemProcessor implements ItemProcessor { @Override public Notice process(Notice item) throws Exception { - // 여기서 필요한 추가적인 처리 로직을 넣을 수 있습니다. - return item; // 예시로 그대로 반환 + return item; } } \ No newline at end of file diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java index 59d3d65c..6af2cfcf 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java @@ -15,11 +15,13 @@ import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.item.ItemReader; import org.springframework.stereotype.Component; +import org.springframework.util.StringUtils; import java.io.IOException; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j @Component @@ -32,6 +34,7 @@ public class NoticelerItemReader implements ItemReader { private int nextIndex = 0; private final String url = "https://dgsw.dge.hs.kr"; private final String additionalUrl = "/dgswh/na/ntt/selectNttList.do?mi=10091723&bbsId=10091723"; + private final String detailAdditionalUrl = "/dgswh/na/ntt/selectNttInfo.do?mi=10091723&bbsId=10091723"; private Member testeacher; // @PostConstruct @@ -57,46 +60,53 @@ public Notice read() throws Exception { public List fetchNoticesArticles() throws IOException { List notices = new ArrayList<>(); + Connection.Response response = Jsoup - .connect(url) + .connect(url + additionalUrl) .method(Connection.Method.GET) .execute(); Map cookies = response.cookies(); + Document document = Jsoup.connect(url + additionalUrl) .cookies(cookies) .get(); - log.info("cookies : {}", cookies); - Elements posts = document.select("td.bbs_tit > a"); - + Elements posts = document.select("tr"); log.info("posts : {}", posts); - for (Element post : posts) { - String nttSn = post.attr("data-id"); - String bbsId = "10091704"; - String mi = "10091704"; - // 공지사항의 상세 URL 생성 - String detailUrl = url + additionalUrl + "&nttSn=" + nttSn; + for (Element post : posts) { + log.info("---------시작------------"); + log.info("post:{}", post); + String nttSn = post.select("tr").select("td").select("a").attr("data-id"); +// String nttSn = post.attr("data-id").trim(); + log.info("href: {}", nttSn); + if (nttSn.isEmpty()) continue; + + String detailUrl = url + detailAdditionalUrl + "&nttSn=" + nttSn; log.info("detailUrl: {}", detailUrl); - // 공지사항 상세 페이지 크롤링 Document detailDocument = Jsoup.connect(detailUrl) .cookies(cookies) .get(); - log.info("detailDocument: {}", detailDocument); - // 제목, 내용, 파일 등 필요한 데이터 추출 - String title = detailDocument.select(".bbs_ViewA").text(); // 제목 - String content = detailDocument.select(".bbsV_cont").text(); // 내용 - String file = detailDocument.select(".fname").text(); // 첨부파일 정보 + Elements detail = detailDocument.select(".bbs_ViewA"); +// log.info("detail : {}", detail); + + Elements titleElement = detail.select("h3"); + String title = (titleElement != null) ? titleElement.text().trim() : "제목 없음"; + String content = detail.select(".bbsV_cont").text().trim(); + String file = detail.select(".fname").text().trim(); - // Notice 객체 생성 후 리스트에 추가 notices.add(new Notice(title, content, NoticeStatus.CREATED, testeacher)); - log.info("Title: " + title); - log.info("Content: " + content); + + log.info("Title: {}", title); + log.info("Content: {}", content); + log.info("file: {}", file); } + return notices; } + } \ No newline at end of file From 2410e5a3af6b07db808116f2a48b944753db41d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Tue, 25 Mar 2025 16:08:44 +0900 Subject: [PATCH 3/9] =?UTF-8?q?chore:=20=EA=B3=B5=EC=A7=80=20=ED=81=AC?= =?UTF-8?q?=EB=A1=A4=EB=A7=81=20=EB=B9=84=EB=8F=99=EA=B8=B0=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dodam/domain/redis/notice/service/NoticeRedisService.java | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java diff --git a/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java new file mode 100644 index 00000000..b30da16a --- /dev/null +++ b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java @@ -0,0 +1,2 @@ +package b1nd.dodam.domain.redis.notice.service;public class NoticeRedisService { +} From a64959042a6daa9440fc1ef5bde1d49796e35260 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Tue, 25 Mar 2025 16:34:03 +0900 Subject: [PATCH 4/9] chore: .. --- .../dodam/domain/redis/notice/service/NoticeRedisService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java index b30da16a..fc07f63f 100644 --- a/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java +++ b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java @@ -1,2 +1,4 @@ -package b1nd.dodam.domain.redis.notice.service;public class NoticeRedisService { +package b1nd.dodam.domain.redis.notice.service; + +public class NoticeRedisService { } From 610276e0b9229cc500d539129b41ac51bf0a6a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Wed, 26 Mar 2025 22:13:25 +0900 Subject: [PATCH 5/9] =?UTF-8?q?refactor:=20=ED=81=AC=EB=A1=A4=EB=A7=81=20?= =?UTF-8?q?=EB=B9=84=EB=8F=99=EA=B8=B0=ED=99=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../notice/application/NoticeUseCase.java | 12 +- .../batch/NoticeItemProcessor.java | 1 + .../batch/NoticeItemWriter.java | 1 + .../batch/NoticelerItemReader.java | 127 +++++++++--------- .../notice/presentation/NoticeController.java | 10 +- .../restapi/support/async/AsyncConfig.java | 12 ++ .../dodam/client/core/WebClientSupport.java | 12 ++ .../notice/service/NoticeRedisService.java | 19 +++ 8 files changed, 112 insertions(+), 82 deletions(-) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java index ee09067b..9a3e21b2 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/application/NoticeUseCase.java @@ -13,8 +13,6 @@ import b1nd.dodam.restapi.notice.application.data.req.GenerateNoticeReq; import b1nd.dodam.restapi.notice.application.data.req.ModifyNoticeReq; import b1nd.dodam.restapi.notice.application.data.res.NoticeRes; -import b1nd.dodam.restapi.notice.infrastructure.batch.BatchJobRunner; -import b1nd.dodam.restapi.notice.infrastructure.batch.NoticelerItemReader; import b1nd.dodam.restapi.support.data.Response; import b1nd.dodam.restapi.support.data.ResponseData; import lombok.RequiredArgsConstructor; @@ -28,21 +26,13 @@ @Component @RequiredArgsConstructor -//@Transactional(rollbackFor = Exception.class) +@Transactional(rollbackFor = Exception.class) public class NoticeUseCase { private final NoticeService noticeService; private final DivisionService divisionService; private final DivisionMemberService divisionMemberService; private final MemberAuthenticationHolder memberAuthenticationHolder; - private final BatchJobRunner batchJobRunner; - private final NoticelerItemReader noticelerItemReader; - - public Response get() throws Exception { -// batchJobRunner.run(); - noticelerItemReader.fetchNoticesArticles(); - return Response.ok("성공"); - } public ResponseData register(GenerateNoticeReq generateNoticeReq) { Member member = memberAuthenticationHolder.current(); diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java index 56e13119..f21f0fc5 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java @@ -11,4 +11,5 @@ public class NoticeItemProcessor implements ItemProcessor { public Notice process(Notice item) throws Exception { return item; } + } \ No newline at end of file diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java index 6fe4e789..e1958b96 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java @@ -17,4 +17,5 @@ public class NoticeItemWriter implements ItemWriter { public void write(Chunk chunk) throws Exception { noticeRepository.saveAll(chunk); } + } diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java index 6af2cfcf..e045c01d 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java @@ -1,9 +1,12 @@ package b1nd.dodam.restapi.notice.infrastructure.batch; +import b1nd.dodam.client.core.WebClientSupport; import b1nd.dodam.domain.rds.member.entity.Member; import b1nd.dodam.domain.rds.member.repository.MemberRepository; import b1nd.dodam.domain.rds.notice.entity.Notice; import b1nd.dodam.domain.rds.notice.enumration.NoticeStatus; +import b1nd.dodam.domain.redis.notice.service.NoticeRedisService; +import b1nd.dodam.restapi.support.async.AsyncConfig; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -11,17 +14,15 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; -import org.jsoup.select.Elements; -import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.item.ItemReader; import org.springframework.stereotype.Component; -import org.springframework.util.StringUtils; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; +import java.util.concurrent.CompletableFuture; @Slf4j @Component @@ -30,83 +31,77 @@ public class NoticelerItemReader implements ItemReader { private final MemberRepository memberRepository; + private final NoticeRedisService noticeRedisService; + private final WebClientSupport webClient; + private final AsyncConfig asyncConfig; + private static final String URL = "https://dgsw.dge.hs.kr"; + private static final String ADDITIONAL_URL = "/dgswh/na/ntt/selectNttList.do?mi=10091723&bbsId=10091723"; + private static final String DETAIL_ADDITIONAL_URL = "/dgswh/na/ntt/selectNttInfo.do?mi=10091723&bbsId=10091723"; private List notices; private int nextIndex = 0; - private final String url = "https://dgsw.dge.hs.kr"; - private final String additionalUrl = "/dgswh/na/ntt/selectNttList.do?mi=10091723&bbsId=10091723"; - private final String detailAdditionalUrl = "/dgswh/na/ntt/selectNttInfo.do?mi=10091723&bbsId=10091723"; - private Member testeacher; - -// @PostConstruct -// public void init() { -// this.testeacher = memberRepository.getById(String.valueOf(testeacher)); -// this.notices = fetchNoticesArticles(); -// } + private Member teacher; + private Map cookies; @PostConstruct public void init() throws IOException { - this.testeacher = memberRepository.getById("re"); - this.notices = fetchNoticesArticles(); + this.teacher = memberRepository.getById("re"); + fetchNoticesArticlesAsync().thenAccept(notices -> this.notices = notices); } @Override - public Notice read() throws Exception { + public Notice read() { if (nextIndex < notices.size()) { return notices.get(nextIndex++); - } else { - return null; } + return null; } - public List fetchNoticesArticles() throws IOException { - List notices = new ArrayList<>(); - - Connection.Response response = Jsoup - .connect(url + additionalUrl) - .method(Connection.Method.GET) - .execute(); - - Map cookies = response.cookies(); - - Document document = Jsoup.connect(url + additionalUrl) - .cookies(cookies) - .get(); - log.info("cookies : {}", cookies); - - Elements posts = document.select("tr"); - log.info("posts : {}", posts); - - for (Element post : posts) { - log.info("---------시작------------"); - log.info("post:{}", post); - String nttSn = post.select("tr").select("td").select("a").attr("data-id"); -// String nttSn = post.attr("data-id").trim(); - log.info("href: {}", nttSn); - if (nttSn.isEmpty()) continue; - - String detailUrl = url + detailAdditionalUrl + "&nttSn=" + nttSn; - log.info("detailUrl: {}", detailUrl); - - Document detailDocument = Jsoup.connect(detailUrl) - .cookies(cookies) - .get(); - - Elements detail = detailDocument.select(".bbs_ViewA"); -// log.info("detail : {}", detail); - - Elements titleElement = detail.select("h3"); - String title = (titleElement != null) ? titleElement.text().trim() : "제목 없음"; - String content = detail.select(".bbsV_cont").text().trim(); - String file = detail.select(".fname").text().trim(); + public CompletableFuture> fetchNoticesArticlesAsync() { + return CompletableFuture.supplyAsync(() -> { + try { + Connection.Response response = Jsoup + .connect(URL + ADDITIONAL_URL) + .method(Connection.Method.GET) + .execute(); + cookies = response.cookies(); + + return fetchDocument(URL + ADDITIONAL_URL, cookies) + .flatMapMany(doc -> Flux.fromIterable(doc.select("tr"))) + .flatMap(this::fetchNoticeDetailAsync) + .collectList() + .block(); + } catch (IOException e) { + throw new RuntimeException(e); + } + }, asyncConfig.batchTaskExecutor()); + } - notices.add(new Notice(title, content, NoticeStatus.CREATED, testeacher)); + private Mono fetchNoticeDetailAsync(Element post) { + String nttSn = post.select("tr").select("td").select("a").attr("data-id"); + if (nttSn.isEmpty()) return Mono.empty(); - log.info("Title: {}", title); - log.info("Content: {}", content); - log.info("file: {}", file); + String cacheKey = "notice:" + nttSn; + if (noticeRedisService.validateNotice(cacheKey)) { + log.info("Skipping cached notice: {}", nttSn); + return Mono.empty(); } - return notices; + return fetchDocument(URL + DETAIL_ADDITIONAL_URL + "&nttSn=" + nttSn, cookies) + .map(doc -> { + String title = doc.select("h3").text().trim(); + String content = doc.select(".bbsV_cont").text().trim(); + String file = doc.select(".fname").text().trim(); + + log.info("title : {}", title); + log.info("content : {}", content); + log.info("file : {}", file); + return new Notice(title, content, NoticeStatus.CREATED, teacher); + }); } -} \ No newline at end of file + private Mono fetchDocument(String url, Map cookies) { + return webClient.batchGet(url, cookies) + .map(Jsoup::parse) + .onErrorReturn(new Document("")); + } +} diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java index c28e0274..8cbef599 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java @@ -38,11 +38,11 @@ public ResponseData> getByStatus( return noticeUseCase.getNotices(keyword, lastId, limit); } - @GetMapping("/craw") - public Response getCraw( - ) throws Exception { - return noticeUseCase.get(); - } +// @GetMapping("/craw") +// public Response getCraw( +// ) throws Exception { +// return noticeUseCase.get(); +// } @GetMapping("/division") public ResponseData> getById( diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java index 1e0d91ef..eb0ffdae 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/support/async/AsyncConfig.java @@ -4,6 +4,7 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Primary; +import org.springframework.core.task.TaskExecutor; import org.springframework.scheduling.annotation.AsyncConfigurer; import org.springframework.scheduling.annotation.EnableAsync; import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; @@ -28,6 +29,17 @@ public Executor asyncExecutor() { return executor; } + @Bean + public TaskExecutor batchTaskExecutor() { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(5); + executor.setMaxPoolSize(10); + executor.setQueueCapacity(100); + executor.setThreadNamePrefix("Batch-"); + executor.initialize(); + return executor; + } + @Override public AsyncUncaughtExceptionHandler getAsyncUncaughtExceptionHandler() { return new AsyncExceptionHandler(); diff --git a/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java b/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java index 28bb4f5a..41f9a941 100644 --- a/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java +++ b/dodam-in-system-available/dodam-client-core/src/main/java/b1nd/dodam/client/core/WebClientSupport.java @@ -11,8 +11,10 @@ import org.springframework.web.reactive.function.client.WebClient; import reactor.core.publisher.Mono; +import java.util.Map; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; @Component @RequiredArgsConstructor @@ -29,6 +31,16 @@ public Mono get(String url, Class responseDtoClass, String... headers) .bodyToMono(responseDtoClass); } + public Mono batchGet(String url, Map cookies) { + return webClient.get() + .uri(url) + .header(HttpHeaders.COOKIE, cookies.entrySet().stream() + .map(entry -> entry.getKey() + "=" + entry.getValue()) + .collect(Collectors.joining("; "))) + .retrieve() + .bodyToMono(String.class); + } + public void post(String url, V body, String... headers) { webClient.post() .uri(url) diff --git a/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java index fc07f63f..c850e95b 100644 --- a/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java +++ b/dodam-system-domain/dodam-domain-redis/src/main/java/b1nd/dodam/domain/redis/notice/service/NoticeRedisService.java @@ -1,4 +1,23 @@ package b1nd.dodam.domain.redis.notice.service; +import lombok.RequiredArgsConstructor; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.stereotype.Service; + +import java.time.Duration; + +@Service +@RequiredArgsConstructor public class NoticeRedisService { + private final StringRedisTemplate redisTemplate; + + public void setNotice(String title, String nttSn){ + String cacheKey = "notice:" + nttSn; + redisTemplate.opsForValue().set(cacheKey, title, Duration.ofDays(1)); + } + + public boolean validateNotice(String cacheKey) { + return redisTemplate.hasKey(cacheKey); + } + } From 064503a07dd6395775cfcaa0016d5508ee84809a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Thu, 27 Mar 2025 15:38:19 +0900 Subject: [PATCH 6/9] =?UTF-8?q?refactor:=20batch=20scheduler=20=EC=A0=81?= =?UTF-8?q?=EC=9A=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../infrastructure/batch/BatchConfig.java | 1 - .../infrastructure/batch/BatchJobRunner.java | 22 ++++++----- .../batch/NoticeItemWriter.java | 2 +- .../batch/NoticelerItemReader.java | 39 +++++++++++++------ 4 files changed, 42 insertions(+), 22 deletions(-) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java index f1db3182..944d2f81 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchConfig.java @@ -7,7 +7,6 @@ import org.springframework.batch.core.job.builder.JobBuilder; import org.springframework.batch.core.repository.JobRepository; import org.springframework.batch.core.step.builder.StepBuilder; -import org.springframework.batch.repeat.RepeatStatus; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.transaction.PlatformTransactionManager; diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java index d735d5cb..d3064586 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java @@ -5,22 +5,26 @@ import org.springframework.batch.core.JobParameters; import org.springframework.batch.core.JobParametersBuilder; import org.springframework.batch.core.launch.JobLauncher; -import org.springframework.boot.CommandLineRunner; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @Component @RequiredArgsConstructor -public class BatchJobRunner implements CommandLineRunner { +public class BatchJobRunner { private final JobLauncher jobLauncher; private final Job job; - @Override - public void run(String... args) throws Exception { - JobParameters jobParameter = new JobParametersBuilder() - .addLong("time", System.currentTimeMillis()) - .toJobParameters(); - - jobLauncher.run(job, jobParameter); + @Scheduled(cron = "0 0/5 * * * ?") + private void executeBatchJob() { + try { + JobParameters jobParameter = new JobParametersBuilder() + .addLong("time", System.currentTimeMillis()) + .toJobParameters(); + jobLauncher.run(job, jobParameter); + } catch (Exception e) { + e.printStackTrace(); + } } + } diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java index e1958b96..8bfa0f86 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemWriter.java @@ -14,7 +14,7 @@ public class NoticeItemWriter implements ItemWriter { private final NoticeRepository noticeRepository; @Override - public void write(Chunk chunk) throws Exception { + public void write(Chunk chunk) { noticeRepository.saveAll(chunk); } diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java index e045c01d..f51e8775 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java @@ -14,6 +14,7 @@ import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; +import org.springframework.batch.core.configuration.annotation.StepScope; import org.springframework.batch.item.ItemReader; import org.springframework.stereotype.Component; import reactor.core.publisher.Flux; @@ -23,10 +24,11 @@ import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; @Slf4j @Component -//@StepScope +@StepScope @RequiredArgsConstructor public class NoticelerItemReader implements ItemReader { @@ -41,17 +43,23 @@ public class NoticelerItemReader implements ItemReader { private int nextIndex = 0; private Member teacher; private Map cookies; + private CompletableFuture> fetchNoticesFuture; @PostConstruct - public void init() throws IOException { + public void init() { this.teacher = memberRepository.getById("re"); - fetchNoticesArticlesAsync().thenAccept(notices -> this.notices = notices); + fetchNoticesFuture = fetchNoticesArticlesAsync(); } @Override public Notice read() { - if (nextIndex < notices.size()) { - return notices.get(nextIndex++); + try { + if (fetchNoticesFuture != null && fetchNoticesFuture.isDone()) { + if (notices == null) notices = fetchNoticesFuture.get(); + if (nextIndex < notices.size()) return notices.get(nextIndex++); + } + } catch (InterruptedException | ExecutionException e) { + log.error("Error while fetching notices", e); } return null; } @@ -88,13 +96,21 @@ private Mono fetchNoticeDetailAsync(Element post) { return fetchDocument(URL + DETAIL_ADDITIONAL_URL + "&nttSn=" + nttSn, cookies) .map(doc -> { - String title = doc.select("h3").text().trim(); - String content = doc.select(".bbsV_cont").text().trim(); - String file = doc.select(".fname").text().trim(); + Element detailElement = doc.select(".bbs_ViewA").first(); + + if (detailElement == null) { + log.error("No detail found for notice: {}", nttSn); + return null; + } + + String title = detailElement.select("h3").text().trim(); + String content = detailElement.select(".bbsV_cont").text().trim(); + String file = detailElement.select(".fname").text().trim(); + + log.info("Notice Title: {}", title); + log.info("Notice Content: {}", content); + log.info("Notice File: {}", file); - log.info("title : {}", title); - log.info("content : {}", content); - log.info("file : {}", file); return new Notice(title, content, NoticeStatus.CREATED, teacher); }); } @@ -104,4 +120,5 @@ private Mono fetchDocument(String url, Map cookies) { .map(Jsoup::parse) .onErrorReturn(new Document("")); } + } From 0daabb1dfc3337ec0ddd740b5502af24890dc9cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Thu, 27 Mar 2025 17:49:33 +0900 Subject: [PATCH 7/9] refactor: spring batch scheduler --- .../infrastructure/batch/BatchJobRunner.java | 2 +- .../batch/NoticelerItemReader.java | 18 ++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java index d3064586..bd29a808 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/BatchJobRunner.java @@ -15,7 +15,7 @@ public class BatchJobRunner { private final JobLauncher jobLauncher; private final Job job; - @Scheduled(cron = "0 0/5 * * * ?") + @Scheduled(cron = "0 0 12 * * ?") private void executeBatchJob() { try { JobParameters jobParameter = new JobParametersBuilder() diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java index f51e8775..65a71906 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java @@ -9,7 +9,6 @@ import b1nd.dodam.restapi.support.async.AsyncConfig; import jakarta.annotation.PostConstruct; import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; import org.jsoup.Connection; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; @@ -26,7 +25,6 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; -@Slf4j @Component @StepScope @RequiredArgsConstructor @@ -59,7 +57,6 @@ public Notice read() { if (nextIndex < notices.size()) return notices.get(nextIndex++); } } catch (InterruptedException | ExecutionException e) { - log.error("Error while fetching notices", e); } return null; } @@ -90,27 +87,16 @@ private Mono fetchNoticeDetailAsync(Element post) { String cacheKey = "notice:" + nttSn; if (noticeRedisService.validateNotice(cacheKey)) { - log.info("Skipping cached notice: {}", nttSn); return Mono.empty(); } return fetchDocument(URL + DETAIL_ADDITIONAL_URL + "&nttSn=" + nttSn, cookies) .map(doc -> { Element detailElement = doc.select(".bbs_ViewA").first(); - - if (detailElement == null) { - log.error("No detail found for notice: {}", nttSn); - return null; - } - String title = detailElement.select("h3").text().trim(); String content = detailElement.select(".bbsV_cont").text().trim(); - String file = detailElement.select(".fname").text().trim(); - - log.info("Notice Title: {}", title); - log.info("Notice Content: {}", content); - log.info("Notice File: {}", file); - +// String file = detailElement.select(".fname").text().trim(); + noticeRedisService.setNotice(title, nttSn); return new Notice(title, content, NoticeStatus.CREATED, teacher); }); } From e31e35bedf3c4eb8445f734b346246fd20fddc06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Thu, 27 Mar 2025 19:45:06 +0900 Subject: [PATCH 8/9] =?UTF-8?q?refactor:=20=EC=A3=BC=EC=84=9D=20=EC=82=AD?= =?UTF-8?q?=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../notice/infrastructure/batch/NoticeItemProcessor.java | 2 +- .../dodam/restapi/notice/presentation/NoticeController.java | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java index f21f0fc5..197d1ea0 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticeItemProcessor.java @@ -8,7 +8,7 @@ public class NoticeItemProcessor implements ItemProcessor { @Override - public Notice process(Notice item) throws Exception { + public Notice process(Notice item) { return item; } diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java index 8cbef599..484cb9d8 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/presentation/NoticeController.java @@ -38,12 +38,6 @@ public ResponseData> getByStatus( return noticeUseCase.getNotices(keyword, lastId, limit); } -// @GetMapping("/craw") -// public Response getCraw( -// ) throws Exception { -// return noticeUseCase.get(); -// } - @GetMapping("/division") public ResponseData> getById( @RequestParam(required = false) Long id, From 4ebf9b500754fce562ad60f234fec4c72f48504f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EA=B9=80=EA=B0=80=EC=98=81?= Date: Thu, 27 Mar 2025 20:00:55 +0900 Subject: [PATCH 9/9] =?UTF-8?q?refactor:=20=EC=A3=BC=EC=84=9D=20=EC=82=AD?= =?UTF-8?q?=EC=A0=9C=20&&=20=ED=95=84=EC=9A=94=EC=97=86=EB=8A=94=20?= =?UTF-8?q?=EC=95=A0=EB=85=B8=ED=85=8C=EC=9D=B4=EC=85=98=20=EC=82=AD?= =?UTF-8?q?=EC=A0=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/b1nd/dodam/DodamRestApiApplication.java | 5 +---- .../notice/infrastructure/batch/NoticelerItemReader.java | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java index e29c9690..d09a5fa8 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/DodamRestApiApplication.java @@ -1,12 +1,9 @@ package b1nd.dodam; -import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -@SpringBootApplication -@EnableBatchProcessing -public class DodamRestApiApplication { +@SpringBootApplication public class DodamRestApiApplication { public static void main(String[] args) { SpringApplication.run(DodamRestApiApplication.class, args); diff --git a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java index 65a71906..0dc70227 100644 --- a/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java +++ b/dodam-application/dodam-rest-api/src/main/java/b1nd/dodam/restapi/notice/infrastructure/batch/NoticelerItemReader.java @@ -90,12 +90,12 @@ private Mono fetchNoticeDetailAsync(Element post) { return Mono.empty(); } + //TODO 홈페이지에서 파일도 크롤링 하기 return fetchDocument(URL + DETAIL_ADDITIONAL_URL + "&nttSn=" + nttSn, cookies) .map(doc -> { Element detailElement = doc.select(".bbs_ViewA").first(); String title = detailElement.select("h3").text().trim(); String content = detailElement.select(".bbsV_cont").text().trim(); -// String file = detailElement.select(".fname").text().trim(); noticeRedisService.setNotice(title, nttSn); return new Notice(title, content, NoticeStatus.CREATED, teacher); });