diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/common/scheduler/CrawlingJobScheduler.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/common/scheduler/CrawlingJobScheduler.kt index 18153b57..153299a2 100644 --- a/idle-batch/src/main/kotlin/com/swm/idle/batch/common/scheduler/CrawlingJobScheduler.kt +++ b/idle-batch/src/main/kotlin/com/swm/idle/batch/common/scheduler/CrawlingJobScheduler.kt @@ -13,7 +13,7 @@ class CrawlingJobScheduler( private val crawlingJobConfig: CrawlingJobConfig, ) { - @Scheduled(cron = "0 30 16 * * *") + @Scheduled(cron = "0 20 17 * * *") fun scheduleJob() { val jobParameters: JobParameters = JobParametersBuilder() .addLong("timestamp", System.currentTimeMillis()) diff --git a/idle-batch/src/main/kotlin/com/swm/idle/batch/util/WorknetCrawler.kt b/idle-batch/src/main/kotlin/com/swm/idle/batch/util/WorknetCrawler.kt index 5dbccefe..1fbadc27 100644 --- a/idle-batch/src/main/kotlin/com/swm/idle/batch/util/WorknetCrawler.kt +++ b/idle-batch/src/main/kotlin/com/swm/idle/batch/util/WorknetCrawler.kt @@ -92,31 +92,38 @@ object WorknetCrawler { logger.warn { "pageCount= " + pageCount } for (i in 1..pageCount) { - if (i >= 2) { - val updatedCrawlingUrl = crawlingUrl - .replace("{yesterday}", yesterday) - .replace(Regex("pageIndex=\\d+"), "pageIndex=${i}") - driver.get(updatedCrawlingUrl) - } - - wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#list1"))) + try { + if (i >= 2) { + val updatedCrawlingUrl = crawlingUrl + .replace("{yesterday}", yesterday) + .replace(Regex("pageIndex=\\d+"), "pageIndex=${i}") + driver.get(updatedCrawlingUrl) + } - crawlPosts(1, JOB_POSTING_COUNT_PER_PAGE, postings) + wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#list1"))) + crawlPosts(1, JOB_POSTING_COUNT_PER_PAGE, postings) + } catch (e: Exception) { + println("크롤링 오류 발생: 페이지 $i - ${e.message}") + } } val lastPageJobPostingCount = jobPostingCount % JOB_POSTING_COUNT_PER_PAGE if (lastPageJobPostingCount > 0) { - val updateCrawlingUrl = crawlingUrl - .replace("{yesterday}", yesterday) - .replace(Regex("pageIndex=\\d+"), "pageIndex=${pageCount + 1}") - driver.get(updateCrawlingUrl) - - wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#list1"))) + try { + val updateCrawlingUrl = crawlingUrl + .replace("{yesterday}", yesterday) + .replace(Regex("pageIndex=\\d+"), "pageIndex=${pageCount + 1}") + driver.get(updateCrawlingUrl) - crawlPosts(1, lastPageJobPostingCount, postings) + wait.until(ExpectedConditions.visibilityOfElementLocated(By.cssSelector("#list1"))) + crawlPosts(1, lastPageJobPostingCount, postings) + } catch (e: Exception) { + println("크롤링 오류 발생: 마지막 페이지 ${pageCount + 1} - ${e.message}") + } } + driver.quit() return postings }