-
Notifications
You must be signed in to change notification settings - Fork 20
feat: postSeeder 추가 #1029
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: postSeeder 추가 #1029
Changes from all commits
478abba
aa22796
5a9777d
c7cb9e1
24cccd9
8e3b7b0
87b4865
95e840f
92db8f1
a899901
baaf724
7e4dcb8
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| package net.causw.app.main.shared.seed; | ||
|
|
||
| public enum ActionType { | ||
| // Whale(70%), Dolphin(25%), Minnow(5%) | ||
| POST(0.70, 0.70 + 0.25, 3.0), | ||
|
|
||
| // Whale(40%), Dolphin(40%), Minnow(20%) | ||
| COMMENT(0.40, 0.40 + 0.40, 2.0), | ||
|
|
||
| // Whale(20%), Dolphin(30%), Minnow(50%) | ||
| LIKE(0.20, 0.20 + 0.30, 1.0); | ||
|
|
||
| // 누적 임계값 (0.0 ~ 1.0) | ||
| public final double whaleThreshold; // 이 값보다 작으면 Whale | ||
| public final double dolphinThreshold; // 이 값보다 작으면 Dolphin, 아니면 Minnow | ||
|
|
||
| // 그룹 내 편중도 (Zipfian exponent) | ||
| // 행동이 어려울수록(Post) 상위 유저 중에서도 더 상위에게 몰리는 경향(3.0)이 심함 | ||
| // 행동이 쉬울수록(Like) 그룹 내에서는 평등하게 퍼짐(1.0) | ||
| public final double skewFactor; | ||
|
|
||
| ActionType(double whaleProb, double cumulativeDolphinProb, double skewFactor) { | ||
| this.whaleThreshold = whaleProb; | ||
| this.dolphinThreshold = cumulativeDolphinProb; | ||
| this.skewFactor = skewFactor; | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| package net.causw.app.main.shared.seed; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| import org.springframework.jdbc.core.JdbcTemplate; | ||
| import org.springframework.transaction.annotation.Transactional; | ||
|
|
||
| import net.causw.app.main.shared.util.DistributionUtils; | ||
| import net.causw.app.main.shared.util.UserSegmenter; | ||
|
|
||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| @RequiredArgsConstructor | ||
| @Slf4j | ||
| public abstract class BasePostSeeder<T> { | ||
| protected final JdbcTemplate jdbcTemplate; | ||
| protected final UserSegmenter userSegmenter; // 유저 유틸 상속 | ||
| protected final int totalCount; | ||
| protected final int batchSize; | ||
|
|
||
| // [Template Method] 실행 흐름을 제어 (final로 오버라이드 방지를 하려고 했으나, AOP 프록시 객체와의 충돌로 제거함) | ||
| @Transactional | ||
| public void seed() { | ||
| log.info("[Seeder] Start seeding {} items for {}", totalCount, this.getClass().getSimpleName()); | ||
| beforeSeeding(); | ||
| long startTime = System.currentTimeMillis(); | ||
|
|
||
| List<T> buffer = new ArrayList<>(); | ||
|
|
||
| for (int i = 0; i < totalCount; i++) { | ||
| // 1. 데이터 생성 위임 (Hook Method) | ||
| T item = createItem(i); | ||
| buffer.add(item); | ||
|
|
||
| // 2. 배치가 차면 DB 투입 | ||
| if (buffer.size() >= batchSize) { | ||
| batchInsert(buffer); | ||
| buffer.clear(); | ||
| } | ||
| } | ||
|
|
||
| // 3. 남은 데이터 처리 | ||
| if (!buffer.isEmpty()) { | ||
| batchInsert(buffer); | ||
| } | ||
|
|
||
| long endTime = System.currentTimeMillis(); | ||
| log.info("[Seeder] Finished. Time taken: {}ms", endTime - startTime); | ||
| } | ||
|
|
||
| protected String pickUser() { | ||
| ActionType type = getActionType(); | ||
| int groupType = DistributionUtils.selectUserGroupType(type); | ||
|
|
||
| // UserSegmenter 내부도 List<String>으로 관리되고 있다고 가정 | ||
| List<String> targetGroup = switch (groupType) { | ||
| case 0 -> userSegmenter.getWhales(); | ||
| case 1 -> userSegmenter.getDolphins(); | ||
| default -> userSegmenter.getActiveMinnows(); | ||
| }; | ||
|
|
||
| return DistributionUtils.pickWeightedRandom(targetGroup, type.skewFactor); | ||
| } | ||
|
|
||
| // [Abstract Methods] 자식들이 반드시 구현해야 할 부분 | ||
| protected abstract T createItem(int index); // 데이터 생성 전략 | ||
|
|
||
| protected abstract void batchInsert(List<T> items); // DB 저장 전략 | ||
|
|
||
| protected abstract ActionType getActionType(); // 어떤 확률 분포를 쓸지 | ||
|
|
||
| protected void beforeSeeding() {} | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| package net.causw.app.main.shared.seed; | ||
|
|
||
| import java.util.List; | ||
|
|
||
| import org.springframework.context.annotation.Profile; | ||
| import org.springframework.stereotype.Component; | ||
| import org.springframework.transaction.annotation.Transactional; | ||
|
|
||
| import net.causw.app.main.domain.community.board.entity.Board; | ||
| import net.causw.app.main.domain.community.board.entity.BoardApply; | ||
| import net.causw.app.main.domain.community.board.entity.BoardApplyStatus; | ||
| import net.causw.app.main.domain.community.board.repository.BoardApplyRepository; | ||
| import net.causw.app.main.domain.community.board.repository.BoardRepository; | ||
| import net.causw.app.main.domain.user.account.entity.user.User; | ||
| import net.causw.app.main.domain.user.account.repository.user.UserRepository; | ||
| import net.causw.global.constant.StaticValue; | ||
|
|
||
| import jakarta.persistence.EntityManager; | ||
| import lombok.RequiredArgsConstructor; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| @Component | ||
| @Profile("seed") | ||
| @RequiredArgsConstructor | ||
| @Slf4j | ||
| public class BoardSeeder { | ||
| private final EntityManager em; | ||
| private final BoardApplyRepository boardApplyRepository; | ||
| private final BoardRepository boardRepository; | ||
| private final UserRepository userRepository; | ||
|
|
||
| @Transactional | ||
| public void seed() { | ||
| if (boardRepository.count() > 0) { | ||
| log.warn("🚫 Seed skipped: boards already exist"); | ||
| return; | ||
| } | ||
|
|
||
| User user = userRepository.findTopBy() | ||
| .orElseThrow(() -> new IllegalStateException("🚫 Seed skipped: User not found for seeding boards")); | ||
|
|
||
| process(user); | ||
| } | ||
|
|
||
| private void process(User user) { | ||
| // 게시판 카테고리 이름 예시 (20개) | ||
| List<String> boardNames = List.of( | ||
| "자유게시판", "익명게시판", "질문게시판", "정보공유", "취업게시판", | ||
| "동아리홍보", "스터디모집", "중고장터", "분실물센터", "학교생활", | ||
| "유머게시판", "건의사항", "컴공게시판", "학사공지", "장학공지", | ||
| "학생회 공지 게시판", "크자회 공지 게시판", "서비스 공지", "졸업생게시판", "새내기게시판"); | ||
|
|
||
| for (String name : boardNames) { | ||
| createActiveBoard(user, name); | ||
| } | ||
|
|
||
| em.flush(); | ||
| em.clear(); | ||
|
|
||
| log.info("✅ Seeded {} boards completed.", boardNames.size()); | ||
| } | ||
|
|
||
| private void createActiveBoard(User user, String name) { | ||
| BoardApply boardApply = BoardApply.of( // 일반 게시판 신청 | ||
| user, | ||
| name, | ||
| null, | ||
| StaticValue.BOARD_NAME_APP_FREE, | ||
| false, | ||
| null); | ||
| boardApply.updateAcceptStatus(BoardApplyStatus.ACCEPTED); | ||
| this.boardApplyRepository.save(boardApply); | ||
| em.persist(boardApply); | ||
|
|
||
| Board newBoard = Board.of( // 일반 게시판 생성 | ||
| boardApply.getBoardName(), | ||
| boardApply.getDescription(), | ||
| boardApply.getCategory(), | ||
| boardApply.getIsAnonymousAllowed(), | ||
| null); | ||
|
|
||
| this.boardRepository.save(newBoard); | ||
| em.persist(newBoard); | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,133 @@ | ||
| package net.causw.app.main.shared.seed; | ||
|
|
||
| import java.sql.PreparedStatement; | ||
| import java.sql.SQLException; | ||
| import java.sql.Timestamp; | ||
| import java.time.LocalDateTime; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import java.util.Random; | ||
| import java.util.UUID; | ||
|
|
||
| import org.springframework.context.annotation.Profile; | ||
| import org.springframework.jdbc.core.BatchPreparedStatementSetter; | ||
| import org.springframework.jdbc.core.JdbcTemplate; | ||
| import org.springframework.stereotype.Component; | ||
|
|
||
| import net.causw.app.main.shared.util.UserSegmenter; | ||
|
|
||
| @Component | ||
| @Profile("seed") | ||
| public class ChildCommentSeeder extends BasePostSeeder<ChildCommentSeeder.ChildCommentItem> { | ||
| private static final Random random = new Random(); | ||
| private final List<ParentCommentInfo> parentComments = new ArrayList<>(); | ||
|
|
||
| public ChildCommentSeeder(JdbcTemplate jdbcTemplate, UserSegmenter userSegmenter) { | ||
| super(jdbcTemplate, userSegmenter, 50_000, 1_000); | ||
| } | ||
|
|
||
| @Override | ||
| protected ActionType getActionType() { | ||
| return ActionType.COMMENT; | ||
| } | ||
|
|
||
| @Override | ||
| protected void beforeSeeding() { | ||
| loadCommentIds(); | ||
| } | ||
|
|
||
| @Override | ||
| protected ChildCommentItem createItem(int index) { | ||
| // 1. 부모 메서드를 통해 확률에 맞는 작성자 선정 | ||
| String writerId = pickUser(); | ||
| // 2. 낙수 효과에 따라 부모 댓글 선정 | ||
| ParentCommentInfo parentInfo = parentComments.get(random.nextInt(parentComments.size())); | ||
| LocalDateTime createdAt = LocalDateTime.now().minusMinutes(totalCount - index); | ||
|
|
||
| return new ChildCommentItem( | ||
| UUID.randomUUID().toString(), | ||
| "Child Comment " + index, | ||
| parentInfo.postId(), | ||
| parentInfo.id, | ||
| writerId, | ||
| createdAt); | ||
| } | ||
|
|
||
| @Override | ||
| protected void batchInsert(List<ChildCommentItem> items) { | ||
| // 1. 대댓글 삽입 | ||
| String sqlChildComment = "INSERT INTO tb_child_comment " + | ||
| "(id, content, parent_comment_id, user_id, is_anonymous, is_deleted, created_at, updated_at) " + | ||
| "VALUES (?, ?, ?, ?, false, false, ?, ?)"; | ||
|
|
||
| jdbcTemplate.batchUpdate(sqlChildComment, new BatchPreparedStatementSetter() { | ||
| @Override | ||
| public void setValues(PreparedStatement ps, int i) throws SQLException { | ||
| ChildCommentItem item = items.get(i); | ||
|
|
||
| ps.setString(1, item.id()); | ||
| ps.setString(2, item.content()); | ||
| ps.setString(3, item.parentId()); | ||
| ps.setString(4, item.writerId()); | ||
| ps.setTimestamp(5, Timestamp.valueOf(item.createdAt())); | ||
| ps.setTimestamp(6, Timestamp.valueOf(item.createdAt())); | ||
| } | ||
|
|
||
| @Override | ||
| public int getBatchSize() { | ||
| return items.size(); | ||
| } | ||
|
Comment on lines
+63
to
+79
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [P5]
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. DB url 설정 시
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
url: jdbc:mysql://localhost:3306/seed_test?rewriteBatchedStatements=true설정은 다음과 같이 옵션만 붙여주면 됩니다. 다만, 데이터가 쌓임에 따라 UUID 설정 방식으로 인해 오히려 Bulk Insert의 성능이 개별 Insert보다 느려지는 결과를 발견했습니다. 이는 병목 지점이 네트워크에서 Disk I/O로 변경됨에 따른 현상으로 보이며, TSID를 도입하여 해결할 수 있다고 하여 해당 라이브러리의 도입을 검토해볼까 합니다! |
||
| }); | ||
|
|
||
| // 2. 알림 삽입 | ||
| String sqlNotification = "INSERT INTO tb_notification " + | ||
| "(id, user_id, content, notice_type, target_id, is_global, created_at, updated_at) " + | ||
| "VALUES (?, ?, ?, 'COMMENT', ?, false, ?, ?)"; | ||
|
|
||
| jdbcTemplate.batchUpdate(sqlNotification, new BatchPreparedStatementSetter() { | ||
| @Override | ||
| public void setValues(PreparedStatement ps, int i) throws SQLException { | ||
| ChildCommentItem item = items.get(i); | ||
|
|
||
| ps.setString(1, UUID.randomUUID().toString()); | ||
| ps.setString(2, item.writerId()); | ||
| ps.setString(3, item.content()); | ||
| ps.setString(4, item.postId()); | ||
| ps.setTimestamp(5, Timestamp.valueOf(item.createdAt())); | ||
| ps.setTimestamp(6, Timestamp.valueOf(item.createdAt())); | ||
| } | ||
|
|
||
| @Override | ||
| public int getBatchSize() { | ||
| return items.size(); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| private void loadCommentIds() { | ||
| String sql = "SELECT id, post_id FROM tb_comment"; | ||
| jdbcTemplate.query(sql, rs -> { | ||
| String id = rs.getString("id"); | ||
| String postId = rs.getString("post_id"); | ||
| parentComments.add(new ParentCommentInfo(id, postId)); | ||
| }); | ||
|
|
||
| if (parentComments.isEmpty()) { | ||
| throw new RuntimeException("❌ 부모 댓글이 없습니다! CommentSeeder를 먼저 실행해주세요."); | ||
| } | ||
| } | ||
|
|
||
| private record ParentCommentInfo( | ||
| String id, | ||
| String postId) { | ||
| } | ||
|
|
||
| public record ChildCommentItem( | ||
| String id, | ||
| String content, | ||
| String postId, | ||
| String parentId, | ||
| String writerId, | ||
| LocalDateTime createdAt) { | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[P5]
시딩 전략을 파레토 법칙, 메트칼프 법칙등을 사용하신게 흥미롭습니다.
어떤 관점에서 해당 전략들이 필요하다고 보셨나요?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
테스트 환경 구축 관련하여 자료 조사를 하다가 테스트 시나리오(데이터)가 실제 환경과 비슷한가가 환경 구축에 있어 중요하다는 글을 봤습니다. 커뮤니티라는 도메인의 데이터를 실제 환경과 맞도록 잘 반영하기 위해서 참고할만한 지표나 법칙들이 있는지 AI와의 대화나 서치 등을 통해 알아보았고, 이를 시딩 전략으로 적용하고자 하였습니다! 해당 시딩 전략을 통해 실제 환경과 비슷한 데이터들을 만들어, 실제로 나타날 수 있는 부하나 성능 병목을 더욱 정확히 측정하고자 하였습니다.