diff --git a/src/main/java/com/sosaw/sosaw/domain/customsound/repository/CustomSoundRepository.java b/src/main/java/com/sosaw/sosaw/domain/customsound/repository/CustomSoundRepository.java index da12adc..69948f3 100644 --- a/src/main/java/com/sosaw/sosaw/domain/customsound/repository/CustomSoundRepository.java +++ b/src/main/java/com/sosaw/sosaw/domain/customsound/repository/CustomSoundRepository.java @@ -1,6 +1,7 @@ package com.sosaw.sosaw.domain.customsound.repository; import com.sosaw.sosaw.domain.customsound.entity.CustomSound; +import com.sosaw.sosaw.domain.customsound.repository.projection.SoundMatchRow; import com.sosaw.sosaw.domain.customsound.web.dto.SoundsRes; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; @@ -24,4 +25,25 @@ public interface CustomSoundRepository extends JpaRepository // 특정 유저의 커스텀 소리만 조회 Optional findByIdAndUserUserId(Long id, Long userId); + + // 유사도가 가장 높은 소리 1가지 추출 + @Query(value = """ + SELECT + c.custom_id AS id, + c.custom_name AS customName, + c.emoji AS emoji, + c.color AS color, + (1 - (c.mfcc <=> (:mfcc)::vector)) AS similarity, + COALESCE(s.alarm_enabled, false) AS alarmEnabled, + COALESCE(s.vibration_level, 0) AS vibration + FROM "custom_sound" c + LEFT JOIN sound_setting s ON s.custom_id = c.custom_id + WHERE c.user_id = :userId + ORDER BY c.mfcc <=> (:mfcc)::vector ASC + LIMIT 1 + """, nativeQuery = true) + Optional findTopMatchByUserIdWithSimilarity( + @Param("userId") Long userId, + @Param("mfcc") String mfccVectorLiteral // "[0.12, -0.03, ...]" 형태의 문자열 + ); } diff --git a/src/main/java/com/sosaw/sosaw/domain/customsound/repository/projection/SoundMatchRow.java b/src/main/java/com/sosaw/sosaw/domain/customsound/repository/projection/SoundMatchRow.java new file mode 100644 index 0000000..ac4e0a8 --- /dev/null +++ b/src/main/java/com/sosaw/sosaw/domain/customsound/repository/projection/SoundMatchRow.java @@ -0,0 +1,13 @@ +package com.sosaw.sosaw.domain.customsound.repository.projection; + +import com.sosaw.sosaw.domain.customsound.entity.enums.Color; + +public interface SoundMatchRow { + Long getId(); + String getCustomName(); + String getEmoji(); + Color getColor(); + Double getSimilarity(); // 0.0 ~ 1.0 (cosine similarity) + Boolean getAlarmEnabled(); // alarm은 boolean + Integer getVibration(); +} diff --git a/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundService.java b/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundService.java index 7b6815a..673f7e8 100644 --- a/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundService.java +++ b/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundService.java @@ -1,9 +1,11 @@ package com.sosaw.sosaw.domain.customsound.service; +import com.sosaw.sosaw.domain.customsound.web.dto.SoundMatchRes; import com.sosaw.sosaw.domain.customsound.web.dto.SoundUploadReq; import com.sosaw.sosaw.domain.customsound.web.dto.SoundsRes; import com.sosaw.sosaw.domain.user.entity.User; import jakarta.validation.Valid; +import org.springframework.web.multipart.MultipartFile; import java.util.List; @@ -15,4 +17,6 @@ public interface CustomSoundService { List getAllSounds(User user); void modify(SoundUploadReq req, Long customSoundId); + + SoundMatchRes match(User user, MultipartFile file); } diff --git a/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundServiceImpl.java b/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundServiceImpl.java index c6e91c6..bda2fdc 100644 --- a/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundServiceImpl.java +++ b/src/main/java/com/sosaw/sosaw/domain/customsound/service/CustomSoundServiceImpl.java @@ -1,42 +1,45 @@ package com.sosaw.sosaw.domain.customsound.service; import com.sosaw.sosaw.domain.customsound.entity.CustomSound; -import com.sosaw.sosaw.domain.customsound.exception.FileProcessFailedException; import com.sosaw.sosaw.domain.customsound.exception.NotFoundSoundException; -import com.sosaw.sosaw.domain.customsound.exception.UnsupportedExtensionException; import com.sosaw.sosaw.domain.customsound.port.AudioFeatureExtractor; import com.sosaw.sosaw.domain.customsound.repository.CustomSoundRepository; +import com.sosaw.sosaw.domain.customsound.web.dto.SoundMatchRes; +import com.sosaw.sosaw.domain.customsound.repository.projection.SoundMatchRow; import com.sosaw.sosaw.domain.customsound.web.dto.SoundUploadReq; import com.sosaw.sosaw.domain.customsound.web.dto.SoundsRes; import com.sosaw.sosaw.domain.soundsetting.entity.SoundSetting; -import com.sosaw.sosaw.domain.soundsetting.entity.enums.SoundKind; +import com.sosaw.sosaw.domain.soundsetting.repository.SoundSettingRepository; import com.sosaw.sosaw.domain.user.entity.User; -import com.sosaw.sosaw.global.integration.fastapi.PythonMFCCService; +import com.sosaw.sosaw.domain.user.repository.UserRepository; import org.springframework.transaction.annotation.Transactional; +import com.sosaw.sosaw.domain.user.exception.UserNotFoundException; +import com.sosaw.sosaw.global.jpa.converter.FloatArrayVectorConverter; + import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.List; -import java.util.Optional; @Service @RequiredArgsConstructor @Slf4j public class CustomSoundServiceImpl implements CustomSoundService{ - private final PythonMFCCService pythonMFCCService; private final CustomSoundRepository customSoundRepository; private final AudioFeatureExtractor audioFeatureExtractor; // 포트 주입 + private final SoundSettingRepository soundSettingRepository; + private final UserRepository userRepository; @Override @Transactional public void upload(SoundUploadReq req, User user) { + userRepository.findById(user.getUserId()) + .orElseThrow(UserNotFoundException::new); + float[] mfcc = audioFeatureExtractor.extractMfcc(req.getFile()); CustomSound sound = CustomSound.toEntity(user, req, mfcc); @@ -58,6 +61,9 @@ public void delete(Long customSoundId) { @Override @Transactional(readOnly = true) public List getAllSounds(User user) { + userRepository.findById(user.getUserId()) + .orElseThrow(UserNotFoundException::new); + return customSoundRepository.findAllByUserId(user.getUserId()); } @@ -70,4 +76,21 @@ public void modify(SoundUploadReq req, Long customSoundId) { sound.replace(req, mfcc); } + @Override + public SoundMatchRes match(User user, MultipartFile file) { + userRepository.findById(user.getUserId()) + .orElseThrow(UserNotFoundException::new); + + float[] mfcc = audioFeatureExtractor.extractMfcc(file); + + String literal = FloatArrayVectorConverter.toLiteral(mfcc); + + SoundMatchRow row = customSoundRepository + .findTopMatchByUserIdWithSimilarity(user.getUserId(), literal) + .orElseThrow(NotFoundSoundException::new); + //TODO: 일상생활 소리 탐지와 사용자 소리 어떤 것을 반환할지, 단일 컷 정확도 기준 필요 + // 일상생활 소리 탐지 부분에서 서비스에서 재공되는 9개의 소리가 아닌 다른 소리 분류시 제외 시켜야 함. + return SoundMatchRes.from(row); + } + } diff --git a/src/main/java/com/sosaw/sosaw/domain/customsound/web/controller/CustomSoundController.java b/src/main/java/com/sosaw/sosaw/domain/customsound/web/controller/CustomSoundController.java index 247f024..f6127f5 100644 --- a/src/main/java/com/sosaw/sosaw/domain/customsound/web/controller/CustomSoundController.java +++ b/src/main/java/com/sosaw/sosaw/domain/customsound/web/controller/CustomSoundController.java @@ -1,6 +1,7 @@ package com.sosaw.sosaw.domain.customsound.web.controller; import com.sosaw.sosaw.domain.customsound.service.CustomSoundService; +import com.sosaw.sosaw.domain.customsound.web.dto.SoundMatchRes; import com.sosaw.sosaw.domain.customsound.web.dto.SoundUploadReq; import com.sosaw.sosaw.domain.customsound.web.dto.SoundsRes; import com.sosaw.sosaw.domain.user.entity.User; @@ -12,6 +13,7 @@ import org.springframework.http.ResponseEntity; import org.springframework.security.core.annotation.AuthenticationPrincipal; import org.springframework.web.bind.annotation.*; +import org.springframework.web.multipart.MultipartFile; import java.util.List; @@ -58,5 +60,15 @@ public ResponseEntity> modify( return ResponseEntity.status(HttpStatus.OK).body(SuccessResponse.ok(null)); } + // 내 소리 탐지 + @PostMapping("/match") + public ResponseEntity> match( + @AuthenticationPrincipal CustomUserDetails userDetails, + @RequestParam("file") MultipartFile file + ){ + SoundMatchRes res = customSoundService.match(userDetails.getUser(), file); + return ResponseEntity.status(HttpStatus.OK).body(SuccessResponse.ok(res)); + } + } diff --git a/src/main/java/com/sosaw/sosaw/domain/customsound/web/dto/SoundMatchRes.java b/src/main/java/com/sosaw/sosaw/domain/customsound/web/dto/SoundMatchRes.java new file mode 100644 index 0000000..6684818 --- /dev/null +++ b/src/main/java/com/sosaw/sosaw/domain/customsound/web/dto/SoundMatchRes.java @@ -0,0 +1,24 @@ +package com.sosaw.sosaw.domain.customsound.web.dto; + +import com.sosaw.sosaw.domain.customsound.entity.enums.Color; +import com.sosaw.sosaw.domain.customsound.repository.projection.SoundMatchRow; + +public record SoundMatchRes( + String soundName, + String emoji, + Color color, + double similarity, + boolean alarmEnabled, + int vibration +) { + public static SoundMatchRes from(SoundMatchRow row){ + return new SoundMatchRes( + row.getCustomName(), + row.getEmoji(), + row.getColor(), + row.getSimilarity(), + row.getAlarmEnabled(), + row.getVibration() + ); + } +} diff --git a/src/main/java/com/sosaw/sosaw/domain/soundsetting/entity/SoundSetting.java b/src/main/java/com/sosaw/sosaw/domain/soundsetting/entity/SoundSetting.java index 040ee17..1ccc533 100644 --- a/src/main/java/com/sosaw/sosaw/domain/soundsetting/entity/SoundSetting.java +++ b/src/main/java/com/sosaw/sosaw/domain/soundsetting/entity/SoundSetting.java @@ -21,7 +21,7 @@ public class SoundSetting extends BaseEntity { // 알람 유무 @Column(name = "alarm_enabled", nullable = false) - private boolean alarmEnabled=false; + private boolean alarmEnabled=true; // 진동 종류 @Column(name = "vibration_level", nullable = false) @@ -62,7 +62,7 @@ public void changeAlarmEnabled(boolean enabled) { public static SoundSetting createForCustom(CustomSound customSound) { SoundSetting setting = SoundSetting.builder() - .alarmEnabled(false) // 기본값 + .alarmEnabled(true) // 기본값 .vibrationLevel(1) // 기본값 .soundKind(SoundKind.CUSTOM) .build(); diff --git a/src/main/java/com/sosaw/sosaw/global/jpa/converter/FloatArrayVectorConverter.java b/src/main/java/com/sosaw/sosaw/global/jpa/converter/FloatArrayVectorConverter.java index 160eb8b..3260939 100644 --- a/src/main/java/com/sosaw/sosaw/global/jpa/converter/FloatArrayVectorConverter.java +++ b/src/main/java/com/sosaw/sosaw/global/jpa/converter/FloatArrayVectorConverter.java @@ -34,7 +34,7 @@ public float[] convertToEntityAttribute(Object dbData) { return parseLiteral(val); } - private static String toLiteral(float[] a) { + public static String toLiteral(float[] a) { StringBuilder sb = new StringBuilder(a.length * 8 + 2); sb.append('['); for (int i = 0; i < a.length; i++) {