diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index bb051c2..fe87762 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -2,7 +2,7 @@ name: Deploy to EC2 on: push: - branches: [ main, fix/package-error ] + branches: [ main, fix/product-api ] pull_request: branches: [ main ] @@ -10,16 +10,41 @@ jobs: deploy: runs-on: ubuntu-latest + env: + DOCKER_BUILDKIT: 1 # BuildKit ν™œμ„±ν™” + steps: - name: Checkout code uses: actions/checkout@v4 + - name: Print start log + run: echo "πŸš€ [START] CI/CD Workflow started!" + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Print cache status + run: echo "πŸ—„οΈ [CACHE] Docker layer cache step completed." + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Print buildx status + run: echo "πŸ”§ [BUILDX] Docker Buildx set up completed." + + - name: Set Docker image tag + id: vars + run: | + echo "🏷️ [TAG] Setting Docker image tag to first 7 chars of GITHUB_SHA." + echo "TAG=${GITHUB_SHA::7}" >> $GITHUB_ENV + echo "TAG=${GITHUB_SHA::7}" - name: Create .env file run: | - echo "πŸ“ Creating .env file..." + echo "πŸ“ [ENV] Creating .env file with secrets." echo "OPENSEARCH_USERNAME=${{ secrets.OPENSEARCH_USERNAME }}" >> .env echo "OPENSEARCH_HOST=${{ secrets.OPENSEARCH_HOST }}" >> .env echo "OPENSEARCH_PORT=${{ secrets.OPENSEARCH_PORT }}" >> .env @@ -31,22 +56,39 @@ jobs: echo "MYSQL_DATABASE=${{ secrets.MYSQL_DATABASE }}" >> .env echo "MYSQL_USERNAME=${{ secrets.MYSQL_USERNAME }}" >> .env echo "MYSQL_PASSWORD=${{ secrets.MYSQL_PASSWORD }}" >> .env + echo "PREFECT_API_URL=${{ secrets.PREFECT_API_URL }}" >> .env + echo "PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }}" >> .env + + - name: Print .env status + run: echo "βœ… [.ENV] .env file created." - name: Login to Docker Hub uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_PASSWORD }} + - name: Print Docker login status + run: echo "πŸ”‘ [DOCKER] Docker Hub login successful." + + - name: Prefect Auth + uses: PrefectHQ/actions-prefect-auth@v1 + with: + prefect-api-key: ${{ secrets.PREFECT_API_KEY }} + prefect-workspace: ${{ secrets.PREFECT_WORKSPACE }} + - name: Print Prefect auth status + run: echo "πŸ”— [PREFECT] Prefect authentication completed." - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . push: true - tags: ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest - cache-from: type=registry,ref=${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest - cache-to: type=inline + tags: | + ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:${{ env.TAG }} + ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest platforms: linux/amd64 + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache - name: Deploy to EC2 uses: appleboy/ssh-action@master @@ -54,45 +96,48 @@ jobs: host: ${{ secrets.EC2_HOST }} username: ubuntu key: ${{ secrets.EC2_SSH_KEY }} - envs: DOCKERHUB_USERNAME + envs: DOCKERHUB_USERNAME, TAG script: | - echo "πŸš€ Starting deployment process..." + echo "πŸš€ [DEPLOY] Starting deployment process on EC2..." - echo "πŸ” Logging into Docker..." + echo "πŸ” [DEPLOY] Logging into Docker Hub on EC2..." docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }} - echo "πŸ“¦ Pulling latest image..." - docker pull ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest + echo "πŸ“¦ [DEPLOY] Pulling latest image..." + docker pull ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:${TAG} - echo "πŸ›‘ Checking if container exists..." + echo "πŸ›‘ [DEPLOY] Checking if old container exists..." if docker container inspect ml-api >/dev/null 2>&1; then - echo "🧼 Stopping and removing old container..." + echo "🧼 [DEPLOY] Stopping and removing old container..." docker stop -t 30 ml-api || true docker rm ml-api || true else - echo "βœ… No existing ml-api container." + echo "βœ… [DEPLOY] No existing ml-api container." fi - echo "🧹 Docker system cleanup (optional)..." + echo "🧹 [DEPLOY] Docker system cleanup (optional)..." docker system prune -f - echo "πŸ“ Validating .env file..." + echo "πŸ“ [DEPLOY] Validating .env file on EC2..." if [ ! -f "/home/ubuntu/machine-learning/.env" ]; then - echo "⚠️ /home/ubuntu/machine-learning/.env not found. Creating empty .env..." + echo "⚠️ [DEPLOY] .env file not found. Creating empty .env..." mkdir -p /home/ubuntu/machine-learning touch /home/ubuntu/machine-learning/.env fi - echo "πŸš€ Starting new container..." + echo "πŸš€ [DEPLOY] Starting new container..." docker run -d \ --name ml-api \ -p 8000:8000 \ --restart unless-stopped \ --env-file /home/ubuntu/machine-learning/.env \ - ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest + ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:${TAG} + + echo "πŸ“‹ [DEPLOY] Container status:" + docker ps -a | grep ml-api || echo "❌ [DEPLOY] ml-api container not found" - echo "πŸ“‹ Container status:" - docker ps -a | grep ml-api || echo "❌ ml-api container not found" + echo "πŸ“„ [DEPLOY] Container logs:" + docker logs ml-api || echo "⚠️ [DEPLOY] Failed to retrieve logs. Container may have exited early." - echo "πŸ“„ Container logs:" - docker logs ml-api || echo "⚠️ Failed to retrieve logs. Container may have exited early." + - name: Print finish log + run: echo "βœ… [FINISH] CI/CD Workflow completed!" diff --git a/Dockerfile b/Dockerfile index bc4aac7..8a84d40 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,40 @@ -# κ²½λŸ‰ 베이슀 이미지 μ‚¬μš© -FROM python:3.12-slim +# 1단계: μ˜μ‘΄μ„±λ§Œ μ„€μΉ˜ν•˜λŠ” λΉŒλ” μŠ€ν…Œμ΄μ§€ +FROM python:3.12-slim AS builder -# μž‘μ—… 디렉토리 μ„€μ • WORKDIR /app -# μ‹œμŠ€ν…œ μ˜μ‘΄μ„± μ΅œμ†Œν™” 및 locale μ„€μ • +# μ‹œμŠ€ν…œ νŒ¨ν‚€μ§€ μ΅œμ†Œν™”, λΉŒλ“œ 툴만 μ„€μΉ˜ RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - build-essential \ - curl \ - && apt-get clean && \ + apt-get install -y --no-install-recommends build-essential && \ rm -rf /var/lib/apt/lists/* -# ν•„μš” νŒ¨ν‚€μ§€ 볡사 및 μ„€μΉ˜ -COPY requirements.txt ./ +# requirements.txt만 볡사 (μΊμ‹œ μ΅œμ ν™”) +COPY requirements.txt . + +# νŒ¨ν‚€μ§€ μ„€μΉ˜ (μΊμ‹œ ν™œμš©) RUN pip install --no-cache-dir -r requirements.txt -# μ†ŒμŠ€ μ½”λ“œ 볡사 -COPY . . +# 2단계: μ‹€ν–‰μš© 베이슀 이미지 +FROM python:3.12-slim + +WORKDIR /app -# ν™˜κ²½λ³€μˆ˜ μ„€μ • +# λΉŒλ”μ—μ„œ νŒ¨ν‚€μ§€μ™€ λ°”μ΄λ„ˆλ¦¬ 볡사 +COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages +COPY --from=builder /usr/local/bin /usr/local/bin + + +# ν™˜κ²½λ³€μˆ˜ ENV PYTHONUNBUFFERED=1 \ PYTHONPATH=/app \ PORT=8000 -# 포트 λ…ΈμΆœ +# μ†ŒμŠ€μ½”λ“œ 볡사 +COPY . . + +# λΆˆν•„μš”ν•œ 파일 볡사 λ°©μ§€ (λ°˜λ“œμ‹œ .dockerignore μ‚¬μš©) +# 예: .git, __pycache__, *.log λ“± + EXPOSE 8000 -# μ‹€ν–‰ λͺ…λ Ήμ–΄ -CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"] \ No newline at end of file diff --git a/api/routers/recommendation.py b/api/routers/recommendation.py index cce7e6e..505ad90 100644 --- a/api/routers/recommendation.py +++ b/api/routers/recommendation.py @@ -1,5 +1,5 @@ from fastapi import APIRouter, HTTPException -from product.repository.recommendation_repository import RecommendationRepository +from product.service.recommendation_service import RecommendationService import logging from config.opensearch_mappings import PRODUCT_MAPPING import datetime @@ -11,11 +11,7 @@ router = APIRouter() # RecommendationService μΈμŠ€ν„΄μŠ€ 생성 -service = RecommendationRepository( - s3_bucket="team6-mlops-bucket", - opensearch_index="recommendations", - mapping=PRODUCT_MAPPING - ) +service = RecommendationService() @router.get("/recommendations/{user_id}") def get_recommendations(user_id: int): @@ -23,7 +19,7 @@ def get_recommendations(user_id: int): μ‚¬μš©μžλ³„ μΆ”μ²œ κ²°κ³Όλ₯Ό 쑰회 """ try: - recommendations = service.get_recommendation_from_opensearch(str(user_id)) + recommendations = service.repository.get_recommendation_from_opensearch(str(user_id)) if not recommendations: raise HTTPException( diff --git a/config/opensearch_mappings.py b/config/opensearch_mappings.py index 9dd47ae..90f257f 100644 --- a/config/opensearch_mappings.py +++ b/config/opensearch_mappings.py @@ -27,18 +27,9 @@ "properties": { "doc_id": {"type": "keyword"}, "user_id": {"type": "keyword"}, - "recommended_item_ids": { - "type": "nested", - "properties": { - "item_id": {"type": "keyword"}, - "score": {"type": "float"} - } - }, + "recommended_item_ids": { "type": "keyword" }, "experiment_id": {"type": "keyword"}, - "run_id": {"type": "keyword"}, - "created_at": {"type": "date"}, - "updated_at": {"type": "date"} + "run_id": {"type": "keyword"} } } } - diff --git a/docker-compose.yaml b/docker-compose.yaml index 7034806..ab27c81 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -11,6 +11,12 @@ services: - OPENSEARCH_PORT=${OPENSEARCH_PORT} - OPENSEARCH_USERNAME=${OPENSEARCH_USERNAME} - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD} + - MYSQL_URL=${MYSQL_URL} + - MYSQL_PORT=${MYSQL_PORT} + - MYSQL_DATABASE=${MYSQL_DATABASE} + - MYSQL_USERNAME=${MYSQL_USERNAME} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + volumes: - .env:/app/.env restart: always diff --git a/prefect.yaml b/prefect.yaml index 6b3355b..15d4bce 100644 --- a/prefect.yaml +++ b/prefect.yaml @@ -2,7 +2,7 @@ pull: - prefect.deployments.steps.git_clone: id: clone-step repository: https://github.com/Team-MoongChi/machine-learning.git - branch: prefect-flows + branch: main - prefect.deployments.steps.pip_install_requirements: requirements_file: requirements.txt diff --git a/product/repository/recommendation_repository.py b/product/repository/recommendation_repository.py index 82653a1..b3b6e26 100644 --- a/product/repository/recommendation_repository.py +++ b/product/repository/recommendation_repository.py @@ -43,6 +43,7 @@ def save_to_s3(self, s3_key: str, recommendation_data: Dict[str, Any]) -> bool: def get_recommendation_from_opensearch(self, user_id: str) -> Dict: """졜근 14일 이내에 μƒμ„±λœ μΆ”μ²œ κ²°κ³Ό 쀑 κ°€μž₯ 졜근 λ¬Έμ„œλ₯Ό OpenSearchμ—μ„œ 쑰회""" + logger.info(f"user_id is {user_id}") try: query = { "query": { @@ -54,8 +55,10 @@ def get_recommendation_from_opensearch(self, user_id: str) -> Dict: "size": 1 } + logger.info(query) + result = self.opensearch.search(query) - print(result) + logger.info(result) hits = result.get("hits", {}).get("hits", []) if hits: @@ -68,6 +71,7 @@ def get_recommendation_from_opensearch(self, user_id: str) -> Dict: try: doc_date = datetime.strptime(date_str, "%Y%m%d") if doc_date >= datetime.now() - timedelta(days=14): + logger.info(source) return { "status": "success", "data": source, diff --git a/product/service/new_user_recommendation_service.py b/product/service/new_user_recommendation_service.py index ba6c28f..9d4a4c7 100644 --- a/product/service/new_user_recommendation_service.py +++ b/product/service/new_user_recommendation_service.py @@ -37,7 +37,7 @@ def __init__(self): # μ €μž₯μ†Œ 및 saver μ€€λΉ„ self.repository = RecommendationRepository( s3_bucket="team6-mlops-bucket", - opensearch_index="recommendations", + opensearch_index="recommendations-v2", mapping=PRODUCT_MAPPING ) self.saver = RecommendationSaver() diff --git a/product/service/recommendation_saver.py b/product/service/recommendation_saver.py index 30c5156..cfecee2 100644 --- a/product/service/recommendation_saver.py +++ b/product/service/recommendation_saver.py @@ -78,9 +78,11 @@ def recommend_for_existing_user( # μΆ”μ²œ μ—”μ§„ μ‹€ν–‰ recommendations = engine.recommend(user_id, top_k=top_k) rec_result = recommendations.to_dict(orient='records') + # S3 ν‚€ 생성 - today = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + now = datetime.datetime.now() + today = now.strftime("%Y%m%d_%H%M%S") doc_id = f"user_{user_id}_{today}" s3_key = f"recommendations/user_{user_id}/product_{today}.json" experiment_id = 2 diff --git a/product/service/recommendation_service.py b/product/service/recommendation_service.py index 5ab2186..05b0ef6 100644 --- a/product/service/recommendation_service.py +++ b/product/service/recommendation_service.py @@ -30,7 +30,7 @@ def __init__(self, bucket: str = "team6-mlops-bucket"): self.engine = None self.repository = RecommendationRepository( s3_bucket="team6-mlops-bucket", - opensearch_index="recommendations", + opensearch_index="recommendations-v2", mapping=PRODUCT_MAPPING ) diff --git a/test/product/test_product_service_pipeline.py b/test/product/test_product_service_pipeline.py index 954106d..b222b2d 100644 --- a/test/product/test_product_service_pipeline.py +++ b/test/product/test_product_service_pipeline.py @@ -9,9 +9,12 @@ def test_delete_storage_data(service, s3_prefix: str): # S3 데이터 μ‚­μ œ deleted_s3 = service.repository.s3.delete_prefix(s3_prefix) print(f"S3 μ‚­μ œλœ 객체 수: {deleted_s3}") - # OpenSearch 데이터 μ‚­μ œ - deleted_os = service.repository.opensearch.delete_all() - print(f"OpenSearch μ‚­μ œλœ λ¬Έμ„œ 수: {deleted_os}") + # # OpenSearch 데이터 μ‚­μ œ + # deleted_os = service.repository.opensearch.delete_all() + # print(f"OpenSearch μ‚­μ œλœ λ¬Έμ„œ 수: {deleted_os}") + + # deleted_index = service.repository.opensearch.delete_index() + # print(f"OpenSearch 인덱슀 μ‚­μ œ κ²°κ³Ό: {deleted_index}") def test_run_full_pipeline(service): """μΆ”μ²œ νŒŒμ΄ν”„λΌμΈ 전체 μ‹€ν–‰ (μ—…λ‘œλ“œ)""" diff --git a/utils/storage/opensearch_manager.py b/utils/storage/opensearch_manager.py index e0e042e..193d5ba 100644 --- a/utils/storage/opensearch_manager.py +++ b/utils/storage/opensearch_manager.py @@ -25,15 +25,15 @@ def create_index(self) -> bool: print(f"Creating index {self.index} if it does not exist...") # 인덱슀 쑴재 μ—¬λΆ€ 확인 if self.client.indices.exists(index=self.index): - logging.info(f"Index {self.index} already exists") + print(f"Index {self.index} already exists") return True response = self.client.indices.create(index=self.index, body=self.mapping) - logging.info(f"Created new index {self.index}") + print(f"Created new index {self.index}") return True except Exception as e: - logging.error(f"Failed to create index {self.index}: {str(e)}") + print(f"Failed to create index {self.index}: {str(e)}") return False # 인덱슀 νƒ€μž… μžλ™ μΆ”λ‘ λΌμ„œ μ—λŸ¬ λ°œμƒ @@ -97,7 +97,13 @@ def delete_all(self) -> int: def delete_index(self) -> bool: """인덱슀 전체 μ‚­μ œ""" try: - response = self.client.indices.delete(index=self.index) - return True + if self.client.indices.exists(index=self.index): + response = self.client.indices.delete(index=self.index) + print(f"[INFO] 인덱슀 '{self.index}' μ‚­μ œ μ™„λ£Œ: {response}") + return response.get("acknowledged", False) + else: + print(f"[INFO] 인덱슀 '{self.index}' μ‘΄μž¬ν•˜μ§€ μ•ŠμŒ.") + return True except Exception as e: + print(f"[ERROR] 인덱슀 μ‚­μ œ μ‹€νŒ¨: {e}") return False \ No newline at end of file