[Fix] Dockerfile 및 CI-CD yaml 수정

.github/workflows/ci-cd.yml

@@ -2,24 +2,49 @@ name: Deploy to EC2
 
 on:
   push:
-    branches: [ main, fix/package-error ]
+    branches: [ main, fix/product-api ]
   pull_request:
     branches: [ main ]
 
 jobs:
   deploy:
     runs-on: ubuntu-latest
 
+    env:
+      DOCKER_BUILDKIT: 1  # BuildKit 활성화
+
     steps:
     - name: Checkout code
       uses: actions/checkout@v4
 
+    - name: Print start log
+      run: echo "🚀 [START] CI/CD Workflow started!"
+
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
+    - name: Print cache status
+      run: echo "🗄️ [CACHE] Docker layer cache step completed."
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
+    - name: Print buildx status
+      run: echo "🔧 [BUILDX] Docker Buildx set up completed."
+
+    - name: Set Docker image tag
+      id: vars
+      run: |
+        echo "🏷️ [TAG] Setting Docker image tag to first 7 chars of GITHUB_SHA."
+        echo "TAG=${GITHUB_SHA::7}" >> $GITHUB_ENV
+        echo "TAG=${GITHUB_SHA::7}"
 
     - name: Create .env file
       run: |
-        echo "📝 Creating .env file..."
+        echo "📝 [ENV] Creating .env file with secrets."
         echo "OPENSEARCH_USERNAME=${{ secrets.OPENSEARCH_USERNAME }}" >> .env
         echo "OPENSEARCH_HOST=${{ secrets.OPENSEARCH_HOST }}" >> .env
         echo "OPENSEARCH_PORT=${{ secrets.OPENSEARCH_PORT }}" >> .env
@@ -31,68 +56,88 @@ jobs:
         echo "MYSQL_DATABASE=${{ secrets.MYSQL_DATABASE }}" >> .env
         echo "MYSQL_USERNAME=${{ secrets.MYSQL_USERNAME }}" >> .env
         echo "MYSQL_PASSWORD=${{ secrets.MYSQL_PASSWORD }}" >> .env
+        echo "PREFECT_API_URL=${{ secrets.PREFECT_API_URL }}" >> .env
+        echo "PREFECT_API_KEY=${{ secrets.PREFECT_API_KEY }}" >> .env
+
+    - name: Print .env status
+      run: echo "✅ [.ENV] .env file created."
 
     - name: Login to Docker Hub
       uses: docker/login-action@v3
       with:
         username: ${{ secrets.DOCKERHUB_USERNAME }}
         password: ${{ secrets.DOCKERHUB_PASSWORD }}
+    - name: Print Docker login status
+      run: echo "🔑 [DOCKER] Docker Hub login successful."
+
+    - name: Prefect Auth
+      uses: PrefectHQ/actions-prefect-auth@v1
+      with:
+        prefect-api-key: ${{ secrets.PREFECT_API_KEY }}
+        prefect-workspace: ${{ secrets.PREFECT_WORKSPACE }}
+    - name: Print Prefect auth status
+      run: echo "🔗 [PREFECT] Prefect authentication completed."
 
     - name: Build and push Docker image
       uses: docker/build-push-action@v5
       with:
         context: .
         push: true
-        tags: ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest
-        cache-from: type=registry,ref=${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest
-        cache-to: type=inline
+        tags: |
+          ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:${{ env.TAG }}
+          ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest
         platforms: linux/amd64
+        cache-from: type=local,src=/tmp/.buildx-cache
+        cache-to: type=local,dest=/tmp/.buildx-cache
 
     - name: Deploy to EC2
       uses: appleboy/ssh-action@master
       with:
         host: ${{ secrets.EC2_HOST }}
         username: ubuntu
         key: ${{ secrets.EC2_SSH_KEY }}
-        envs: DOCKERHUB_USERNAME
+        envs: DOCKERHUB_USERNAME, TAG
         script: |
-          echo "🚀 Starting deployment process..."
+          echo "🚀 [DEPLOY] Starting deployment process on EC2..."
 
-          echo "🔐 Logging into Docker..."
+          echo "🔐 [DEPLOY] Logging into Docker Hub on EC2..."
           docker login -u ${{ secrets.DOCKERHUB_USERNAME }} -p ${{ secrets.DOCKERHUB_PASSWORD }}
 
-          echo "📦 Pulling latest image..."
-          docker pull ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest
+          echo "📦 [DEPLOY] Pulling latest image..."
+          docker pull ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:${TAG}
 
-          echo "🛑 Checking if container exists..."
+          echo "🛑 [DEPLOY] Checking if old container exists..."
           if docker container inspect ml-api >/dev/null 2>&1; then
-            echo "🧼 Stopping and removing old container..."
+            echo "🧼 [DEPLOY] Stopping and removing old container..."
             docker stop -t 30 ml-api || true
             docker rm ml-api || true
           else
-            echo "✅ No existing ml-api container."
+            echo "✅ [DEPLOY] No existing ml-api container."
           fi
 
-          echo "🧹 Docker system cleanup (optional)..."
+          echo "🧹 [DEPLOY] Docker system cleanup (optional)..."
           docker system prune -f
 
-          echo "📝 Validating .env file..."
+          echo "📝 [DEPLOY] Validating .env file on EC2..."
           if [ ! -f "/home/ubuntu/machine-learning/.env" ]; then
-            echo "⚠️ /home/ubuntu/machine-learning/.env not found. Creating empty .env..."
+            echo "⚠️ [DEPLOY] .env file not found. Creating empty .env..."
             mkdir -p /home/ubuntu/machine-learning
             touch /home/ubuntu/machine-learning/.env
           fi
 
-          echo "🚀 Starting new container..."
+          echo "🚀 [DEPLOY] Starting new container..."
           docker run -d \
             --name ml-api \
             -p 8000:8000 \
             --restart unless-stopped \
             --env-file /home/ubuntu/machine-learning/.env \
-            ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:latest
+            ${{ secrets.DOCKERHUB_USERNAME }}/ml-api:${TAG}
+
+          echo "📋 [DEPLOY] Container status:"
+          docker ps -a | grep ml-api || echo "❌ [DEPLOY] ml-api container not found"
 
-          echo "📋 Container status:"
-          docker ps -a | grep ml-api || echo "❌ ml-api container not found"
+          echo "📄 [DEPLOY] Container logs:"
+          docker logs ml-api || echo "⚠️ [DEPLOY] Failed to retrieve logs. Container may have exited early."
 
-          echo "📄 Container logs:"
-          docker logs ml-api || echo "⚠️ Failed to retrieve logs. Container may have exited early."
+    - name: Print finish log
+      run: echo "✅ [FINISH] CI/CD Workflow completed!"

Dockerfile

@@ -1,31 +1,40 @@
-# 경량 베이스 이미지 사용
-FROM python:3.12-slim
+# 1단계: 의존성만 설치하는 빌더 스테이지
+FROM python:3.12-slim AS builder
 
-# 작업 디렉토리 설정
 WORKDIR /app
 
-# 시스템 의존성 최소화 및 locale 설정
+# 시스템 패키지 최소화, 빌드 툴만 설치
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends \
-    build-essential \
-    curl \
-    && apt-get clean && \
+    apt-get install -y --no-install-recommends build-essential && \
     rm -rf /var/lib/apt/lists/*
 
-# 필요 패키지 복사 및 설치
-COPY requirements.txt ./
+# requirements.txt만 복사 (캐시 최적화)
+COPY requirements.txt .
+
+# 패키지 설치 (캐시 활용)
 RUN pip install --no-cache-dir -r requirements.txt
 
-# 소스 코드 복사
-COPY . .
+# 2단계: 실행용 베이스 이미지
+FROM python:3.12-slim
+
+WORKDIR /app
 
-# 환경변수 설정
+# 빌더에서 패키지와 바이너리 복사
+COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/python3.12/site-packages
+COPY --from=builder /usr/local/bin /usr/local/bin
+
+
+# 환경변수
 ENV PYTHONUNBUFFERED=1 \
     PYTHONPATH=/app \
     PORT=8000
 
-# 포트 노출
+# 소스코드 복사
+COPY . .
+
+# 불필요한 파일 복사 방지 (반드시 .dockerignore 사용)
+# 예: .git, __pycache__, *.log 등
+
 EXPOSE 8000
 
-# 실행 명령어
-CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"]

api/routers/recommendation.py

@@ -1,5 +1,5 @@
 from fastapi import APIRouter, HTTPException
-from product.repository.recommendation_repository import RecommendationRepository
+from product.service.recommendation_service import RecommendationService
 import logging
 from config.opensearch_mappings import PRODUCT_MAPPING
 import datetime
@@ -11,19 +11,15 @@
 router = APIRouter()
 
 # RecommendationService 인스턴스 생성
-service = RecommendationRepository(
-        s3_bucket="team6-mlops-bucket",
-        opensearch_index="recommendations",
-        mapping=PRODUCT_MAPPING
-    )
+service = RecommendationService()
 
 @router.get("/recommendations/{user_id}")
 def get_recommendations(user_id: int):
     """
     사용자별 추천 결과를 조회
     """
     try:
-        recommendations = service.get_recommendation_from_opensearch(str(user_id))
+        recommendations = service.repository.get_recommendation_from_opensearch(str(user_id))
 
         if not recommendations:
             raise HTTPException(

config/opensearch_mappings.py

@@ -27,18 +27,9 @@
         "properties": {
             "doc_id": {"type": "keyword"},
             "user_id": {"type": "keyword"},
-            "recommended_item_ids": {
-                "type": "nested",
-                "properties": {
-                    "item_id": {"type": "keyword"},
-                    "score": {"type": "float"}
-                }
-            },
+            "recommended_item_ids": { "type": "keyword" },
             "experiment_id": {"type": "keyword"},
-            "run_id": {"type": "keyword"},
-            "created_at": {"type": "date"},
-            "updated_at": {"type": "date"}
+            "run_id": {"type": "keyword"}
         }
     }
 }
-

docker-compose.yaml

@@ -11,6 +11,12 @@ services:
       - OPENSEARCH_PORT=${OPENSEARCH_PORT}
       - OPENSEARCH_USERNAME=${OPENSEARCH_USERNAME}
       - OPENSEARCH_PASSWORD=${OPENSEARCH_PASSWORD}
+      - MYSQL_URL=${MYSQL_URL}
+      - MYSQL_PORT=${MYSQL_PORT}
+      - MYSQL_DATABASE=${MYSQL_DATABASE}
+      - MYSQL_USERNAME=${MYSQL_USERNAME}
+      - MYSQL_PASSWORD=${MYSQL_PASSWORD}
+
     volumes:
       - .env:/app/.env
     restart: always

prefect.yaml

@@ -2,7 +2,7 @@ pull:
   - prefect.deployments.steps.git_clone:
       id: clone-step
       repository: https://github.com/Team-MoongChi/machine-learning.git
-      branch: prefect-flows
+      branch: main
 
   - prefect.deployments.steps.pip_install_requirements:
       requirements_file: requirements.txt

product/repository/recommendation_repository.py

@@ -43,6 +43,7 @@ def save_to_s3(self, s3_key: str, recommendation_data: Dict[str, Any]) -> bool:
 
     def get_recommendation_from_opensearch(self, user_id: str) -> Dict:
             """최근 14일 이내에 생성된 추천 결과 중 가장 최근 문서를 OpenSearch에서 조회"""
+            logger.info(f"user_id is {user_id}")
             try:
                 query = {
                     "query": {
@@ -54,8 +55,10 @@ def get_recommendation_from_opensearch(self, user_id: str) -> Dict:
                     "size": 1
                 }
 
+                logger.info(query)
+
                 result = self.opensearch.search(query)
-                print(result)
+                logger.info(result)
                 hits = result.get("hits", {}).get("hits", [])
 
                 if hits:
@@ -68,6 +71,7 @@ def get_recommendation_from_opensearch(self, user_id: str) -> Dict:
                         try:
                             doc_date = datetime.strptime(date_str, "%Y%m%d")
                             if doc_date >= datetime.now() - timedelta(days=14):
+                                logger.info(source)
                                 return {
                                     "status": "success",
                                     "data": source,

product/service/new_user_recommendation_service.py

@@ -37,7 +37,7 @@ def __init__(self):
         # 저장소 및 saver 준비
         self.repository = RecommendationRepository(
             s3_bucket="team6-mlops-bucket",
-            opensearch_index="recommendations",
+            opensearch_index="recommendations-v2",
             mapping=PRODUCT_MAPPING
         )
         self.saver = RecommendationSaver()

product/service/recommendation_saver.py

@@ -78,9 +78,11 @@ def recommend_for_existing_user(
         # 추천 엔진 실행
         recommendations = engine.recommend(user_id, top_k=top_k)
         rec_result = recommendations.to_dict(orient='records')
+
 
         # S3 키 생성
-        today = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+        now = datetime.datetime.now()
+        today = now.strftime("%Y%m%d_%H%M%S")
         doc_id = f"user_{user_id}_{today}"
         s3_key = f"recommendations/user_{user_id}/product_{today}.json"
         experiment_id = 2

product/service/recommendation_service.py

@@ -30,7 +30,7 @@ def __init__(self, bucket: str = "team6-mlops-bucket"):
         self.engine = None
         self.repository = RecommendationRepository(
             s3_bucket="team6-mlops-bucket",
-            opensearch_index="recommendations",
+            opensearch_index="recommendations-v2",
             mapping=PRODUCT_MAPPING
         )
 

test/product/test_product_service_pipeline.py

@@ -9,9 +9,12 @@ def test_delete_storage_data(service, s3_prefix: str):
     # S3 데이터 삭제
     deleted_s3 = service.repository.s3.delete_prefix(s3_prefix)
     print(f"S3 삭제된 객체 수: {deleted_s3}")
-    # OpenSearch 데이터 삭제
-    deleted_os = service.repository.opensearch.delete_all()
-    print(f"OpenSearch 삭제된 문서 수: {deleted_os}")
+    # # OpenSearch 데이터 삭제
+    # deleted_os = service.repository.opensearch.delete_all()
+    # print(f"OpenSearch 삭제된 문서 수: {deleted_os}")
+
+    # deleted_index = service.repository.opensearch.delete_index()
+    # print(f"OpenSearch 인덱스 삭제 결과: {deleted_index}")
 
 def test_run_full_pipeline(service):
     """추천 파이프라인 전체 실행 (업로드)"""