bot_api.py

import os
import requests
from flask import Flask, request, Response
from botbuilder.core import BotFrameworkAdapter, BotFrameworkAdapterSettings, TurnContext
from botbuilder.schema import Activity

import speech_recognition as sr

from pydub import AudioSegment
from io import BytesIO
from asyncio import run
from sentence_transformers import SentenceTransformer
from transformers import BartTokenizer, BartForConditionalGeneration
from datasets import load_from_disk

app = Flask(__name__)

# Set up the adapter
SETTINGS = BotFrameworkAdapterSettings("", "")  # Microsoft App ID and Password
ADAPTER = BotFrameworkAdapter(SETTINGS)

# Load environment variables
os.environ['FLASK_API_URL'] = 'http://localhost:5001/api/messages'  # URL for Flask API deployment

# Initialize RAG components (import this from the updated RAG model initialization code)
bart_model_path = 'bart_model'
os.environ['TRANSFORMERS_CACHE'] = bart_model_path

# Load the pretrain information
tokenizer = BartTokenizer.from_pretrained(bart_model_path, cache_dir=os.environ['TRANSFORMERS_CACHE'])
generator = BartForConditionalGeneration.from_pretrained(bart_model_path, cache_dir=os.environ['TRANSFORMERS_CACHE'])
print("Tokenizer and model loaded successfully.")

# Database loading
dataset_path = 'book_dataset'
index_path = 'book_faiss_index'
dataset = load_from_disk(dataset_path)
dataset.load_faiss_index(index_name='embeddings', file=index_path)

# Running the model
embedding_model = SentenceTransformer('distilbert-base-nli-stsb-mean-tokens')

def generate_recommendation(query):
    # Encode the query
    query_embedding = embedding_model.encode([query])
    
    # Retrieve relevant documents
    scores, retrieved_examples = dataset.get_nearest_examples('embeddings', query_embedding, k=5)
    retrieved_texts = " ".join(retrieved_examples['text'])
    
    # Generate response using BART
    inputs = tokenizer(retrieved_texts, return_tensors="pt", max_length=1024, truncation=True)
    outputs = generator.generate(inputs['input_ids'], max_length=100, num_beams=5, early_stopping=True)
    recommendation = tokenizer.decode(outputs[0], skip_special_tokens=True)
    
    return recommendation

@app.route("/api/messages", methods=["POST"])
def messages():
    if "application/json" in request.headers["Content-Type"]:
        json_message = request.json
    else:
        return Response(status=415)

    activity = Activity().deserialize(json_message)
    auth_header = request.headers["Authorization"] if "Authorization" in request.headers else ""

    async def bot_logic(turn_context: TurnContext):
        if turn_context.activity.attachments and len(turn_context.activity.attachments) > 0:

            # Process audio attachment
            attachment = turn_context.activity.attachments[0]
            response = requests.get(attachment.content_url)
            audio_bytes = BytesIO(response.content)
            response_text = process_audio_attachment(audio_bytes)
            await turn_context.send_activity(Activity(type="message", text=response_text))
        else:
            # Process text input
            # Classify user intent
            intent = call_flask_api("1", turn_context.activity.text)
            print(intent)
            # User want recommendation and give details
            if "Case 1" in intent:
                rag_response_text = generate_recommendation(intent.split(":")[1])
                final_recommendation = call_flask_api("3", rag_response_text)
                await turn_context.send_activity(Activity(type="message", text=final_recommendation))

            # User want recommendation but not giving any details
            elif intent == "Case 2":
                await turn_context.send_activity(Activity(type="message", text="Can you give me some details about the book that you like? The main point? Author? Publisher?"))
            
            # Normal chat
            else:
                await turn_context.send_activity(Activity(type="message", text=call_flask_api("2", turn_context.activity.text))) 

    try:
        task = ADAPTER.process_activity(activity, auth_header, bot_logic)
        if not task:
            return Response(status=401)
        run(task)  # This ensures that the async task is executed
    except Exception as exception:
        raise exception

    return Response(status=200)

def call_flask_api(case, user_input):
    """ Send user input to the Flask API and return the response """
    response = requests.post('http://localhost:5001/api/messages', json={'case': case, 'text': user_input})
    if response.status_code == 200:
        return response.json()['text']
    else:
        pass

def process_audio_attachment(audio_bytes):
    """ Process the audio file and return the recognized text """
    recognizer = sr.Recognizer()
    try:
        audio = sr.AudioFile(audio_bytes)
        with audio as source:
            audio_data = recognizer.record(source)
        
        # Get the text from the audio
        text = recognizer.recognize_google(audio_data)
        
        # Classify user intent
        intent = call_flask_api("1", text)

        print(intent)
        # User want recommendation and give details
        if intent == "Case 1":
            rag_response_text = generate_recommendation(text)
            final_recommendation = call_flask_api("3", rag_response_text)
            return final_recommendation

        # User want recommendation but not giving any details
        elif intent == "Case 2":
            return "Can you give me some details about the book that you like? The main point? Author? Publisher?"
        
        # Normal chat
        else:
            return call_flask_api("2", text)
        
    except sr.UnknownValueError:
        return 'Speech Recognition could not understand the audio'
    except sr.RequestError as e:
        return f'Could not request results from Speech Recognition service; {e}'
    except Exception as e:
        return str(e)

if __name__ == '__main__':
    app.run(port=3978)  # Use port 3978 for local development