fix: merging dev changes to main branch (#1883)

Signed-off-by: dependabot[bot] <[email protected]>
Co-authored-by: Ajit Padhi <[email protected]>
Co-authored-by: Pavan-Microsoft <[email protected]>
Co-authored-by: Ross Smith <[email protected]>
Co-authored-by: gpickett <[email protected]>
Co-authored-by: Francia Riesco <[email protected]>
Co-authored-by: Francia Riesco <[email protected]>
Co-authored-by: Prajwal D C <[email protected]>
Co-authored-by: Harmanpreet-Microsoft <[email protected]>
Co-authored-by: UtkarshMishra-Microsoft <[email protected]>
Co-authored-by: Priyanka-Microsoft <[email protected]>
Co-authored-by: Prasanjeet-Microsoft <[email protected]>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Kiran-Siluveru-Microsoft <[email protected]>
Co-authored-by: Prashant-Microsoft <[email protected]>
Co-authored-by: Rohini-Microsoft <[email protected]>
Co-authored-by: Avijit-Microsoft <[email protected]>
Co-authored-by: RaviKiran-Microsoft <[email protected]>
Co-authored-by: Somesh Joshi <[email protected]>
Co-authored-by: Himanshi Agrawal <[email protected]>
Co-authored-by: pradeepjha-microsoft <[email protected]>
Co-authored-by: Harmanpreet Kaur <[email protected]>
Co-authored-by: Bangarraju-Microsoft <[email protected]>
Co-authored-by: Harsh-Microsoft <[email protected]>
Co-authored-by: Kanchan-Microsoft <[email protected]>
Co-authored-by: Cristopher Coronado <[email protected]>
Co-authored-by: Cristopher Coronado Moreira <[email protected]>
Co-authored-by: Vamshi-Microsoft <[email protected]>
Co-authored-by: Thanusree-Microsoft <[email protected]>
Co-authored-by: Niraj Chaudhari (Persistent Systems Inc) <[email protected]>
Co-authored-by: Rohini-Microsoft <[email protected]>

Author: 29 people

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
             return response.data[0]?.number ?? "";
           retries: 3
       - name: Comment coverage
-        uses: MishaKav/pytest-coverage-comment@81882822c5b22af01f91bd3eacb1cefb6ad73dc2
+        uses: MishaKav/pytest-coverage-comment@13d3c18e21895566c746187c9ea74736372e5e91
         with:
           pytest-xml-coverage-path: coverage.xml
           junitxml-path: coverage-junit.xml

.github/workflows/comment_coverage.yml

@@ -1,29 +1,29 @@
 name: Test Automation CWYD
 on:
-  push:
-    branches:
-      - main
-      - dev
-    paths:
-      - 'tests/e2e-test/**'
-  schedule:
-    - cron: '0 13 * * 3'  # Runs at 1 PM UTC once a week on Wednesday
-  workflow_dispatch:
+    workflow_call:
+      inputs:
+        web_url:
+          required: true
+          type: string
+        admin_url:
+          required: true
+          type: string
 
-env:
-    web_url: ${{ vars.CWYD_WEB_URL }}
-    admin_url: ${{ vars.CWYD_ADMIN_URL }}
-    accelerator_name: "Chat with your Data"
 
 jobs:
   test:
     runs-on: ubuntu-latest
+    env:
+      web_url: ${{ inputs.web_url }}
+      admin_url: ${{ inputs.admin_url }}
+      accelerator_name: "Chat with your Data"
+
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
 
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.13'
 

.github/workflows/test-automation.yml

@@ -57,14 +57,114 @@ azd-login: ## 🔑 Login to Azure with azd and a SPN
 	@echo -e "\e[34m$@\e[0m" || true
 	@azd auth login --client-id ${AZURE_CLIENT_ID} --client-secret ${AZURE_CLIENT_SECRET} --tenant-id ${AZURE_TENANT_ID}
 
-deploy: azd-login ## 🚀 Deploy everything to Azure
+azd-login: ## 🔑 Login to Azure with azd and a SPN
+	@echo -e "\e[34m$@\e[0m" || true
+	@azd auth login --client-id ${AZURE_CLIENT_ID} --client-secret ${AZURE_CLIENT_SECRET} --tenant-id ${AZURE_TENANT_ID}
+
+# Fixed Makefile section for deploy target
+# Fixed Makefile section for deploy target
+deploy: azd-login ## Deploy everything to Azure
 	@echo -e "\e[34m$@\e[0m" || true
 	@azd env new ${AZURE_ENV_NAME}
-	@azd env set AZURE_APP_SERVICE_HOSTING_MODEL code --no-prompt
+
+	# Provision and deploy
 	@azd provision --no-prompt
-	@azd deploy web --no-prompt
-	@azd deploy function --no-prompt
+	@azd deploy web --no-prompt || true
+	@azd deploy function --no-prompt || true
 	@azd deploy adminweb --no-prompt
+	@azd env get-values > .env.temp
+	@cat .env.temp
+
+	@sleep 30
+	@azd show --output json > deploy_output.json || echo "{}" > deploy_output.json
+	@echo "=== deploy_output.json contents ==="
+	@cat deploy_output.json | jq . || cat deploy_output.json
+
+	# Extract URLs
+	@echo "=== Extracting URLs using multiple methods ==="
+	@azd show 2>&1 | tee full_deployment_output.log
+	@jq -r '.services.web?.project?.hostedEndpoints?[0]?.url // ""' deploy_output.json > frontend_url.txt || echo "" > frontend_url.txt
+	@jq -r '.services.adminweb?.project?.hostedEndpoints?[0]?.url // ""' deploy_output.json > admin_url.txt || echo "" > admin_url.txt
+	@grep -oE "https://app-[a-zA-Z0-9-]*\.azurewebsites\.net/" full_deployment_output.log | grep -v admin | head -1 >> frontend_url.txt || true
+	@grep -oE "https://app-[a-zA-Z0-9-]*-admin\.azurewebsites\.net/" full_deployment_output.log | head -1 >> admin_url.txt || true
+	@sort frontend_url.txt | uniq | grep -v '^$$' | head -1 > frontend_url_clean.txt && mv frontend_url_clean.txt frontend_url.txt || echo "" > frontend_url.txt
+	@sort admin_url.txt | uniq | grep -v '^$$' | head -1 > admin_url_clean.txt && mv admin_url_clean.txt admin_url.txt || echo "" > admin_url.txt
+
+	@echo "=== URL Extraction Results ==="
+	@FRONTEND_URL=$$(cat frontend_url.txt 2>/dev/null | tr -d '\n\r' | xargs); \
+	ADMIN_URL=$$(cat admin_url.txt 2>/dev/null | tr -d '\n\r' | xargs); \
+	echo "Frontend URL: $$FRONTEND_URL"; \
+	echo "Admin URL: $$ADMIN_URL"
+
+	@echo "=== Final Deployment Status ==="
+	@echo "Frontend URL:" && cat frontend_url.txt || echo "Not available"
+	@echo "Admin URL:" && cat admin_url.txt || echo "Not available"
+	@echo ""
+	@echo "🚀 Deployment completed!"
+	@echo "⏰ Authentication will be disabled via GitHub Actions pipeline."
+	@echo "🔄 Check the pipeline logs for authentication disable status."
+	@echo "=== Extracting PostgreSQL Host Endpoint ==="
+		@azd env get-values > .env.temp 2>/dev/null || echo "" > .env.temp
+
+		@PG_HOST_VAL=$$(grep '^AZURE_POSTGRESQL_HOST_NAME=' .env.temp | cut -d'=' -f2 | tr -d '"' | xargs); \
+		if [ -z "$$PG_HOST_VAL" ]; then \
+			echo "❌ PostgreSQL host not found in .env.temp. Using fallback localhost"; \
+			PG_HOST_VAL="localhost"; \
+		else \
+			echo "✅ PostgreSQL host extracted from .env.temp: $$PG_HOST_VAL"; \
+		fi; \
+		echo "$$PG_HOST_VAL" > pg_host.txt
+
+
+
+
+
+	@echo "=== PostgreSQL Configuration ==="
+	@echo "Username: admintest (hardcoded)"
+	@echo "Database: postgres (hardcoded)"
+	@echo "Port: 5432 (hardcoded)"
+	@echo "Host: $$(cat pg_host.txt 2>/dev/null || echo 'Not available')"
+	@echo "Password: Initial_0524 (hardcoded)"
+
+# Helper target to check current authentication status
+check-auth:
+	@echo "=== Checking Authentication Status ==="
+	@FRONTEND_URL=$$(cat frontend_url.txt 2>/dev/null | tr -d '\n\r' | xargs); \
+	ADMIN_URL=$$(cat admin_url.txt 2>/dev/null | tr -d '\n\r' | xargs); \
+	if [ -n "$$FRONTEND_URL" ]; then \
+		echo "Testing Frontend: $$FRONTEND_URL"; \
+		HTTP_CODE=$$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 "$$FRONTEND_URL" 2>/dev/null || echo "000"); \
+		echo "Frontend HTTP Status: $$HTTP_CODE"; \
+	fi; \
+	if [ -n "$$ADMIN_URL" ]; then \
+		echo "Testing Admin: $$ADMIN_URL"; \
+		HTTP_CODE=$$(curl -s -o /dev/null -w "%{http_code}" --connect-timeout 10 "$$ADMIN_URL" 2>/dev/null || echo "000"); \
+		echo "Admin HTTP Status: $$HTTP_CODE"; \
+	fi
+
+# Helper target to manually disable authentication (for debugging)
+disable-auth-manual:
+	@echo "=== Manually Disabling Authentication ==="
+	@echo "This target requires Azure CLI to be logged in manually"
+	@FRONTEND_URL=$$(cat frontend_url.txt 2>/dev/null | tr -d '\n\r' | xargs); \
+	ADMIN_URL=$$(cat admin_url.txt 2>/dev/null | tr -d '\n\r' | xargs); \
+	export FRONTEND_WEBSITE_URL="$$FRONTEND_URL"; \
+	export ADMIN_WEBSITE_URL="$$ADMIN_URL"; \
+	if [ -f "disable_auth.sh" ]; then \
+		chmod +x disable_auth.sh && ./disable_auth.sh; \
+	else \
+		echo "ERROR: disable_auth.sh not found"; \
+		exit 1; \
+	fi
+
+disable-auth-fixed:
+	@echo "=== Using Fixed Authentication Disable Script ==="
+	@if [ -f "disable_auth_fixed.sh" ]; then \
+		chmod +x disable_auth_fixed.sh && ./disable_auth_fixed.sh; \
+	else \
+		echo "ERROR: disable_auth_fixed.sh not found"; \
+		exit 1; \
+	fi
 
 destroy: azd-login ## 🧨 Destroy everything in Azure
 	@echo -e "\e[34m$@\e[0m" || true

Makefile

@@ -166,7 +166,7 @@ def __load_config(self, **kwargs) -> None:
                 "Unsupported DATABASE_TYPE. Please set DATABASE_TYPE to 'CosmosDB' or 'PostgreSQL'."
             )
 
-        self.AZURE_AUTH_TYPE = os.getenv("AZURE_AUTH_TYPE", "keys")
+        self.AZURE_AUTH_TYPE = os.getenv("AZURE_AUTH_TYPE", "rbac")
         # Azure OpenAI
         self.AZURE_OPENAI_RESOURCE = os.getenv("AZURE_OPENAI_RESOURCE", "")
         # Fetch AZURE_OPENAI_MODEL_INFO from environment
@@ -234,6 +234,7 @@ def __load_config(self, **kwargs) -> None:
         self.AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION = os.getenv(
             "AZURE_COMPUTER_VISION_VECTORIZE_IMAGE_MODEL_VERSION", "2023-04-15"
         )
+        self.FUNCTION_KEY = os.getenv("FUNCTION_KEY", "")
 
         # Initialize Azure keys based on authentication type and environment settings.
         # When AZURE_AUTH_TYPE is "rbac", azure keys are None or an empty string.
@@ -242,6 +243,7 @@ def __load_config(self, **kwargs) -> None:
             self.AZURE_OPENAI_API_KEY = ""
             self.AZURE_SPEECH_KEY = None
             self.AZURE_COMPUTER_VISION_KEY = None
+            self.FUNCTION_KEY = self.secretHelper.get_secret("FUNCTION_KEY")
         else:
             self.AZURE_SEARCH_KEY = self.secretHelper.get_secret("AZURE_SEARCH_KEY")
             self.AZURE_OPENAI_API_KEY = self.secretHelper.get_secret(
@@ -269,7 +271,6 @@ def __load_config(self, **kwargs) -> None:
         os.environ["OPENAI_API_VERSION"] = self.OPENAI_API_VERSION
         # Azure Functions - Batch processing
         self.BACKEND_URL = os.getenv("BACKEND_URL", "http://localhost:7071")
-        self.FUNCTION_KEY = os.getenv("FUNCTION_KEY")
         self.AzureWebJobsStorage = os.getenv("AzureWebJobsStorage", "")
         self.DOCUMENT_PROCESSING_QUEUE_NAME = os.getenv(
             "DOCUMENT_PROCESSING_QUEUE_NAME", "doc-processing"

code/backend/batch/utilities/helpers/env_helper.py

@@ -209,6 +209,11 @@ def conversation_with_data(conversation: Request, env_helper: EnvHelper):
                                 env_helper.AZURE_SEARCH_CONTENT_VECTOR_COLUMN
                             ],
                             "title_field": env_helper.AZURE_SEARCH_TITLE_COLUMN or None,
+                            "source_field": env_helper.AZURE_SEARCH_SOURCE_COLUMN
+                            or None,
+                            "text_field": env_helper.AZURE_SEARCH_TEXT_COLUMN or None,
+                            "layoutText_field": env_helper.AZURE_SEARCH_LAYOUT_TEXT_COLUMN
+                            or None,
                             "url_field": env_helper.AZURE_SEARCH_FIELDS_METADATA
                             or None,
                             "filepath_field": (
@@ -521,7 +526,6 @@ def speech_config():
             if response.status_code == 200:
                 return {
                     "token": response.text,
-                    "key": speech_key,
                     "region": env_helper.AZURE_SPEECH_SERVICE_REGION,
                     "languages": env_helper.AZURE_SPEECH_RECOGNIZER_LANGUAGES,
                 }