Add Performance test for gen3-embedding

.github/workflows/load_tests.yaml

@@ -19,12 +19,19 @@ on:
           - TestIndexdDrsEndpoint
           - TestMetadataCreateAndQuery
           - TestSheepdogImportClinicalMetadata
+          - TestGen3EmbeddingBulkRetrievalSmall
+          - TestGen3EmbeddingBulkRetrievalMedium
+          - TestGen3EmbeddingPublishEmbeddings
+          - TestGen3EmbeddingSearchKnownCollection
+          - TestGen3EmbeddingSearchUnKnownCollection
+          - TestGen3EmbeddingLoadData
           - ALL
         default: ALL
       RELEASE_VERSION:
         description: 'Release version'
         required: true
 
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -50,7 +57,7 @@ jobs:
               TEST_SUITE="${{ github.event.inputs.TEST_SUITE }}"
               RELEASE_VERSION="${{ github.event.inputs.RELEASE_VERSION }}"
             else
-              TEST_SUITE="ALL"
+              TEST_SUITE="TestGen3EmbeddingLoadData"
               RELEASE_VERSION=""
             fi
 
@@ -72,12 +79,15 @@ jobs:
           BRANCH: ${{ github.event.pull_request.head.ref }}
           PR_NUM: ${{ github.event.pull_request.number }}
           RUN_NUM: ${{ github.run_number }}
+          ATTEMPT_NUM: ${{ github.run_attempt }}
           SLACK_CHANNEL: ${{ secrets.CI_SLACK_CHANNEL_ID }}
           MTLS_CRT: ${{ secrets.MTLS_CRT }}
           MTLS_KEY: ${{ secrets.MTLS_KEY }}
           TEST_SUITE: ${{ needs.setup.outputs.TEST_SUITE }}
           RELEASE_VERSION: ${{ needs.setup.outputs.RELEASE_VERSION }}
           EKS_CLUSTER_NAME : ${{ secrets.EKS_CLUSTER_NAME }}
+          HELM_BRANCH: 'feat/add-gen3-embeddings'
+          CHUNK_ID: "0"
 
 
         steps:
@@ -108,7 +118,7 @@ jobs:
             uses: actions/checkout@v4
             with:
               repository: uc-cdis/gen3-gitops
-              ref: 'master'
+              ref: 'chore/add_gen3_embedding_ci'
               path: gen3-gitops-ci
 
           - name: Get commit time
@@ -201,17 +211,10 @@ jobs:
             if: ${{ env.SKIP_TESTS != 'true' }}
             # Change the HOSTNAME with full domain name
             run: |
-              NAMESPACE=ci-perf
-              HOSTNAME=perf.ci.planx-pla.net
+              NAMESPACE=ci-perf-embedding
+              HOSTNAME=perf-embedding.ci.planx-pla.net
               echo "NAMESPACE=$NAMESPACE" >> $GITHUB_ENV
               echo "HOSTNAME=$HOSTNAME" >> $GITHUB_ENV
-              namespace_exists=$(kubectl get namespace "$NAMESPACE" --no-headers --ignore-not-found)
-              if [ -n "$namespace_exists" ]; then
-                echo "Namespace $NAMESPACE exists, performing teardown..."
-                helm delete $NAMESPACE -n $NAMESPACE || true
-                kubectl delete ns $NAMESPACE
-              fi
-              kubectl create namespace $NAMESPACE
 
           - name: Download Thor repo_list.txt file
             if: ${{ env.SKIP_TESTS != 'true' }}
@@ -220,6 +223,12 @@ jobs:
               THOR_REPO_LIST_PATH="$(pwd)/repo_list.txt"
               echo "THOR_REPO_LIST_PATH=$THOR_REPO_LIST_PATH" >> $GITHUB_ENV
 
+          # - name: Download embedding tests file
+          #   if: ${{ env.SKIP_TESTS != 'true' }}
+          #   run: |
+          #     aws s3 cp s3://ci-embedding-test/expr_hist_summ_as_tsvs.zip ./test_data/embedding.zip
+          #     unzip -o ./test_data/embedding.zip -d ./test_data/embedding/
+
           - name: Prepare CI environment
             id: prep_ci_env
             if: ${{ env.SKIP_TESTS != 'true' }}
@@ -268,7 +277,7 @@ jobs:
             continue-on-error: true  # if this fails, we still need to run clean-up steps
             run: |
               DATE=$(date +%Y%m%d)
-              aws s3 sync ./allure-report s3://ci-allure-reports/load-tests/$DATE/$RUN_NUM
+              aws s3 sync ./allure-report s3://ci-allure-reports/load-tests/$DATE/$RUN_NUM/$ATTEMPT_NUM
               if [ $? -ne 0 ]; then
                 echo "PR_ERROR_MSG=Failed to upload allure report to s3 bucket" >> $GITHUB_ENV
               fi

.gitignore

@@ -26,3 +26,4 @@ gen3-integration-tests/test_data/gen3_workflow/nf-canary/
 
 gen3-load-tests/load_testing_output/
 gen3-load-tests/test_data/generated_metadata_service_template/
+gen3-load-tests/test_data/embedding/

gen3-load-tests/conftest.py

@@ -36,52 +36,58 @@ def pytest_configure(config):
 
     LOAD_TESTING_OUTPUT_PATH.mkdir(parents=True, exist_ok=True)
 
-    setup.perform_pre_load_testing_setup()
-
-
-@pytest.hookimpl(hookwrapper=True)
-def pytest_runtest_logreport(report):
-    yield
-
-    if report.when != "call":
-        return
-
-    test_nodeid = report.nodeid
-    start_time = datetime.fromtimestamp(report.start)
-    file_name = test_nodeid.split("::")[-1].replace("test_", "").replace("_", "-")
-    output_path = LOAD_TESTING_OUTPUT_PATH / f"{file_name}.json"
-    output = json.loads(output_path.read_text())
-    message = {
-        "run_date": str(start_time.date()),
-        "run_num": os.getenv("RUN_NUM"),
-        "release_version": os.getenv("RELEASE_VERSION"),
-        "test_suite": test_nodeid.split("::")[1],
-        "test_case": test_nodeid.split("::")[-1],
-        "result": report.outcome,
-        "checks_fails": output["metrics"]["checks"]["fails"],
-        "checks_passes": output["metrics"]["checks"]["passes"],
-        "checks_value": output["metrics"]["checks"]["value"],
-        "http_req_duration_avg": output["metrics"]["http_req_duration"]["avg"],
-        "http_req_duration_min": output["metrics"]["http_req_duration"]["min"],
-        "http_req_duration_med": output["metrics"]["http_req_duration"]["med"],
-        "http_req_duration_max": output["metrics"]["http_req_duration"]["max"],
-        "http_req_duration_p90": output["metrics"]["http_req_duration"]["p(90)"],
-        "http_req_duration_p95": output["metrics"]["http_req_duration"]["p(95)"],
-        "data_sent_count": output["metrics"]["data_sent"]["count"],
-        "data_sent_rate": output["metrics"]["data_sent"]["rate"],
-        "iterations_count": output["metrics"]["iterations"]["count"],
-        "iterations_rate": output["metrics"]["iterations"]["rate"],
-    }
-
-    try:
-        sqs = boto3.client("sqs")
-        queue_url = (
-            "https://sqs.us-east-1.amazonaws.com/707767160287/load-test-metrics-sqs"
-        )
-        response = sqs.send_message(QueueUrl=queue_url, MessageBody=json.dumps(message))
-        logger.info(f"[SQS MESSAGE SENT] MessageId: {response['MessageId']}")
-    except Exception as e:
-        logger.error(f"[SQS SEND ERROR] {e}")
+    # setup.perform_pre_load_testing_setup()
+
+
+# @pytest.hookimpl(hookwrapper=True)
+# def pytest_runtest_logreport(report):
+#     yield
+
+#     if report.when != "call":
+#         return
+
+#     test_nodeid = report.nodeid
+#     start_time = datetime.fromtimestamp(report.start)
+#     file_name = test_nodeid.split("::")[-1].replace("test_", "").replace("_", "-")
+#     output_path = LOAD_TESTING_OUTPUT_PATH / f"{file_name}.json"
+#     output = json.loads(output_path.read_text())
+#     message = {
+#         "run_date": str(start_time.date()),
+#         "run_num": os.getenv("RUN_NUM"),
+#         "attempt_num": os.getenv("ATTEMPT_NUM"),
+#         "release_version": os.getenv("RELEASE_VERSION"),
+#         "test_suite": test_nodeid.split("::")[1],
+#         "test_case": test_nodeid.split("::")[-1],
+#         "result": report.outcome,
+#         "checks_fails": output["metrics"]["checks"]["fails"],
+#         "checks_passes": output["metrics"]["checks"]["passes"],
+#         "checks_value": output["metrics"]["checks"]["value"],
+#         "http_req_duration_avg": output["metrics"]["http_req_duration"]["avg"],
+#         "http_req_duration_min": output["metrics"]["http_req_duration"]["min"],
+#         "http_req_duration_med": output["metrics"]["http_req_duration"]["med"],
+#         "http_req_duration_max": output["metrics"]["http_req_duration"]["max"],
+#         "http_req_duration_p90": output["metrics"]["http_req_duration"]["p(90)"],
+#         "http_req_duration_p95": output["metrics"]["http_req_duration"]["p(95)"],
+#         "data_sent_count": output["metrics"]["data_sent"]["count"],
+#         "data_sent_rate": output["metrics"]["data_sent"]["rate"],
+#         "iterations_count": output["metrics"]["iterations"]["count"],
+#         "iterations_rate": output["metrics"]["iterations"]["rate"],
+#     }
+
+#     try:
+#         sqs = boto3.client("sqs")
+#         if not os.getenv("RELEASE_VERSION"):
+#             # If RELEASE_VERSION is not provided, send the data to non-release table
+#             queue_url = "https://sqs.us-east-1.amazonaws.com/707767160287/load-test-metrics-non-release-sqs"
+#         else:
+#             # Else send it to the table with release data
+#             queue_url = (
+#                 "https://sqs.us-east-1.amazonaws.com/707767160287/load-test-metrics-sqs"
+#             )
+#         response = sqs.send_message(QueueUrl=queue_url, MessageBody=json.dumps(message))
+#         logger.info(f"[SQS MESSAGE SENT] MessageId: {response['MessageId']}")
+#     except Exception as e:
+#         logger.error(f"[SQS SEND ERROR] {e}")
 
 
 def pytest_unconfigure(config):

gen3-load-tests/gen3_ci/load_test_metrics/load-test-metrics-non-release-cronjob.yaml

@@ -0,0 +1,134 @@
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: load-test-metrics-non-release
+  namespace: default
+spec:
+  schedule: "*/120 * * * *"
+  successfulJobsHistoryLimit: 2
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    spec:
+      backoffLimit: 2
+      template:
+        spec:
+          serviceAccountName: ci-metrics-sa
+          restartPolicy: OnFailure
+          containers:
+          - name: load-test-metrics-non-release
+            image: quay.io/cdis/ci_metrics:master
+            env:
+              - name: METRICS_DB_DSN
+                valueFrom:
+                  secretKeyRef:
+                    name: ci-metrics-db
+                    key: "METRICS_DB_DSN"
+            command:
+              - /bin/sh
+              - -c
+              - |
+                #!/bin/bash
+                set -euo pipefail
+
+                QUEUE_URL="https://sqs.us-east-1.amazonaws.com/707767160287/load-test-metrics-non-release-sqs"
+                REGION="${AWS_REGION:-us-east-1}"  # fallback if AWS_REGION not set
+
+                while true; do
+                    RESPONSE=$(aws sqs receive-message \
+                        --queue-url "$QUEUE_URL" \
+                        --max-number-of-messages 10 \
+                        --wait-time-seconds 10 \
+                        --region "$REGION")
+
+                    MESSAGES=$(echo "$RESPONSE" | jq -c '.Messages[]?')
+
+                    if [ -z "$MESSAGES" ]; then
+                        echo "No more messages found. Exiting."
+                        break
+                    fi
+
+                    echo "$MESSAGES" | while read -r message; do
+                        RECEIPT_HANDLE=$(echo "$message" | jq -r '.ReceiptHandle')
+                        BODY=$(echo "$message" | jq -r '.Body')
+
+                        echo "Processing message: $BODY"
+
+                        RUN_DATE=$(echo "$BODY" | jq -r '.run_date')
+                        RUN_NUM=$(echo "$BODY" | jq -r '.run_num')
+                        ATTEMPT_NUM=$(echo "$BODY" | jq -r '.attempt_num')
+                        RELEASE_VERSION=$(echo "$BODY" | jq -r '.release_version')
+                        SUITE=$(echo "$BODY" | jq -r '.test_suite')
+                        CASE=$(echo "$BODY" | jq -r '.test_case')
+                        RESULT=$(echo "$BODY" | jq -r '.result')
+                        CHECKS_FAILS=$(echo "$BODY" | jq -r '.checks_fails')
+                        CHECKS_PASSES=$(echo "$BODY" | jq -r '.checks_passes')
+                        CHECKS_VALUE=$(echo "$BODY" | jq -r '.checks_value')
+                        HTTP_REQ_DURATION_AVG=$(echo "$BODY" | jq -r '.http_req_duration_avg')
+                        HTTP_REQ_DURATION_MIN=$(echo "$BODY" | jq -r '.http_req_duration_min')
+                        HTTP_REQ_DURATION_MED=$(echo "$BODY" | jq -r '.http_req_duration_med')
+                        HTTP_REQ_DURATION_MAX=$(echo "$BODY" | jq -r '.http_req_duration_max')
+                        HTTP_REQ_DURATION_P90=$(echo "$BODY" | jq -r '.http_req_duration_p90')
+                        HTTP_REQ_DURATION_P95=$(echo "$BODY" | jq -r '.http_req_duration_p95')
+                        DATA_SENT_COUNT=$(echo "$BODY" | jq -r '.data_sent_count')
+                        DATA_SENT_RATE=$(echo "$BODY" | jq -r '.data_sent_rate')
+                        ITERATIONS_COUNT=$(echo "$BODY" | jq -r '.iterations_count')
+                        ITERATIONS_RATE=$(echo "$BODY" | jq -r '.iterations_rate')
+
+
+                        psql "$METRICS_DB_DSN" <<EOF
+                INSERT INTO load_test_metrics_non_release_data (
+                    run_date,
+                    run_num,
+                    attempt_num,
+                    release_version,
+                    test_suite,
+                    test_case,
+                    result,
+                    checks_fails,
+                    checks_passes,
+                    checks_value,
+                    http_req_duration_avg,
+                    http_req_duration_min,
+                    http_req_duration_med,
+                    http_req_duration_max,
+                    http_req_duration_p90,
+                    http_req_duration_p95,
+                    data_sent_count,
+                    data_sent_rate,
+                    iterations_count,
+                    iterations_rate
+                ) VALUES (
+                    '$RUN_DATE',
+                    $RUN_NUM,
+                    $ATTEMPT_NUM,
+                    '$RELEASE_VERSION',
+                    '$SUITE',
+                    '$CASE',
+                    '$RESULT',
+                    $CHECKS_FAILS,
+                    $CHECKS_PASSES,
+                    $CHECKS_VALUE,
+                    $HTTP_REQ_DURATION_AVG,
+                    $HTTP_REQ_DURATION_MIN,
+                    $HTTP_REQ_DURATION_MED,
+                    $HTTP_REQ_DURATION_MAX,
+                    $HTTP_REQ_DURATION_P90,
+                    $HTTP_REQ_DURATION_P95,
+                    $DATA_SENT_COUNT,
+                    $DATA_SENT_RATE,
+                    $ITERATIONS_COUNT,
+                    $ITERATIONS_RATE
+                );
+                EOF
+
+
+                        aws sqs delete-message \
+                            --queue-url "$QUEUE_URL" \
+                            --receipt-handle "$RECEIPT_HANDLE" \
+                            --region "$REGION"
+
+                        echo "Message processed and deleted."
+                    done
+                done
+
+                echo "Finished processing all messages."