diff --git a/.ci/scripts/test_backend.sh b/.ci/scripts/test_backend.sh
index 8bf2236e4e6..300751aca7b 100755
--- a/.ci/scripts/test_backend.sh
+++ b/.ci/scripts/test_backend.sh
@@ -58,8 +58,6 @@ if [[ "$FLOW" == *vulkan* ]]; then
 fi
 
 if [[ "$FLOW" == *arm* ]]; then
-    PYTEST_RETRY_ARGS=(--reruns 2 --reruns-delay 1)
-
     # Setup ARM deps.
     if [[ "$FLOW" == *vgf* ]]; then
         .ci/scripts/setup-arm-baremetal-tools.sh --enable-mlsdk-deps --install-mlsdk-deps-with-pip
@@ -96,6 +94,10 @@ CMAKE_ARGS="$EXTRA_BUILD_ARGS" ${CONDA_RUN_CMD} $SETUP_SCRIPT --build-tool cmake
 GOLDEN_DIR="${ARTIFACT_DIR}/golden-artifacts"
 export GOLDEN_ARTIFACTS_DIR="${GOLDEN_DIR}"
 
+if [[ "$FLOW" == *arm* && "$SUITE" == "operators" ]]; then
+    PYTEST_RETRY_ARGS=(--reruns 1 --reruns-delay 1)
+fi
+
 EXIT_CODE=0
 PYTEST_ARGS=(-c /dev/null -n auto)
 if [[ ${#PYTEST_RETRY_ARGS[@]} -gt 0 ]]; then
diff --git a/backends/arm/test/test_arm_baremetal.sh b/backends/arm/test/test_arm_baremetal.sh
index ad8cd8b7d3a..1d6613f89de 100755
--- a/backends/arm/test/test_arm_baremetal.sh
+++ b/backends/arm/test/test_arm_baremetal.sh
@@ -48,7 +48,7 @@ fi
 TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}"
 
 EXCLUDE_TARGET_EXPR="(not u55) and (not u85) and (not tosa) and (not _vgf_)"
-PYTEST_RETRY_ARGS=(--reruns 2 --reruns-delay 1)
+PYTEST_OP_RETRY_ARGS=(--reruns 1 --reruns-delay 1)
 
 all() { # Run all tests
     # This will list all lines in this file that is starting with test_ remove () { and add this script name in
@@ -81,7 +81,7 @@ test_pytest_ops_no_target() {
     echo "${TEST_SUITE_NAME}: Run pytest ops for target-less tests"
 
     # Run arm baremetal pytest tests without target
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10 backends/arm/test/ --ignore=backends/arm/test/models -k "${EXCLUDE_TARGET_EXPR}"
+    pytest "${PYTEST_OP_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10 backends/arm/test/ --ignore=backends/arm/test/models -k "${EXCLUDE_TARGET_EXPR}"
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -92,7 +92,7 @@ test_pytest_models_no_target() {
     source backends/arm/scripts/install_models_for_test.sh
 
     # Run arm baremetal pytest tests without FVP
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k "${EXCLUDE_TARGET_EXPR}"
+    pytest --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k "${EXCLUDE_TARGET_EXPR}"
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -102,7 +102,7 @@ test_pytest_models_no_target() {
 test_pytest_ops_tosa() {
     echo "${TEST_SUITE_NAME}: Run pytest ops for TOSA"
 
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10 backends/arm/test/ --ignore=backends/arm/test/models -k tosa
+    pytest "${PYTEST_OP_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10 backends/arm/test/ --ignore=backends/arm/test/models -k tosa
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -112,7 +112,7 @@ test_pytest_models_tosa() {
     # Install model dependencies for pytest
     source backends/arm/scripts/install_models_for_test.sh
 
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k tosa
+    pytest --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k tosa
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -135,7 +135,7 @@ test_pytest_ops_ethos_u55() {
     backends/arm/scripts/build_executorch.sh
     backends/arm/test/setup_testing.sh
 
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10  backends/arm/test/ --ignore=backends/arm/test/models -k u55
+    pytest "${PYTEST_OP_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10  backends/arm/test/ --ignore=backends/arm/test/models -k u55
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -148,7 +148,7 @@ test_pytest_models_ethos_u55() {
     # Install model dependencies for pytest
     source backends/arm/scripts/install_models_for_test.sh
 
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u55
+    pytest --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u55
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -189,7 +189,7 @@ test_pytest_ops_ethos_u85() {
     backends/arm/test/setup_testing.sh
 
     # Run arm baremetal pytest tests with FVP
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10  backends/arm/test/ --ignore=backends/arm/test/models -k u85
+    pytest "${PYTEST_OP_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10  backends/arm/test/ --ignore=backends/arm/test/models -k u85
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -202,7 +202,7 @@ test_pytest_models_ethos_u85() {
     # Install model dependencies for pytest
     source backends/arm/scripts/install_models_for_test.sh
 
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u85
+    pytest --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k u85
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -236,7 +236,7 @@ test_pytest_ops_vkml() {
 
     source backends/arm/test/setup_testing_vkml.sh
 
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10  backends/arm/test/ \
+    pytest "${PYTEST_OP_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=10  backends/arm/test/ \
             --ignore=backends/arm/test/models -k _vgf_
     echo "${TEST_SUITE_NAME}: PASS"
 }
@@ -249,7 +249,7 @@ test_pytest_models_vkml() {
     # Install model dependencies for pytest
     source backends/arm/scripts/install_models_for_test.sh
 
-    pytest "${PYTEST_RETRY_ARGS[@]}" --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k _vgf_
+    pytest --verbose --color=yes --numprocesses=auto --durations=0 backends/arm/test/models -k _vgf_
     echo "${TEST_SUITE_NAME}: PASS"
 }
 
@@ -296,7 +296,6 @@ test_smaller_stories_llama() {
     # Get path to source directory
     pytest \
     -c /dev/null \
-    "${PYTEST_RETRY_ARGS[@]}" \
     --verbose \
     --color=yes \
     --numprocesses=auto \