diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5219b9826bd6..440ad2b75072 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -119,7 +119,7 @@ jobs: - name: Run pytest tests with coverage run: | - pytest -n auto --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term + pytest tests/integration/challenges/basic_abilities/test_write_file.py python tests/integration/challenges/utils/build_current_score.py env: CI: true @@ -127,9 +127,6 @@ jobs: AGENT_MODE: ${{ vars.AGENT_MODE }} AGENT_TYPE: ${{ vars.AGENT_TYPE }} - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 - - name: Update cassette submodule to push target if push event if: ${{ github.event_name == 'push' }} run: | @@ -177,7 +174,7 @@ jobs: # Check if there are any changes if ! git diff-index --quiet HEAD; then - git commit -m "Auto-update cassettes after merging PR #$pr_number" + git commit -m "Auto-update cassettes after merging PR #${{ github.event.pull_request.number }}" git push -f origin HEAD:refs/heads/$new_branch else @@ -193,10 +190,67 @@ jobs: echo "DIFF_EXISTS=false" >> $GITHUB_ENV fi + - name: Beat Challenges + if: ${{ env.DIFF_EXISTS == 'true' }} + run: | + pytest -n auto tests/integration/challenges/memory/test_memory_challenge_c.py tests/integration/challenges/basic_abilities/test_write_file.py --beat-challenges --cov=autogpt --cov-report term-missing --cov-branch --cov-report xml --cov-report term + python tests/integration/challenges/utils/build_current_score.py + env: + CI: true + PROXY: ${{ secrets.PROXY }} + AGENT_MODE: ${{ vars.AGENT_MODE }} + AGENT_TYPE: ${{ vars.AGENT_TYPE }} + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + + - name: Beat challenges - Update cassette submodule to submodule branch if PR event + if: ${{ env.DIFF_EXISTS == 'true' }} + run: | + new_branch="${{ github.event.pull_request.user.login }}-${{ github.event.pull_request.head.ref }}" + + cd tests/Auto-GPT-test-cassettes + git config --global user.name "Auto-GPT-Bot" + git config --global user.email "github-bot@agpt.co" + git add . + + # Check if there are any changes + if ! git diff-index --quiet HEAD; then + git commit -m "Auto-update cassettes after merging PR #${{ github.event.pull_request.number }} (Beat challenges)" + git push -f origin HEAD:refs/heads/$new_branch + + else + echo "No changes to commit" + exit 0 + fi + + - name: Create Pull Request + if: ${{ env.DIFF_EXISTS == 'true' }} + run: | + current_date_time=$(date +'%Y%m%d%H%M%S') + new_branch="update-score-${{ github.event.pull_request.number }}-$current_date_time" + + git config --global user.name "Auto-GPT-Bot" + git config --global user.email "github-bot@agpt.co" + git add tests/integration/challenges/current_score.json + if ! git diff-index --quiet HEAD; then + git commit -m "Update score" + git checkout -b $new_branch + git push origin $new_branch + + echo ${{ secrets.PAT_REVIEW }} | gh auth login --with-token + gh pr create --title "Update Score In Pull Request Number ${{ github.event.pull_request.number }}" \ + --head "$new_branch" \ + --base "${{ github.event.pull_request.head.ref }}" \ + --body "This pull request updates the current score of Auto-GPT. Please check the files changed and merge the pull request." \ + --repo ${{ github.event.pull_request.head.repo.full_name }} + else + echo "The current score didn't change." + fi + - name: Apply or remove behaviour change label and comment if: ${{ github.event_name == 'pull_request_target' }} run: | - PR_NUMBER=${{ github.event.pull_request.number }} TOKEN=${{ secrets.PAT_REVIEW }} REPO=${{ github.repository }} @@ -205,15 +259,15 @@ jobs: curl -X POST \ -H "Authorization: Bearer $TOKEN" \ -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels \ + https://api.github.com/repos/$REPO/issues/${{ github.event.pull_request.number }}/labels \ -d '{"labels":["behaviour change"]}' echo $TOKEN | gh auth login --with-token - gh api repos/$REPO/issues/$PR_NUMBER/comments -X POST -F body="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged." + gh api repos/$REPO/issues/${{ github.event.pull_request.number }}/comments -X POST -F body="You changed AutoGPT's behaviour. The cassettes have been updated and will be merged to the submodule when this Pull Request gets merged." else echo "Removing label..." curl -X DELETE \ -H "Authorization: Bearer $TOKEN" \ -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/repos/$REPO/issues/$PR_NUMBER/labels/behaviour%20change + https://api.github.com/repos/$REPO/issues/${{ github.event.pull_request.number }}/labels/behaviour%20change fi diff --git a/.gitmodules b/.gitmodules index 55d09f842c8f..528c9ca4ae18 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,4 +1,4 @@ [submodule "tests/Auto-GPT-test-cassettes"] path = tests/Auto-GPT-test-cassettes url = https://github.com/Significant-Gravitas/Auto-GPT-test-cassettes - branch = master \ No newline at end of file + branch = ci-test-1 diff --git a/autogpt/llm/utils/__init__.py b/autogpt/llm/utils/__init__.py index 756c4bd55de4..fab1910b33c2 100644 --- a/autogpt/llm/utils/__init__.py +++ b/autogpt/llm/utils/__init__.py @@ -96,7 +96,9 @@ def _wrapped(*args, **kwargs): user_warned = True except APIError as e: - if (e.http_status not in [502, 429]) or (attempt == num_attempts): + if (e.http_status not in [502, 429]) or ( + attempt == num_attempts + ): raise backoff = backoff_base ** (attempt + 2) diff --git a/tests/Auto-GPT-test-cassettes b/tests/Auto-GPT-test-cassettes index be280df43d6a..a04dac13efe2 160000 --- a/tests/Auto-GPT-test-cassettes +++ b/tests/Auto-GPT-test-cassettes @@ -1 +1 @@ -Subproject commit be280df43d6a23b8074d9cba10d18ed8724a54c9 +Subproject commit a04dac13efe263f363458bccdcef330bede7e12a diff --git a/tests/integration/agent_factory.py b/tests/integration/agent_factory.py index 30d9cc13b2aa..1446ca0f23ab 100644 --- a/tests/integration/agent_factory.py +++ b/tests/integration/agent_factory.py @@ -81,7 +81,7 @@ def writer_agent(agent_test_config, memory_none: NoMemory, workspace: Workspace) ai_role="an AI designed to use the write_to_file command to write 'Hello World' into a file named \"hello_world.txt\" and then use the task_complete command to complete the task.", ai_goals=[ "Use the write_to_file command to write 'Hello World' into a file named \"hello_world.txt\".", - "Use the task_complete command to complete the task.", + "Use the task_complete command to complete the task!", "Do not use any other commands.", ], ) @@ -116,7 +116,7 @@ def memory_management_agent(agent_test_config, memory_json_file, workspace: Work ai_config = AIConfig( ai_name="Follow-Instructions-GPT", - ai_role="an AI designed to read the instructions_1.txt file using the read_file method and follow the instructions in the file.", + ai_role="an AI designed to read the instructions_1.txt file using the read_file method and follow the instructions in the file!", ai_goals=[ "Use the command read_file to read the instructions_1.txt file", "Follow the instructions in the instructions_1.txt file", diff --git a/tests/integration/challenges/basic_abilities/test_browse_website.py b/tests/integration/challenges/basic_abilities/test_browse_website.py index 09e5ab2200a4..93e3ee241807 100644 --- a/tests/integration/challenges/basic_abilities/test_browse_website.py +++ b/tests/integration/challenges/basic_abilities/test_browse_website.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from tests.integration.challenges.challenge_decorator.challenge_decorator import ( @@ -15,7 +16,7 @@ @challenge def test_browse_website( browser_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, level_to_run: int, ) -> None: diff --git a/tests/integration/challenges/basic_abilities/test_write_file.py b/tests/integration/challenges/basic_abilities/test_write_file.py index cbbad514b6b6..393dbfd05d85 100644 --- a/tests/integration/challenges/basic_abilities/test_write_file.py +++ b/tests/integration/challenges/basic_abilities/test_write_file.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -17,7 +18,7 @@ @challenge def test_write_file( writer_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/current_score.json b/tests/integration/challenges/current_score.json index 726613991d48..123b1b991b75 100644 --- a/tests/integration/challenges/current_score.json +++ b/tests/integration/challenges/current_score.json @@ -1,47 +1,13 @@ { "basic_abilities": { - "browse_website": { - "max_level": 1, - "max_level_beaten": 1 - }, "write_file": { "max_level": 1, "max_level_beaten": 1 } }, - "debug_code": { - "debug_code_challenge_a": { - "max_level": 1, - "max_level_beaten": 1 - } - }, - "information_retrieval": { - "information_retrieval_challenge_a": { - "max_level": 3, - "max_level_beaten": 1 - }, - "information_retrieval_challenge_b": { - "max_level": 1, - "max_level_beaten": 1 - } - }, - "kubernetes": { - "kubernetes_template_challenge_a": { - "max_level": 1, - "max_level_beaten": null - } - }, "memory": { - "memory_challenge_a": { - "max_level": 3, - "max_level_beaten": 3 - }, - "memory_challenge_b": { - "max_level": 5, - "max_level_beaten": null - }, "memory_challenge_c": { - "max_level": 5, + "max_level": 1, "max_level_beaten": 1 } } diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py index 6b970e8b227d..2f61fef3902a 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py +++ b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_a.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.commands.file_operations import read_file from autogpt.config import Config @@ -19,7 +20,7 @@ def test_information_retrieval_challenge_a( information_retrieval_agents: Agent, monkeypatch: pytest.MonkeyPatch, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, config: Config, level_to_run: int, ) -> None: diff --git a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py index feac95a0f646..6461e13fbf4c 100644 --- a/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py +++ b/tests/integration/challenges/information_retrieval/test_information_retrieval_challenge_b.py @@ -1,6 +1,7 @@ import contextlib import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -20,7 +21,7 @@ def test_information_retrieval_challenge_b( get_nobel_prize_agent: Agent, monkeypatch: pytest.MonkeyPatch, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, level_to_run: int, config: Config, ) -> None: diff --git a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py b/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py index 5fd280ac4bbb..ad658bae9183 100644 --- a/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py +++ b/tests/integration/challenges/kubernetes/test_kubernetes_template_challenge_a.py @@ -1,5 +1,6 @@ import pytest import yaml +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file @@ -21,6 +22,7 @@ def test_kubernetes_template_challenge_a( monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, + patched_api_requestor: MockerFixture, ) -> None: """ Test the challenge_a function in a given agent by mocking user inputs diff --git a/tests/integration/challenges/memory/test_memory_challenge_a.py b/tests/integration/challenges/memory/test_memory_challenge_a.py index 8919bf58c0eb..08f461bdd86d 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_a.py +++ b/tests/integration/challenges/memory/test_memory_challenge_a.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -15,7 +16,7 @@ @challenge def test_memory_challenge_a( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/memory/test_memory_challenge_b.py b/tests/integration/challenges/memory/test_memory_challenge_b.py index 5c28b330a3b4..c82763129aca 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_b.py +++ b/tests/integration/challenges/memory/test_memory_challenge_b.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -17,7 +18,7 @@ @challenge def test_memory_challenge_b( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/challenges/memory/test_memory_challenge_c.py b/tests/integration/challenges/memory/test_memory_challenge_c.py index 23c0217d6135..ab8ece105262 100644 --- a/tests/integration/challenges/memory/test_memory_challenge_c.py +++ b/tests/integration/challenges/memory/test_memory_challenge_c.py @@ -1,4 +1,5 @@ import pytest +from pytest_mock import MockerFixture from autogpt.agent import Agent from autogpt.commands.file_operations import read_file, write_to_file @@ -18,7 +19,7 @@ @challenge def test_memory_challenge_c( memory_management_agent: Agent, - patched_api_requestor: None, + patched_api_requestor: MockerFixture, monkeypatch: pytest.MonkeyPatch, config: Config, level_to_run: int, diff --git a/tests/integration/memory/test_json_file_memory.py b/tests/integration/memory/test_json_file_memory.py index 9134a06969f1..4de5819daabd 100644 --- a/tests/integration/memory/test_json_file_memory.py +++ b/tests/integration/memory/test_json_file_memory.py @@ -80,7 +80,9 @@ def test_json_memory_get(config: Config, memory_item: MemoryItem, mock_get_embed @pytest.mark.vcr @requires_api_key("OPENAI_API_KEY") -def test_json_memory_get_relevant(config: Config, patched_api_requestor: None) -> None: +def test_json_memory_get_relevant( + config: Config, patched_api_requestor: MockerFixture +) -> None: index = JSONFileMemory(config) mem1 = MemoryItem.from_text_file("Sample text", "sample.txt") mem2 = MemoryItem.from_text_file("Grocery list:\n- Pancake mix", "groceries.txt")