From 7444ae3a8260f31a4ade256fe5b141aae9838cb9 Mon Sep 17 00:00:00 2001 From: tazlin Date: Sat, 17 Feb 2024 09:44:30 -0500 Subject: [PATCH] ci: preliminary ci support (#368) * ci: preliminary ci support * style: fix whitespace * ci: run some horde_sdk tests too * ci: fix use local testing API for SDK tests * ci: show all output for `pytest` invocations * ci: skip model list checking for now * style: fix --- .github/workflows/maintests.yml | 84 +++++++++++++++++++++ .gitignore | 8 +- horde/apis/models/v2.py | 2 + horde/classes/base/processing_generation.py | 2 +- horde/database/threads.py | 3 + horde/exceptions.py | 1 + horde/flask.py | 1 + horde/model_reference.py | 1 + horde/patreon.py | 2 + requirements.dev.txt | 1 + requirements.txt | 4 +- tests/__init__.py | 0 tests/conftest.py | 45 +++++++++++ tests/test_alchemy.py | 28 +++---- tests/test_image.py | 24 +++--- tests/test_text.py | 24 +++--- 16 files changed, 184 insertions(+), 46 deletions(-) create mode 100644 .github/workflows/maintests.yml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py diff --git a/.github/workflows/maintests.yml b/.github/workflows/maintests.yml new file mode 100644 index 00000000..18fe06a4 --- /dev/null +++ b/.github/workflows/maintests.yml @@ -0,0 +1,84 @@ +name: AI-Horde main tests + +on: + push: + branches: + - main + paths: + - '**.py' + - '**.json' + - 'tox.ini' + - '.github/workflows/maintests.yml' + - '.github/workflows/prtests.yml' + - '.github/workflows/release.yml' + +jobs: + runner-job: + runs-on: ubuntu-latest + # runs-on: self-hosted + env: + POSTGRES_URL: "localhost:5432/horde_test" + POSTGRES_PASS: "postgres" + PGPASSWORD: "postgres" + REDIS_IP: "localhost" + REDIS_SERVERS: '["localhost"]' + USE_SQLITE: 0 + ADMINS: '["test_user#1"]' + R2_TRANSIENT_ACCOUNT: ${{ secrets.R2_TRANSIENT_ACCOUNT }} + R2_PERMANENT_ACCOUNT: ${{ secrets.R2_PERMANENT_ACCOUNT }} + SHARED_AWS_ACCESS_ID: ${{ secrets.SHARED_AWS_ACCESS_ID }} + SHARED_AWS_ACCESS_KEY: ${{ secrets.SHARED_AWS_ACCESS_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + KUDOS_TRUST_THRESHOLD: 100 + AI_HORDE_DEV_URL: "http://localhost:7001/api/" # For horde_sdk tests + + services: + postgres: + image: postgres:15.6-bullseye + env: + POSTGRES_PASSWORD: postgres + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + + redis: + image: redis + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379 + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + # cache: 'pip' + - run: python -m pip install --upgrade pip wheel setuptools + - name: Install and run lint/format checkers + run: | + python -m pip install -r requirements.dev.txt + black --check . + ruff . + - name: Install and run tests + run: | + python -m pip install -r requirements.txt + psql -h localhost -U postgres -c "CREATE DATABASE horde_test;" + python server.py -vvvvi --horde stable & + sleep 5 + curl -X POST --data-raw 'username=test_user' http://localhost:7001/register | grep -Po '

\K.*(?=<\/p>)' > tests/apikey.txt + export AI_HORDE_DEV_APIKEY=$(cat tests/apikey.txt) + pytest tests/ -s + python -m pip download --no-deps --no-binary :all: horde_sdk + tar -xvf horde_sdk-*.tar.gz + cd horde_sdk**/ + pytest tests/ --ignore-glob=*api_calls.py --ignore-glob=*test_model_meta.py -s diff --git a/.gitignore b/.gitignore index dfbe880d..52aa5150 100644 --- a/.gitignore +++ b/.gitignore @@ -136,4 +136,10 @@ horde.log horde*.bz2 horde.db /.idea -/boto3oeo.py \ No newline at end of file +/boto3oeo.py + + +apikey.txt +.vscode/ + +horde_sdk** diff --git a/horde/apis/models/v2.py b/horde/apis/models/v2.py index 0e443287..774e2d19 100644 --- a/horde/apis/models/v2.py +++ b/horde/apis/models/v2.py @@ -1,6 +1,8 @@ from flask_restx import fields, reqparse + from horde.exceptions import KNOWN_RC + class Parsers: def __init__(self): self.generate_parser = reqparse.RequestParser() diff --git a/horde/classes/base/processing_generation.py b/horde/classes/base/processing_generation.py index 7b32c3ac..87adf0db 100644 --- a/horde/classes/base/processing_generation.py +++ b/horde/classes/base/processing_generation.py @@ -25,7 +25,7 @@ class ProcessingGeneration(db.Model): id = db.Column(uuid_column_type(), primary_key=True, default=get_db_uuid) procgen_type = db.Column(db.String(30), nullable=False, index=True) generation = db.Column(db.Text) - gen_metadata = db.Column(json_column_type, nullable=False) + gen_metadata = db.Column(json_column_type, nullable=True) model = db.Column(db.String(255), default="", nullable=False) seed = db.Column(db.BigInteger, default=0, nullable=False) diff --git a/horde/database/threads.py b/horde/database/threads.py index 29f6c70e..3d25ef42 100644 --- a/horde/database/threads.py +++ b/horde/database/threads.py @@ -295,6 +295,9 @@ def prune_stats(): def store_patreon_members(): api_client = patreon.API(os.getenv("PATREON_CREATOR_ACCESS_TOKEN")) # campaign_id = api_client.get_campaigns(10).data()[0].id() + if api_client is None: + logger.error("Failed to get patreon API client") + return cursor = None members = [] while True: diff --git a/horde/exceptions.py b/horde/exceptions.py index 51c6660d..e69add03 100644 --- a/horde/exceptions.py +++ b/horde/exceptions.py @@ -111,6 +111,7 @@ "Locked", ] + class BadRequest(wze.BadRequest): def __init__(self, message, log=None, rc="BadRequest"): self.specific = message diff --git a/horde/flask.py b/horde/flask.py index 44b53cd6..595312bf 100644 --- a/horde/flask.py +++ b/horde/flask.py @@ -22,6 +22,7 @@ HORDE.config["SQLALCHEMY_ENGINE_OPTIONS"] = { "pool_size": 50, "max_overflow": -1, + # "pool_pre_ping": True, } HORDE.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False db = SQLAlchemy(HORDE) diff --git a/horde/model_reference.py b/horde/model_reference.py index bd583c73..bca3d646 100644 --- a/horde/model_reference.py +++ b/horde/model_reference.py @@ -130,3 +130,4 @@ def has_nsfw_models(self, model_names): model_reference = ModelReference(3600, None) +model_reference.call_function() diff --git a/horde/patreon.py b/horde/patreon.py index 8205a8a6..3588ac1f 100644 --- a/horde/patreon.py +++ b/horde/patreon.py @@ -17,6 +17,8 @@ def call_function(self): # logger.debug(self.patrons) except (TypeError, AttributeError): logger.warning("Patreon cache could not be retrieved from redis. Leaving existing cache.") + except Exception as e: + logger.error(f"Error retrieving patreon cache from redis: {e}") def is_patron(self, user_id): return user_id in self.patrons diff --git a/requirements.dev.txt b/requirements.dev.txt index cc82e1d3..5c0814a6 100644 --- a/requirements.dev.txt +++ b/requirements.dev.txt @@ -1,4 +1,5 @@ black==23.12.1 ruff==0.1.13 +pytest==8.0.0 tox~=4.12.1 horde_sdk>=0.7.29 diff --git a/requirements.txt b/requirements.txt index cda993a0..3166d7bd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ werkzeug~=2.2.2 Flask~=2.2.2 flask-restx flask_limiter~=2.8.1 -Flask-Caching +Flask-Caching waitress~=2.1.2 requests >= 2.27 Markdown~=3.4.1 @@ -28,4 +28,4 @@ git+https://github.com/Patreon/patreon-python.git torch emoji semver >= 3.0.2 -numpy ~= 1.24.1 # better_profanity fails on later versions of numpy \ No newline at end of file +numpy ~= 1.24.1 # better_profanity fails on later versions of numpy diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..c449802d --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,45 @@ +import pathlib + +import pytest +import requests + + +@pytest.fixture +def CIVERSION() -> str: + return "0.1.1" + + +@pytest.fixture +def HORDE_URL() -> str: + return "localhost:7001" + + +@pytest.fixture +def api_key() -> str: + key_file = pathlib.Path(__file__).parent / "apikey.txt" + if key_file.exists(): + return key_file.read_text().strip() + + raise ValueError("No api key file found") + + +@pytest.fixture(autouse=True) +def increase_kudos(api_key: str, HORDE_URL: str, CIVERSION: str) -> None: + headers = {"apikey": api_key, "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625", "user_id": "1"} + + payload_set_to_mod = { + "trusted": True, + "moderator": True, + } + + response_set_to_mod = requests.put(f"http://{HORDE_URL}/api/v2/users/1", json=payload_set_to_mod, headers=headers) + + assert response_set_to_mod.ok, response_set_to_mod.text + + payload_set_kudos = { + "kudos": 10000, + } + + response_kudos = requests.put(f"http://{HORDE_URL}/api/v2/users/1", json=payload_set_kudos, headers=headers) + + assert response_kudos.ok, response_kudos.text diff --git a/tests/test_alchemy.py b/tests/test_alchemy.py index 5ece8b25..e5ad4150 100644 --- a/tests/test_alchemy.py +++ b/tests/test_alchemy.py @@ -1,20 +1,16 @@ import requests -CIVERSION = "0.1.1" -HORDE_URL = "dev.stablehorde.net" -TEST_MODELS = ["elinas/chronos-70b-v2"] - -def test_simple_alchemy() -> None: - headers = {"apikey": "2bc5XkMeLAWiN9O5s7bhfg", "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"} # ci/cd user +def test_simple_alchemy(api_key: str, HORDE_URL: str, CIVERSION: str) -> None: + headers = {"apikey": api_key, "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"} # ci/cd user async_dict = { "forms": [ {"name": "caption"}, ], "source_image": "https://github.com/Haidra-Org/AI-Horde/blob/main/icon.png?raw=true", } - async_req = requests.post(f"https://{HORDE_URL}/api/v2/interrogate/async", json=async_dict, headers=headers) - assert async_req.ok + async_req = requests.post(f"http://{HORDE_URL}/api/v2/interrogate/async", json=async_dict, headers=headers) + assert async_req.ok, async_req.text async_results = async_req.json() req_id = async_results["id"] # print(async_results) @@ -25,27 +21,27 @@ def test_simple_alchemy() -> None: "max_tiles": 96, } try: - pop_req = requests.post(f"https://{HORDE_URL}/api/v2/interrogate/pop", json=pop_dict, headers=headers) + pop_req = requests.post(f"http://{HORDE_URL}/api/v2/interrogate/pop", json=pop_dict, headers=headers) except Exception: - requests.delete(f"https://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers) + requests.delete(f"http://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers) raise - assert pop_req.ok + assert pop_req.ok, pop_req.text pop_results = pop_req.json() # print(json.dumps(pop_results, indent=4)) job_id = pop_results["forms"][0]["id"] - assert job_id is not None + assert job_id is not None, pop_results submit_dict = { "id": job_id, "result": {"caption": "Test"}, "state": "ok", } - submit_req = requests.post(f"https://{HORDE_URL}/api/v2/interrogate/submit", json=submit_dict, headers=headers) - assert submit_req.ok + submit_req = requests.post(f"http://{HORDE_URL}/api/v2/interrogate/submit", json=submit_dict, headers=headers) + assert submit_req.ok, submit_req.text submit_results = submit_req.json() assert submit_results["reward"] > 0 - retrieve_req = requests.get(f"https://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers) - assert retrieve_req.ok + retrieve_req = requests.get(f"http://{HORDE_URL}/api/v2/interrogate/status/{req_id}", headers=headers) + assert retrieve_req.ok, retrieve_req.text retrieve_results = retrieve_req.json() # print(json.dumps(retrieve_results,indent=4)) assert len(retrieve_results["forms"]) == 1 diff --git a/tests/test_image.py b/tests/test_image.py index 27a29552..6d4898ed 100644 --- a/tests/test_image.py +++ b/tests/test_image.py @@ -1,12 +1,10 @@ import requests -CIVERSION = "0.1.1" -HORDE_URL = "dev.stablehorde.net" TEST_MODELS = ["Fustercluck", "AlbedoBase XL (SDXL)"] -def test_simple_image_gen() -> None: - headers = {"apikey": "2bc5XkMeLAWiN9O5s7bhfg", "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"} # ci/cd user +def test_simple_image_gen(api_key: str, HORDE_URL: str, CIVERSION: str) -> None: + headers = {"apikey": api_key, "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"} # ci/cd user async_dict = { "prompt": "a horde of cute stable robots in a sprawling server room repairing a massive mainframe", "nsfw": True, @@ -22,8 +20,8 @@ def test_simple_image_gen() -> None: "models": TEST_MODELS, "loras": [{"name": "247778", "is_version": True}], } - async_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/async", json=async_dict, headers=headers) - assert async_req.ok + async_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/async", json=async_dict, headers=headers) + assert async_req.ok, async_req.text async_results = async_req.json() req_id = async_results["id"] # print(async_results) @@ -40,25 +38,25 @@ def test_simple_image_gen() -> None: "allow_controlnet": True, "allow_lora": True, } - pop_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/pop", json=pop_dict, headers=headers) - assert pop_req.ok + pop_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/pop", json=pop_dict, headers=headers) + assert pop_req.ok, pop_req.text pop_results = pop_req.json() # print(json.dumps(pop_results, indent=4)) job_id = pop_results["id"] - assert job_id is not None + assert job_id is not None, pop_results submit_dict = { "id": job_id, "generation": "R2", "state": "ok", "seed": 0, } - submit_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/submit", json=submit_dict, headers=headers) - assert submit_req.ok + submit_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/submit", json=submit_dict, headers=headers) + assert submit_req.ok, submit_req.text submit_results = submit_req.json() assert submit_results["reward"] > 0 - retrieve_req = requests.get(f"https://{HORDE_URL}/api/v2/generate/status/{req_id}", headers=headers) - assert retrieve_req.ok + retrieve_req = requests.get(f"http://{HORDE_URL}/api/v2/generate/status/{req_id}", headers=headers) + assert retrieve_req.ok, retrieve_req.text retrieve_results = retrieve_req.json() # print(json.dumps(retrieve_results,indent=4)) assert len(retrieve_results["generations"]) == 1 diff --git a/tests/test_text.py b/tests/test_text.py index e16f764b..3b9fed0d 100644 --- a/tests/test_text.py +++ b/tests/test_text.py @@ -1,12 +1,10 @@ import requests -CIVERSION = "0.1.1" -HORDE_URL = "dev.stablehorde.net" TEST_MODELS = ["elinas/chronos-70b-v2"] -def test_simple_text_gen() -> None: - headers = {"apikey": "2bc5XkMeLAWiN9O5s7bhfg", "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"} # ci/cd user +def test_simple_text_gen(api_key: str, HORDE_URL: str, CIVERSION: str) -> None: + headers = {"apikey": api_key, "Client-Agent": f"aihorde_ci_client:{CIVERSION}:(discord)db0#1625"} # ci/cd user async_dict = { "prompt": "a horde of cute stable robots in a sprawling server room repairing a massive mainframe", "trusted_workers": True, @@ -15,8 +13,8 @@ def test_simple_text_gen() -> None: "temperature": 1, "models": TEST_MODELS, } - async_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/text/async", json=async_dict, headers=headers) - assert async_req.ok + async_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/text/async", json=async_dict, headers=headers) + assert async_req.ok, async_req.text async_results = async_req.json() req_id = async_results["id"] # print(async_results) @@ -28,25 +26,25 @@ def test_simple_text_gen() -> None: "max_context_length": 4096, "max_length": 512, } - pop_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/text/pop", json=pop_dict, headers=headers) - assert pop_req.ok + pop_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/text/pop", json=pop_dict, headers=headers) + assert pop_req.ok, pop_req.text pop_results = pop_req.json() # print(json.dumps(pop_results, indent=4)) job_id = pop_results["id"] - assert job_id is not None + assert job_id is not None, pop_results submit_dict = { "id": job_id, "generation": "test ", "state": "ok", "seed": 0, } - submit_req = requests.post(f"https://{HORDE_URL}/api/v2/generate/text/submit", json=submit_dict, headers=headers) - assert submit_req.ok + submit_req = requests.post(f"http://{HORDE_URL}/api/v2/generate/text/submit", json=submit_dict, headers=headers) + assert submit_req.ok, submit_req.text submit_results = submit_req.json() assert submit_results["reward"] > 0 - retrieve_req = requests.get(f"https://{HORDE_URL}/api/v2/generate/text/status/{req_id}", headers=headers) - assert retrieve_req.ok + retrieve_req = requests.get(f"http://{HORDE_URL}/api/v2/generate/text/status/{req_id}", headers=headers) + assert retrieve_req.ok, retrieve_req.text retrieve_results = retrieve_req.json() # print(json.dumps(retrieve_results,indent=4)) assert len(retrieve_results["generations"]) == 1