diff --git a/docker-compose.override.yml b/docker-compose.override.yml index c8c9785e..709ca36d 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -37,15 +37,15 @@ services: networks: - vaas - default - celery-scheduler: - entrypoint: ['/wait-for-it.sh', 'uwsgi:3030', '-t', '60', '--', '/entrypoint-celery-scheduler.sh' ] - env_file: ./envs/dev.env - volumes: - - ./vaas:/home/app/vaas - - ./plugins:/home/app/plugins - networks: - - vaas - - default + # celery-scheduler: + # entrypoint: ['/wait-for-it.sh', 'uwsgi:3030', '-t', '60', '--', '/entrypoint-celery-scheduler.sh' ] + # env_file: ./envs/dev.env + # volumes: + # - ./vaas:/home/app/vaas + # - ./plugins:/home/app/plugins + # networks: + # - vaas + # - default # Varnishes & nginx nginx-0: diff --git a/docker-compose.yml b/docker-compose.yml index f439a760..ac446a16 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,13 +28,28 @@ services: - 3030:3030 depends_on: - redis + - rabbitmq - mysql + rabbitmq: + image: rabbitmq:3.11-management + healthcheck: + test: ["CMD", "rabbitmqctl", "status"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 5s + environment: + - RABBITMQ_DEFAULT_VHOST=myvhost + ports: + - 5672:5672 + - 15672:15672 celery-worker: build: context: ./ dockerfile: Dockerfile depends_on: - redis + - rabbitmq - mysql celery-cron-worker: build: @@ -42,6 +57,7 @@ services: dockerfile: Dockerfile depends_on: - redis + - rabbitmq - mysql celery-routes-test: build: @@ -49,11 +65,12 @@ services: dockerfile: Dockerfile depends_on: - redis + - rabbitmq - mysql - celery-scheduler: - build: - context: ./ - dockerfile: Dockerfile - depends_on: - - redis - - mysql + # celery-scheduler: + # build: + # context: ./ + # dockerfile: Dockerfile + # depends_on: + # - redis + # - mysql diff --git a/envs/dev.env b/envs/dev.env index cb9fc5d4..4614128a 100644 --- a/envs/dev.env +++ b/envs/dev.env @@ -2,4 +2,4 @@ DEBUG=True TEMPLATE_DEBUG=True DJANGO_SETTINGS_MODULE=vaas.settings.docker -PROMETHEUS_ENABLE=True +PROMETHEUS_ENABLE=False diff --git a/vaas/requirements/base.txt b/vaas/requirements/base.txt index a19fc95b..f72b5b64 100644 --- a/vaas/requirements/base.txt +++ b/vaas/requirements/base.txt @@ -14,7 +14,7 @@ django-environ==0.4.5 lockfile==0.12.2 django-taggit==1.3.0 # newer versions requires incompatible kombu -celery[redis]==5.2.3 +# celery[redis]==5.2.3 celery-beatx requests>=2.25.1 GitPython==3.1.35 @@ -30,8 +30,11 @@ python-json-logger sentry-sdk==1.14.0 # newer versions are incompatible with our fork of redis-py: https://github.com/allegro/redis-py -kombu==5.2.3 -redis==4.4.4 +# kombu==5.2.3 +# redis==4.4.4 +amqp==5.1.1 +celery==5.3 + django_admin_bootstrapped @ git+https://github.com/allegro/django-admin-bootstrapped@2.5.9.1#egg=django_admin_bootstrapped python_varnish @ git+https://github.com/allegro/python-varnish@0.2.2#egg=python_varnish django_ace @ git+https://github.com/allegro/django-ace@v1.2.5.2#egg=django_ace diff --git a/vaas/vaas/settings/base.py b/vaas/vaas/settings/base.py index 0cb72713..3971a35e 100644 --- a/vaas/vaas/settings/base.py +++ b/vaas/vaas/settings/base.py @@ -279,22 +279,23 @@ def generate_redis_url(hostname: str, port: int, db_number: int, password: Optio return f'redis://{hostname}:{port}/{db_number}' -REDIS_HOSTNAME = env.str('REDIS_HOSTNAME', default='redis') -REDIS_PORT = env.int('REDIS_PORT', default=6379) -BROKER_DB_NUMBER = env.int('BROKER_DB_NUMBER', default=0) -CELERY_RESULT_DB_NUMBER = env.int('CELERY_RESULT_DB_NUMBER', default=1) -REDIS_PASSWORD = env.str('REDIS_PASSWORD', default=None) -REDIS_BACKEND_HEALTH_CHECK_INTERVAL_SEC = env.int('REDIS_BACKEND_HEALTH_CHECK_INTERVAL_SEC', default=60) -REDIS_SOCKET_KEEPALIVE = env.int('REDIS_SOCKET_KEEPALIVE', default=True) -REDIS_RETRY_ON_TIMEOUT = env.int('REDIS_RETRY_ON_TIMEOUT', default=True) -REDIS_SOCKET_CONNECT_TIMEOUT = env.int('REDIS_SOCKET_CONNECT_TIMEOUT', default=5) -REDIS_SOCKET_TIMEOUT = env.int('REDIS_SOCKET_TIMEOUT', default=120) - -BROKER_URL = generate_redis_url( - hostname=REDIS_HOSTNAME, port=REDIS_PORT, db_number=BROKER_DB_NUMBER, password=REDIS_PASSWORD) -CELERY_RESULT_BACKEND = generate_redis_url( - hostname=REDIS_HOSTNAME, port=REDIS_PORT, db_number=CELERY_RESULT_DB_NUMBER, password=REDIS_PASSWORD) - +# REDIS_HOSTNAME = env.str('REDIS_HOSTNAME', default='redis') +# REDIS_PORT = env.int('REDIS_PORT', default=6379) +# BROKER_DB_NUMBER = env.int('BROKER_DB_NUMBER', default=0) +# CELERY_RESULT_DB_NUMBER = env.int('CELERY_RESULT_DB_NUMBER', default=1) +# REDIS_PASSWORD = env.str('REDIS_PASSWORD', default=None) +# REDIS_BACKEND_HEALTH_CHECK_INTERVAL_SEC = env.int('REDIS_BACKEND_HEALTH_CHECK_INTERVAL_SEC', default=60) +# REDIS_SOCKET_KEEPALIVE = env.int('REDIS_SOCKET_KEEPALIVE', default=True) +# REDIS_RETRY_ON_TIMEOUT = env.int('REDIS_RETRY_ON_TIMEOUT', default=True) +# REDIS_SOCKET_CONNECT_TIMEOUT = env.int('REDIS_SOCKET_CONNECT_TIMEOUT', default=5) +# REDIS_SOCKET_TIMEOUT = env.int('REDIS_SOCKET_TIMEOUT', default=120) + +# BROKER_URL = generate_redis_url( +# hostname=REDIS_HOSTNAME, port=REDIS_PORT, db_number=BROKER_DB_NUMBER, password=REDIS_PASSWORD) +# CELERY_RESULT_BACKEND = generate_redis_url( +# hostname=REDIS_HOSTNAME, port=REDIS_PORT, db_number=CELERY_RESULT_DB_NUMBER, password=REDIS_PASSWORD) +BROKER_URL = 'pyamqp://guest:guest@rabbitmq:5672/myvhost' +CELERY_RESULT_BACKEND = 'rpc://guest:guest@rabbitmq:5672/myvhost' ROUTES_LEFT_CONDITIONS = {} for condition in env.json('ROUTES_LEFT_CONDITIONS', default=[ diff --git a/vaas/vaas/settings/celery.py b/vaas/vaas/settings/celery.py index 93000bff..89fc0227 100644 --- a/vaas/vaas/settings/celery.py +++ b/vaas/vaas/settings/celery.py @@ -18,28 +18,28 @@ def debug_task(self): print('Request: {0!r}'.format(self.request)) -app.conf.beatx_store = settings.BROKER_URL -app.conf.beat_max_loop_interval = settings.CELERY_BEAT_MAX_LOOP_INTERVAL -app.conf.beat_schedule = { - 'refresh_backend_statuses': { - 'task': 'vaas.monitor.tasks.refresh_backend_statuses', - 'schedule': settings.BACKEND_STATUSES_UPDATE_INTERVAL_SECONDS, - }, -} - -# After redis conenction troubles 'Connection closed by server / Connection by peer' we allow to re-queued -# task which was executed when failure occurred. "We know what we are doing." -# https://docs.celeryq.dev/en/latest/userguide/configuration.html#task-reject-on-worker-lost -app.conf.task_reject_on_worker_lost = settings.CELERY_TASK_REJECT_ON_WORKER_LOST - -# For better handle redis ConenctionError exception we give possibility to configure keepalive and connect_timeout parameters -# https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-socket-keepalive -app.conf.redis_socket_keepalive = settings.REDIS_SOCKET_KEEPALIVE -# https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-retry-on-timeout -app.conf.redis_retry_on_timeout = settings.REDIS_RETRY_ON_TIMEOUT -# https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-socket-connect-timeout -app.conf.redis_socket_connect_timeout = settings.REDIS_SOCKET_CONNECT_TIMEOUT -# https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-socket-timeout -app.conf.redis_socket_timeout = settings.REDIS_SOCKET_TIMEOUT -# https://docs.celeryq.dev/en/stable/userguide/configuration.html?highlight=redis_retry_on_timeout#redis-backend-health-check-interval -app.conf.redis_backend_health_check_interval = settings.REDIS_BACKEND_HEALTH_CHECK_INTERVAL_SEC +# app.conf.beatx_store = settings.BROKER_URL +# app.conf.beat_max_loop_interval = settings.CELERY_BEAT_MAX_LOOP_INTERVAL +# app.conf.beat_schedule = { +# 'refresh_backend_statuses': { +# 'task': 'vaas.monitor.tasks.refresh_backend_statuses', +# 'schedule': settings.BACKEND_STATUSES_UPDATE_INTERVAL_SECONDS, +# }, +# } + +# # After redis conenction troubles 'Connection closed by server / Connection by peer' we allow to re-queued +# # task which was executed when failure occurred. "We know what we are doing." +# # https://docs.celeryq.dev/en/latest/userguide/configuration.html#task-reject-on-worker-lost +# app.conf.task_reject_on_worker_lost = settings.CELERY_TASK_REJECT_ON_WORKER_LOST + +# # For better handle redis ConenctionError exception we give possibility to configure keepalive and connect_timeout parameters +# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-socket-keepalive +# app.conf.redis_socket_keepalive = settings.REDIS_SOCKET_KEEPALIVE +# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-retry-on-timeout +# app.conf.redis_retry_on_timeout = settings.REDIS_RETRY_ON_TIMEOUT +# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-socket-connect-timeout +# app.conf.redis_socket_connect_timeout = settings.REDIS_SOCKET_CONNECT_TIMEOUT +# # https://docs.celeryq.dev/en/stable/userguide/configuration.html#redis-socket-timeout +# app.conf.redis_socket_timeout = settings.REDIS_SOCKET_TIMEOUT +# # https://docs.celeryq.dev/en/stable/userguide/configuration.html?highlight=redis_retry_on_timeout#redis-backend-health-check-interval +# app.conf.redis_backend_health_check_interval = settings.REDIS_BACKEND_HEALTH_CHECK_INTERVAL_SEC