[DOP-21350] Find a reason for cov decrease

.github/workflows/scheduler-tests.yml

@@ -0,0 +1,79 @@
+name: Scheduler Tests
+on:
+  workflow_call:
+
+env:
+  DEFAULT_PYTHON: '3.12'
+
+jobs:
+  tests:
+    name: Run Scheduler tests
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Cache jars
+        uses: actions/cache@v4
+        with:
+          path: ./cached_jars
+          key: ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-scheduler
+          restore-keys: |
+            ${{ runner.os }}-python-${{ env.DEFAULT_PYTHON }}-test-scheduler
+            ${{ runner.os }}-python-
+
+      - name: Build Worker Image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          tags: mtsrus/syncmaster-worker:${{ github.sha }}
+          target: test
+          file: docker/Dockerfile.worker
+          load: true
+          cache-from: mtsrus/syncmaster-worker:develop
+
+      - name: Docker compose up
+        run: |
+          docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
+          docker compose -f docker-compose.test.yml --profile worker up -d --wait --wait-timeout 200
+        env:
+          WORKER_IMAGE_TAG: ${{ github.sha }}
+
+      # This is important, as coverage is exported after receiving SIGTERM
+      - name: Run Scheduler Tests
+        run: |
+          docker compose -f ./docker-compose.test.yml --profile worker exec -T worker coverage run -m pytest -vvv -s -m "worker and scheduler_integration"
+
+      - name: Dump worker logs on failure
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+        with:
+          images: mtsrus/syncmaster-worker
+          dest: ./logs
+
+      - name: Shutdown
+        if: always()
+        run: |
+          docker compose -f docker-compose.test.yml --profile all down -v --remove-orphans
+
+      - name: Upload worker logs
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: worker-logs-scheduler
+          path: logs/*
+
+      - name: Upload coverage results
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-scheduler
+          path: reports/*
+          # https://github.com/actions/upload-artifact/issues/602
+          include-hidden-files: true

.github/workflows/tests.yml

@@ -2,7 +2,7 @@ name: Run All Tests
 on:
   push:
     branches:
-      - develop
+      - feature/DOP-21350
   pull_request:
     branches-ignore:
       - master
@@ -32,6 +32,10 @@ jobs:
     name: S3 tests
     uses: ./.github/workflows/s3-tests.yml
 
+  scheduler_tests:
+    name: Scheduler tests
+    uses: ./.github/workflows/scheduler-tests.yml
+
   unit_tests:
     name: Unit tests
     uses: ./.github/workflows/unit-test.yml

.github/workflows/unit-test.yml

@@ -56,7 +56,7 @@ jobs:
         run: |
           source .env.local
           poetry run python -m syncmaster.db.migrations upgrade head
-          poetry run coverage run -m pytest -vvv -s -m backend
+          poetry run coverage run -m pytest -vvv -s -m "backend or scheduler"
 
       - name: Shutdown
         if: always()

docker-compose.test.yml

@@ -51,6 +51,25 @@ services:
         condition: service_healthy
     profiles: [backend, all]
 
+  scheduler:
+    image: mtsrus/syncmaster-backend:${BACKEND_IMAGE_TAG:-test}
+    restart: unless-stopped
+    build:
+      dockerfile: docker/Dockerfile.backend
+      context: .
+      target: test
+    env_file: .env.docker
+    volumes:
+      - ./syncmaster:/app/syncmaster
+      - ./pyproject.toml:/app/pyproject.toml
+    depends_on:
+      db:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+    entrypoint: [python, -m, syncmaster.scheduler]
+    profiles: [scheduler, all]
+
   worker:
     image: mtsrus/syncmaster-worker:${WORKER_IMAGE_TAG:-test}
     restart: unless-stopped
@@ -59,6 +78,7 @@ services:
       context: .
       target: test
     command: --loglevel=info -Q test_queue
+    entrypoint: [python, -m, celery, -A, tests.test_integration.celery_test, worker, --max-tasks-per-child=1]
     env_file: .env.docker
     volumes:
       - ./syncmaster:/app/syncmaster
@@ -71,7 +91,7 @@ services:
         condition: service_healthy
       rabbitmq:
         condition: service_healthy
-    profiles: [worker, s3, oracle, hdfs, hive, all]
+    profiles: [worker, scheduler, s3, oracle, hdfs, hive, all]
 
   test-postgres:
     image: postgres

docs/changelog/next_release/114.feature.rst

@@ -0,0 +1 @@
+Implement a scheduler to run celery tasks on a schedule. This can be done via the `Transfer` table by fields `is_scheduled` and `schedule` (cron-like expression). The Run model now has a `type` field with options `MANUAL` and `SCHEDULED`.