Remove worker (and fix readmen + dockercompose.yml); add sync_plans and pulls_sync tasks; finish gh webhook handler. (codecov#144)

Author: pierce-m

.gitmodules

@@ -1,3 +0,0 @@
-[submodule "worker"]
-	path = worker
-	url = https://github.com/codecov/worker

Makefile

@@ -3,9 +3,6 @@ ssh_private_key = `cat ~/.ssh/codecov-io_rsa`
 build:
 	docker build -f Dockerfile . -t codecov/api:latest --build-arg SSH_PRIVATE_KEY="${ssh_private_key}"
 
-standalone: build
-	$(MAKE) -C worker build
-
 test:
 	python -m pytest --cov=./
 

README.md

@@ -7,11 +7,9 @@ A private Django REST Framework API intended to serve codecov's front end.
 
 ### Building
 
-This project contains a makefile. To build in standalone mode, run:
+This project contains a makefile. To build the docker image:
 
-    make standalone
-
-To build with codecovio (as a submodule) the `build` make target is included
+    make build
 
 `requirements.txt` is used in the base image. If you make changes to `requirements.txt` you will need to rebuild.
 
@@ -25,7 +23,7 @@ To start the service, do
 
 Utilizing its own database provides a convenient way for the REST API to provide its own helpful seeds and migrations for active development without potentially destroying/modifying your development database for codecov.io.
 
-Once running, the api will be available at `http://localhost:8000`
+Once running, the api will be available at `http://localhost:5100`
 
 ### Running with codecov.io
 

docker-compose.yml

@@ -35,20 +35,6 @@ services:
       - redis-volume:/data
     networks:
       - codecovapi
-  worker-new:
-    image: codecov/worker
-    build: ./worker
-    working_dir: /app/
-    entrypoint: sh worker.sh
-    volumes:
-      - ./worker:/app/
-    environment:
-      - CELERY_BROKER_URL=redis://redis:6379/0
-      - CELERY_RESULT_BACKEND=redis://redis:6379/0
-    networks:
-      - codecovapi
-    depends_on:
-      - redis
 
 volumes:
   postgres-volume:

services/task.py

@@ -35,6 +35,15 @@ def notify(self, repoid, commitid):
             kwargs=dict(repoid=repoid, commitid=commitid)
         ).apply_async()
 
+    def pulls_sync(self, repoid, pullid):
+        self._create_signature(
+            'app.tasks.pulls.Sync',
+            kwargs=dict(
+                repoid=repoid,
+                pullid=pullid,
+            )
+        ).apply_async()
+
     def refresh(self, ownerid, username, sync_teams=True, sync_repos=True, using_integration=False):
         """
         !!!
@@ -69,3 +78,13 @@ def refresh(self, ownerid, username, sync_teams=True, sync_repos=True, using_int
             ))
 
         return chain(*chain_to_call).apply_async()
+
+    def sync_plans(self, sender=None, account=None, action=None):
+        self._create_signature(
+            'app.tasks.ghm_sync_plans.SyncPlans',
+            kwargs=dict(
+                sender=sender,
+                account=account,
+                action=action
+            )
+        )

webhook_handlers/constants.py

@@ -15,6 +15,8 @@ class GitHubWebhookEvents:
     INSTALLATION = "installation"
     INSTALLATION_REPOSITORIES = "installation_repositories"
     ORGANIZATION = "organization"
+    MARKETPLACE_PURCHASE = "marketplace_purchase"
+    MARKETPLACE_SUBSCRIPTION = "marketplace_subscription"
 
     repository_events = [PULL_REQUEST, DELETE, PUSH, PUBLIC, STATUS, REPOSITORY]
 

webhook_handlers/tests/test_github.py

@@ -1,5 +1,8 @@
 import uuid
 import pytest
+import hmac
+from hashlib import sha1
+import json
 
 from unittest.mock import patch, call
 
@@ -11,6 +14,7 @@
 from core.models import Repository
 from codecov_auth.models import Owner
 from codecov_auth.tests.factories import OwnerFactory
+from utils.config import get_config
 
 from webhook_handlers.constants import GitHubHTTPHeaders, GitHubWebhookEvents, WebhookHandlerErrorMessages
 
@@ -22,7 +26,15 @@ def _post_event_data(self, event, data={}):
             **{
                 GitHubHTTPHeaders.EVENT: event,
                 GitHubHTTPHeaders.DELIVERY_TOKEN: uuid.UUID(int=5),
-                GitHubHTTPHeaders.SIGNATURE: 0
+                GitHubHTTPHeaders.SIGNATURE: 'sha1='+hmac.new(
+                    get_config(
+                        "github",
+                        'webhook_secret',
+                        default=b'testixik8qdauiab1yiffydimvi72ekq'
+                    ),
+                    json.dumps(data, separators=(',', ':')).encode('utf-8'),
+                    digestmod=sha1
+                ).hexdigest(),
             },
             data=data,
             format="json"
@@ -371,9 +383,26 @@ def test_pull_request_exits_early_if_repo_not_active(self):
         assert response.status_code == status.HTTP_200_OK
         assert response.data == WebhookHandlerErrorMessages.SKIP_NOT_ACTIVE
 
-    @pytest.mark.xfail
-    def test_pull_request_triggers_pulls_sync_task_for_valid_actions(self):
-        assert False
+    @patch('services.task.TaskService.pulls_sync')
+    def test_pull_request_triggers_pulls_sync_task_for_valid_actions(self, pulls_sync_mock):
+        pull = PullFactory(repository=self.repo)
+
+        valid_actions = ["opened", "closed", "reopened", "synchronize"]
+
+        for action in valid_actions:
+            response = self._post_event_data(
+                event=GitHubWebhookEvents.PULL_REQUEST,
+                data={
+                    "repository": {
+                        "id": self.repo.service_id
+                    },
+                    "action": action,
+                    "number": pull.pullid
+                }
+            )
+
+        pulls_sync_mock.assert_has_calls([call(repoid=self.repo.repoid, pullid=pull.pullid)] * len(valid_actions))
+
 
     def test_pull_request_updates_title_if_edited(self):
         pull = PullFactory(repository=self.repo)
@@ -552,3 +581,42 @@ def test_membership_with_removed_action_removes_user_from_org(self):
         user.refresh_from_db()
 
         assert org.ownerid not in user.organizations
+
+    @patch('services.task.TaskService.sync_plans')
+    def test_marketplace_subscription_triggers_sync_plans_task(self, sync_plans_mock):
+        sender = {
+            "id": 545,
+            "login": "[email protected]"
+        }
+        action = "purchased"
+        account = {
+            "type": "Organization",
+            "id": 54678,
+            "login": "username"
+        }
+        response = self._post_event_data(
+            event=GitHubWebhookEvents.MARKETPLACE_PURCHASE,
+            data={
+                "action": action,
+                "sender": sender,
+                "marketplace_purchase": {
+                    "account": account
+                }
+            }
+        )
+
+        sync_plans_mock.assert_called_once_with(sender=sender, account=account, action=action)
+
+    def test_signature_validation(self):
+        response = self.client.post(
+            reverse("github-webhook"),
+            **{
+                GitHubHTTPHeaders.EVENT: '',
+                GitHubHTTPHeaders.DELIVERY_TOKEN: uuid.UUID(int=5),
+                GitHubHTTPHeaders.SIGNATURE: 0
+            },
+            data={},
+            format="json"
+        )
+
+        assert response.status_code == status.HTTP_403_FORBIDDEN

webhook_handlers/views/github.py

@@ -1,16 +1,20 @@
 import logging
 import re
+import hmac
+from hashlib import sha1
 
 from rest_framework.views import APIView
 from rest_framework.permissions import AllowAny
 from rest_framework.response import Response
 from rest_framework import status
+from rest_framework.exceptions import PermissionDenied
 
 from core.models import Repository, Branch, Commit, Pull
 from codecov_auth.models import Owner
 from services.archive import ArchiveService
 from services.redis import get_redis_connection
 from services.task import TaskService
+from utils.config import get_config
 
 from webhook_handlers.constants import GitHubHTTPHeaders, GitHubWebhookEvents, WebhookHandlerErrorMessages
 
@@ -32,7 +36,18 @@ class GithubWebhookHandler(APIView):
     redis = get_redis_connection()
 
     def validate_signature(self, request):
-        pass
+        sig = 'sha1='+hmac.new(
+            get_config(
+                "github",
+                'webhook_secret',
+                default=b'testixik8qdauiab1yiffydimvi72ekq'
+            ),
+            request.body,
+            digestmod=sha1
+        ).hexdigest()
+
+        if sig != request.META.get(GitHubHTTPHeaders.SIGNATURE):
+            raise PermissionDenied()
 
     def unhandled_webhook_event(self, request, *args, **kwargs):
         return Response(data=WebhookHandlerErrorMessages.UNSUPPORTED_EVENT)
@@ -141,7 +156,7 @@ def pull_request(self, request, *args, **kwargs):
         action, pullid = request.data.get("action"), request.data.get("number")
 
         if action in ["opened", "closed", "reopened", "synchronize"]:
-            pass # TODO: should trigger pulls.sync task
+            TaskService().pulls_sync(repoid=repo.repoid, pullid=pullid)
         elif action == "edited":
             Pull.objects.filter(
                 repository=repo, pullid=pullid
@@ -205,11 +220,25 @@ def organization(self, request, *args, **kwargs):
 
         return Response()
 
+    def _handle_marketplace_events(self, request, *args, **kwargs):
+        TaskService().sync_plans(
+            sender=request.data["sender"],
+            account=request.data["marketplace_purchase"]["account"],
+            action=request.data["action"]
+        )
+        return Response()
+
+    def marketplace_subscription(self, request, *args, **kwargs):
+        return self._handle_marketplace_events(request, *args, **kwargs)
+
+    def marketplace_purchase(self, request, *args, **kwargs):
+        return self._handle_marketplace_events(request, *args, **kwargs)
+
     def post(self, request, *args, **kwargs):
         self.validate_signature(request)
 
-        self.event = self.request.META.get(GitHubHTTPHeaders.EVENT)
-        log.info("GitHub Webhook Handler invoked with: %s", self.event.upper())
-        handler = getattr(self, self.event, self.unhandled_webhook_event)
+        event = self.request.META.get(GitHubHTTPHeaders.EVENT)
+        log.info("GitHub Webhook Handler invoked with: %s", event.upper())
+        handler = getattr(self, event, self.unhandled_webhook_event)
 
         return handler(request, *args, **kwargs)