Merge pull request #25 from redhataccess/ap_CCS-4538_cache_clear_pr

CCS-4538: Provide an endpoint in Git2Pantheon that would handle cache…

README.md

@@ -43,5 +43,7 @@ flask run
 ```
 
 8. The swagger docs can be found at:  
-http://localhost:5000/apidocs/
+http://localhost:5000/apidocs/  
+
+_Note_: _Please don't try to run cache clear API_
 

git2pantheon/__init__.py

@@ -6,6 +6,7 @@
 from flasgger import Swagger
 from .api.upload import api_blueprint, executor
 import atexit
+from .helpers import EnvironmentVariablesHelper
 
 
 def create_app():
@@ -15,11 +16,23 @@ def create_app():
         EXECUTOR_MAX_WORKERS="1",
         EXECUTOR_PROPAGATE_EXCEPTIONS=True
     )
+    # check if required vars are available
+    EnvironmentVariablesHelper.check_vars(['PANTHEON_SERVER', 'UPLOADER_PASSWORD','UPLOADER_USER'])
+    try:
+        EnvironmentVariablesHelper.check_vars(['AKAMAI_HOST', 'DRUPAL_HOST', 'AKAMAI_ACCESS_TOKEN',
+                                               'AKAMAI_CLIENT_SECRET','AKAMAI_CLIENT_TOKEN'])
+    except Exception as e:
+        print(
+            'Environment variable(s) for cache clear not present.'
+            'Details={0}Please ignore if you are running on local server'.format(
+                str(e)))
+
     app.config.from_mapping(
         PANTHEON_SERVER=os.environ['PANTHEON_SERVER'],
         UPLOADER_PASSWORD=os.environ['UPLOADER_PASSWORD'],
         UPLOADER_USER=os.environ['UPLOADER_USER']
     )
+
     gunicorn_error_logger = logging.getLogger('gunicorn.error')
     app.logger.handlers.extend(gunicorn_error_logger.handlers)
     logging.basicConfig(level=logging.DEBUG)

git2pantheon/api-docs/cache_clear_api.yaml

@@ -0,0 +1,53 @@
+Clears the drupal and akamai cache
+---
+tags:
+  - Clears the drupal and akamai cache.
+
+parameters:
+  - in: body
+    name: body
+    description: 'The structure containing list of URLs of modules and assemblies whose cache has to be cleared'
+    required: true
+    schema:
+      $ref: '#/definitions/CacheClearData'
+
+reponses:
+  200:
+    description: 'The status of upload corresponding to the key'
+    schema:
+      $ref: '#/definitions/UploaderKey'
+  400:
+    description: 'Invalid content error'
+    schema:
+      $ref: '#/definitions/Error'
+  500:
+    description: 'Internal server error'
+    schema:
+      $ref: '#/definitions/Error'
+
+
+definitions:
+  CacheClearData:
+    type: object
+    properties:
+      assemblies:
+        type: array
+        items:
+          type: string
+      modules:
+        type: array
+        items:
+          type: string
+  Error:
+    type: object
+    properties:
+      code:
+        type: string
+        description: 'HTTP status code of the error'
+      message:
+        type: string
+        description: 'Error message'
+      details:
+        type: string
+        description: 'Error details'
+

git2pantheon/api/__init__.py

@@ -2,8 +2,16 @@
 from flask_executor import Executor
 from flask_cors import CORS
 from git2pantheon.utils import ApiError
+from git2pantheon.clients.akamai.akamai_rest_client import AkamaiCachePurgeClient
+from git2pantheon.clients.drupal.drupal_rest_client import DrupalClient
+import os
 
 executor = Executor()
+akamai_purge_client = AkamaiCachePurgeClient(host=os.getenv('AKAMAI_HOST'),
+                                             client_token=os.getenv('AKAMAI_CLIENT_TOKEN'),
+                                             client_secret=os.getenv('AKAMAI_CLIENT_SECRET'),
+                                             access_token=os.getenv('AKAMAI_ACCESS_TOKEN'))
+drupal_client = DrupalClient(os.getenv('DRUPAL_HOST'))
 api_blueprint = Blueprint('api', __name__, url_prefix='/api/')
 CORS(api_blueprint)
 

git2pantheon/api/upload.py

@@ -14,14 +14,19 @@
 from pantheon_uploader import pantheon
 from . import api_blueprint
 from . import executor
+from . import drupal_client
+from . import akamai_purge_client
 from .. import utils
-from ..helpers import FileHelper, GitHelper, MessageHelper
+from ..helpers import FileHelper, GitHelper, MessageHelper, CacheObjectHelper
 from ..messaging import broker
 from ..models.request_models import RepoSchema
 from ..models.response_models import Status
 from ..utils import ApiError, get_docs_path_for
 from flask import current_app
+from decorest import HTTPErrorWrapper
 
+ASSEMBLIES = "assemblies"
+MODULES = "modules"
 logger = logging.getLogger(__name__)
 
 
@@ -44,7 +49,7 @@ def push_repo():
 
 def clone_repo(repo_name, repo_url, branch):
     try:
-        
+
         MessageHelper.publish(repo_name + "-clone",
                               json.dumps(dict(current_status="cloning", details="Cloning repo " + repo_name + "")))
         logger.info("Cloning repo=" + repo_url + " and branch=" + branch)
@@ -81,8 +86,8 @@ def upload_repo(cloned_repo, channel_name):
     try:
         pantheon.start_process(numeric_level=10, pw=current_app.config['UPLOADER_PASSWORD'],
                                user=current_app.config['UPLOADER_USER'],
-                               server=current_app.config['PANTHEON_SERVER'],  directory=cloned_repo.working_dir,
-                               use_broker=True, channel=channel_name, broker_host= os.getenv('REDIS_SERVICE') )
+                               server=current_app.config['PANTHEON_SERVER'], directory=cloned_repo.working_dir,
+                               use_broker=True, channel=channel_name, broker_host=os.getenv('REDIS_SERVICE'))
     except Exception as e:
         logger.error("Upload failed due to error=" + str(e))
         MessageHelper.publish(channel_name,
@@ -105,7 +110,7 @@ def info():
 def status():
     status_data, clone_data = get_upload_data()
     current_status = get_current_status(clone_data, status_data)
-    logger.debug("current status="+current_status)
+    logger.debug("current status=" + current_status)
     status_message = Status(clone_status=clone_data.get('current_status', ""),
                             current_status=current_status,
                             file_type=status_data.get('type_uploading', ""),
@@ -120,8 +125,8 @@ def status():
 
 
 def get_current_status(clone_data, status_data):
-    logger.debug('upload status data='+json.dumps(status_data))
-    logger.debug('clone status data='+json.dumps(clone_data))
+    logger.debug('upload status data=' + json.dumps(status_data))
+    logger.debug('clone status data=' + json.dumps(clone_data))
     return status_data.get('current_status') if status_data.get('current_status', "") != "" else clone_data[
         'current_status']
 
@@ -147,7 +152,7 @@ def get_request_data():
 
 
 def reset_if_exists(repo_name):
-    MessageHelper.publish(repo_name +"-clone", json.dumps(dict(current_status='')))
+    MessageHelper.publish(repo_name + "-clone", json.dumps(dict(current_status='')))
     MessageHelper.publish(repo_name, json.dumps(dict(current_status='')))
 
 
@@ -157,7 +162,7 @@ def progress_update():
     status_data, clone_data = get_upload_data()
     # status_progress: UploadStatus = upload_status_from_dict(status_data)
     if "server" in status_data and status_data["server"]["response_code"] and not 200 <= int(status_data["server"][
-                                                                                               "response_code"]) <= 400:
+                                                                                                 "response_code"]) <= 400:
         return jsonify(
             dict(
                 server_status=status_data["server"]["response_code"],
@@ -265,3 +270,52 @@ def progress_update_resources():
     return jsonify(
         response_dict
     ), 200
+
+
+@swag_from(get_docs_path_for('cache_clear_api.yaml'))
+@api_blueprint.route('/cache/clear', methods=['POST'])
+def clear_cache():
+    data = get_request_data()
+    cache_clear_result = {
+        "drupal_result_assemblies":{},
+        "drupal_result_modules": {}
+    }
+
+    try:
+        clear_drupal_cache(data, cache_clear_result)
+        clear_akamai_cache(data, cache_clear_result)
+    except Exception as e:
+        logger.error("Exception occurred while trying to clear  cache with error=" + str(e))
+        raise ApiError("Upstream Server Error", 503, details=str(e))
+    return jsonify(cache_clear_result)
+
+
+def drupal_cache_clear_bulk(cache_clear_result, cache_req_data):
+    if "assemblies" in cache_req_data:
+        cache_clear_result["drupal_result_assemblies"] = drupal_client.purge_cache_assembly("assemblies")
+
+    if "modules" in cache_req_data:
+        cache_clear_result["drupal_result_modules"] = drupal_client.purge_cache_assembly("modules")
+
+
+def clear_drupal_cache(data, cache_clear_result, bulk_clear=False):
+    cache_req_data = CacheObjectHelper.get_drupal_req_data(data)
+    if bulk_clear:
+        drupal_cache_clear_bulk(cache_clear_result, cache_req_data)
+        return
+    drupal_cache_clear_individual(cache_clear_result, cache_req_data)
+
+
+def drupal_cache_clear_individual(cache_clear_result, cache_req_data):
+    if ASSEMBLIES in cache_req_data:
+        for guid in cache_req_data[ASSEMBLIES]:
+            cache_clear_result["drupal_result_assemblies"][str(guid)] = drupal_client.purge_cache_assembly(guid)
+    if MODULES in cache_req_data:
+        for guid in cache_req_data[MODULES]:
+            cache_clear_result["drupal_result_modules"][str(guid)] = (drupal_client.purge_cache_module(guid))
+
+
+def clear_akamai_cache(data, cache_clear_result):
+    cache_req_data = CacheObjectHelper.get_akamai_req_object(data)
+    cache_clear_result['akamai_result'] = akamai_purge_client.purge(cache_req_data)
+

git2pantheon/clients/akamai/akamai_rest_client.py

@@ -0,0 +1,19 @@
+import requests, logging, json
+from ..client import RestClient
+from akamai.edgegrid import EdgeGridAuth
+
+logger = logging.getLogger(__name__)
+
+
+class AkamaiCachePurgeClient:
+    def __init__(self,host, client_token, client_secret, access_token):
+        self.session: requests.Session = requests.Session()
+        self.session.auth = EdgeGridAuth(client_token=client_token, client_secret=client_secret,
+                                         access_token=access_token)
+        self.host = host
+        self.akamai_rest_client = RestClient(auth_session=self.session, verbose=True,
+                                                         base_url=self.host)
+
+    def purge(self, purge_obj, action='delete'):
+        logger.info('Adding %s request to the queue for %s' % (action, json.dumps(purge_obj)))
+        return self.akamai_rest_client.post('/ccu/v3/delete/url', json.dumps(purge_obj))

git2pantheon/clients/client.py

@@ -0,0 +1,57 @@
+from urllib import parse
+
+import json
+import logging
+import requests
+
+logger = logging.getLogger(__name__)
+
+
+class RestClient:
+    def __init__(self, auth_session, verbose, base_url):
+        self.auth_session: requests.Session = auth_session
+        self.verbose = verbose
+        self.base_url = base_url
+        self.errors = self.init_errors_dict()
+
+    def join_path(self, path):
+        return parse.urljoin(self.base_url, path)
+
+    def get(self, endpoint, params=None):
+        response = self.auth_session.get(self.join_path(endpoint), params=params)
+        self.process_response(endpoint, response)
+        return response.json()
+
+    def log_verbose(self, endpoint, response):
+        if self.verbose:
+            logger.info(
+                'status=' + str(response.status_code) + ' for endpoint=' + endpoint +
+                ' with content type=' + response.headers['content-type']
+            )
+            logger.info("response body=" + json.dumps(response.json(), indent=2))
+
+    def post(self, endpoint, body, params=None):
+        headers = {"content-type": "application/json"}
+        response = self.auth_session.post(self.join_path(endpoint), data=body, headers=headers, params=params)
+        self.process_response(endpoint, response)
+        return response.json()
+
+    def process_response(self, endpoint, response):
+        self.check_error(response, endpoint)
+        self.log_verbose(endpoint, response)
+
+    @staticmethod
+    def init_errors_dict():
+        return {
+            404: "Call to {URI} failed with a 404 result\n with details: {details}\n",
+            403: "Call to {URI} failed with a 403 result\n with details: {details}\n",
+            401: "Call to {URI} failed with a 401 result\n with details: {details}\n",
+            400: "Call to {URI} failed with a 400 result\n with details: {details}\n"
+        }
+
+    def check_error(self, response, endpoint):
+        if not 200 >= response.status_code >= 400:
+            return
+        message = self.errors.get(response.status_code)
+        if message:
+            raise Exception(message.format(URI=self.join_path(endpoint), details=response.json()))

git2pantheon/clients/drupal/drupal_rest_client.py

@@ -0,0 +1,30 @@
+from decorest import RestClient, GET, on, header, POST, body
+
+
+class DrupalClient(RestClient):
+    def __init__(self, *args, **kwargs):
+        super(DrupalClient, self).__init__(*args, **kwargs)
+
+    @GET('/api/cache_clear/topic/{guid}')
+    @header('accept', 'application/json')
+    def purge_cache_module(self, guid):
+        """Purge the drupal cache for module"""
+
+    @GET('/api/cache_clear/guide/{guid}')
+    @header('accept', 'application/json')
+    def purge_cache_assembly(self, guid):
+        """Purge the drupal cache for module"""
+
+    @POST('/api/cache_clear/topic')
+    @header('content-type', 'application/json')
+    @header('accept', 'application/json')
+    @body('ids')
+    def purge_cache_module_bulk(self, ids):
+        """Purge the drupal cache for module"""
+
+    @GET('/api/cache_clear/guide')
+    @header('content-type', 'application/json')
+    @header('accept', 'application/json')
+    @body('ids')
+    def purge_cache_assembly_bulk(self, ids):
+        """Purge the drupal cache for module"""