From 39d5b5bf2fc96882c2b15e6ea6c8d3360bf0d54f Mon Sep 17 00:00:00 2001 From: "A.P. Rajshekhar" Date: Fri, 23 Jul 2021 13:01:23 +0530 Subject: [PATCH 1/2] CCS-4538: Provide an endpoint in Git2Pantheon that would handle cache clearing for both Drupal and Akamai. --- README.md | 4 +- git2pantheon/__init__.py | 13 ++++ git2pantheon/api-docs/cache_clear_api.yaml | 53 ++++++++++++++ git2pantheon/api/__init__.py | 8 +++ git2pantheon/api/upload.py | 72 ++++++++++++++++--- git2pantheon/clients/__init__.py | 0 git2pantheon/clients/akamai/__init__.py | 0 .../clients/akamai/akamai_rest_client.py | 19 +++++ git2pantheon/clients/client.py | 57 +++++++++++++++ git2pantheon/clients/drupal/__init__.py | 0 .../clients/drupal/drupal_rest_client.py | 30 ++++++++ git2pantheon/helpers.py | 53 +++++++++++++- requirements.txt | 4 ++ setup.py | 4 ++ 14 files changed, 304 insertions(+), 13 deletions(-) create mode 100644 git2pantheon/api-docs/cache_clear_api.yaml create mode 100644 git2pantheon/clients/__init__.py create mode 100644 git2pantheon/clients/akamai/__init__.py create mode 100644 git2pantheon/clients/akamai/akamai_rest_client.py create mode 100644 git2pantheon/clients/client.py create mode 100644 git2pantheon/clients/drupal/__init__.py create mode 100644 git2pantheon/clients/drupal/drupal_rest_client.py diff --git a/README.md b/README.md index 47f47bc..3b64b1b 100644 --- a/README.md +++ b/README.md @@ -43,5 +43,7 @@ flask run ``` 8. The swagger docs can be found at: -http://localhost:5000/apidocs/ +http://localhost:5000/apidocs/ + +_Note_: _Please don't try to run cache clear API_ diff --git a/git2pantheon/__init__.py b/git2pantheon/__init__.py index 05b9162..cd67300 100644 --- a/git2pantheon/__init__.py +++ b/git2pantheon/__init__.py @@ -6,6 +6,7 @@ from flasgger import Swagger from .api.upload import api_blueprint, executor import atexit +from .helpers import EnvironmentVariablesHelper def create_app(): @@ -15,11 +16,23 @@ def create_app(): EXECUTOR_MAX_WORKERS="1", EXECUTOR_PROPAGATE_EXCEPTIONS=True ) + # check if required vars are available + EnvironmentVariablesHelper.check_vars(['PANTHEON_SERVER', 'UPLOADER_PASSWORD','UPLOADER_USER']) + try: + EnvironmentVariablesHelper.check_vars(['AKAMAI_HOST', 'DRUPAL_HOST', 'AKAMAI_ACCESS_TOKEN', + 'AKAMAI_CLIENT_SECRET','AKAMAI_CLIENT_TOKEN']) + except Exception as e: + print( + 'Environment variable(s) for cache clear not present.' + 'Details={0}Please ignore if you are running on local server'.format( + str(e))) + app.config.from_mapping( PANTHEON_SERVER=os.environ['PANTHEON_SERVER'], UPLOADER_PASSWORD=os.environ['UPLOADER_PASSWORD'], UPLOADER_USER=os.environ['UPLOADER_USER'] ) + gunicorn_error_logger = logging.getLogger('gunicorn.error') app.logger.handlers.extend(gunicorn_error_logger.handlers) logging.basicConfig(level=logging.DEBUG) diff --git a/git2pantheon/api-docs/cache_clear_api.yaml b/git2pantheon/api-docs/cache_clear_api.yaml new file mode 100644 index 0000000..9222e74 --- /dev/null +++ b/git2pantheon/api-docs/cache_clear_api.yaml @@ -0,0 +1,53 @@ +Clears the drupal and akamai cache +--- +tags: + - Clears the drupal and akamai cache. + +parameters: + - in: body + name: body + description: 'The structure containing list of URLs of modules and assemblies whose cache has to be cleared' + required: true + schema: + $ref: '#/definitions/CacheClearData' + +reponses: + 200: + description: 'The status of upload corresponding to the key' + schema: + $ref: '#/definitions/UploaderKey' + 400: + description: 'Invalid content error' + schema: + $ref: '#/definitions/Error' + 500: + description: 'Internal server error' + schema: + $ref: '#/definitions/Error' + + +definitions: + CacheClearData: + type: object + properties: + assemblies: + type: array + items: + type: string + modules: + type: array + items: + type: string + Error: + type: object + properties: + code: + type: string + description: 'HTTP status code of the error' + message: + type: string + description: 'Error message' + details: + type: string + description: 'Error details' + diff --git a/git2pantheon/api/__init__.py b/git2pantheon/api/__init__.py index 533f01b..93b3352 100644 --- a/git2pantheon/api/__init__.py +++ b/git2pantheon/api/__init__.py @@ -2,8 +2,16 @@ from flask_executor import Executor from flask_cors import CORS from git2pantheon.utils import ApiError +from git2pantheon.clients.akamai.akamai_rest_client import AkamaiCachePurgeClient +from git2pantheon.clients.drupal.drupal_rest_client import DrupalClient +import os executor = Executor() +akamai_purge_client = AkamaiCachePurgeClient(host=os.getenv('AKAMAI_HOST'), + client_token=os.getenv('AKAMAI_CLIENT_TOKEN'), + client_secret=os.getenv('AKAMAI_CLIENT_SECRET'), + access_token=os.getenv('AKAMAI_ACCESS_TOKEN')) +drupal_client = DrupalClient(os.getenv('DRUPAL_HOST')) api_blueprint = Blueprint('api', __name__, url_prefix='/api/') CORS(api_blueprint) diff --git a/git2pantheon/api/upload.py b/git2pantheon/api/upload.py index ffd7fbb..4bbb590 100644 --- a/git2pantheon/api/upload.py +++ b/git2pantheon/api/upload.py @@ -14,14 +14,19 @@ from pantheon_uploader import pantheon from . import api_blueprint from . import executor +from . import drupal_client +from . import akamai_purge_client from .. import utils -from ..helpers import FileHelper, GitHelper, MessageHelper +from ..helpers import FileHelper, GitHelper, MessageHelper, CacheObjectHelper from ..messaging import broker from ..models.request_models import RepoSchema from ..models.response_models import Status from ..utils import ApiError, get_docs_path_for from flask import current_app +from decorest import HTTPErrorWrapper +ASSEMBLIES = "assemblies" +MODULES = "modules" logger = logging.getLogger(__name__) @@ -44,7 +49,7 @@ def push_repo(): def clone_repo(repo_name, repo_url, branch): try: - + MessageHelper.publish(repo_name + "-clone", json.dumps(dict(current_status="cloning", details="Cloning repo " + repo_name + ""))) logger.info("Cloning repo=" + repo_url + " and branch=" + branch) @@ -81,8 +86,8 @@ def upload_repo(cloned_repo, channel_name): try: pantheon.start_process(numeric_level=10, pw=current_app.config['UPLOADER_PASSWORD'], user=current_app.config['UPLOADER_USER'], - server=current_app.config['PANTHEON_SERVER'], directory=cloned_repo.working_dir, - use_broker=True, channel=channel_name, broker_host= os.getenv('REDIS_SERVICE') ) + server=current_app.config['PANTHEON_SERVER'], directory=cloned_repo.working_dir, + use_broker=True, channel=channel_name, broker_host=os.getenv('REDIS_SERVICE')) except Exception as e: logger.error("Upload failed due to error=" + str(e)) MessageHelper.publish(channel_name, @@ -105,7 +110,7 @@ def info(): def status(): status_data, clone_data = get_upload_data() current_status = get_current_status(clone_data, status_data) - logger.debug("current status="+current_status) + logger.debug("current status=" + current_status) status_message = Status(clone_status=clone_data.get('current_status', ""), current_status=current_status, file_type=status_data.get('type_uploading', ""), @@ -120,8 +125,8 @@ def status(): def get_current_status(clone_data, status_data): - logger.debug('upload status data='+json.dumps(status_data)) - logger.debug('clone status data='+json.dumps(clone_data)) + logger.debug('upload status data=' + json.dumps(status_data)) + logger.debug('clone status data=' + json.dumps(clone_data)) return status_data.get('current_status') if status_data.get('current_status', "") != "" else clone_data[ 'current_status'] @@ -147,7 +152,7 @@ def get_request_data(): def reset_if_exists(repo_name): - MessageHelper.publish(repo_name +"-clone", json.dumps(dict(current_status=''))) + MessageHelper.publish(repo_name + "-clone", json.dumps(dict(current_status=''))) MessageHelper.publish(repo_name, json.dumps(dict(current_status=''))) @@ -157,7 +162,7 @@ def progress_update(): status_data, clone_data = get_upload_data() # status_progress: UploadStatus = upload_status_from_dict(status_data) if "server" in status_data and status_data["server"]["response_code"] and not 200 <= int(status_data["server"][ - "response_code"]) <= 400: + "response_code"]) <= 400: return jsonify( dict( server_status=status_data["server"]["response_code"], @@ -265,3 +270,52 @@ def progress_update_resources(): return jsonify( response_dict ), 200 + + +@swag_from(get_docs_path_for('cache_clear_api.yaml')) +@api_blueprint.route('/cache/clear', methods=['POST']) +def clear_cache(): + data = get_request_data() + cache_clear_result = { + "drupal_result_assemblies":{}, + "drupal_result_modules": {} + } + + try: + clear_drupal_cache(data, cache_clear_result) + clear_akamai_cache(data,cache_clear_result) + except Exception as e: + logger.error("Exception occurred while trying to clear cache with error=" + str(e)) + raise ApiError("Upstream Server Error", 503, details=str(e)) + return jsonify(cache_clear_result) + + +def drupal_cache_clear_bulk(cache_clear_result, cache_req_data): + if "assemblies" in cache_req_data: + cache_clear_result["drupal_result_assemblies"] = drupal_client.purge_cache_assembly("assemblies") + + if "modules" in cache_req_data: + cache_clear_result["drupal_result_modules"] = drupal_client.purge_cache_assembly("modules") + + +def clear_drupal_cache(data, cache_clear_result, bulk_clear=False): + cache_req_data = CacheObjectHelper.get_drupal_req_data(data) + if bulk_clear: + drupal_cache_clear_bulk(cache_clear_result, cache_req_data) + return + drupal_cache_clear_individual(cache_clear_result, cache_req_data) + + +def drupal_cache_clear_individual(cache_clear_result, cache_req_data): + if ASSEMBLIES in cache_req_data: + for guid in cache_req_data[ASSEMBLIES]: + cache_clear_result["drupal_result_assemblies"][str(guid)] = drupal_client.purge_cache_assembly(guid) + if MODULES in cache_req_data: + for guid in cache_req_data[MODULES]: + cache_clear_result["drupal_result_modules"][str(guid)] = (drupal_client.purge_cache_module(guid)) + + +def clear_akamai_cache(data, cache_clear_result): + cache_req_data = CacheObjectHelper.get_akamai_req_object(data) + cache_clear_result['akamai_result'] = akamai_purge_client.purge(cache_req_data) + diff --git a/git2pantheon/clients/__init__.py b/git2pantheon/clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/git2pantheon/clients/akamai/__init__.py b/git2pantheon/clients/akamai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/git2pantheon/clients/akamai/akamai_rest_client.py b/git2pantheon/clients/akamai/akamai_rest_client.py new file mode 100644 index 0000000..c360f9f --- /dev/null +++ b/git2pantheon/clients/akamai/akamai_rest_client.py @@ -0,0 +1,19 @@ +import requests, logging, json +from ..client import RestClient +from akamai.edgegrid import EdgeGridAuth + +logger = logging.getLogger(__name__) + + +class AkamaiCachePurgeClient: + def __init__(self,host, client_token, client_secret, access_token): + self.session: requests.Session = requests.Session() + self.session.auth = EdgeGridAuth(client_token=client_token, client_secret=client_secret, + access_token=access_token) + self.host = host + self.akamai_rest_client = RestClient(auth_session=self.session, verbose=True, + base_url=self.host) + + def purge(self, purge_obj, action='delete'): + logger.info('Adding %s request to the queue for %s' % (action, json.dumps(purge_obj))) + return self.akamai_rest_client.post('/ccu/v3/delete/url', json.dumps(purge_obj)) \ No newline at end of file diff --git a/git2pantheon/clients/client.py b/git2pantheon/clients/client.py new file mode 100644 index 0000000..8fe226a --- /dev/null +++ b/git2pantheon/clients/client.py @@ -0,0 +1,57 @@ +from urllib import parse + +import json +import logging +import requests + +logger = logging.getLogger(__name__) + + +class RestClient: + def __init__(self, auth_session, verbose, base_url): + self.auth_session: requests.Session = auth_session + self.verbose = verbose + self.base_url = base_url + self.errors = self.init_errors_dict() + + def join_path(self, path): + return parse.urljoin(self.base_url, path) + + def get(self, endpoint, params=None): + response = self.auth_session.get(self.join_path(endpoint), params=params) + self.process_response(endpoint, response) + return response.json() + + def log_verbose(self, endpoint, response): + if self.verbose: + logger.info( + 'status=' + str(response.status_code) + ' for endpoint=' + endpoint + + ' with content type=' + response.headers['content-type'] + ) + logger.info("response body=" + json.dumps(response.json(), indent=2)) + + def post(self, endpoint, body, params=None): + headers = {"content-type": "application/json"} + response = self.auth_session.post(self.join_path(endpoint), data=body, headers=headers, params=params) + self.process_response(endpoint, response) + return response.json() + + def process_response(self, endpoint, response): + self.check_error(response, endpoint) + self.log_verbose(endpoint, response) + + @staticmethod + def init_errors_dict(): + return { + 404: "Call to {URI} failed with a 404 result\n with details: {details}\n", + 403: "Call to {URI} failed with a 403 result\n with details: {details}\n", + 401: "Call to {URI} failed with a 401 result\n with details: {details}\n", + 400: "Call to {URI} failed with a 400 result\n with details: {details}\n" + } + + def check_error(self, response, endpoint): + if not 200 >= response.status_code >= 400: + return + message = self.errors.get(response.status_code) + if message: + raise Exception(message.format(URI=self.join_path(endpoint), details=response.json())) diff --git a/git2pantheon/clients/drupal/__init__.py b/git2pantheon/clients/drupal/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/git2pantheon/clients/drupal/drupal_rest_client.py b/git2pantheon/clients/drupal/drupal_rest_client.py new file mode 100644 index 0000000..4b2923f --- /dev/null +++ b/git2pantheon/clients/drupal/drupal_rest_client.py @@ -0,0 +1,30 @@ +from decorest import RestClient, GET, on, header, POST, body + + +class DrupalClient(RestClient): + def __init__(self, *args, **kwargs): + super(DrupalClient, self).__init__(*args, **kwargs) + + @GET('/api/cache_clear/topic/{guid}') + @header('accept', 'application/json') + def purge_cache_module(self, guid): + """Purge the drupal cache for module""" + + @GET('/api/cache_clear/guide/{guid}') + @header('accept', 'application/json') + def purge_cache_assembly(self, guid): + """Purge the drupal cache for module""" + + @POST('/api/cache_clear/topic') + @header('content-type', 'application/json') + @header('accept', 'application/json') + @body('ids') + def purge_cache_module_bulk(self, ids): + """Purge the drupal cache for module""" + + @GET('/api/cache_clear/guide') + @header('content-type', 'application/json') + @header('accept', 'application/json') + @body('ids') + def purge_cache_assembly_bulk(self, ids): + """Purge the drupal cache for module""" diff --git a/git2pantheon/helpers.py b/git2pantheon/helpers.py index 6854f3c..31a8943 100644 --- a/git2pantheon/helpers.py +++ b/git2pantheon/helpers.py @@ -44,6 +44,7 @@ def validate_git_url(cls, url): """ return giturlparse.validate(url) + class ProgressHelper(RemoteProgress): def line_dropped(self, line): logger.info(line) @@ -73,10 +74,10 @@ def publish(cls, key, message): :return: """ try: - logger.info("Publishing key="+key+" with message="+message) + logger.info("Publishing key=" + key + " with message=" + message) broker.set(key, message) except Exception as e: - logger.error('Could not publish state due to'+str(e)) + logger.error('Could not publish state due to' + str(e)) @classmethod def unpublish(cls, key): @@ -87,5 +88,51 @@ class JsonEncoder(JSONEncoder): """ Helper for encoding objects into JSON """ + def default(self, object_to_serialize): - return object_to_serialize.__dict__ \ No newline at end of file + return object_to_serialize.__dict__ + + +class CacheObjectHelper: + components = ["assemblies", "modules"] + + @classmethod + def get_akamai_req_object(cls, purge_data: dict): + purge_request_body = dict() + urls = [] + for component in cls.components: + if component in purge_data: + for url in purge_data[component]: + urls.append(url) + purge_request_body['objects'] = urls + return purge_request_body + + @classmethod + def get_drupal_req_data(cls, purge_data): + purge_request_body = dict() + ids = [] + for component in cls.components: + if component in purge_data: + for url in purge_data[component]: + ids.append(url.split('/')[-1]) + purge_request_body[component] = list(ids) + # reset ids + ids.clear() + return purge_request_body + + +class EnvironmentVariablesHelper: + """Verifies if the environment variables are present or not""" + @classmethod + def validate_required_vars(cls, env_vars=[]): + cls.check_vars(env_vars) + + @classmethod + def check_vars(cls, env_vars): + for var in env_vars: + if var not in os.environ: + raise Exception("The variable=" + var + " is not present as an environment variable") + + @classmethod + def check_non_required_vars(cls, env_vars=[]): + cls.check_vars(env_vars) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ca2963f..adebbf1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,8 @@ marshmallow==3.10.0 Flask-Executor==0.9.4 giturlparse==0.10.0 PyYAML==5.4.1 +edgegrid-python==1.0.10 +decorest==0.0.6 +requests +requests-toolbelt==0.9.1 git+https://github.com/redhataccess/pantheon-uploader.git \ No newline at end of file diff --git a/setup.py b/setup.py index ffc5339..4cf8aa5 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,10 @@ 'marshmallow>=3.9.1', 'gitpython>=3.1.11', 'gunicorn', + 'edgegrid-python>=1.0.10', + 'decorest>=0.0.6', + 'requests', + 'requests-toolbelt>=0.9.1' 'pantheon-uploader @ git+https://github.com/redhataccess/pantheon-uploader.git@master#egg=pantheon-uploader-0.2' ], dependency_links=['https://github.com/redhataccess/pantheon-uploader/tarball/master#egg=pantheon-uploader'], From 51706a6876045125c9f77c693ed0bb01fc1027fb Mon Sep 17 00:00:00 2001 From: nitish-sharma Date: Tue, 30 Nov 2021 20:15:50 +0530 Subject: [PATCH 2/2] - requirement file updated with git dependency - formatting issue --- git2pantheon/api/upload.py | 2 +- requirements.txt | 5 +++-- setup.py | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/git2pantheon/api/upload.py b/git2pantheon/api/upload.py index 4bbb590..77cb427 100644 --- a/git2pantheon/api/upload.py +++ b/git2pantheon/api/upload.py @@ -283,7 +283,7 @@ def clear_cache(): try: clear_drupal_cache(data, cache_clear_result) - clear_akamai_cache(data,cache_clear_result) + clear_akamai_cache(data, cache_clear_result) except Exception as e: logger.error("Exception occurred while trying to clear cache with error=" + str(e)) raise ApiError("Upstream Server Error", 503, details=str(e)) diff --git a/requirements.txt b/requirements.txt index adebbf1..559469e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flasgger==0.9.5 Flask==1.1.2 -git2pantheon~=0.1 +#git2pantheon~=0.1 Werkzeug==1.0.1 redis==3.5.3 setuptools==54.2.0 @@ -14,4 +14,5 @@ edgegrid-python==1.0.10 decorest==0.0.6 requests requests-toolbelt==0.9.1 -git+https://github.com/redhataccess/pantheon-uploader.git \ No newline at end of file +git +git+https://github.com/redhataccess/pantheon-uploader.git diff --git a/setup.py b/setup.py index 4cf8aa5..47f4b6d 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,8 @@ 'edgegrid-python>=1.0.10', 'decorest>=0.0.6', 'requests', - 'requests-toolbelt>=0.9.1' + 'requests-toolbelt>=0.9.1', + 'git', 'pantheon-uploader @ git+https://github.com/redhataccess/pantheon-uploader.git@master#egg=pantheon-uploader-0.2' ], dependency_links=['https://github.com/redhataccess/pantheon-uploader/tarball/master#egg=pantheon-uploader'],