diff --git a/.coveragerc b/.coveragerc index cc14a39..d6c8955 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ source = apployer/ [report] exclude_lines = def __repr__ # TODO set 100 -fail_under = 91 +fail_under = 92 # we won't work on those files and they will be eventually obsolete omit = diff --git a/.pylintrc b/.pylintrc index c3c05fe..cbf7378 100644 --- a/.pylintrc +++ b/.pylintrc @@ -9,7 +9,7 @@ # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" -disable=W1620,E1607,I0021,E1601,W1618,W1613,W1607,E1604,W1611,W1632,W1628,W1637,W1625,W1609,W1638,W1621,W1640,W1635,E1605,W1623,W0704,W1619,W1622,W1614,W1604,W1606,E1606,W1636,W1629,I0020,W1610,W1603,W1601,W1624,W1602,E1602,W1633,W1634,W1630,E1603,E1608,W1605,W1617,W1639,W1615,W1612,W1626,W1627,W1616,W1608,too-few-public-methods,fixme,locally-disabled,file-ignored +disable=too-few-public-methods,fixme,locally-disabled,file-ignored [REPORTS] diff --git a/apployer/app_compare.py b/apployer/app_compare.py new file mode 100644 index 0000000..1172cd9 --- /dev/null +++ b/apployer/app_compare.py @@ -0,0 +1,165 @@ +# +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Comparing an application from appstack to its counterpart in the live environment. +""" + +import logging +import pkg_resources + +import datadiff +from requests.structures import CaseInsensitiveDict + +from . import cf_api +from . import cf_cli + +_log = logging.getLogger(__name__) # pylint: disable=invalid-name + + +def should_update(app): + """Checks whether an application should be updated (or created from scratch) in the live + Cloud Foundry environment. + + Args: + app (`apployer.appstack.AppConfig`): An application. + + Returns: + bool: True if the app should be pushed, False otherwise. + """ + try: + app_guid = cf_cli.get_app_guid(app.name) + except cf_cli.CommandFailedError as ex: + _log.debug(str(ex)) + _log.info("Failed to get GUID of app %s. Assuming it doesn't exist yet. " + "Will need to push it...", app.name) + return True + app_summary = cf_api.get_app_summary(app_guid) + app_properties = app.app_properties + + appstack_version, live_env_version = _get_app_versions(app_properties, app_summary) + if appstack_version > live_env_version: + _log.info("Appstack's version of the app (%s) is higher than in the live env (%s). " + "Will update...", appstack_version, live_env_version) + return True + elif appstack_version < live_env_version: + _log.info("Appstack's version of the app (%s) is lower than in the live env (%s). " + "Won't push, because that would downgrade the app...", + appstack_version, live_env_version) + return False + + return _properties_differ(app_properties, app_summary) + + +def _get_app_versions(app_properties, app_summary): + """ + Args: + app_properties (dict): Application's properties from appstack. + app_summary (dict): Application's properties from Cloud Foundry. + + Returns: + (`pkg_resources.SetuptoolsVersion`, `pkg_resources.SetuptoolsVersion`): Tuple in form: + (app_version_in_appstack, app_version_in_live_env). + If version isn't found, then the lowest possible version will be returned. + """ + appstack_version = app_properties.get('env', {}).get('VERSION', '') + live_env_version = app_summary.get('environment_json', {}).get('VERSION', '') + # empty version string will be parsed to the lowest possible version + return (pkg_resources.parse_version(appstack_version), + pkg_resources.parse_version(live_env_version)) + + +def _properties_differ(app_properties, app_summary): + """Compares application's properties from appstack to those taken from a live environment. + + Args: + app_properties (dict): Application's properties from appstack. + app_summary (dict): Application's properties from Cloud Foundry. + + Returns: + bool: True if one of the properties present in `app_properties` is different in + `app_summary`. False otherwise. + """ + for key, value in app_properties.items(): + if key == 'env': + if not _dict_is_part_of(app_summary['environment_json'], value): + _log.info("Differences in application's env:\n%s", + datadiff.diff(app_summary['environment_json'], value, + fromfile='live env', tofile='appstack')) + return True + elif key in ('disk_quota', 'memory'): + # Values in the manifest will be strings and have suffix M, MB, G or GB, + # while values in summary will be ints specifying the number of megabytes. + megabytes_in_properties = _normalize_to_megabytes(value) + if megabytes_in_properties != app_summary[key]: + _log.info("Difference in application's %s field: %s (live env) vs. %s (appstack).", + key, app_summary[key], megabytes_in_properties) + return True + elif key == 'services': + summary_services = [service['name'] for service in app_summary[key]] + if not set(value).issubset(set(summary_services)): + _log.info("Difference in application's services: \n%s", + datadiff.diff(app_summary[key], value, + fromfile='live env', tofile='appstack')) + return True + elif key == 'host': + summary_hosts = [route['host'] for route in app_summary['routes']] + if value not in summary_hosts: + _log.info("Application's hosts in live env don't contain %s.", value) + return True + else: + if value != app_summary[key]: + _log.info("Difference in application's %s field: %s (live env) vs. %s (appstack).", + key, app_summary[key], value) + return True + return False + + +def _dict_is_part_of(dict_a, dict_b): + """ + Checks whether dict_b is a part of dict_a. + That is if dict_b is dict_a, just with some (or none) keys removed. + + Args: + dict_a (dict): + dict_b (dict): + + Return: + bool: True if dict_a contains dict_b. + """ + dict_a, dict_b = CaseInsensitiveDict(dict_a), CaseInsensitiveDict(dict_b) + for key, value in dict_b.items(): + if key not in dict_a or dict_a[key] != value: + return False + return True + + +def _normalize_to_megabytes(value): + """ + Args: + value (str): A string with a number and a size suffix, like 2GB or 300M. + + Returns: + int: Number of megabytes represented by the `value`. + """ + if value.endswith('M'): + return int(value[:-1]) + elif value.endswith('MB'): + return int(value[:-2]) + elif value.endswith('G'): + return int(value[:-1]) * 1024 + elif value.endswith('GB'): + return int(value[:-2]) * 1024 diff --git a/apployer/cf_api.py b/apployer/cf_api.py index 639cf17..114f07b 100644 --- a/apployer/cf_api.py +++ b/apployer/cf_api.py @@ -66,6 +66,7 @@ def delete_service_binding(binding): raise cf_cli.CommandFailedError('Failed to delete a service binding. CF response: {}' .format(cmd_output)) + def get_app_name(app_guid): """ Args: @@ -78,6 +79,18 @@ def get_app_name(app_guid): return app_desctiption['entity']['name'] +def get_app_summary(app_guid): + """Gets a summary of the application's state. This includes its full configuration. + + Args: + app_guid (str): Application's GUID. + + Returns: + dict: Application's summary. Contains fields like "services", "memory", "buildpack", etc. + """ + return _cf_curl_get('/v2/apps/{}/summary'.format(app_guid)) + + def get_upsi_credentials(service_guid): """Gets the credentials (configuration) of a user-provided service instance. diff --git a/apployer/cf_cli.py b/apployer/cf_cli.py index b577c1a..45e32c9 100644 --- a/apployer/cf_cli.py +++ b/apployer/cf_cli.py @@ -91,7 +91,7 @@ def bind_service(app_name, instance_name): def buildpacks(): """ Returns: - list[`BuildpackDescription`]: A list of buildpacks on the environment. + list[`BuildpackDescription`]: A list of buildpacks in the environment. """ out = get_command_output([CF, 'buildpacks']) buildpack_lines = out.splitlines()[3:] @@ -216,6 +216,18 @@ def env(app_name): return get_command_output([CF, 'env', app_name]) +def get_app_guid(app_name): + """ + Args: + app_name (str): Name of a service instace (can be a user-provided one). + + Returns: + str: GUID of the application that can be used in API calls. + """ + cmd_output = get_command_output([CF, 'app', '--guid', app_name]) + return cmd_output.split()[0] + + def get_service_guid(service_name): """ Args: @@ -287,6 +299,16 @@ def service(service_name): return get_command_output([CF, 'service', service_name]) +def service_brokers(): + """ + Returns: + set[str]: A set of brokers (their names) in the environment. + """ + out = get_command_output([CF, 'service-brokers']) + broker_lines = out.splitlines()[3:] + return set([line.split()[0] for line in broker_lines]) + + def update_buildpack(buildpack_name, buildpack_path): """Updates a buildpack. diff --git a/apployer/deployer.py b/apployer/deployer.py index c3f4813..e44cc9a 100644 --- a/apployer/deployer.py +++ b/apployer/deployer.py @@ -27,11 +27,9 @@ from zipfile import ZipFile import datadiff -from pkg_resources import parse_version import yaml -import apployer.app_file as app_file -from apployer import cf_cli, cf_api +from apployer import cf_cli, cf_api, app_file, app_compare, dry_run from .cf_cli import CommandFailedError _log = logging.getLogger(__name__) #pylint: disable=invalid-name @@ -45,10 +43,36 @@ DEPLOYER_OUTPUT = 'apployer_out' -# TODO add option of dry run that switches all cf_cli commands to stubs -def deploy_appstack(cf_login_data, filled_appstack, artifacts_path, push_strategy): +def deploy_appstack(cf_login_data, filled_appstack, artifacts_path, push_strategy, is_dry_run): """Deploys the appstack to Cloud Foundry. + Args: + cf_login_data (`apployer.cf_cli.CfInfo`): Credentials and addresses needed to log into + Cloud Foundry. + filled_appstack (`apployer.appstack.AppStack`): Expanded appstack filled with configuration + extracted from a live TAP environment. + artifacts_path (str): Path to a directory containing application artifacts (zips). + push_strategy (str): Strategy for pushing applications. + is_dry_run (bool): Is this a dry run? If set to True, no changes (except for creating org + and space) will be introduced to targeted Cloud Foundry. + """ + global cf_cli, register_in_application_broker #pylint: disable=C0103,W0603,W0601 + if is_dry_run: + normal_cf_cli = cf_cli + cf_cli = dry_run.get_dry_run_cf_cli() + normal_register_in_app_broker = register_in_application_broker + register_in_application_broker = dry_run.get_dry_function(register_in_application_broker) + try: + _do_deploy(cf_login_data, filled_appstack, artifacts_path, push_strategy) + finally: + if is_dry_run: + cf_cli = normal_cf_cli + register_in_application_broker = normal_register_in_app_broker + + +def _do_deploy(cf_login_data, filled_appstack, artifacts_path, push_strategy): + """Actual heavy lifting of deployment. + Args: cf_login_data (`apployer.cf_cli.CfInfo`): Credentials and addresses needed to log into Cloud Foundry. @@ -70,15 +94,12 @@ def deploy_appstack(cf_login_data, filled_appstack, artifacts_path, push_strateg for buildpack in filled_appstack.buildpacks: setup_buildpack(buildpack, artifacts_path) + names_to_apps = {app.name: app for app in filled_appstack.apps} + for app in filled_appstack.apps: app_deployer = AppDeployer(app, DEPLOYER_OUTPUT) affected_apps = app_deployer.deploy(artifacts_path, push_strategy) apps_to_restart.extend(affected_apps) - - _restart_apps(filled_appstack, apps_to_restart) - - names_to_apps = {app.name: app for app in filled_appstack.apps} - for app in filled_appstack.apps: if app.register_in: # FIXME this universal mechanism is kind of pointless, because we can only do # registering in application-broker. Even we made "register.sh" in the registrator app @@ -91,11 +112,12 @@ def deploy_appstack(cf_login_data, filled_appstack, artifacts_path, push_strateg filled_appstack.domain, DEPLOYER_OUTPUT, artifacts_path) - + _restart_apps(filled_appstack, apps_to_restart) _log.info('DEPLOYMENT FINISHED') -def register_in_application_broker(registered_app, application_broker, app_domain, +def register_in_application_broker(registered_app, # pylint: disable=function-redefined + application_broker, app_domain, unpacked_apps_dir, artifacts_location): """Registers an application in another application that provides some special functionality. E.g. there's the application-broker app that registers another application as a broker. @@ -149,13 +171,12 @@ def setup_broker(broker): """ _log.info('Setting up broker %s...', broker.name) broker_args = [broker.name, broker.auth_username, broker.auth_password, broker.url] - try: - cf_cli.update_service_broker(*broker_args) - except CommandFailedError as ex: - _log.debug(str(ex)) - _log.info("Updating a broker (%s) failed, assuming it doesn't exist yet. " - "Gonna create it now...", broker.name) + if broker.name not in cf_cli.service_brokers(): + _log.info("Broker %s doesn't exist. Gonna create it now...", broker.name) cf_cli.create_service_broker(*broker_args) + else: + _log.info("Broker %s exists. Will update it...", broker.name) + cf_cli.update_service_broker(*broker_args) _enable_broker_access(broker) @@ -291,6 +312,7 @@ def deploy(self): try: service_guid = cf_cli.get_service_guid(service_name) _log.info('User provided service %s has GUID %s.', service_name, service_guid) + return self._update(service_guid) except CommandFailedError as ex: _log.debug(str(ex)) _log.info("Failed to get GUID of user provided service %s, assuming it doesn't exist " @@ -300,8 +322,6 @@ def deploy(self): _log.debug('Created user provided service %s.', service_name) return [] - return self._update(service_guid) - def _update(self, service_guid): """Updates the service if it's different in the appstack and in the live environment. @@ -351,7 +371,6 @@ class AppDeployer(object): FILLED_MANIFEST = 'filled_manifest.yml' - # TODO it should throw some error on push that can be handled by the overall procedure. def __init__(self, app, output_path): self.app = app self.output_path = output_path @@ -431,45 +450,26 @@ def _push_app(self, artifacts_location, push_strategy): _log.info("No need to push app %s, it's already up-to-date...", self.app.name) def _check_push_needed(self, push_strategy): - _log.debug('Checking whether to push app %s...', self.app.name) - try: - if push_strategy == PUSH_ALL_STRATEGY: - _log.debug('Will push app %s because strategy is PUSH_ALL.', self.app.name) - else: - live_app_version = self._get_app_version() - # empty version string will be parsed to the lowest possible version - appstack_app_version = self.app.app_properties.get('env', {}).get('VERSION', '') - if parse_version(live_app_version) >= parse_version(appstack_app_version): - _log.debug("App's version (%s) in the live environment isn't lower than the " - "one in filled appstack (%s).\nWon't push app %s", - live_app_version, appstack_app_version, self.app.name) - return False - else: - _log.debug("App's version (%s) in the in filled appstack is higher than in " - "live environment (%s).\nWill push app %s", - appstack_app_version, live_app_version, self.app.name) - except (CommandFailedError, AppVersionNotFoundError) as ex: - _log.debug(str(ex)) - _log.debug("Getting app (%s) version failed. Will push the app because of that.", - self.app.name) - return True + """Checks whether an application should be pushed to Cloud Foundry. + If strategy is set to "PUSH_ALL" then the app should be pushed. + If strategy is set to "UPGRADE" then the application will be pushed when: - def _get_app_version(self): - app_env = cf_cli.env(self.app.name) - try: - app_version_line = next(line for line in app_env.splitlines() - if line.startswith('VERSION:')) - except StopIteration: - raise AppVersionNotFoundError( - "Can't determine the version of app {}. VERSION environment variable not found." - .format(self.app.name)) - return app_version_line.split()[1] + - app doesn't yet exist in the environment + - live environment has older version of the app + - live environment has current version of the app, but different values for properties + found in the appstack app (additional properties found only in live env don't matter) + Args: + push_strategy (str): Strategy for pushing the application. -class AppVersionNotFoundError(Exception): - """ - 'VERSION' environment variable wasn't present for an application. - """ + Returns: + bool: True if app should be pushed, False otherwise. + """ + if push_strategy == PUSH_ALL_STRATEGY: + _log.info('Will push app %s because strategy is PUSH_ALL.', self.app.name) + return True + else: + return app_compare.should_update(self.app) def _prepare_org_and_space(cf_login_data): diff --git a/apployer/dry_run.py b/apployer/dry_run.py new file mode 100644 index 0000000..a618921 --- /dev/null +++ b/apployer/dry_run.py @@ -0,0 +1,77 @@ +# +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Providing an elegant option of a dry run for the Apployer. +""" + +import inspect +import logging +import types + +from apployer import cf_cli + +_log = logging.getLogger(__name__) #pylint: disable=invalid-name + + +def get_dry_run_cf_cli(): + """Providing a module with functions having identical signatures as functions in cf_cli. + Functions that don't introduce any changes to Cloud Foundry (with the exception of those that + create the org and space) will remain, others will just log their names and parameters.""" + function_exceptions = ['login', 'buildpacks', 'create_org', 'create_space', 'env', + 'get_app_guid', 'get_service_guid', 'oauth_token', 'service', 'api', + 'auth', 'target', 'get_command_output'] + return provide_dry_run_module(cf_cli, function_exceptions) + + +def provide_dry_run_module(module, exceptions): + """Provides a new module with functions having identical signatures as functions in the + provided module. Public functions (the ones not starting with "_") in the new module do nothing + but log their parameters and the name of the original function. + Non-public functions act the same like in the old module. + + Args: + module (`types.ModuleType`): A module to create a "dry-run" version of. + exceptions (`list[str]`): A list of functions that shouldn't be modified. They will retain + their normal behavior and effects in the new module. + """ + module_functions = [member for member in inspect.getmembers(module) + if inspect.isfunction(member[1])] + dry_run_module = types.ModuleType('dry_run_' + module.__name__) + + exceptions_set = set(exceptions) + for name, function in module_functions: + if name.startswith('_') or name in exceptions_set: + setattr(dry_run_module, name, function) + else: + setattr(dry_run_module, name, get_dry_function(function)) + return dry_run_module + + +def get_dry_function(function): + """Returns a substitute for a given function. The substitute logs the name of the original + function and the parameters given to it. + + Args: + function (types.FunctionType): Function to be substituted. + + Returns: + types.FunctionType: The substitute function. + """ + def _wrapper(*args, **kwargs): + _log.info('DRY RUN: calling %s with arguments %s', + function.__name__, inspect.getcallargs(function, *args, **kwargs)) + return _wrapper diff --git a/apployer/fetcher/cdh_utilities.py b/apployer/fetcher/cdh_utilities.py index d682878..6aea65d 100644 --- a/apployer/fetcher/cdh_utilities.py +++ b/apployer/fetcher/cdh_utilities.py @@ -32,6 +32,7 @@ import paramiko import yaml import requests +import xml.etree.ElementTree as ET GENERATE_KEYTAB_SCRIPT = """#!/bin/sh @@ -231,11 +232,11 @@ def get_all_deployments_conf(self): result['krb5_base64'] = self.generate_base64_for_file('/etc/krb5.conf', self._cdh_manager_ip) result['kerberos_cacert'] = self.generate_base64_for_file('/var/krb5kdc/cacert.pem', self._cdh_manager_ip) - #sentry_service = helper.get_service_from_cdh('SENTRY') - result['sentry_port'] = '' #helper.get_entry(sentry_service, 'port') - result['sentry_address'] = '' #helper.get_host(sentry_service) - result['sentry_keytab_value'] = '' #self.generate_keytab('hive/sys') - result['auth_gateway_profile'] = 'cloud,zookeeper-auth-gateway,hdfs-auth-gateway,kerberos-hgm-auth-gateway' + sentry_service = helper.get_service_from_cdh('SENTRY') + result['sentry_port'] = helper.get_entry(sentry_service, 'sentry_service_server_rpc_port') + result['sentry_address'] = helper.get_host(sentry_service) + result['sentry_keytab_value'] = self.generate_keytab('hive/sys') + result['auth_gateway_profile'] = 'cloud,sentry-auth-gateway,zookeeper-auth-gateway,hdfs-auth-gateway,kerberos-hgm-auth-gateway,yarn-auth-gateway,hbase-auth-gateway' hgm_service = helper.get_service_from_cdh('HADOOPGROUPSMAPPING') result['hgm_adress'] = 'http://' + helper.get_host(hgm_service, 'HADOOPGROUPSMAPPING-HADOOPGROUPSMAPPING_RESTSERVER') + ':' \ + helper.get_entry_from_group(hgm_service, 'rest_port', 'HADOOPGROUPSMAPPING-HADOOPGROUPSMAPPING_RESTSERVER-BASE') @@ -251,13 +252,27 @@ def get_all_deployments_conf(self): result['hgm_keytab_value'] = '' result['krb5_base64'] = '' result['kerberos_cacert'] = '' - result['auth_gateway_profile'] = 'cloud,zookeeper-auth-gateway,hdfs-auth-gateway,https-hgm-auth-gateway' + result['auth_gateway_profile'] = 'cloud,zookeeper-auth-gateway,hdfs-auth-gateway,https-hgm-auth-gateway,yarn-auth-gateway,hbase-auth-gateway' hgm_service = helper.get_service_from_cdh('HADOOPGROUPSMAPPING') result['hgm_adress'] = 'https://' + helper.get_host(hgm_service, 'HADOOPGROUPSMAPPING-HADOOPGROUPSMAPPING_RESTSERVER') + ':'\ + helper.get_entry_from_group(hgm_service, 'rest_port', 'HADOOPGROUPSMAPPING-HADOOPGROUPSMAPPING_RESTSERVER-BASE') result['hgm_password'] = helper.get_entry_from_group(hgm_service, 'basic_auth_pass', 'HADOOPGROUPSMAPPING-HADOOPGROUPSMAPPING_RESTSERVER-BASE') result['hgm_username'] = helper.get_entry_from_group(hgm_service, 'basic_auth_user', 'HADOOPGROUPSMAPPING-HADOOPGROUPSMAPPING_RESTSERVER-BASE') + result['cloudera_address'] = result['cloudera_manager_internal_host'] + result['cloudera_port'] = 7180 + result['cloudera_user'] = self._cdh_manager_user + result['cloudera_password'] = self._cdh_manager_password + + oozie_server = helper.get_service_from_cdh('OOZIE') + result['oozie_server'] = 'http://' + helper.get_host(oozie_server) + ':' + helper.get_entry(oozie_server, 'oozie_http_port') + + yarn = helper.get_service_from_cdh('YARN') + result['job_tracker'] = helper.get_host(yarn) + ':' + helper.get_entry(yarn, 'yarn_resourcemanager_address') + + sqoop_client = helper.get_service_from_cdh('SQOOP_CLIENT') + result['metastore'] = self._get_property_value(helper.get_entry(sqoop_client, 'sqoop-conf/sqoop-site.xml_client_config_safety_valve'), 'sqoop.metastore.client.autoconnect.url') + master_nodes = self.extract_nodes_info('cdh-master', deployments_settings) for i, node in enumerate(master_nodes): result['master_node_host_' + str(i+1)] = node['hostname'] @@ -268,11 +283,18 @@ def get_all_deployments_conf(self): result['import_hadoop_conf_hdfs'] = self.get_client_config_for_service('HDFS') result['import_hadoop_conf_hbase'] = self.get_client_config_for_service('HBASE') result['import_hadoop_conf_yarn'] = self.get_client_config_for_service('YARN') + result['import_hadoop_conf_hive'] = self.get_client_config_for_service('HIVE') return result # helpful methods + def _get_property_value(self, config, key): + properties = ET.fromstring('' + config + '') + for property in properties: + if property.find('name').text == key: + return property.find('value').text + def _find_item_by_attr_value(self, attr_value, attr_name, array_with_dicts): return next(item for item in array_with_dicts if item[attr_name] == attr_value) diff --git a/apployer/main.py b/apployer/main.py index 7412461..1722534 100644 --- a/apployer/main.py +++ b/apployer/main.py @@ -114,6 +114,12 @@ def expand(appstack_file, artifacts_location, expanded_appstack_location): "'UPGRADE': deploy everything that doesn't exist in the environment or is in " "lower version on the environment than in the filled appstack.\n" "'PUSH_ALL': deploy everything from filled appstack.'") +@click.option('--dry-run', is_flag=True, + help="Does a dry run of the deployment. No changes will be introduced to the " + "Cloud Foundry environment, except for creating org and space if those don't " + "already exist. " + "Each action that the deployment would perform is logged.") + def deploy( #pylint: disable=too-many-arguments artifacts_location, cf_api_endpoint, @@ -125,7 +131,8 @@ def deploy( #pylint: disable=too-many-arguments filled_appstack, expanded_appstack, appstack, - push_strategy): + push_strategy, + dry_run): """ Deploy the whole appstack. This should be run from environment's bastion to reduce chance of errors. @@ -147,7 +154,7 @@ def deploy( #pylint: disable=too-many-arguments org=cf_org, space=cf_space) filled_appstack = _get_filled_appstack(appstack, expanded_appstack, filled_appstack, fetcher_config, artifacts_location) - deploy_appstack(cf_info, filled_appstack, artifacts_location, push_strategy) + deploy_appstack(cf_info, filled_appstack, artifacts_location, push_strategy, dry_run) _log.info('Deployment time: %s', _seconds_to_time(time.time() - start_time)) @@ -217,12 +224,13 @@ def _seconds_to_time(seconds): # TODO primary -# --dry-run (deployer functions should be in a class, some methods need to be overwritten, -# need to check for existance first before trying to update/create services) -# make installable and testable for py26 (no networkx) -# Brokers serving instances that hold no data (like all WSSB brokers) can be marked as "recreatable" -# or something. Then, they could be recreated and rebound to apps when WSSB configuration changes. -# Right now it won't happen, because there's no universal way of recreating a service instance. +# Make installable and testable for py26. This will require to make an "extra feature" (see +# setuptools docs on "declaring extras") out of appstack expansion. +# Only dependency for this feature would be networkx. +# Brokers serving instances that hold no data (like all WSSB brokers) can be marked +# as "recreatable" or something. Then, they could be recreated and rebound to apps when WSSB +# configuration changes. Right now it won't happen, because there's no universal way of +# recreating a service instance. # add app timeout parameter (default should be 180 seconds), think about enxanced timeout and # differentiation between continuous crash and timeout # make all docstrings conform to Google standard diff --git a/appstack.yml b/appstack.yml index bb1c3df..a7e39d1 100644 --- a/appstack.yml +++ b/appstack.yml @@ -8,7 +8,7 @@ apps: - name: auth-gateway order: 0 push_options: - post_command: curl -X PUT -v auth-gateway.{{ run_domain }}/organizations/$(cf org {{ core_org_name }} --guid) + post_command: 'curl -X PUT -H "Authorization: $(cf oauth-token | grep bearer)" -v auth-gateway.{{ run_domain }}/organizations/$(cf org {{ core_org_name }} --guid);curl -X PUT -H "Authorization: $(cf oauth-token | grep bearer)" -v auth-gateway.{{ run_domain }}/organizations/$(cf org {{ core_org_name }} --guid)/users/$(cf oauth-token | grep bearer | cut -d"." -f2 | base64 -d | jq .user_id | tr -d \")' app_properties: buildpack: java_buildpack env: @@ -16,6 +16,7 @@ apps: HDFS_KEYTAB: '{{ auth_gateway_keytab_value }}' HGM_PRINCIPAL: '{{ hgm_principal }}{{ authgateway_principals_suffix if kerberos_host != '' }}' HGM_PRINCIPAL_KEYTAB: '{{ hgm_keytab_value }}' + HBASE_PROVIDED_ZIP: '{{ import_hadoop_conf_hbase }}' HGM_URL: '{{ hgm_adress }}' HGM_USERNAME: '{{ hgm_username }}' HGM_PASSWORD: '{{ hgm_password }}' @@ -30,6 +31,10 @@ apps: KRB_REALM: "{{ kerberos_realm if kerberos_host != '' }}" KRB_USER: "{{ kerberos_username }}" HADOOP_PROVIDED_ZIP: '{{ import_hadoop_conf_hdfs }}' + CLOUDERA_USER: "{{ cloudera_user }}" + CLOUDERA_PASSWORD: "{{ cloudera_password }}" + CLOUDERA_ADDRESS: "{{ cloudera_address }}" + CLOUDERA_PORT: "{{ cloudera_port }}" ZK_CLUSTER_URL: '{{ master_node_host_1 }}:2181,{{ master_node_host_2 }}:2181,{{ master_node_host_3 }}:2181' - name: auth-proxy @@ -52,6 +57,7 @@ apps: # BROKERS - name: application-broker + order: 2 app_properties: buildpack: go_buildpack env: @@ -69,6 +75,7 @@ apps: url: http://app-dependency-discoverer.{{ apps_domain }} - name: gearpump-broker + order: -1 app_properties: env: BASE_GUID: 1ebaebc7-e344-4cb9-ad18-e29901386e04 @@ -87,11 +94,11 @@ apps: auth_username: '{{ gearpump_broker_user }}' auth_password: '{{ gearpump_broker_user_pass }}' -#To be uncommented after finishing: DPNG-4038 -#- name: h2o-broker -# app_properties: -# env: -# USER_PASSWORD: '{{ h2o_broker_user_pass }}' +- name: h2o-broker + app_properties: + env: + USER_PASSWORD: '{{ h2o_broker_user_pass }}' +# Do not register as a broker to prevent enabling h2o service. Remove after DPNG-6678. # broker_config: # name: h2o # url: https://h2o-broker.{{ apps_domain }} @@ -138,7 +145,7 @@ apps: auth_password: '{{ hbase_broker_user_pass }}' service_instances: - name: hbase-for-atk - plan: shared + plan: bare - name: zookeeper-broker app_properties: @@ -170,6 +177,8 @@ apps: plan: shared - name: zookeeper-gearpump plan: shared + - name: zookeeper-for-hive + plan: shared - name: yarn-broker app_properties: @@ -194,6 +203,25 @@ apps: - name: yarn-gearpump plan: bare +- name: hive-broker + app_properties: + env: + HIVE_SERVER2_THRIFT_BIND_HOST: '{{ namenode_internal_host }}' + HIVE_SERVER2_THRIFT_PORT: 10000 + HIVE_SUPERUSER: '{{ sentry_superuser }}{{ authgateway_principals_suffix if kerberos_host != '' }}' + HIVE_SUPERUSER_KEYTAB: '{{ sentry_keytab_value }}' + HADOOP_PROVIDED_ZIP: '{{ import_hadoop_conf_hive }}' + USER_PASSWORD: '{{ hive_broker_user_pass }}' + SYSTEM_USER: "{{ broker_system_user }}" + SYSTEM_USER_PASSWORD: "{{ broker_system_user_password }}" + broker_config: + name: hive + url: http://hive-broker.{{ apps_domain }} + auth_username: '{{ hive_broker_user }}' + auth_password: '{{ hive_broker_user_pass }}' + service_instances: + - name: hive-instance-shared + plan: shared # APPS IN APPLICATION BROKER - name: gateway @@ -341,6 +369,17 @@ apps: credentials: url: http://metadata.{{ apps_domain }} +- name: workflow-scheduler + app_properties: + env: + JOB_TRACKER: '{{ job_tracker }}' + OOZIE_API_URL: '{{ oozie_server }}' + SQOOP_METASTORE: '{{ metastore }}' + user_provided_services: + - name: workflow-scheduler + credentials: + host: http://workflow-scheduler.{{ apps_domain }} + - name: user-management user_provided_services: - name: user-management @@ -359,10 +398,18 @@ apps: credentials: host: http://model-catalog.{{ apps_domain }} +- name: platform-snapshot + user_provided_services: + - name: platform-snapshot + credentials: + host: http://platform-snapshot.{{ apps_domain }} + - name: platform-context app_properties: env: CF_CLI_VERSION: '{{ cf_cli_version }}' + PLATFORM_VERSION: '{{ platform_version }}' + PLATFORM_COREORG: '{{ core_org_name }}' user_provided_services: - name: platformcontext credentials: @@ -429,7 +476,7 @@ apps: SERVICE_PLAN_NAME: shared TAGS: kerberos service_broker_names: kerberos - CREDENTIALS: '{"kdc": "{{ kerberos_host }}", "krealm": "{{ kerberos_realm if kerberos_host != "" }}", "kcacert": "{{ kerberos_cacert }}", "kuser": "{{ kerberos_username }}", "kpassword": "{{ kerberos_password if kerberos_host != "" }}", "enabled": {{ "true" if kerberos_host != "" else "false"}} }' + CREDENTIALS: '{"kdc": "{{ kerberos_host if kerberos_host != "" else " " }}", "krealm": "{{ kerberos_realm if kerberos_host != "" else " " }}", "kcacert": "{{ kerberos_cacert }}", "kuser": "{{ kerberos_username }}", "kpassword": "{{ kerberos_password if kerberos_host != "" else " " }}", "enabled": {{ "true" if kerberos_host != "" else "false"}} }' VERSION: '{{ wssb_version }}' broker_config: name: kerberos @@ -569,6 +616,12 @@ brokers: # - name: cassandra-example # plan: v3.3 # label: cassandra + - name: platform-snapshot-db + plan: free + label: postgresql93 + - name: workflow-scheduler-db + plan: free + label: postgresql93 - name: postgres-for-atk plan: free label: postgresql93 diff --git a/templates/template_variables.yml b/templates/template_variables.yml index bd096d6..e7c3626 100644 --- a/templates/template_variables.yml +++ b/templates/template_variables.yml @@ -21,6 +21,9 @@ CDH_cluster_name: CDH-cluster # username of the Cloud Foundry admin cf_admin_username: admin +# version of TAP +platform_version: 0.7.0 + # version of CF CLI compatiple with the platform cf_cli_version: 6.12.4 @@ -159,6 +162,12 @@ yarm_broker_user: user # password that the YARN broker will use yarn_broker_user_pass: little-gold-bubbles-heavenly +# username that the HIVE broker will use +hive_broker_user: user + +# password that the HIVE broker will use +hive_broker_user_pass: little-platinum-bubbles-heavenly + # username that the KERBEROS broker will use kerberos_broker_user: admin @@ -232,6 +241,7 @@ h2o_provisioner_port: import_hadoop_conf_hbase: import_hadoop_conf_hdfs: import_hadoop_conf_yarn: +import_hadoop_conf_hive: # *** set only for Kerberos-enabled deployment *** # hostname of the Kerberos Key Distribution Center, should be the same as CDH Manager machine diff --git a/apployer/cf_cli_dry_run.py b/tests/fake_cf_outputs.py similarity index 57% rename from apployer/cf_cli_dry_run.py rename to tests/fake_cf_outputs.py index 6c06ce7..fa7f63c 100644 --- a/apployer/cf_cli_dry_run.py +++ b/tests/fake_cf_outputs.py @@ -14,10 +14,34 @@ # limitations under the License. # +GET_ENV_SUCCESS = """ +Getting env variables for app data-catalog in org seedorg / space seedspace as michal.bultrowicz@intel.com... +OK + +System-Provided: +{ + "VCAP_SERVICES": { + +} + +{ + "VCAP_APPLICATION": { + "application_name": "FAKYFAKE" + } +} + +User-Provided: +LOG_LEVEL: INFO +VERSION: 6.6.6 + +No running env variables have been set + +No staging env variables have been set + """ -Providing a class with static functions having identical signatures as functions in cf_cli. -Functions that don't introduce any changes to Cloud Foundry will remain, others will just log what -they would normally do. -All of this is to elegantly provide an option of a dry run for deployer. +GET_SERVICE_INFO_SUCCESS = """ + +Service instance: bla +Service: user-provided """ diff --git a/tests/fake_cli_outputs.py b/tests/fake_cli_outputs.py deleted file mode 100644 index a38ca46..0000000 --- a/tests/fake_cli_outputs.py +++ /dev/null @@ -1,31 +0,0 @@ -GET_ENV_SUCCESS = """ -Getting env variables for app data-catalog in org seedorg / space seedspace as michal.bultrowicz@intel.com... -OK - -System-Provided: -{ - "VCAP_SERVICES": { - -} - -{ - "VCAP_APPLICATION": { - "application_name": "FAKYFAKE" - } -} - -User-Provided: -LOG_LEVEL: INFO -VERSION: 6.6.6 - -No running env variables have been set - -No staging env variables have been set - -""" - -GET_SERVICE_INFO_SUCCESS = """ - -Service instance: bla -Service: user-provided -""" diff --git a/tests/fake_module.py b/tests/fake_module.py index 982b3b9..38c9304 100644 --- a/tests/fake_module.py +++ b/tests/fake_module.py @@ -20,6 +20,7 @@ FUNC_D_RETURN = "I'm the output of a function that shouldn't be replaced." +FUNC_E_RETURN = "I'm the output of another function that shouldn't be replaced." def some_function_a(param_a_1, param_a_2): @@ -39,4 +40,4 @@ def some_function_d(): def _some_function_e(): - print("I'm a protected function.") + return FUNC_E_RETURN diff --git a/tests/test_app_compare.py b/tests/test_app_compare.py new file mode 100644 index 0000000..4b62630 --- /dev/null +++ b/tests/test_app_compare.py @@ -0,0 +1,156 @@ +# +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from mock import MagicMock +import pytest + +from apployer import app_compare +from apployer import cf_cli +from apployer.appstack import AppConfig + + +@pytest.fixture +def mock_cf_cli(monkeypatch): + mock_cf = MagicMock() + monkeypatch.setattr('apployer.app_compare.cf_cli', mock_cf) + mock_cf.CommandFailedError = cf_cli.CommandFailedError + return mock_cf + + +@pytest.fixture +def mock_cf_api(monkeypatch): + cf_api = MagicMock() + monkeypatch.setattr('apployer.app_compare.cf_api', cf_api) + return cf_api + + +@pytest.fixture +def fake_app_summary(): + return { + "name": "some-app", + "routes": [ + { + "guid": "0bb42a5b-2dbf-439a-852e-ea8c686199e7", + "host": "someapp", + "domain": { + "guid": "7e8c9473-ae02-4567-beb2-8dd67104d3d0", + "name": "example.com" + } + } + ], + "running_instances": 1, + "services": [ + { + "guid": "15d10bb0-6b77-44f3-be20-c51857addc3f", + "name": "service_1" + }, + { + "guid": "f7892fac-419f-4d6c-b0c6-6759a619ab87", + "name": "service_2" + } + ], + "buildpack": "python_buildpack", + "environment_json": { + "SOME_VALUE": "FAKE", + "SOME_OTHER_VALUE": "not present in app config" + }, + "some_additional_field": True, + "memory": 64, + "instances": 2, + "disk_quota": 1024, + "command": "python fake_fake/fake_app.py", + } + + +@pytest.fixture +def app(): + """App config that is consistent with values in `fake_app_summary`.""" + return AppConfig('some-app', app_properties={ + 'buildpack': 'python_buildpack', + 'command': 'python fake_fake/fake_app.py', + 'disk_quota': '1G', + 'env': {'SOME_VALUE': 'FAKE'}, + 'instances': 2, + 'memory': '64M', + 'services': ['service_1', 'service_2'], + 'host': 'someapp' + }) + + +@pytest.mark.parametrize('appstack_version, summary_version, should_update', [ + (None, None, False), + (None, '0.0.1', False), + ('0.0.1', None, True), + ('0.1.2', '0.1.2', False), + ('0.1.3', '0.1.2', True), + ('0.1.2', '0.1.3', False), +]) +def test_should_update(mock_cf_cli, mock_cf_api, fake_app_summary, app, + appstack_version, summary_version, should_update): + if appstack_version: + app.app_properties['env']['VERSION'] = appstack_version + if summary_version: + fake_app_summary['environment_json']['VERSION'] = summary_version + app_guid = 'some-fake-guid' + mock_cf_cli.get_app_guid.return_value = app_guid + mock_cf_api.get_app_summary.return_value = fake_app_summary + + assert app_compare.should_update(app) == should_update + + mock_cf_cli.get_app_guid.assert_called_with(app.name) + mock_cf_api.get_app_summary.assert_called_with(app_guid) + + +def test_should_update_no_app_in_live_env(mock_cf_cli, app): + mock_cf_cli.get_app_guid.side_effect = cf_cli.CommandFailedError + assert app_compare.should_update(app) + + +@pytest.mark.parametrize('app_properties, app_summary, are_different', [ + ({'buildpack': 'python_buildpack'}, {'buildpack': 'python_buildpack'}, False), + ({'buildpack': 'python_buildpack'}, {'buildpack': 'python_buildpack', 'asdasd': 1}, False), + ({'buildpack': 'python_buildpack'}, {'buildpack': 'bla_buildpack'}, True), + ({'instances': 2}, {'instances': 3}, True), + ({'memory': '64M'}, {'memory': 65}, True), + ({'memory': '64M'}, {'memory': 64}, False), + ({'disk_quota': '2G'}, {'disk_quota': 2048}, False), + ({'env': {'bla': 1}}, {'environment_json': {'bla': 2}}, True), + ({'host': 'abc'}, {'routes': [{'host': 'qwe'}]}, True), + ({'host': 'abc'}, {'routes': [{'host': 'qwe'}, {'host': 'abc'}]}, False), + ({'services': ['service_1', 'service_2']}, {'services': [{'name': 'service_1'}]}, True) +]) +def test_compare(app_properties, app_summary, are_different): + assert app_compare._properties_differ(app_properties, app_summary) == are_different + + +@pytest.mark.parametrize('dict_a, dict_b, is_part_of', [ + ({'a': 'b'}, {'a': 'b'}, True), + ({'a': 'b'}, {'a': 'b', 'c': 1}, False), + ({'a': 'b', 'c': 1}, {'a': 'b'}, True), + ({'a': 'c'}, {'a': 'b'}, False), +]) +def test_dict_is_part_of(dict_a, dict_b, is_part_of): + assert app_compare._dict_is_part_of(dict_a, dict_b) == is_part_of + + +@pytest.mark.parametrize('value_string, megabytes', [ + ('400M', 400), + ('2MB', 2), + ('4G', 4096), + ('1GB', 1024), +]) +def test_normalize_to_megabytes(value_string, megabytes): + assert app_compare._normalize_to_megabytes(value_string) == megabytes diff --git a/tests/test_cf_api.py b/tests/test_cf_api.py index 232341c..d30e89e 100644 --- a/tests/test_cf_api.py +++ b/tests/test_cf_api.py @@ -204,3 +204,11 @@ def test_get_app_name(mock_popen): mock_popen.set_command("cf curl /v2/apps/{}".format(app_guid), stdout=app_description) assert cf_api.get_app_name(app_guid) == app_name + + +def test_get_app_summary(mock_popen): + app_guid = 'some-fake-guid' + fake_app_summary = '{"bla": "something", "ble": true}\n' + mock_popen.set_command("cf curl /v2/apps/{}/summary".format(app_guid), stdout=fake_app_summary) + + assert cf_api.get_app_summary(app_guid) == {'bla': 'something', 'ble': True} diff --git a/tests/test_cf_cli.py b/tests/test_cf_cli.py index 9bca1aa..6f55e55 100644 --- a/tests/test_cf_cli.py +++ b/tests/test_cf_cli.py @@ -17,7 +17,7 @@ from mock import call import pytest -from .fake_cli_outputs import GET_ENV_SUCCESS, GET_SERVICE_INFO_SUCCESS +from .fake_cf_outputs import GET_ENV_SUCCESS, GET_SERVICE_INFO_SUCCESS from apployer import cf_cli from apployer.cf_cli import CommandFailedError, CfInfo @@ -206,6 +206,18 @@ def test_create_space(mock_popen): cf_cli.create_space(space_name, org_name) +def test_get_brokers(mock_popen): + cmd_output = """Getting service brokers as admin... + +name url +application-broker http://application-broker.example.com +cdh http://cdh-broker.example.com +""" + mock_popen.set_command('cf service-brokers', stdout=cmd_output) + + assert cf_cli.service_brokers() == {'application-broker', 'cdh'} + + def test_get_buildpacks(mock_popen): cmd_output = """Getting buildpacks... @@ -233,10 +245,12 @@ def test_get_oauth_token(mock_popen): assert cf_cli.oauth_token() == 'bearer abcdf.1233456789.fdcba' -def test_get_service_guid(mock_popen): - service_guid = '8b89a54b-b292-49eb-a8c4-2396ec038120' - cmd_output = service_guid + '\n' - service_name = 'some-service' - mock_popen.set_command('cf service --guid {}'.format(service_name), stdout=cmd_output) +def test_get_app_and_service_guid(mock_popen): + guid = '8b89a54b-b292-49eb-a8c4-2396ec038120' + cmd_output = guid + '\n' + name = 'something' + mock_popen.set_command('cf service --guid {}'.format(name), stdout=cmd_output) + mock_popen.set_command('cf app --guid {}'.format(name), stdout=cmd_output) - assert cf_cli.get_service_guid(service_name) == service_guid + assert cf_cli.get_app_guid(name) == guid + assert cf_cli.get_service_guid(name) == guid diff --git a/tests/test_cf_cli_dry_run.py b/tests/test_cf_cli_dry_run.py deleted file mode 100644 index ec8fca4..0000000 --- a/tests/test_cf_cli_dry_run.py +++ /dev/null @@ -1,33 +0,0 @@ -# -# Copyright (c) 2016 Intel Corporation -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# from apployer import cf_cli_dry_run -# from . import fake_module -# -# -# def test_provide_dry_run_module(): -# exceptions = ['some_function_d'] -# dry_run_fake_module = cf_cli_dry_run.provide_dry_run_module(fake_module, exceptions) -# -# # provide_dry_run_module needs to create a logger for the fake module and we need to capture it -# -# handler = logging.StreamHandler(sys.stdout) -# handler.setFormatter(log_formatter) -# -# project_logger = logging.getLogger(apployer.__name__) -# project_logger.setLevel(level) -# project_logger.addHandler(handler) - diff --git a/tests/test_deployer.py b/tests/test_deployer.py index 19d272b..8e0cfe7 100644 --- a/tests/test_deployer.py +++ b/tests/test_deployer.py @@ -27,7 +27,6 @@ ServiceInstance) from apployer.cf_cli import CommandFailedError, CfInfo, BuildpackDescription -from .fake_cli_outputs import GET_ENV_SUCCESS from .utils import get_appstack_resource from .test_cf_api import SERVICE_BINDING @@ -96,21 +95,6 @@ def test_app_deploy(app_deployer, mock_upsi_deployer, mock_setup_broker): mock_setup_broker.assert_called_with(broker) -def test_get_app_version(app_deployer, mock_cf_cli): - app_version = '6.6.6' - mock_cf_cli.env.return_value = GET_ENV_SUCCESS - - assert app_deployer._get_app_version() == app_version - mock_cf_cli.env.assert_called_once_with(app_deployer.app.name) - - -def test_get_app_version_fail(app_deployer, mock_cf_cli): - mock_cf_cli.env.return_value = 'some fake output \n without any version' - - with pytest.raises(deployer.AppVersionNotFoundError): - app_deployer._get_app_version() - - def test_prepare_app(artifacts_location, app_deployer): prepared_app_path = app_deployer.prepare(artifacts_location) @@ -128,25 +112,22 @@ def test_prepare_app_no_artifact(app_deployer): app_deployer.prepare('/some/fake/location') -@pytest.mark.parametrize('live_version, appstack_version, strategy, push_needed', [ - ('0.0.1', '0.0.2', deployer.UPGRADE_STRATEGY, True), - ('0.0.3', '0.0.2', deployer.UPGRADE_STRATEGY, False), - ('0.0.1', '0.0.2', deployer.PUSH_ALL_STRATEGY, True), - ('0.0.3', '0.0.2', deployer.PUSH_ALL_STRATEGY, True), -]) -def test_check_app_push_needed(live_version, appstack_version, strategy, push_needed): - app = AppConfig('bla', app_properties={'env': {'VERSION': appstack_version}}) +def test_check_app_push_needed_push_all(): + app = AppConfig('bla') app_deployer = deployer.AppDeployer(app, 'some-fake-path') - app_deployer._get_app_version = lambda: live_version - assert app_deployer._check_push_needed(strategy) == push_needed + assert app_deployer._check_push_needed(deployer.PUSH_ALL_STRATEGY) -def test_check_app_push_needed_get_version_fail(): - app_deployer = deployer.AppDeployer(AppConfig('bla'), 'some-fake-path') - app_deployer._get_app_version = MagicMock(side_effect=CommandFailedError) +def test_check_app_push_needed_upgrade(monkeypatch): + mock_should_update = MagicMock(return_value=False) + monkeypatch.setattr('apployer.deployer.app_compare.should_update', mock_should_update) + app = AppConfig('bla') + app_deployer = deployer.AppDeployer(app, 'some-fake-path') + + assert not app_deployer._check_push_needed(deployer.UPGRADE_STRATEGY) - assert app_deployer._check_push_needed(deployer.UPGRADE_STRATEGY) + mock_should_update.assert_called_with(app) @pytest.fixture @@ -209,19 +190,21 @@ def mock_enable_broker_access(monkeypatch): def test_setup_broker_new(broker, mock_enable_broker_access, mock_setup_service, mock_cf_cli): - mock_cf_cli.update_service_broker.side_effect = CommandFailedError + mock_cf_cli.service_brokers.return_value = {'no', 'broker', 'that', 'we', 'want'} deployer.setup_broker(broker) mock_cf_cli.create_service_broker.assert_called_with(broker.name, broker.auth_username, broker.auth_password, broker.url) mock_enable_broker_access.assert_called_with(broker) + assert mock_cf_cli.service_brokers.call_args_list assert mock.call(broker, broker.service_instances[0]) in mock_setup_service.call_args_list assert mock.call(broker, broker.service_instances[1]) in mock_setup_service.call_args_list def test_setup_broker_update(mock_cf_cli, mock_setup_service, mock_enable_broker_access): broker = BrokerConfig('some-name', 'https://some-name.example.com', 'username', 'password') + mock_cf_cli.service_brokers.return_value = {broker.name} deployer.setup_broker(broker) @@ -429,8 +412,8 @@ def test_deploy_appstack(monkeypatch, mock_upsi_deployer, mock_setup_broker): mock_register_in_app_broker) # act - deployer.deploy_appstack(cf_login_data, appstack, - artifacts_path, deployer.UPGRADE_STRATEGY) + deployer.deploy_appstack(cf_login_data, appstack, artifacts_path, + deployer.UPGRADE_STRATEGY, False) # assert mock_prep_org_and_space.assert_called_with(cf_login_data) @@ -450,6 +433,22 @@ def test_deploy_appstack(monkeypatch, mock_upsi_deployer, mock_setup_broker): deployer.DEPLOYER_OUTPUT, artifacts_path) +def test_deploy_appstack_dry_run(monkeypatch): + fake_cf_login, fake_appstack, fake_artifacts_path, fake_strategy = 1, 2, 3, 4 + mock_do_deploy = MagicMock() + monkeypatch.setattr('apployer.deployer._do_deploy', mock_do_deploy) + real_cf_cli = deployer.cf_cli + real_register_in_app_broker = deployer.register_in_application_broker + + deployer.deploy_appstack(fake_cf_login, fake_appstack, fake_artifacts_path, + fake_strategy, True) + + mock_do_deploy.assert_called_with(fake_cf_login, fake_appstack, + fake_artifacts_path, fake_strategy) + assert deployer.cf_cli is real_cf_cli + assert deployer.register_in_application_broker is real_register_in_app_broker + + def test_register_in_app_broker(monkeypatch, mock_check_call): # arrange app_env = {'display_name': 'blabla', diff --git a/tests/test_dry_run.py b/tests/test_dry_run.py new file mode 100644 index 0000000..9e0aecb --- /dev/null +++ b/tests/test_dry_run.py @@ -0,0 +1,47 @@ +# +# Copyright (c) 2016 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from mock import MagicMock + +from apployer import dry_run +from . import fake_module + + +def test_provide_dry_run_module(monkeypatch): + exceptions = ['some_function_d'] + mock_dry_run_log = MagicMock() + monkeypatch.setattr('apployer.dry_run._log', mock_dry_run_log) + + dry_run_fake_module = dry_run.provide_dry_run_module(fake_module, exceptions) + dry_run_fake_module.some_function_a('param 1', 'param 2') + dry_run_fake_module.some_function_b(1) + dry_run_fake_module.some_function_c() + assert dry_run_fake_module.some_function_d() == fake_module.FUNC_D_RETURN + assert dry_run_fake_module._some_function_e() == fake_module.FUNC_E_RETURN + + assert len(mock_dry_run_log.info.call_args_list) == 3 + first_call = mock_dry_run_log.info.call_args_list[0] + log_func_name = 'some_function_a' + log_func_params = {'param_a_1': 'param 1', 'param_a_2': 'param 2'} + assert first_call[0][0].count('%s') == 2 + assert first_call[0][1] == log_func_name + assert first_call[0][2] == log_func_params + + +def test_get_dry_run_cf_cli(): + dry_run_cf_cli = dry_run.get_dry_run_cf_cli() + dry_run_cf_cli.restart('some-fake-app') + dry_run_cf_cli.create_user_provided_service('some-fake-upsi', {'a': 'b'}) \ No newline at end of file