From 1804d98dfb8dcb2f338e37f57a9841959ca5ade7 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 22 Jun 2023 03:31:55 +0200 Subject: [PATCH 01/34] netapp_ontap: Initialize New formula for managing NetApp ONTAP devices. This is intended to replace the LUN/disk management parts in the messy orchestration modules. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 95 +++++++++++++++++++++ netapp_ontap-formula/_utils/ontap_config.py | 32 +++++++ 2 files changed, 127 insertions(+) create mode 100644 netapp_ontap-formula/_modules/ontap.py create mode 100644 netapp_ontap-formula/_utils/ontap_config.py diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py new file mode 100644 index 00000000..58fb9474 --- /dev/null +++ b/netapp_ontap-formula/_modules/ontap.py @@ -0,0 +1,95 @@ +""" +Salt execution module for maging ONTAP based NetApp storage systems using Ansible +Copyright (C) 2023 SUSE LLC + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import logging + +log = logging.getLogger(__name__) + +# https://stackoverflow.com/a/9808122 +def _find(key, value): + for k, v in value.items(): + if k == key: + yield v + elif isinstance(v, dict): + for result in find(key, v): + yield result + elif isinstance(v, list): + for d in v: + for result in find(key, d): + yield result + +def _config(): + return __utils__['ontap_config.config']() + +def _call(host, certificate, key, rundir, playbook, extravars={}, descend=[]): + host, colon, port = host.rpartition(':') + varmap = {'ontap_host': host, 'ontap_port': int(port), 'ontap_crt': certificate, 'ontap_key': key} + if extravars: + varmap.update(extravars) + log.debug(f'ontap_ansible: executing {playbook} with {varmap} in {rundir}') + out = __salt__['ansible.playbooks']( + playbook=playbook, rundir=rundir, extra_vars=varmap) + + plays = out.get('plays', []) + plays_len = len(plays) + if not plays_len: + log.error(f'ontap_ansible: returned with no plays') + return False + if plays_len > 0: + log.warning(f'ontap_ansible: discarding {plays_len} additional plays') + play0 = plays[0] + + tasks = play0.get('tasks', []) + tasks_len = len(tasks) + if not tasks_len: + log.error(f'ontap_ansible: play returned with no tasks') + return False + if tasks_len > 0: + log.warning(f'ontap_ansible: discarding {tasks_len} additional tasks') + task0 = tasks[0] + + task = task0.get('hosts').get('localhost') + if task is None: + log.error(f'ontap_ansible: unable to parse task - ensure it executed locally') + return False + + if descend: + if isinstance(descend, str): + descend = [descend] + for level in descend: + gain = task.get(level) + if gain is None: + break + if gain is not None: + log.debug(f'ontap_ansible: found artifact for {level}') + task = gain + + return task + + +def get_lun(uuid=None): + varmap = _config() + if uuid: + varmap.update({'extravars': {'ontap_lun_uuid': uuid}}) + playbook = 'fetch-lun_restit' + else: + playbook = 'fetch-luns_restit' + descend = ['response', 'records'] + varmap.update({'playbook': f'playbooks/{playbook}.yml', 'descend': descend}) + result = _call(**varmap) + return result diff --git a/netapp_ontap-formula/_utils/ontap_config.py b/netapp_ontap-formula/_utils/ontap_config.py new file mode 100644 index 00000000..f236ac4d --- /dev/null +++ b/netapp_ontap-formula/_utils/ontap_config.py @@ -0,0 +1,32 @@ +""" +Salt utility module for providing functions used by other ONTAP related modules +Copyright (C) 2023 SUSE LLC + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import logging + +log = logging.getLogger(__name__) + +def config(): + config = __pillar__.get('netapp_ontap', {}) + host = config.get('host') + certificate = config.get('certificate') + key = config.get('key') + rundir = config.get('rundir') + if None in [host, certificate, key, rundir]: + log.error('netapp_ontap: configuration is incomplete!') + return False + return {'host': host, 'certificate': certificate, 'key': key, 'rundir': rundir} From 07ac3e33cb0801ccd3df8c7ce58225a4ff4846ca Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 22 Jun 2023 04:29:26 +0200 Subject: [PATCH 02/34] netapp_ontap: Support LUN query by comment Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index 58fb9474..a3c79db6 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -82,14 +82,27 @@ def _call(host, certificate, key, rundir, playbook, extravars={}, descend=[]): return task -def get_lun(uuid=None): +def get_lun(comment=None, uuid=None): + if (comment is not None) and (uuid is not None): + log.error(f'Only a single filter may be specified') + raise ValueError('Only a single filter may be specified') varmap = _config() - if uuid: - varmap.update({'extravars': {'ontap_lun_uuid': uuid}}) + extravars = None + + if comment: + playbook = 'fetch-lun-by-comment_restit' + extravars = {'extravars': {'ontap_lun_comment': comment}} + elif uuid: playbook = 'fetch-lun_restit' - else: + extravars = {'extravars': {'ontap_lun_uuid': uuid}} + + if extravars is None: playbook = 'fetch-luns_restit' + else: + varmap.update(extravars) + descend = ['response', 'records'] varmap.update({'playbook': f'playbooks/{playbook}.yml', 'descend': descend}) + result = _call(**varmap) return result From 1f66c0867ee656ce8644a1c85d60169c7bf1a463 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Fri, 23 Jun 2023 15:55:27 +0200 Subject: [PATCH 03/34] netapp_ontap: Support LUN deletions; parse results Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 72 ++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index a3c79db6..fbe45c33 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -17,6 +17,7 @@ """ import logging +import re log = logging.getLogger(__name__) @@ -81,6 +82,40 @@ def _call(host, certificate, key, rundir, playbook, extravars={}, descend=[]): return task +def _result(result): + log.debug(f'ontap_ansible: parsing result: {result}') + + error = result.get('error_message') + status = result.get('status_code') + changed = result.get('changed') + response = result.get('response') + method = result.get('invocation', {}).get('module_args', {}).get('method') + if response is not None and 'num_records' in response: + records = response['num_records'] + elif method == 'POST': + # API does not return a record number for any creation calls, we cannot tell how many items changed + records = None + else: + # API does not return a record number if a DELETE call did not yield any deletions + records = 0 + + res = {} + + if status >= 400 and error: + __context__["retcode"] = 2 + res = {'error': error, 'result': False} + if 200 <= status < 300: + res = {'result': True} + + if res: + resmap = {'status': status} + if records is not None: + resmap.update({'changed': records}) + resmap.update(res) + return resmap + + log.warning('ontap_ansible: dumping unknown result') + return result def get_lun(comment=None, uuid=None): if (comment is not None) and (uuid is not None): @@ -106,3 +141,40 @@ def get_lun(comment=None, uuid=None): result = _call(**varmap) return result + +# https://stackoverflow.com/a/60708339 +# based on https://stackoverflow.com/a/42865957/2002471 +units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40} +def _parse_size(size): + size = size.upper() + #print("parsing size ", size) + if not re.match(r' ', size): + size = re.sub(r'([KMGT]?B)', r' \1', size) + number, unit = [string.strip() for string in size.split()] + return int(float(number)*units[unit]) + +def provision_lun(name, size, lunid, volume, vserver): + varmap = _config() + size = _parse_size(size) + varmap.update({'playbook': 'playbooks/deploy-lun_restit.yml', 'extravars': {'ontap_comment': name, 'ontap_lun_id': lunid, 'ontap_volume': volume, 'ontap_vserver': vserver, 'ontap_size': size}}) + result = _call(**varmap) + return _result(result) + +def _delete_lun(name=None, volume=None, uuid=None): + if (name is None or volume is None) and (uuid is None): + log.error('Specify either name and volume or uuid') + raise ValueError('Specify either name and volume or uuid') + varmap = _config() + if name and volume: + extravars = {'ontap_volume': volume, 'ontap_lun_name': name} + elif uuid: + extravars = {'ontap_lun_uuid': uuid} + varmap.update({'playbook': 'playbooks/delete-lun_restit.yml', 'extravars': extravars}) + result = _call(**varmap) + return _result(result) + +def delete_lun_name(name, volume): + return _delete_lun(name, volume) + +def delete_lun_uuid(uuid): + return _delete_lun(uuid=uuid) From ad369b257811bc5f9aad509826c410826603c714 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Fri, 23 Jun 2023 20:30:21 +0200 Subject: [PATCH 04/34] netapp_ontap: Support LUN mappings Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 29 ++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index fbe45c33..3a250065 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -178,3 +178,32 @@ def delete_lun_name(name, volume): def delete_lun_uuid(uuid): return _delete_lun(uuid=uuid) + +def get_lun_mapping(comment): + query = get_lun(comment) + resmap = {} + for lun in query: + log.debug(f'netapp_ontap: parsing LUN {lun}') + name = lun.get('name') + mapped = lun.get('status', {}).get('mapped') + resmap.update({name: mapped}) + if None in resmap: + log.error('netapp_ontap: invalid LUN mapping map') + return resmap + +def _path(volume, name): + return f'/vol/{volume}/{name}' + +def map_lun(name, lunid, volume, vserver, igroup): + varmap = _config() + path = _path(volume, name) + varmap.update({'playbook': 'playbooks/map-lun_restit.yml', 'extravars': {'ontap_lun_id': lunid, 'ontap_lun_path': path, 'ontap_vserver': vserver, 'ontap_igroup': igroup}}) + result = _call(**varmap) + return _result(result) + +def unmap_lun(name, volume, igroup): + varmap = _config() + path = _path(volume, name) + varmap.update({'playbook': 'playbooks/unmap-lun_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_igroup': igroup}}) + result = _call(**varmap) + return _result(result) From 912edfc4c6acdb3c6fc33c510a1320646ba09383 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 01:58:00 +0200 Subject: [PATCH 05/34] netapp_ontap: Improve Ansible logic Handle executions with multiple tasks. Facilitate more REST operations in results parsing. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 64 ++++++++++++++++---------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index 3a250065..856ac2b8 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -37,7 +37,10 @@ def _find(key, value): def _config(): return __utils__['ontap_config.config']() -def _call(host, certificate, key, rundir, playbook, extravars={}, descend=[]): +def _path(volume, name): + return f'/vol/{volume}/{name}' + +def _call(host, certificate, key, rundir, playbook, extravars={}, descend=[], single_task=True): host, colon, port = host.rpartition(':') varmap = {'ontap_host': host, 'ontap_port': int(port), 'ontap_crt': certificate, 'ontap_key': key} if extravars: @@ -60,40 +63,51 @@ def _call(host, certificate, key, rundir, playbook, extravars={}, descend=[]): if not tasks_len: log.error(f'ontap_ansible: play returned with no tasks') return False - if tasks_len > 0: + if tasks_len > 0 and single_task: log.warning(f'ontap_ansible: discarding {tasks_len} additional tasks') - task0 = tasks[0] - - task = task0.get('hosts').get('localhost') - if task is None: - log.error(f'ontap_ansible: unable to parse task - ensure it executed locally') - return False - - if descend: - if isinstance(descend, str): - descend = [descend] - for level in descend: - gain = task.get(level) - if gain is None: - break - if gain is not None: - log.debug(f'ontap_ansible: found artifact for {level}') - task = gain - return task + mytasks = [] + for task in tasks: + mytask = task.get('hosts').get('localhost') + if mytask is None: + log.error(f'ontap_ansible: unable to parse task - ensure it executed locally') + return False + + if descend: + if isinstance(descend, str): + descend = [descend] + for level in descend: + gain = mytask.get(level) + if gain is None: + break + if gain is not None: + log.debug(f'ontap_ansible: found artifact for {level}') + mytask = gain + + if single_task: + break + mytasks.append(mytask) + + if single_task: + return mytask + return mytasks def _result(result): log.debug(f'ontap_ansible: parsing result: {result}') + if isinstance(result, bool): + log.error(f'ontap_ansible: result seems like a failed execution, refusing to parse') + return False + error = result.get('error_message') status = result.get('status_code') - changed = result.get('changed') + records = result.get('changed') response = result.get('response') method = result.get('invocation', {}).get('module_args', {}).get('method') if response is not None and 'num_records' in response: records = response['num_records'] elif method == 'POST': - # API does not return a record number for any creation calls, we cannot tell how many items changed + # API does not return a record number for any creation calls, we cannot tell how many items records records = None else: # API does not return a record number if a DELETE call did not yield any deletions @@ -104,15 +118,17 @@ def _result(result): if status >= 400 and error: __context__["retcode"] = 2 res = {'error': error, 'result': False} - if 200 <= status < 300: + if method in set(['DELETE', 'POST']) and 200 <= status < 300: res = {'result': True} if res: resmap = {'status': status} if records is not None: - resmap.update({'changed': records}) + resmap.update({'records': records}) resmap.update(res) return resmap + elif status == 200: + return response log.warning('ontap_ansible: dumping unknown result') return result From e2f2c00e186f6a7fe295dfaf0b3b0496dad6e02f Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 01:59:09 +0200 Subject: [PATCH 06/34] netapp_ontap: Support "next free LUN" query Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index 856ac2b8..6f6166c1 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -133,7 +133,7 @@ def _result(result): log.warning('ontap_ansible: dumping unknown result') return result -def get_lun(comment=None, uuid=None): +def get_lun(comment=None, uuid=None, get_next_free=False): if (comment is not None) and (uuid is not None): log.error(f'Only a single filter may be specified') raise ValueError('Only a single filter may be specified') @@ -155,6 +155,10 @@ def get_lun(comment=None, uuid=None): descend = ['response', 'records'] varmap.update({'playbook': f'playbooks/{playbook}.yml', 'descend': descend}) + if get_next_free and comment is None and uuid is None: + result = _call(**varmap, single_task=False) + next_free = result[5].get('ansible_facts', {}).get('lun_id') + return result[0], next_free result = _call(**varmap) return result From 8caa153478ea04aabed2dda10fc3c55af091fb33 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 02:02:26 +0200 Subject: [PATCH 07/34] netapp_ontap: Support LUN provisioning with comments Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index 6f6166c1..60a52c9c 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -173,10 +173,13 @@ def _parse_size(size): number, unit = [string.strip() for string in size.split()] return int(float(number)*units[unit]) -def provision_lun(name, size, lunid, volume, vserver): +def provision_lun(name, size, volume, vserver, comment=None): varmap = _config() size = _parse_size(size) - varmap.update({'playbook': 'playbooks/deploy-lun_restit.yml', 'extravars': {'ontap_comment': name, 'ontap_lun_id': lunid, 'ontap_volume': volume, 'ontap_vserver': vserver, 'ontap_size': size}}) + path = _path(volume, name) + varmap.update({'playbook': 'playbooks/deploy-lun_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_volume': volume, 'ontap_vserver': vserver, 'ontap_size': size}}) + if comment is not None: + varmap['extravars'].update({'ontap_comment': comment}) result = _call(**varmap) return _result(result) From e7a6448db4dcf15dab030de584954334026c3c3d Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 02:02:49 +0200 Subject: [PATCH 08/34] netapp_ontap: Support LUN patching Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index 60a52c9c..126568f8 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -183,6 +183,15 @@ def provision_lun(name, size, volume, vserver, comment=None): result = _call(**varmap) return _result(result) +# to-do: support property updates other than size changes +def update_lun(name, size, volume, vserver): + varmap = _config() + size = _parse_size(size) + path = _path(volume, name) + varmap.update({'playbook': 'playbooks/patch-lun_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_vserver': vserver, 'ontap_size': size}}) + result = _call(**varmap) + return _result(result) + def _delete_lun(name=None, volume=None, uuid=None): if (name is None or volume is None) and (uuid is None): log.error('Specify either name and volume or uuid') From d3d41fc102f0bf58b2a799a4970d694aca5ac449 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 02:03:27 +0200 Subject: [PATCH 09/34] netapp_ontap: Update LUN mapping query functions Split into two functions, one to query whether the LUN is mapped at all, and onne to query details about the mapping. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index 126568f8..ffe96878 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -211,8 +211,14 @@ def delete_lun_name(name, volume): def delete_lun_uuid(uuid): return _delete_lun(uuid=uuid) -def get_lun_mapping(comment): - query = get_lun(comment) +def get_lun_mapped(comment=None, lun_result=None): + if (comment is None) and (lun_result is None): + log.error('Specify either a comment or existing LUN output') + raise ValueError('Specify a comment') + if comment is not None: + query = get_lun(comment) + elif lun_result is not None: + query = lun_result resmap = {} for lun in query: log.debug(f'netapp_ontap: parsing LUN {lun}') @@ -223,8 +229,12 @@ def get_lun_mapping(comment): log.error('netapp_ontap: invalid LUN mapping map') return resmap -def _path(volume, name): - return f'/vol/{volume}/{name}' +def get_lun_mapping(name, volume, igroup): + varmap = _config() + path = _path(volume, name) + varmap.update({'playbook': 'playbooks/get_lun_mapping_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_igroup': igroup}}) + result = _call(**varmap) + return _result(result) def map_lun(name, lunid, volume, vserver, igroup): varmap = _config() From c0d398847118297bbba66ab6fe5a068fdaaefff2 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 02:04:15 +0200 Subject: [PATCH 10/34] netapp_ontap: Add present and mapped states Initial state module. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_states/ontap.py | 248 ++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 netapp_ontap-formula/_states/ontap.py diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py new file mode 100644 index 00000000..a613f786 --- /dev/null +++ b/netapp_ontap-formula/_states/ontap.py @@ -0,0 +1,248 @@ +""" +Salt state module for managing LUNs using the ONTAP Ansible collection +Copyright (C) 2023 SUSE LLC + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import logging +log = logging.getLogger(__name__) + +# Source: https://stackoverflow.com/a/14996816 +suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] +def _humansize(nbytes): + i = 0 + while nbytes >= 1024 and i < len(suffixes)-1: + nbytes /= 1024. + i += 1 + f = ('%.2f' % nbytes).rstrip('0').rstrip('.') + return '%s%s' % (f, suffixes[i]) + +# https://stackoverflow.com/a/60708339 +# based on https://stackoverflow.com/a/42865957/2002471 +units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40} +def _parse_size(size): + size = size.upper() + #print("parsing size ", size) + if not re.match(r' ', size): + size = re.sub(r'([KMGT]?B)', r' \1', size) + number, unit = [string.strip() for string in size.split()] + return int(float(number)*units[unit]) + +def lun_present(name, comment, size, volume, vserver, lunid=None, igroup=None): + path = f'/vol/{volume}/{name}' + ret = {'name': path, 'result': False, 'changes': {}, 'comment': ''} + size_ok = False + map_ok = False + if not None in [lunid, igroup]: + do_map = True + else: + do_map = False + + def _size(details, human=False): + size = details.get('space', {}).get('size') + if size is not None and human: + return _humansize(size) + return size + + # FIXME drop mapping logic from lun_present in favor of lun_mapped + def _map(name, lunid, volume, vserver, igroup): + ok = False + map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) + if map_out.get('result', False) and map_out.get('status') == 201: + comment = f'Mapped LUN to ID {lunid}' + ok = True + # consider another get_lun to validate .. given the queries being expensive in time, it should be combined with the resize validation + else: + comment = 'LUN mapping failed' + return comment, ok + + query = __salt__['ontap.get_lun'](get_next_free=True) + luns = query[0] + next_free = query[1] + + for lun in luns: + lun_path = lun.get('name') + lun_comment = lun.get('comment') + lun_uuid = lun.get('uuid') + if lun_comment == comment or lun_path == path: + log.debug(f'netapp_ontap: found existing LUN {name}') + if lun_uuid is None: + log.error(f'netapp_ontap: found LUN with no UUID') + lun_details = __salt__['ontap.get_lun'](uuid=lun_uuid) + lun_size = _size(lun_details[0], True) + # lun_size_human = needed? + lun_mapping = __salt__['ontap.get_lun_mapped'](lun_result=lun_details) + lun_mapped = lun_mapping.get(name) + # lun_id = needed? + if lun_size == size: + comment_size = f'Size {size} matches' + size_ok = True + elif lun_size != size: + if __opts__['test']: + comment_size = f'Would resize LUN to {size}' + else: + __salt__['ontap.update_lun'](name, size, volume, vserver) + lun2_details = __salt__['ontap.get_lun'](uuid=lun_uuid) + lun2_size = _size(lun2_details[0], True) + comment_size = f'LUN from {lun_size} to {size}' + if lun2_size != lun_size and lun2_size == size: + comment_size = f'Sucessfully resized {comment_size}' + size_ok = True + elif lun2_size == lun_size: + comment_size = f'Failed to resize {comment_size}, it is still {lun2_size}' + else: + comment_size = f'Unexpected outcome while resizing {comment_size}' + + if not do_map: + comment_mapping = None + map_ok = True + else: + if lun_mapped: + comment_mapping = 'Already mapped' + map_ok = True + else: + map_out = _map(name, lunid, volume, vserver, igroup) + comment_mapping = map_out[0] + map_ok = map_out[1] + + #map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) + #if map_out.get('result', False) and map_out.get('status') == 201: + # comment_mapping = f'Mapped LUN to ID {lunid}' + # map_ok = True + # # consider another get_lun to validate .. given the queries being expensive in time, it should be combined with the resize validation + #else: + # comment_mapping = 'LUN mapping failed' + + comment_base = 'LUN is already present' + if size_ok and map_ok: + ret['result'] = True + retcomment = f'{comment_base}; {comment_size}' + if comment_mapping is not None: + retcomment = f'{retcomment}, {comment_mapping}' + if __opts__['test']: + ret['result'] = None + ret['comment'] = retcomment + return ret + + if __opts__['test']: + ret['comment'] = 'Would provision LUN' + ret['result'] = None + return ret + + __salt__['ontap.provision_lun'](name, size, volume, vserver, comment) + if do_map: + map_out = _map(name, lunid, volume, vserver, igroup) + comment_mapping = map_out[0] + map_ok = map_out[1] + lun2_details = __salt__['ontap.get_lun'](comment) + lun2_size = _size(lun2_details) + # FIXME changes dict + + if lun2_details.get('name') == path: + ret['result'] = True + comment_path = f'{path} created' + else: + ret['result'] = False + comment_path = f'{path} not properly created' + + if lun2_size == size: + ret['result'] = True + comment_size = f'with size {size}' + else: + ret['result'] = False + comment_size = f'with mismatching size {lun2_size}' + + comment = f'LUN {comment_path} {comment_size}.' + + if do_map: + if map_ok: + ret['result'] = True + comment_mapping = f'mapped to ID {lunid}' + else: + ret['result'] = False + comment_mapping = f'mapping to ID {lunid} failed' + comment = f'{comment} LUN {comment_mapping}.' + + ret['comment'] = comment + return ret + +def lun_mapped(name, lunid, volume, vserver, igroup): + path = f'/vol/{volume}/{name}' + ret = {'name': path, 'result': False, 'changes': {}, 'comment': ''} + + mapping_out = __salt__['ontap.get_lun_mapping'](name, volume, igroup) + log.debug(f'netapp_ontap: mapping result: {mapping_out}') + current_igroups = [] + current_svms = [] + records = mapping_out.get('num_records') + do_igroup = True + do_svm = True + do_map = False + if records == 0: + do_map = True + if __opts__['test']: + comment = 'Would create mapping' + elif records is not None and records > 0: + for mapping in mapping_out.get('records', []): + this_igroup = mapping.get('igroup', {}).get('name') + if this_igroup is None: + log.error(f'netapp_ontap: unable to determine igroup in mapping result') + else: + if this_igroup not in current_igroups: + current_igroups.append(this_igroup) + this_svm = mapping.get('svm', {}).get('name') + if this_svm is None: + log.error(f'netapp_ontap: unable to determine svm in mapping result') + else: + if this_svm not in current_svms: + current_svms.append(this_svm) + + comment_igroup = f' to igroup {igroup}' + if igroup in current_igroups: + do_igroup = False + + comment_svm = f' in SVM {vserver}' + if vserver in current_svms: + do_svm = False + + comments = f'{comment_igroup}{comment_svm}' + already = False + if do_map or do_igroup or do_svm: + if __opts__['test']: + comment = f'Would map ID {lunid}{comments}' + elif not do_igroup or not do_svm: + comment = f'Already mapped{comment_igroup}{comment_svm}' + ret['result'] = True + else: + log.error('Unhandled mapping state') + comment = 'Something weird happened' + + if __opts__['test']: + ret['result'] = None + if __opts__['test'] or ret['result'] is True: + ret['comment'] = comment + return ret + + map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) + if map_out.get('result', False) and map_out.get('status') == 201: + comment = f'Mapped LUN to ID {lunid} in igroup {igroup}' + ret['result'] = True + else: + comment = 'LUN mapping failed' + ret['result'] = False + + ret['comment'] = comment + return ret + From 9c47cc31cfd928cced308a8e085c6d1017feafd3 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 02:16:27 +0200 Subject: [PATCH 11/34] netapp_ontap: Import Ansible Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/ansible/README.md | 1 + .../ansible/playbooks/delete-lun_restit.yml | 11 ++++++ .../ansible/playbooks/deploy-lun_restit.yml | 11 ++++++ .../playbooks/fetch-lun-by-comment_restit.yml | 12 +++++++ .../ansible/playbooks/fetch-lun_restit.yml | 12 +++++++ .../ansible/playbooks/fetch-luns_restit.yml | 12 +++++++ .../playbooks/get-lun-mapping_restit.yml | 11 ++++++ .../ansible/playbooks/map-lun_restit.yml | 11 ++++++ .../ansible/playbooks/patch-lun_restit.yml | 11 ++++++ .../ansible/playbooks/unmap-lun_restit.yml | 11 ++++++ .../ansible/tasks/create_lun_restit.yml | 22 ++++++++++++ .../ansible/tasks/delete_lun_restit.yml | 16 +++++++++ .../tasks/delete_lun_restit_by_path.yml | 17 +++++++++ .../ansible/tasks/get_lun_by_name_restit.yml | 17 +++++++++ .../ansible/tasks/get_lun_mapping_restit.yml | 18 ++++++++++ .../ansible/tasks/get_lun_restit.yml | 17 +++++++++ .../ansible/tasks/get_luns_restit.yml | 36 +++++++++++++++++++ .../ansible/tasks/map_lun_restit.yml | 24 +++++++++++++ .../ansible/tasks/patch_lun_restit.yml | 25 +++++++++++++ .../ansible/tasks/unmap_lun_restit.yml | 19 ++++++++++ 20 files changed, 314 insertions(+) create mode 100644 netapp_ontap-formula/ansible/README.md create mode 100644 netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/create_lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml create mode 100644 netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/get_lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/get_luns_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/map_lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml create mode 100644 netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml diff --git a/netapp_ontap-formula/ansible/README.md b/netapp_ontap-formula/ansible/README.md new file mode 100644 index 00000000..a741ed6e --- /dev/null +++ b/netapp_ontap-formula/ansible/README.md @@ -0,0 +1 @@ +The Ansible playbooks and tasks are intended to be run by the Salt `ontap` module, however can be run directly if the variables are correctly set. diff --git a/netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml new file mode 100644 index 00000000..28cdd2f3 --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml @@ -0,0 +1,11 @@ +--- +- hosts: localhost + name: Delete LUN + connection: local + gather_facts: False + + tasks: + - name: Delete + block: + - name: Delete LUN + import_tasks: "{{ '../tasks/delete_lun_restit.yml' if ontap_lun_uuid is defined else '../tasks/delete_lun_restit_by_path.yml' }}" diff --git a/netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml new file mode 100644 index 00000000..4272ecc7 --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml @@ -0,0 +1,11 @@ +--- +- hosts: localhost + name: Create LUN + connection: local + gather_facts: False + + tasks: + - name: Deploy + block: + - name: Create LUN + import_tasks: '../tasks/create_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml b/netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml new file mode 100644 index 00000000..1066e744 --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml @@ -0,0 +1,12 @@ +--- +- hosts: localhost + name: Get LUN + connection: local + gather_facts: False + + tasks: + - name: Gather details + block: + - name: Query LUN + import_tasks: '../tasks/get_lun_by_name_restit.yml' + diff --git a/netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml new file mode 100644 index 00000000..7f21d124 --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml @@ -0,0 +1,12 @@ +--- +- hosts: localhost + name: Query LUN + connection: local + gather_facts: False + + tasks: + - name: Gather details + block: + - name: Query LUN + import_tasks: '../tasks/get_lun_restit.yml' + diff --git a/netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml b/netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml new file mode 100644 index 00000000..455cbcad --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml @@ -0,0 +1,12 @@ +--- +- hosts: localhost + name: Query LUNs + connection: local + gather_facts: False + + tasks: + - name: Gather details + block: + - name: Query LUNs + import_tasks: '../tasks/get_luns_restit.yml' + diff --git a/netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml b/netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml new file mode 100644 index 00000000..f1d2cdca --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml @@ -0,0 +1,11 @@ +--- +- hosts: localhost + name: Create LUN + connection: local + gather_facts: False + + tasks: + - name: Deploy + block: + - name: Map LUN + import_tasks: '../tasks/get_lun_mapping_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml new file mode 100644 index 00000000..71d87bf0 --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml @@ -0,0 +1,11 @@ +--- +- hosts: localhost + name: Create LUN + connection: local + gather_facts: False + + tasks: + - name: Deploy + block: + - name: Map LUN + import_tasks: '../tasks/map_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml new file mode 100644 index 00000000..aaba4cdc --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml @@ -0,0 +1,11 @@ +--- +- hosts: localhost + name: Patch LUN + connection: local + gather_facts: False + + tasks: + - name: Deploy + block: + - name: Patch LUN + import_tasks: '../tasks/patch_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml new file mode 100644 index 00000000..1528a014 --- /dev/null +++ b/netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml @@ -0,0 +1,11 @@ +--- +- hosts: localhost + name: Create LUN + connection: local + gather_facts: False + + tasks: + - name: Deploy + block: + - name: Map LUN + import_tasks: '../tasks/unmap_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/tasks/create_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/create_lun_restit.yml new file mode 100644 index 00000000..6cb5f8ec --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/create_lun_restit.yml @@ -0,0 +1,22 @@ +--- +- name: Create LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + http_port: '{{ ontap_port }}' + use_rest: always + validate_certs: false + api: storage/luns + method: POST + body: + name: '{{ ontap_lun_path }}' + comment: "{{ ontap_comment if ontap_comment is defined else 'Provisioned by Ansible'}}" + svm: '{{ ontap_vserver }}' + space: + size: '{{ ontap_size }}' + os_type: linux + register: lun_create_info + +- debug: var=lun_create_info +- assert: { that: lun_create_info.status_code==201 } diff --git a/netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml new file mode 100644 index 00000000..c4442f61 --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml @@ -0,0 +1,16 @@ +--- +- name: Delete LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + http_port: '{{ ontap_port }}' + use_rest: always + validate_certs: false + api: 'storage/luns' + method: DELETE + query: + uuid: '{{ ontap_lun_uuid }}' + register: lun_delete_info + +- assert: { that: lun_delete_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml b/netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml new file mode 100644 index 00000000..57a2685f --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml @@ -0,0 +1,17 @@ +--- +- name: Delete LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + http_port: '{{ ontap_port }}' + use_rest: always + validate_certs: false + api: 'storage/luns' + method: DELETE + query: + name: '/vol/{{ ontap_volume }}/{{ ontap_lun_name }}' + register: lun_delete_info + +- debug: var=lun_delete_info +- assert: { that: lun_delete_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml b/netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml new file mode 100644 index 00000000..caab4469 --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml @@ -0,0 +1,17 @@ +--- +- name: Get LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + http_port: '{{ ontap_port }}' + use_rest: always + validate_certs: false + wait_for_completion: true + api: 'storage/luns' + query: + comment: '{{ ontap_lun_comment }}' + fields: 'space,status.mapped' + register: lun_info + +- assert: { that: lun_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml b/netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml new file mode 100644 index 00000000..a0ae00a2 --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml @@ -0,0 +1,18 @@ +--- +- name: Map LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + http_port: '{{ ontap_port }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + use_rest: always + validate_certs: false + api: protocols/san/lun-maps + query: + igroup: '{{ ontap_igroup }}' + lun: '{{ ontap_lun_path }}' + register: lun_map_out + +- debug: var=lun_map_out +- assert: { that: lun_map_out.status_code==200 } + diff --git a/netapp_ontap-formula/ansible/tasks/get_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/get_lun_restit.yml new file mode 100644 index 00000000..30e4efb3 --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/get_lun_restit.yml @@ -0,0 +1,17 @@ +--- +- name: Get LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + http_port: '{{ ontap_port }}' + use_rest: always + validate_certs: false + wait_for_completion: true + api: 'storage/luns' + query: + uuid: '{{ ontap_lun_uuid }}' + fields: 'space,status.mapped' + register: lun_info + +- assert: { that: lun_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/get_luns_restit.yml b/netapp_ontap-formula/ansible/tasks/get_luns_restit.yml new file mode 100644 index 00000000..3a21f4b5 --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/get_luns_restit.yml @@ -0,0 +1,36 @@ +--- +- name: Get LUNs + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + http_port: '{{ ontap_port }}' + use_rest: always + validate_certs: false + wait_for_completion: true + api: storage/luns + query: + fields: comment + register: lun_info +- debug: var=lun_info +- assert: { that: lun_info.status_code==200 } + +- name: Store LUN dump + local_action: + module: copy + content: '{{ lun_info }}' + dest: /dev/shm/luns.out + +# I'm sure there's a native way to do this... +- name: Get highest LUN + ansible.builtin.shell: + cmd: "grep -Po 'lun\\K(\\d+)' /dev/shm/luns.out | sort -nu | tail -n1" + register: lun_grep + +- name: Define LUN number + set_fact: + lun_id: '{{ lun_grep.stdout | int + 1 }}' + +- name: Dump gathered variables + ansible.builtin.debug: + msg: 'LUN ID defined as {{ lun_id }}' diff --git a/netapp_ontap-formula/ansible/tasks/map_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/map_lun_restit.yml new file mode 100644 index 00000000..28a5d69e --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/map_lun_restit.yml @@ -0,0 +1,24 @@ +--- +- name: Map LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + http_port: '{{ ontap_port }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + use_rest: always + validate_certs: false + api: protocols/san/lun-maps + method: POST + body: + svm: + name: '{{ ontap_vserver }}' + igroup: + name: '{{ ontap_igroup }}' + lun: + name: '{{ ontap_lun_path }}' + logical_unit_number: '{{ ontap_lun_id }}' + register: lun_map_out + +- debug: var=lun_map_out +- assert: { that: lun_map_out.status_code==201 } + diff --git a/netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml new file mode 100644 index 00000000..8931da3f --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml @@ -0,0 +1,25 @@ +--- +- name: Patch LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + http_port: '{{ ontap_port }}' + use_rest: always + validate_certs: false + api: storage/luns + method: PATCH + query: + name: '{{ ontap_lun_path }}' + body: + space: + size: '{{ ontap_size }}' + register: lun_patch_info + +- debug: var=lun_patch_info +- assert: { that: lun_patch_info.status_code==200 } + +- local_action: + module: copy + content: '{{ lun_patch_info }}' + dest: /tmp/lun_patch.out diff --git a/netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml new file mode 100644 index 00000000..41dbd6c5 --- /dev/null +++ b/netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml @@ -0,0 +1,19 @@ +--- +- name: Map LUN + netapp.ontap.na_ontap_restit: + hostname: '{{ ontap_host }}' + http_port: '{{ ontap_port }}' + cert_filepath: '{{ ontap_crt }}' + key_filepath: '{{ ontap_key }}' + use_rest: always + validate_certs: false + api: protocols/san/lun-maps + method: DELETE + query: + igroup: '{{ ontap_igroup }}' + lun: '{{ ontap_lun_path }}' + register: lun_unmap_out + +- debug: var=lun_unmap_out +- assert: { that: lun_unmap_out.status_code==200 } + From 3376eae5482e9ed74e3ee872ac3021df087a6a1b Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 08:13:59 +0200 Subject: [PATCH 12/34] netapp_ontap: Add native Python client Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap_native.py | 264 ++++++++++++++++++ 1 file changed, 264 insertions(+) create mode 100644 netapp_ontap-formula/_modules/ontap_native.py diff --git a/netapp_ontap-formula/_modules/ontap_native.py b/netapp_ontap-formula/_modules/ontap_native.py new file mode 100644 index 00000000..6211ae2c --- /dev/null +++ b/netapp_ontap-formula/_modules/ontap_native.py @@ -0,0 +1,264 @@ +""" +Salt execution module for maging ONTAP based NetApp storage systems using Ansible +Copyright (C) 2023 SUSE LLC + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . +""" + +import logging +import re + +from netapp_ontap.resources import Igroup, Lun, LunMap, Svm + +log = logging.getLogger(__name__) + +def _config(): + return __utils__['ontap_config.config']() + +def __virtual__(): + try: + from netapp_ontap import config as netapp_config + from netapp_ontap import NetAppRestError, HostConnection + except ImportError as err: + return (False, 'The netapp_ontap library is not available') + + config = _config() + config_host = config['host'] + verify = config.get('verify', False) + host, colon, port = config_host.rpartition(':') + netapp_config.CONNECTION = HostConnection(host, port=port, cert=config['certificate'], key=config['key'], verify=verify) + netapp_config.RAISE_API_ERRORS = False + + return True + +def _path(volume, name): + return f'/vol/{volume}/{name}' + +def _result(result): + log.debug(f'ontap_ansible: parsing result: {result}') + + error = result.is_err + status = result.http_response.status_code + data = result.http_response.json() + if 'error' in data: + message = data['error']['message'] + else: + message = result.http_response.text + + res = {} + + if status >= 400 and error: + __context__["retcode"] = 2 + res = {'result': False, 'message': message} + if 200 <= status < 300: + res = {'result': True} + + if res: + resmap = {'status': status} + resmap.update(res) + return resmap + + log.warning('ontap_ansible: dumping unknown result') + return result + +def _strip(resource, inners=[]): + resource_dict = resource.to_dict() + del resource_dict['_links'] + for inner in inners: + del resource_dict[inner]['_links'] + return resource_dict + +# Source: https://stackoverflow.com/a/14996816 +suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] +def _humansize(nbytes): + i = 0 + while nbytes >= 1024 and i < len(suffixes)-1: + nbytes /= 1024. + i += 1 + f = ('%.2f' % nbytes).rstrip('0').rstrip('.') + return '%s%s' % (f, suffixes[i]) + +def get_lun(comment=None, path=None, uuid=None, human=True): + args = [comment, path, uuid] + argcount = args.count(None) + if 1 > argcount < 3: + log.error(f'Only a single filter may be specified') + raise ValueError('Only a single filter may be specified') + fields = 'space.size,status.mapped' + result = [] + + def _handle(resource): + resource.get(fields=fields) + resource_stripped = _strip(resource) + if human: + resource_stripped['space']['size'] = _humansize(resource_stripped['space']['size']) + result.append(resource_stripped) + + if comment: + for resource in Lun.get_collection(**{'comment': comment}): + _handle(resource) + elif path: + for resource in Lun.get_collection(**{'name': path}): + _handle(resource) + elif uuid: + resource = Lun(uuid=uuid) + _handle(resource) + else: + for resource in Lun.get_collection(): + _handle(resource) + + return result + +def get_next_free(igroup): + numbers = [] + for resource in LunMap.get_collection(igroup=igroup, fields="logical_unit_number"): + numbers.append(resource.logical_unit_number) + return max(numbers)+1 + +# https://stackoverflow.com/a/60708339 +# based on https://stackoverflow.com/a/42865957/2002471 +units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40} +def _parse_size(size): + size = size.upper() + if not re.match(r' ', size): + size = re.sub(r'([KMGT]?B)', r' \1', size) + number, unit = [string.strip() for string in size.split()] + return int(float(number)*units[unit]) + +def provision_lun(name, size, volume, vserver, comment=None): + size = _parse_size(size) + path = _path(volume, name) + + resource = Lun() + resource.svm = Svm(name=vserver) + resource.name = path + resource.os_type = 'linux' + resource.space = {'size': size} + + if comment is not None: + resource.comment = comment + + result = resource.post() + return _result(result) + +# to-do: support property updates other than size changes +def update_lun(uuid, size): + size = _parse_size(size) + + resource = Lun(uuid=uuid) + resource.space = {'size': size} + result = resource.patch() + return _result(result) + +def _delete_lun(name=None, volume=None, uuid=None): + if (name is None or volume is None) and (uuid is None): + log.error('Specify either name and volume or uuid') + raise ValueError('Specify either name and volume or uuid') + if name and volume: + path = _path(volume, name) + resources = get_lun(path=path) + log.debug(f'netapp_ontap: resources to delete: {resources}') + found = len(resources) + if found > 1: + log.error('Refusing to delete multiple resources') + return({'result': False, 'message': 'Found more than one matching LUN, aborting deletion'}) + if found == 0: + return({'result': None, 'message': 'Did not find any matching LUN\'s'}) + resource = Lun(uuid=resources[0]['uuid']) + elif uuid: + resource = Lun(uuid=uuid) + + result = resource.delete() + return _result(result) + +def delete_lun_name(name, volume): + return _delete_lun(name, volume) + +def delete_lun_uuid(uuid): + return _delete_lun(uuid=uuid) + +def get_lun_mapped(comment=None, lun_result=None): + if (comment is None) and (lun_result is None): + log.error('Specify either a comment or existing LUN output') + raise ValueError('Specify a comment') + if comment is not None: + query = get_lun(comment) + elif lun_result is not None: + query = lun_result + resmap = {} + for lun in query: + log.debug(f'netapp_ontap: parsing LUN {lun}') + name = lun.get('name') + mapped = lun.get('status', {}).get('mapped') + resmap.update({name: mapped}) + if None in resmap: + log.error('netapp_ontap: invalid LUN mapping map') + return resmap + +def get_igroup_uuid(igroup): + resource = Igroup(name=igroup) + resource.get() + uuid = resource.uuid + return uuid + +def get_lun_mapping(name, volume, igroup): + path = _path(volume, name) + result = [] + + igroup_uuid = get_igroup_uuid(igroup) + luns = get_lun(path=path) + log.debug(f'netapp_ontap: found luns: {luns}') + for resource in luns: + lun_uuid = resource['uuid'] + mapresource = LunMap(**{'igroup.uuid': igroup_uuid, 'lun.uuid': lun_uuid}) + mrs = mapresource.get_collection(fields='logical_unit_number') + # FIXME get() fails, saying more than one item is found, and get_collection() returns dozens of completely unrelated entries + # the loop below is a workaround discarding all the bogus entries + for mr in mrs: + mr_stripped = _strip(mr, ['igroup', 'lun', 'svm']) + if mr_stripped['lun']['uuid'] == lun_uuid and mr_stripped['igroup']['uuid'] == igroup_uuid: + log.debug(f'netapp_ontap: elected {mr_stripped}') + result.append(mr_stripped) + + return result + +def map_lun(name, lunid, volume, vserver, igroup): + path = _path(volume, name) + + resource = LunMap() + resource.svm = Svm(name=vserver) + resource.igroup = Igroup(name=igroup) + resource.lun = Lun(name=path) + resource.logical_unit_number = lunid + result = resource.post() + + return _result(result) + +def unmap_lun(name, volume, igroup): + path = _path(volume, name) + results = [] + + mappings = get_lun_mapping(name, volume, igroup) + log.debug(f'netapp_ontap: parsing mappings: {mappings}') + for mapping in mappings: + igroup_uuid = mapping['igroup']['uuid'] + lun_uuid = mapping['lun']['uuid'] + resource = LunMap(**{'igroup.uuid': igroup_uuid, 'lun.uuid': lun_uuid}) + result = resource.delete() + results.append(_result(result)) + + if len(mappings) == 1: + return results[0] + + return results From bc7b084f5d88ce251cde40523ce46b3d8c0be7d9 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sat, 24 Jun 2023 08:56:14 +0200 Subject: [PATCH 13/34] netapp_ontap: Use native Python client for states Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_states/ontap.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index a613f786..65f2f092 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -59,7 +59,7 @@ def _size(details, human=False): # FIXME drop mapping logic from lun_present in favor of lun_mapped def _map(name, lunid, volume, vserver, igroup): ok = False - map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) + map_out = __salt__['ontap_native.map_lun'](name, lunid, volume, vserver, igroup) if map_out.get('result', False) and map_out.get('status') == 201: comment = f'Mapped LUN to ID {lunid}' ok = True @@ -68,9 +68,10 @@ def _map(name, lunid, volume, vserver, igroup): comment = 'LUN mapping failed' return comment, ok - query = __salt__['ontap.get_lun'](get_next_free=True) - luns = query[0] - next_free = query[1] + query = __salt__['ontap_native.get_lun']() + #luns = query[0] + luns = query + next_free = __salt__['ontap_native.get_next_free']('wilde') # drop this for lun in luns: lun_path = lun.get('name') @@ -80,10 +81,10 @@ def _map(name, lunid, volume, vserver, igroup): log.debug(f'netapp_ontap: found existing LUN {name}') if lun_uuid is None: log.error(f'netapp_ontap: found LUN with no UUID') - lun_details = __salt__['ontap.get_lun'](uuid=lun_uuid) + lun_details = __salt__['ontap_native.get_lun'](uuid=lun_uuid, human=False) lun_size = _size(lun_details[0], True) # lun_size_human = needed? - lun_mapping = __salt__['ontap.get_lun_mapped'](lun_result=lun_details) + lun_mapping = __salt__['ontap_native.get_lun_mapped'](lun_result=lun_details) lun_mapped = lun_mapping.get(name) # lun_id = needed? if lun_size == size: @@ -93,8 +94,8 @@ def _map(name, lunid, volume, vserver, igroup): if __opts__['test']: comment_size = f'Would resize LUN to {size}' else: - __salt__['ontap.update_lun'](name, size, volume, vserver) - lun2_details = __salt__['ontap.get_lun'](uuid=lun_uuid) + __salt__['ontap_native.update_lun'](lun_uuid, size) + lun2_details = __salt__['ontap_native.get_lun'](uuid=lun_uuid, human=False) lun2_size = _size(lun2_details[0], True) comment_size = f'LUN from {lun_size} to {size}' if lun2_size != lun_size and lun2_size == size: @@ -117,7 +118,7 @@ def _map(name, lunid, volume, vserver, igroup): comment_mapping = map_out[0] map_ok = map_out[1] - #map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) + #map_out = __salt__['ontap_native.map_lun'](name, lunid, volume, vserver, igroup) #if map_out.get('result', False) and map_out.get('status') == 201: # comment_mapping = f'Mapped LUN to ID {lunid}' # map_ok = True @@ -141,12 +142,12 @@ def _map(name, lunid, volume, vserver, igroup): ret['result'] = None return ret - __salt__['ontap.provision_lun'](name, size, volume, vserver, comment) + __salt__['ontap_native.provision_lun'](name, size, volume, vserver, comment) if do_map: map_out = _map(name, lunid, volume, vserver, igroup) comment_mapping = map_out[0] map_ok = map_out[1] - lun2_details = __salt__['ontap.get_lun'](comment) + lun2_details = __salt__['ontap_native.get_lun'](comment)[0] lun2_size = _size(lun2_details) # FIXME changes dict @@ -182,7 +183,7 @@ def lun_mapped(name, lunid, volume, vserver, igroup): path = f'/vol/{volume}/{name}' ret = {'name': path, 'result': False, 'changes': {}, 'comment': ''} - mapping_out = __salt__['ontap.get_lun_mapping'](name, volume, igroup) + mapping_out = __salt__['ontap_native.get_lun_mapping'](name, volume, igroup) log.debug(f'netapp_ontap: mapping result: {mapping_out}') current_igroups = [] current_svms = [] @@ -235,7 +236,7 @@ def lun_mapped(name, lunid, volume, vserver, igroup): ret['comment'] = comment return ret - map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) + map_out = __salt__['ontap_native.map_lun'](name, lunid, volume, vserver, igroup) if map_out.get('result', False) and map_out.get('status') == 201: comment = f'Mapped LUN to ID {lunid} in igroup {igroup}' ret['result'] = True From cf27374fd4f32b8934108bc26f5a9a12f0f995b4 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 22:48:28 +0200 Subject: [PATCH 14/34] netapp_ontap: minor cleanup Add some comments, make igroup filter in get_lun_mapped() optional, show comment in get_lun() output, drop single-use _config() function. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap_native.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap_native.py b/netapp_ontap-formula/_modules/ontap_native.py index 6211ae2c..02bca44b 100644 --- a/netapp_ontap-formula/_modules/ontap_native.py +++ b/netapp_ontap-formula/_modules/ontap_native.py @@ -23,9 +23,6 @@ log = logging.getLogger(__name__) -def _config(): - return __utils__['ontap_config.config']() - def __virtual__(): try: from netapp_ontap import config as netapp_config @@ -33,7 +30,7 @@ def __virtual__(): except ImportError as err: return (False, 'The netapp_ontap library is not available') - config = _config() + config = __utils__['ontap_config.config']() config_host = config['host'] verify = config.get('verify', False) host, colon, port = config_host.rpartition(':') @@ -89,19 +86,21 @@ def _humansize(nbytes): f = ('%.2f' % nbytes).rstrip('0').rstrip('.') return '%s%s' % (f, suffixes[i]) +# to-do: make fields adjustable def get_lun(comment=None, path=None, uuid=None, human=True): args = [comment, path, uuid] argcount = args.count(None) if 1 > argcount < 3: log.error(f'Only a single filter may be specified') raise ValueError('Only a single filter may be specified') - fields = 'space.size,status.mapped' + fields = 'comment,space.size,status.mapped' result = [] def _handle(resource): resource.get(fields=fields) resource_stripped = _strip(resource) if human: + # transform LUN size to a more readable format resource_stripped['space']['size'] = _humansize(resource_stripped['space']['size']) result.append(resource_stripped) @@ -115,6 +114,7 @@ def _handle(resource): resource = Lun(uuid=uuid) _handle(resource) else: + # no filter specified, fetch all LUNs for resource in Lun.get_collection(): _handle(resource) @@ -188,6 +188,8 @@ def delete_lun_name(name, volume): def delete_lun_uuid(uuid): return _delete_lun(uuid=uuid) +# to-do: allow filter by path=None and uuid +# to-do: potentially move this to get_luns_mapped() and make a separate get_lun_mapped() returning a single entry def get_lun_mapped(comment=None, lun_result=None): if (comment is None) and (lun_result is None): log.error('Specify either a comment or existing LUN output') @@ -216,16 +218,23 @@ def get_lun_mapping(name, volume, igroup): path = _path(volume, name) result = [] - igroup_uuid = get_igroup_uuid(igroup) + if igroup is not None: + log.debug('netapp_ontap: filtering by igroup') + igroup_uuid = get_igroup_uuid(igroup) luns = get_lun(path=path) log.debug(f'netapp_ontap: found luns: {luns}') for resource in luns: lun_uuid = resource['uuid'] - mapresource = LunMap(**{'igroup.uuid': igroup_uuid, 'lun.uuid': lun_uuid}) + filterdict = {'lun.uuid': lun_uuid} + if igroup is not None: + filterdict.update({'igroup.uuid': igroup_uuid}) + mapresource = LunMap(**filterdict) mrs = mapresource.get_collection(fields='logical_unit_number') # FIXME get() fails, saying more than one item is found, and get_collection() returns dozens of completely unrelated entries # the loop below is a workaround discarding all the bogus entries - for mr in mrs: + mymrs = list(mrs) + log.debug(f'netapp_ontap: found mappings: {mymrs}') + for mr in mymrs: mr_stripped = _strip(mr, ['igroup', 'lun', 'svm']) if mr_stripped['lun']['uuid'] == lun_uuid and mr_stripped['igroup']['uuid'] == igroup_uuid: log.debug(f'netapp_ontap: elected {mr_stripped}') From aeed60cd21a5f0600c4b23cd5ecf274424f00435 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 22:50:27 +0200 Subject: [PATCH 15/34] netapp_ontap: add docstrings Add basic explanations to the functiosn in ontap_native. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap_native.py | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap_native.py b/netapp_ontap-formula/_modules/ontap_native.py index 02bca44b..9c849f58 100644 --- a/netapp_ontap-formula/_modules/ontap_native.py +++ b/netapp_ontap-formula/_modules/ontap_native.py @@ -43,6 +43,11 @@ def _path(volume, name): return f'/vol/{volume}/{name}' def _result(result): + """ + Transforms API results to a common format + Used for DELETE/PATCH/POST output, not for GET + result = the output to parse + """ log.debug(f'ontap_ansible: parsing result: {result}') error = result.is_err @@ -88,6 +93,9 @@ def _humansize(nbytes): # to-do: make fields adjustable def get_lun(comment=None, path=None, uuid=None, human=True): + """ + Queries one or more LUN(s), depending on whether filters are set + """ args = [comment, path, uuid] argcount = args.count(None) if 1 > argcount < 3: @@ -121,6 +129,9 @@ def _handle(resource): return result def get_next_free(igroup): + """ + Returns the next free LUN ID for the specfied initiator group + """ numbers = [] for resource in LunMap.get_collection(igroup=igroup, fields="logical_unit_number"): numbers.append(resource.logical_unit_number) @@ -137,6 +148,9 @@ def _parse_size(size): return int(float(number)*units[unit]) def provision_lun(name, size, volume, vserver, comment=None): + """ + Create a new LUN + """ size = _parse_size(size) path = _path(volume, name) @@ -154,6 +168,9 @@ def provision_lun(name, size, volume, vserver, comment=None): # to-do: support property updates other than size changes def update_lun(uuid, size): + """ + Change values of an existing LUN + """ size = _parse_size(size) resource = Lun(uuid=uuid) @@ -162,6 +179,10 @@ def update_lun(uuid, size): return _result(result) def _delete_lun(name=None, volume=None, uuid=None): + """ + Meta-function for deleting a LUN + Currently, the individual targeting functions need to be used + """ if (name is None or volume is None) and (uuid is None): log.error('Specify either name and volume or uuid') raise ValueError('Specify either name and volume or uuid') @@ -183,14 +204,24 @@ def _delete_lun(name=None, volume=None, uuid=None): return _result(result) def delete_lun_name(name, volume): + """ + Delete a single LUN based on its name and volume + """ return _delete_lun(name, volume) def delete_lun_uuid(uuid): + """ + Delete a single LUN based on its UUID + """ return _delete_lun(uuid=uuid) # to-do: allow filter by path=None and uuid # to-do: potentially move this to get_luns_mapped() and make a separate get_lun_mapped() returning a single entry def get_lun_mapped(comment=None, lun_result=None): + """ + Assess whether LUNs named by a comment are mapped + For more efficient programmatic use, an existing get_lun() output can be fed + """ if (comment is None) and (lun_result is None): log.error('Specify either a comment or existing LUN output') raise ValueError('Specify a comment') @@ -209,12 +240,18 @@ def get_lun_mapped(comment=None, lun_result=None): return resmap def get_igroup_uuid(igroup): + """ + Return the UUID of a single initiator group + """ resource = Igroup(name=igroup) resource.get() uuid = resource.uuid return uuid -def get_lun_mapping(name, volume, igroup): +def get_lun_mappings(name, volume, igroup=None): + """ + Return details about LUN mappings + """ path = _path(volume, name) result = [] @@ -243,6 +280,9 @@ def get_lun_mapping(name, volume, igroup): return result def map_lun(name, lunid, volume, vserver, igroup): + """ + Map a LUN to an initiator group + """ path = _path(volume, name) resource = LunMap() @@ -254,7 +294,10 @@ def map_lun(name, lunid, volume, vserver, igroup): return _result(result) -def unmap_lun(name, volume, igroup): +def unmap_luns(name, volume, igroup): + """ + Remove LUNs from an initiator group + """ path = _path(volume, name) results = [] From c55c9122e23a1a11a536aac2d905cc0dbc7ae134 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 22:51:44 +0200 Subject: [PATCH 16/34] netapp_ontap: allow single get_lun_mapping result Split into get_lun_mapping() and get_lun_mappings(). Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap_native.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap_native.py b/netapp_ontap-formula/_modules/ontap_native.py index 9c849f58..787d1a63 100644 --- a/netapp_ontap-formula/_modules/ontap_native.py +++ b/netapp_ontap-formula/_modules/ontap_native.py @@ -279,6 +279,19 @@ def get_lun_mappings(name, volume, igroup=None): return result +def get_lun_mapping(name, volume, igroup): + """ + Return details about a single LUN mapping + """ + results = get_lun_mappings(name, volume, igroup) + lr = len(results) + if lr == 1: + return results[0] + if lr > 1: + log.error(f'netapp_ontap: found {lr} results, but expected only one') + return None + return {} + def map_lun(name, lunid, volume, vserver, igroup): """ Map a LUN to an initiator group @@ -301,7 +314,7 @@ def unmap_luns(name, volume, igroup): path = _path(volume, name) results = [] - mappings = get_lun_mapping(name, volume, igroup) + mappings = get_lun_mappings(name, volume, igroup) log.debug(f'netapp_ontap: parsing mappings: {mappings}') for mapping in mappings: igroup_uuid = mapping['igroup']['uuid'] @@ -310,7 +323,4 @@ def unmap_luns(name, volume, igroup): result = resource.delete() results.append(_result(result)) - if len(mappings) == 1: - return results[0] - return results From 95fc1eaf041e6a5fcbc63374e8e04a9c2c1c9436 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 22:53:24 +0200 Subject: [PATCH 17/34] netapp_ontap: simplify lun_mapped Drop useless logic for multiple mapping results. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_states/ontap.py | 49 +++++---------------------- 1 file changed, 9 insertions(+), 40 deletions(-) diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index 65f2f092..f3b508fb 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -185,54 +185,23 @@ def lun_mapped(name, lunid, volume, vserver, igroup): mapping_out = __salt__['ontap_native.get_lun_mapping'](name, volume, igroup) log.debug(f'netapp_ontap: mapping result: {mapping_out}') - current_igroups = [] - current_svms = [] - records = mapping_out.get('num_records') - do_igroup = True - do_svm = True - do_map = False - if records == 0: - do_map = True - if __opts__['test']: - comment = 'Would create mapping' - elif records is not None and records > 0: - for mapping in mapping_out.get('records', []): - this_igroup = mapping.get('igroup', {}).get('name') - if this_igroup is None: - log.error(f'netapp_ontap: unable to determine igroup in mapping result') - else: - if this_igroup not in current_igroups: - current_igroups.append(this_igroup) - this_svm = mapping.get('svm', {}).get('name') - if this_svm is None: - log.error(f'netapp_ontap: unable to determine svm in mapping result') - else: - if this_svm not in current_svms: - current_svms.append(this_svm) - - comment_igroup = f' to igroup {igroup}' - if igroup in current_igroups: - do_igroup = False - comment_svm = f' in SVM {vserver}' - if vserver in current_svms: - do_svm = False - - comments = f'{comment_igroup}{comment_svm}' - already = False - if do_map or do_igroup or do_svm: + comment_details = f' to igroup {igroup} in SVM {vserver}' + current_igroup = mapping_out.get('igroup', {}).get('name') + current_vserver = mapping_out.get('svm', {}).get('name') + if not mapping_out or igroup != current_igroup or vserver != current_vserver: if __opts__['test']: - comment = f'Would map ID {lunid}{comments}' - elif not do_igroup or not do_svm: - comment = f'Already mapped{comment_igroup}{comment_svm}' + comment = f'Would map ID {lunid}{comment_details}' + elif mapping_out and igroup == current_igroup or vserver == current_svm: + comment = f'Already mapped{comment_details}' ret['result'] = True else: log.error('Unhandled mapping state') comment = 'Something weird happened' - if __opts__['test']: - ret['result'] = None if __opts__['test'] or ret['result'] is True: + if __opts__['test']: + ret['result'] = None ret['comment'] = comment return ret From b90902d3b30fce339f59ef81706ed366d20cd64f Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 22:54:50 +0200 Subject: [PATCH 18/34] netapp_ontap: implement lun_unmapped Add unmap_lun() for unmapping of a single LUN and the matching lun_unmapped() state function. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap_native.py | 15 +++++++++++ netapp_ontap-formula/_states/ontap.py | 25 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/netapp_ontap-formula/_modules/ontap_native.py b/netapp_ontap-formula/_modules/ontap_native.py index 787d1a63..c722f063 100644 --- a/netapp_ontap-formula/_modules/ontap_native.py +++ b/netapp_ontap-formula/_modules/ontap_native.py @@ -324,3 +324,18 @@ def unmap_luns(name, volume, igroup): results.append(_result(result)) return results + +def unmap_lun(name, volume, igroup): + """ + Remove a single LUN from an initiator group + Not implemented! Might unmap multiple LUNs. + """ + # to-do: abort if more than one LUN is going to be affected, consider using get_lun_mapping() + results = unmap_luns(name, volume, igroup) + reslen = len(results) + if reslen > 1: + log.warning(f'Unmapped {reslen} LUNs, but expected only one') + if reslen == 0: + return {} + + return results[0] diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index f3b508fb..c61ac256 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -216,3 +216,28 @@ def lun_mapped(name, lunid, volume, vserver, igroup): ret['comment'] = comment return ret +def lun_unmapped(name, volume, igroup): + path = f'/vol/{volume}/{name}' + ret = {'name': path, 'result': True, 'changes': {}, 'comment': ''} + + if __opts__['test']: + result = __salt__['ontap_native.get_lun_mapping'](name, volume, igroup) + ret['result'] = None + else: + result = __salt__['ontap_native.unmap_lun'](name, volume, igroup) + rr = result.get('result', True) + rs = result.get('status') + log.debug(f'result: {result}') + + if __opts__['test'] and result: + comment = f'Would unmap LUN' + elif not result: + comment = 'Nothing to unmap' + elif rr is True and rs == 200: + comment = f'Unmapped LUN' + elif rr is False or rs != 200: + comment = f'Unmapping failed' + ret['result'] = False + + ret['comment'] = comment + return ret From 4741fbf1a3bdc25feab016eddac0256ec3408f8d Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 22:57:13 +0200 Subject: [PATCH 19/34] netapp_ontap: add README Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/README.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 netapp_ontap-formula/README.md diff --git a/netapp_ontap-formula/README.md b/netapp_ontap-formula/README.md new file mode 100644 index 00000000..c07634d4 --- /dev/null +++ b/netapp_ontap-formula/README.md @@ -0,0 +1,7 @@ +# Salt states and modules for managing NetApp ONTAP storage appliances + +_Work in progress ..._ + +## Available states + +_to do ..._ From 407a8c34fef41445dc0ecfb8e17b2fc856d3415b Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 23:01:32 +0200 Subject: [PATCH 20/34] netapp_ontap: replace ontap with ontap_native The Ansible based "ontap" execution module is not pursued anymore. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 364 +++++++++++------- netapp_ontap-formula/_modules/ontap_native.py | 341 ---------------- netapp_ontap-formula/_states/ontap.py | 28 +- 3 files changed, 241 insertions(+), 492 deletions(-) delete mode 100644 netapp_ontap-formula/_modules/ontap_native.py diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index ffe96878..c722f063 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -19,199 +19,209 @@ import logging import re +from netapp_ontap.resources import Igroup, Lun, LunMap, Svm + log = logging.getLogger(__name__) -# https://stackoverflow.com/a/9808122 -def _find(key, value): - for k, v in value.items(): - if k == key: - yield v - elif isinstance(v, dict): - for result in find(key, v): - yield result - elif isinstance(v, list): - for d in v: - for result in find(key, d): - yield result - -def _config(): - return __utils__['ontap_config.config']() +def __virtual__(): + try: + from netapp_ontap import config as netapp_config + from netapp_ontap import NetAppRestError, HostConnection + except ImportError as err: + return (False, 'The netapp_ontap library is not available') + + config = __utils__['ontap_config.config']() + config_host = config['host'] + verify = config.get('verify', False) + host, colon, port = config_host.rpartition(':') + netapp_config.CONNECTION = HostConnection(host, port=port, cert=config['certificate'], key=config['key'], verify=verify) + netapp_config.RAISE_API_ERRORS = False + + return True def _path(volume, name): return f'/vol/{volume}/{name}' -def _call(host, certificate, key, rundir, playbook, extravars={}, descend=[], single_task=True): - host, colon, port = host.rpartition(':') - varmap = {'ontap_host': host, 'ontap_port': int(port), 'ontap_crt': certificate, 'ontap_key': key} - if extravars: - varmap.update(extravars) - log.debug(f'ontap_ansible: executing {playbook} with {varmap} in {rundir}') - out = __salt__['ansible.playbooks']( - playbook=playbook, rundir=rundir, extra_vars=varmap) - - plays = out.get('plays', []) - plays_len = len(plays) - if not plays_len: - log.error(f'ontap_ansible: returned with no plays') - return False - if plays_len > 0: - log.warning(f'ontap_ansible: discarding {plays_len} additional plays') - play0 = plays[0] - - tasks = play0.get('tasks', []) - tasks_len = len(tasks) - if not tasks_len: - log.error(f'ontap_ansible: play returned with no tasks') - return False - if tasks_len > 0 and single_task: - log.warning(f'ontap_ansible: discarding {tasks_len} additional tasks') - - mytasks = [] - for task in tasks: - mytask = task.get('hosts').get('localhost') - if mytask is None: - log.error(f'ontap_ansible: unable to parse task - ensure it executed locally') - return False - - if descend: - if isinstance(descend, str): - descend = [descend] - for level in descend: - gain = mytask.get(level) - if gain is None: - break - if gain is not None: - log.debug(f'ontap_ansible: found artifact for {level}') - mytask = gain - - if single_task: - break - mytasks.append(mytask) - - if single_task: - return mytask - return mytasks - def _result(result): + """ + Transforms API results to a common format + Used for DELETE/PATCH/POST output, not for GET + result = the output to parse + """ log.debug(f'ontap_ansible: parsing result: {result}') - if isinstance(result, bool): - log.error(f'ontap_ansible: result seems like a failed execution, refusing to parse') - return False - - error = result.get('error_message') - status = result.get('status_code') - records = result.get('changed') - response = result.get('response') - method = result.get('invocation', {}).get('module_args', {}).get('method') - if response is not None and 'num_records' in response: - records = response['num_records'] - elif method == 'POST': - # API does not return a record number for any creation calls, we cannot tell how many items records - records = None + error = result.is_err + status = result.http_response.status_code + data = result.http_response.json() + if 'error' in data: + message = data['error']['message'] else: - # API does not return a record number if a DELETE call did not yield any deletions - records = 0 + message = result.http_response.text res = {} if status >= 400 and error: __context__["retcode"] = 2 - res = {'error': error, 'result': False} - if method in set(['DELETE', 'POST']) and 200 <= status < 300: + res = {'result': False, 'message': message} + if 200 <= status < 300: res = {'result': True} if res: resmap = {'status': status} - if records is not None: - resmap.update({'records': records}) resmap.update(res) return resmap - elif status == 200: - return response log.warning('ontap_ansible: dumping unknown result') return result -def get_lun(comment=None, uuid=None, get_next_free=False): - if (comment is not None) and (uuid is not None): +def _strip(resource, inners=[]): + resource_dict = resource.to_dict() + del resource_dict['_links'] + for inner in inners: + del resource_dict[inner]['_links'] + return resource_dict + +# Source: https://stackoverflow.com/a/14996816 +suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] +def _humansize(nbytes): + i = 0 + while nbytes >= 1024 and i < len(suffixes)-1: + nbytes /= 1024. + i += 1 + f = ('%.2f' % nbytes).rstrip('0').rstrip('.') + return '%s%s' % (f, suffixes[i]) + +# to-do: make fields adjustable +def get_lun(comment=None, path=None, uuid=None, human=True): + """ + Queries one or more LUN(s), depending on whether filters are set + """ + args = [comment, path, uuid] + argcount = args.count(None) + if 1 > argcount < 3: log.error(f'Only a single filter may be specified') raise ValueError('Only a single filter may be specified') - varmap = _config() - extravars = None + fields = 'comment,space.size,status.mapped' + result = [] + + def _handle(resource): + resource.get(fields=fields) + resource_stripped = _strip(resource) + if human: + # transform LUN size to a more readable format + resource_stripped['space']['size'] = _humansize(resource_stripped['space']['size']) + result.append(resource_stripped) if comment: - playbook = 'fetch-lun-by-comment_restit' - extravars = {'extravars': {'ontap_lun_comment': comment}} + for resource in Lun.get_collection(**{'comment': comment}): + _handle(resource) + elif path: + for resource in Lun.get_collection(**{'name': path}): + _handle(resource) elif uuid: - playbook = 'fetch-lun_restit' - extravars = {'extravars': {'ontap_lun_uuid': uuid}} - - if extravars is None: - playbook = 'fetch-luns_restit' + resource = Lun(uuid=uuid) + _handle(resource) else: - varmap.update(extravars) - - descend = ['response', 'records'] - varmap.update({'playbook': f'playbooks/{playbook}.yml', 'descend': descend}) + # no filter specified, fetch all LUNs + for resource in Lun.get_collection(): + _handle(resource) - if get_next_free and comment is None and uuid is None: - result = _call(**varmap, single_task=False) - next_free = result[5].get('ansible_facts', {}).get('lun_id') - return result[0], next_free - result = _call(**varmap) return result +def get_next_free(igroup): + """ + Returns the next free LUN ID for the specfied initiator group + """ + numbers = [] + for resource in LunMap.get_collection(igroup=igroup, fields="logical_unit_number"): + numbers.append(resource.logical_unit_number) + return max(numbers)+1 + # https://stackoverflow.com/a/60708339 # based on https://stackoverflow.com/a/42865957/2002471 units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40} def _parse_size(size): size = size.upper() - #print("parsing size ", size) if not re.match(r' ', size): size = re.sub(r'([KMGT]?B)', r' \1', size) number, unit = [string.strip() for string in size.split()] return int(float(number)*units[unit]) def provision_lun(name, size, volume, vserver, comment=None): - varmap = _config() + """ + Create a new LUN + """ size = _parse_size(size) path = _path(volume, name) - varmap.update({'playbook': 'playbooks/deploy-lun_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_volume': volume, 'ontap_vserver': vserver, 'ontap_size': size}}) + + resource = Lun() + resource.svm = Svm(name=vserver) + resource.name = path + resource.os_type = 'linux' + resource.space = {'size': size} + if comment is not None: - varmap['extravars'].update({'ontap_comment': comment}) - result = _call(**varmap) + resource.comment = comment + + result = resource.post() return _result(result) # to-do: support property updates other than size changes -def update_lun(name, size, volume, vserver): - varmap = _config() +def update_lun(uuid, size): + """ + Change values of an existing LUN + """ size = _parse_size(size) - path = _path(volume, name) - varmap.update({'playbook': 'playbooks/patch-lun_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_vserver': vserver, 'ontap_size': size}}) - result = _call(**varmap) + + resource = Lun(uuid=uuid) + resource.space = {'size': size} + result = resource.patch() return _result(result) def _delete_lun(name=None, volume=None, uuid=None): + """ + Meta-function for deleting a LUN + Currently, the individual targeting functions need to be used + """ if (name is None or volume is None) and (uuid is None): log.error('Specify either name and volume or uuid') raise ValueError('Specify either name and volume or uuid') - varmap = _config() if name and volume: - extravars = {'ontap_volume': volume, 'ontap_lun_name': name} + path = _path(volume, name) + resources = get_lun(path=path) + log.debug(f'netapp_ontap: resources to delete: {resources}') + found = len(resources) + if found > 1: + log.error('Refusing to delete multiple resources') + return({'result': False, 'message': 'Found more than one matching LUN, aborting deletion'}) + if found == 0: + return({'result': None, 'message': 'Did not find any matching LUN\'s'}) + resource = Lun(uuid=resources[0]['uuid']) elif uuid: - extravars = {'ontap_lun_uuid': uuid} - varmap.update({'playbook': 'playbooks/delete-lun_restit.yml', 'extravars': extravars}) - result = _call(**varmap) + resource = Lun(uuid=uuid) + + result = resource.delete() return _result(result) def delete_lun_name(name, volume): + """ + Delete a single LUN based on its name and volume + """ return _delete_lun(name, volume) def delete_lun_uuid(uuid): + """ + Delete a single LUN based on its UUID + """ return _delete_lun(uuid=uuid) +# to-do: allow filter by path=None and uuid +# to-do: potentially move this to get_luns_mapped() and make a separate get_lun_mapped() returning a single entry def get_lun_mapped(comment=None, lun_result=None): + """ + Assess whether LUNs named by a comment are mapped + For more efficient programmatic use, an existing get_lun() output can be fed + """ if (comment is None) and (lun_result is None): log.error('Specify either a comment or existing LUN output') raise ValueError('Specify a comment') @@ -229,23 +239,103 @@ def get_lun_mapped(comment=None, lun_result=None): log.error('netapp_ontap: invalid LUN mapping map') return resmap -def get_lun_mapping(name, volume, igroup): - varmap = _config() +def get_igroup_uuid(igroup): + """ + Return the UUID of a single initiator group + """ + resource = Igroup(name=igroup) + resource.get() + uuid = resource.uuid + return uuid + +def get_lun_mappings(name, volume, igroup=None): + """ + Return details about LUN mappings + """ path = _path(volume, name) - varmap.update({'playbook': 'playbooks/get_lun_mapping_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_igroup': igroup}}) - result = _call(**varmap) - return _result(result) + result = [] + + if igroup is not None: + log.debug('netapp_ontap: filtering by igroup') + igroup_uuid = get_igroup_uuid(igroup) + luns = get_lun(path=path) + log.debug(f'netapp_ontap: found luns: {luns}') + for resource in luns: + lun_uuid = resource['uuid'] + filterdict = {'lun.uuid': lun_uuid} + if igroup is not None: + filterdict.update({'igroup.uuid': igroup_uuid}) + mapresource = LunMap(**filterdict) + mrs = mapresource.get_collection(fields='logical_unit_number') + # FIXME get() fails, saying more than one item is found, and get_collection() returns dozens of completely unrelated entries + # the loop below is a workaround discarding all the bogus entries + mymrs = list(mrs) + log.debug(f'netapp_ontap: found mappings: {mymrs}') + for mr in mymrs: + mr_stripped = _strip(mr, ['igroup', 'lun', 'svm']) + if mr_stripped['lun']['uuid'] == lun_uuid and mr_stripped['igroup']['uuid'] == igroup_uuid: + log.debug(f'netapp_ontap: elected {mr_stripped}') + result.append(mr_stripped) + + return result + +def get_lun_mapping(name, volume, igroup): + """ + Return details about a single LUN mapping + """ + results = get_lun_mappings(name, volume, igroup) + lr = len(results) + if lr == 1: + return results[0] + if lr > 1: + log.error(f'netapp_ontap: found {lr} results, but expected only one') + return None + return {} def map_lun(name, lunid, volume, vserver, igroup): - varmap = _config() + """ + Map a LUN to an initiator group + """ path = _path(volume, name) - varmap.update({'playbook': 'playbooks/map-lun_restit.yml', 'extravars': {'ontap_lun_id': lunid, 'ontap_lun_path': path, 'ontap_vserver': vserver, 'ontap_igroup': igroup}}) - result = _call(**varmap) + + resource = LunMap() + resource.svm = Svm(name=vserver) + resource.igroup = Igroup(name=igroup) + resource.lun = Lun(name=path) + resource.logical_unit_number = lunid + result = resource.post() + return _result(result) -def unmap_lun(name, volume, igroup): - varmap = _config() +def unmap_luns(name, volume, igroup): + """ + Remove LUNs from an initiator group + """ path = _path(volume, name) - varmap.update({'playbook': 'playbooks/unmap-lun_restit.yml', 'extravars': {'ontap_lun_path': path, 'ontap_igroup': igroup}}) - result = _call(**varmap) - return _result(result) + results = [] + + mappings = get_lun_mappings(name, volume, igroup) + log.debug(f'netapp_ontap: parsing mappings: {mappings}') + for mapping in mappings: + igroup_uuid = mapping['igroup']['uuid'] + lun_uuid = mapping['lun']['uuid'] + resource = LunMap(**{'igroup.uuid': igroup_uuid, 'lun.uuid': lun_uuid}) + result = resource.delete() + results.append(_result(result)) + + return results + +def unmap_lun(name, volume, igroup): + """ + Remove a single LUN from an initiator group + Not implemented! Might unmap multiple LUNs. + """ + # to-do: abort if more than one LUN is going to be affected, consider using get_lun_mapping() + results = unmap_luns(name, volume, igroup) + reslen = len(results) + if reslen > 1: + log.warning(f'Unmapped {reslen} LUNs, but expected only one') + if reslen == 0: + return {} + + return results[0] diff --git a/netapp_ontap-formula/_modules/ontap_native.py b/netapp_ontap-formula/_modules/ontap_native.py deleted file mode 100644 index c722f063..00000000 --- a/netapp_ontap-formula/_modules/ontap_native.py +++ /dev/null @@ -1,341 +0,0 @@ -""" -Salt execution module for maging ONTAP based NetApp storage systems using Ansible -Copyright (C) 2023 SUSE LLC - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program. If not, see . -""" - -import logging -import re - -from netapp_ontap.resources import Igroup, Lun, LunMap, Svm - -log = logging.getLogger(__name__) - -def __virtual__(): - try: - from netapp_ontap import config as netapp_config - from netapp_ontap import NetAppRestError, HostConnection - except ImportError as err: - return (False, 'The netapp_ontap library is not available') - - config = __utils__['ontap_config.config']() - config_host = config['host'] - verify = config.get('verify', False) - host, colon, port = config_host.rpartition(':') - netapp_config.CONNECTION = HostConnection(host, port=port, cert=config['certificate'], key=config['key'], verify=verify) - netapp_config.RAISE_API_ERRORS = False - - return True - -def _path(volume, name): - return f'/vol/{volume}/{name}' - -def _result(result): - """ - Transforms API results to a common format - Used for DELETE/PATCH/POST output, not for GET - result = the output to parse - """ - log.debug(f'ontap_ansible: parsing result: {result}') - - error = result.is_err - status = result.http_response.status_code - data = result.http_response.json() - if 'error' in data: - message = data['error']['message'] - else: - message = result.http_response.text - - res = {} - - if status >= 400 and error: - __context__["retcode"] = 2 - res = {'result': False, 'message': message} - if 200 <= status < 300: - res = {'result': True} - - if res: - resmap = {'status': status} - resmap.update(res) - return resmap - - log.warning('ontap_ansible: dumping unknown result') - return result - -def _strip(resource, inners=[]): - resource_dict = resource.to_dict() - del resource_dict['_links'] - for inner in inners: - del resource_dict[inner]['_links'] - return resource_dict - -# Source: https://stackoverflow.com/a/14996816 -suffixes = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] -def _humansize(nbytes): - i = 0 - while nbytes >= 1024 and i < len(suffixes)-1: - nbytes /= 1024. - i += 1 - f = ('%.2f' % nbytes).rstrip('0').rstrip('.') - return '%s%s' % (f, suffixes[i]) - -# to-do: make fields adjustable -def get_lun(comment=None, path=None, uuid=None, human=True): - """ - Queries one or more LUN(s), depending on whether filters are set - """ - args = [comment, path, uuid] - argcount = args.count(None) - if 1 > argcount < 3: - log.error(f'Only a single filter may be specified') - raise ValueError('Only a single filter may be specified') - fields = 'comment,space.size,status.mapped' - result = [] - - def _handle(resource): - resource.get(fields=fields) - resource_stripped = _strip(resource) - if human: - # transform LUN size to a more readable format - resource_stripped['space']['size'] = _humansize(resource_stripped['space']['size']) - result.append(resource_stripped) - - if comment: - for resource in Lun.get_collection(**{'comment': comment}): - _handle(resource) - elif path: - for resource in Lun.get_collection(**{'name': path}): - _handle(resource) - elif uuid: - resource = Lun(uuid=uuid) - _handle(resource) - else: - # no filter specified, fetch all LUNs - for resource in Lun.get_collection(): - _handle(resource) - - return result - -def get_next_free(igroup): - """ - Returns the next free LUN ID for the specfied initiator group - """ - numbers = [] - for resource in LunMap.get_collection(igroup=igroup, fields="logical_unit_number"): - numbers.append(resource.logical_unit_number) - return max(numbers)+1 - -# https://stackoverflow.com/a/60708339 -# based on https://stackoverflow.com/a/42865957/2002471 -units = {"B": 1, "KB": 2**10, "MB": 2**20, "GB": 2**30, "TB": 2**40} -def _parse_size(size): - size = size.upper() - if not re.match(r' ', size): - size = re.sub(r'([KMGT]?B)', r' \1', size) - number, unit = [string.strip() for string in size.split()] - return int(float(number)*units[unit]) - -def provision_lun(name, size, volume, vserver, comment=None): - """ - Create a new LUN - """ - size = _parse_size(size) - path = _path(volume, name) - - resource = Lun() - resource.svm = Svm(name=vserver) - resource.name = path - resource.os_type = 'linux' - resource.space = {'size': size} - - if comment is not None: - resource.comment = comment - - result = resource.post() - return _result(result) - -# to-do: support property updates other than size changes -def update_lun(uuid, size): - """ - Change values of an existing LUN - """ - size = _parse_size(size) - - resource = Lun(uuid=uuid) - resource.space = {'size': size} - result = resource.patch() - return _result(result) - -def _delete_lun(name=None, volume=None, uuid=None): - """ - Meta-function for deleting a LUN - Currently, the individual targeting functions need to be used - """ - if (name is None or volume is None) and (uuid is None): - log.error('Specify either name and volume or uuid') - raise ValueError('Specify either name and volume or uuid') - if name and volume: - path = _path(volume, name) - resources = get_lun(path=path) - log.debug(f'netapp_ontap: resources to delete: {resources}') - found = len(resources) - if found > 1: - log.error('Refusing to delete multiple resources') - return({'result': False, 'message': 'Found more than one matching LUN, aborting deletion'}) - if found == 0: - return({'result': None, 'message': 'Did not find any matching LUN\'s'}) - resource = Lun(uuid=resources[0]['uuid']) - elif uuid: - resource = Lun(uuid=uuid) - - result = resource.delete() - return _result(result) - -def delete_lun_name(name, volume): - """ - Delete a single LUN based on its name and volume - """ - return _delete_lun(name, volume) - -def delete_lun_uuid(uuid): - """ - Delete a single LUN based on its UUID - """ - return _delete_lun(uuid=uuid) - -# to-do: allow filter by path=None and uuid -# to-do: potentially move this to get_luns_mapped() and make a separate get_lun_mapped() returning a single entry -def get_lun_mapped(comment=None, lun_result=None): - """ - Assess whether LUNs named by a comment are mapped - For more efficient programmatic use, an existing get_lun() output can be fed - """ - if (comment is None) and (lun_result is None): - log.error('Specify either a comment or existing LUN output') - raise ValueError('Specify a comment') - if comment is not None: - query = get_lun(comment) - elif lun_result is not None: - query = lun_result - resmap = {} - for lun in query: - log.debug(f'netapp_ontap: parsing LUN {lun}') - name = lun.get('name') - mapped = lun.get('status', {}).get('mapped') - resmap.update({name: mapped}) - if None in resmap: - log.error('netapp_ontap: invalid LUN mapping map') - return resmap - -def get_igroup_uuid(igroup): - """ - Return the UUID of a single initiator group - """ - resource = Igroup(name=igroup) - resource.get() - uuid = resource.uuid - return uuid - -def get_lun_mappings(name, volume, igroup=None): - """ - Return details about LUN mappings - """ - path = _path(volume, name) - result = [] - - if igroup is not None: - log.debug('netapp_ontap: filtering by igroup') - igroup_uuid = get_igroup_uuid(igroup) - luns = get_lun(path=path) - log.debug(f'netapp_ontap: found luns: {luns}') - for resource in luns: - lun_uuid = resource['uuid'] - filterdict = {'lun.uuid': lun_uuid} - if igroup is not None: - filterdict.update({'igroup.uuid': igroup_uuid}) - mapresource = LunMap(**filterdict) - mrs = mapresource.get_collection(fields='logical_unit_number') - # FIXME get() fails, saying more than one item is found, and get_collection() returns dozens of completely unrelated entries - # the loop below is a workaround discarding all the bogus entries - mymrs = list(mrs) - log.debug(f'netapp_ontap: found mappings: {mymrs}') - for mr in mymrs: - mr_stripped = _strip(mr, ['igroup', 'lun', 'svm']) - if mr_stripped['lun']['uuid'] == lun_uuid and mr_stripped['igroup']['uuid'] == igroup_uuid: - log.debug(f'netapp_ontap: elected {mr_stripped}') - result.append(mr_stripped) - - return result - -def get_lun_mapping(name, volume, igroup): - """ - Return details about a single LUN mapping - """ - results = get_lun_mappings(name, volume, igroup) - lr = len(results) - if lr == 1: - return results[0] - if lr > 1: - log.error(f'netapp_ontap: found {lr} results, but expected only one') - return None - return {} - -def map_lun(name, lunid, volume, vserver, igroup): - """ - Map a LUN to an initiator group - """ - path = _path(volume, name) - - resource = LunMap() - resource.svm = Svm(name=vserver) - resource.igroup = Igroup(name=igroup) - resource.lun = Lun(name=path) - resource.logical_unit_number = lunid - result = resource.post() - - return _result(result) - -def unmap_luns(name, volume, igroup): - """ - Remove LUNs from an initiator group - """ - path = _path(volume, name) - results = [] - - mappings = get_lun_mappings(name, volume, igroup) - log.debug(f'netapp_ontap: parsing mappings: {mappings}') - for mapping in mappings: - igroup_uuid = mapping['igroup']['uuid'] - lun_uuid = mapping['lun']['uuid'] - resource = LunMap(**{'igroup.uuid': igroup_uuid, 'lun.uuid': lun_uuid}) - result = resource.delete() - results.append(_result(result)) - - return results - -def unmap_lun(name, volume, igroup): - """ - Remove a single LUN from an initiator group - Not implemented! Might unmap multiple LUNs. - """ - # to-do: abort if more than one LUN is going to be affected, consider using get_lun_mapping() - results = unmap_luns(name, volume, igroup) - reslen = len(results) - if reslen > 1: - log.warning(f'Unmapped {reslen} LUNs, but expected only one') - if reslen == 0: - return {} - - return results[0] diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index c61ac256..ab2dcb31 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -59,7 +59,7 @@ def _size(details, human=False): # FIXME drop mapping logic from lun_present in favor of lun_mapped def _map(name, lunid, volume, vserver, igroup): ok = False - map_out = __salt__['ontap_native.map_lun'](name, lunid, volume, vserver, igroup) + map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) if map_out.get('result', False) and map_out.get('status') == 201: comment = f'Mapped LUN to ID {lunid}' ok = True @@ -68,10 +68,10 @@ def _map(name, lunid, volume, vserver, igroup): comment = 'LUN mapping failed' return comment, ok - query = __salt__['ontap_native.get_lun']() + query = __salt__['ontap.get_lun']() #luns = query[0] luns = query - next_free = __salt__['ontap_native.get_next_free']('wilde') # drop this + next_free = __salt__['ontap.get_next_free']('wilde') # drop this for lun in luns: lun_path = lun.get('name') @@ -81,10 +81,10 @@ def _map(name, lunid, volume, vserver, igroup): log.debug(f'netapp_ontap: found existing LUN {name}') if lun_uuid is None: log.error(f'netapp_ontap: found LUN with no UUID') - lun_details = __salt__['ontap_native.get_lun'](uuid=lun_uuid, human=False) + lun_details = __salt__['ontap.get_lun'](uuid=lun_uuid, human=False) lun_size = _size(lun_details[0], True) # lun_size_human = needed? - lun_mapping = __salt__['ontap_native.get_lun_mapped'](lun_result=lun_details) + lun_mapping = __salt__['ontap.get_lun_mapped'](lun_result=lun_details) lun_mapped = lun_mapping.get(name) # lun_id = needed? if lun_size == size: @@ -94,8 +94,8 @@ def _map(name, lunid, volume, vserver, igroup): if __opts__['test']: comment_size = f'Would resize LUN to {size}' else: - __salt__['ontap_native.update_lun'](lun_uuid, size) - lun2_details = __salt__['ontap_native.get_lun'](uuid=lun_uuid, human=False) + __salt__['ontap.update_lun'](lun_uuid, size) + lun2_details = __salt__['ontap.get_lun'](uuid=lun_uuid, human=False) lun2_size = _size(lun2_details[0], True) comment_size = f'LUN from {lun_size} to {size}' if lun2_size != lun_size and lun2_size == size: @@ -118,7 +118,7 @@ def _map(name, lunid, volume, vserver, igroup): comment_mapping = map_out[0] map_ok = map_out[1] - #map_out = __salt__['ontap_native.map_lun'](name, lunid, volume, vserver, igroup) + #map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) #if map_out.get('result', False) and map_out.get('status') == 201: # comment_mapping = f'Mapped LUN to ID {lunid}' # map_ok = True @@ -142,12 +142,12 @@ def _map(name, lunid, volume, vserver, igroup): ret['result'] = None return ret - __salt__['ontap_native.provision_lun'](name, size, volume, vserver, comment) + __salt__['ontap.provision_lun'](name, size, volume, vserver, comment) if do_map: map_out = _map(name, lunid, volume, vserver, igroup) comment_mapping = map_out[0] map_ok = map_out[1] - lun2_details = __salt__['ontap_native.get_lun'](comment)[0] + lun2_details = __salt__['ontap.get_lun'](comment)[0] lun2_size = _size(lun2_details) # FIXME changes dict @@ -183,7 +183,7 @@ def lun_mapped(name, lunid, volume, vserver, igroup): path = f'/vol/{volume}/{name}' ret = {'name': path, 'result': False, 'changes': {}, 'comment': ''} - mapping_out = __salt__['ontap_native.get_lun_mapping'](name, volume, igroup) + mapping_out = __salt__['ontap.get_lun_mapping'](name, volume, igroup) log.debug(f'netapp_ontap: mapping result: {mapping_out}') comment_details = f' to igroup {igroup} in SVM {vserver}' @@ -205,7 +205,7 @@ def lun_mapped(name, lunid, volume, vserver, igroup): ret['comment'] = comment return ret - map_out = __salt__['ontap_native.map_lun'](name, lunid, volume, vserver, igroup) + map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) if map_out.get('result', False) and map_out.get('status') == 201: comment = f'Mapped LUN to ID {lunid} in igroup {igroup}' ret['result'] = True @@ -221,10 +221,10 @@ def lun_unmapped(name, volume, igroup): ret = {'name': path, 'result': True, 'changes': {}, 'comment': ''} if __opts__['test']: - result = __salt__['ontap_native.get_lun_mapping'](name, volume, igroup) + result = __salt__['ontap.get_lun_mapping'](name, volume, igroup) ret['result'] = None else: - result = __salt__['ontap_native.unmap_lun'](name, volume, igroup) + result = __salt__['ontap.unmap_lun'](name, volume, igroup) rr = result.get('result', True) rs = result.get('status') log.debug(f'result: {result}') From 149b8bdfc1c6840dde262f099c75743cee2ce4c3 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 23:02:28 +0200 Subject: [PATCH 21/34] netapp_ontap: drop Ansible related files These are no longer used, as the Salt modules now communicate with the ONTAP Python library natively. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/ansible/README.md | 1 - .../ansible/playbooks/delete-lun_restit.yml | 11 ------ .../ansible/playbooks/deploy-lun_restit.yml | 11 ------ .../playbooks/fetch-lun-by-comment_restit.yml | 12 ------- .../ansible/playbooks/fetch-lun_restit.yml | 12 ------- .../ansible/playbooks/fetch-luns_restit.yml | 12 ------- .../playbooks/get-lun-mapping_restit.yml | 11 ------ .../ansible/playbooks/map-lun_restit.yml | 11 ------ .../ansible/playbooks/patch-lun_restit.yml | 11 ------ .../ansible/playbooks/unmap-lun_restit.yml | 11 ------ .../ansible/tasks/create_lun_restit.yml | 22 ------------ .../ansible/tasks/delete_lun_restit.yml | 16 --------- .../tasks/delete_lun_restit_by_path.yml | 17 --------- .../ansible/tasks/get_lun_by_name_restit.yml | 17 --------- .../ansible/tasks/get_lun_mapping_restit.yml | 18 ---------- .../ansible/tasks/get_lun_restit.yml | 17 --------- .../ansible/tasks/get_luns_restit.yml | 36 ------------------- .../ansible/tasks/map_lun_restit.yml | 24 ------------- .../ansible/tasks/patch_lun_restit.yml | 25 ------------- .../ansible/tasks/unmap_lun_restit.yml | 19 ---------- 20 files changed, 314 deletions(-) delete mode 100644 netapp_ontap-formula/ansible/README.md delete mode 100644 netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/create_lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/get_lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/get_luns_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/map_lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml delete mode 100644 netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml diff --git a/netapp_ontap-formula/ansible/README.md b/netapp_ontap-formula/ansible/README.md deleted file mode 100644 index a741ed6e..00000000 --- a/netapp_ontap-formula/ansible/README.md +++ /dev/null @@ -1 +0,0 @@ -The Ansible playbooks and tasks are intended to be run by the Salt `ontap` module, however can be run directly if the variables are correctly set. diff --git a/netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml deleted file mode 100644 index 28cdd2f3..00000000 --- a/netapp_ontap-formula/ansible/playbooks/delete-lun_restit.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: localhost - name: Delete LUN - connection: local - gather_facts: False - - tasks: - - name: Delete - block: - - name: Delete LUN - import_tasks: "{{ '../tasks/delete_lun_restit.yml' if ontap_lun_uuid is defined else '../tasks/delete_lun_restit_by_path.yml' }}" diff --git a/netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml deleted file mode 100644 index 4272ecc7..00000000 --- a/netapp_ontap-formula/ansible/playbooks/deploy-lun_restit.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: localhost - name: Create LUN - connection: local - gather_facts: False - - tasks: - - name: Deploy - block: - - name: Create LUN - import_tasks: '../tasks/create_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml b/netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml deleted file mode 100644 index 1066e744..00000000 --- a/netapp_ontap-formula/ansible/playbooks/fetch-lun-by-comment_restit.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- hosts: localhost - name: Get LUN - connection: local - gather_facts: False - - tasks: - - name: Gather details - block: - - name: Query LUN - import_tasks: '../tasks/get_lun_by_name_restit.yml' - diff --git a/netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml deleted file mode 100644 index 7f21d124..00000000 --- a/netapp_ontap-formula/ansible/playbooks/fetch-lun_restit.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- hosts: localhost - name: Query LUN - connection: local - gather_facts: False - - tasks: - - name: Gather details - block: - - name: Query LUN - import_tasks: '../tasks/get_lun_restit.yml' - diff --git a/netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml b/netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml deleted file mode 100644 index 455cbcad..00000000 --- a/netapp_ontap-formula/ansible/playbooks/fetch-luns_restit.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- hosts: localhost - name: Query LUNs - connection: local - gather_facts: False - - tasks: - - name: Gather details - block: - - name: Query LUNs - import_tasks: '../tasks/get_luns_restit.yml' - diff --git a/netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml b/netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml deleted file mode 100644 index f1d2cdca..00000000 --- a/netapp_ontap-formula/ansible/playbooks/get-lun-mapping_restit.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: localhost - name: Create LUN - connection: local - gather_facts: False - - tasks: - - name: Deploy - block: - - name: Map LUN - import_tasks: '../tasks/get_lun_mapping_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml deleted file mode 100644 index 71d87bf0..00000000 --- a/netapp_ontap-formula/ansible/playbooks/map-lun_restit.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: localhost - name: Create LUN - connection: local - gather_facts: False - - tasks: - - name: Deploy - block: - - name: Map LUN - import_tasks: '../tasks/map_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml deleted file mode 100644 index aaba4cdc..00000000 --- a/netapp_ontap-formula/ansible/playbooks/patch-lun_restit.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: localhost - name: Patch LUN - connection: local - gather_facts: False - - tasks: - - name: Deploy - block: - - name: Patch LUN - import_tasks: '../tasks/patch_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml b/netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml deleted file mode 100644 index 1528a014..00000000 --- a/netapp_ontap-formula/ansible/playbooks/unmap-lun_restit.yml +++ /dev/null @@ -1,11 +0,0 @@ ---- -- hosts: localhost - name: Create LUN - connection: local - gather_facts: False - - tasks: - - name: Deploy - block: - - name: Map LUN - import_tasks: '../tasks/unmap_lun_restit.yml' diff --git a/netapp_ontap-formula/ansible/tasks/create_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/create_lun_restit.yml deleted file mode 100644 index 6cb5f8ec..00000000 --- a/netapp_ontap-formula/ansible/tasks/create_lun_restit.yml +++ /dev/null @@ -1,22 +0,0 @@ ---- -- name: Create LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - http_port: '{{ ontap_port }}' - use_rest: always - validate_certs: false - api: storage/luns - method: POST - body: - name: '{{ ontap_lun_path }}' - comment: "{{ ontap_comment if ontap_comment is defined else 'Provisioned by Ansible'}}" - svm: '{{ ontap_vserver }}' - space: - size: '{{ ontap_size }}' - os_type: linux - register: lun_create_info - -- debug: var=lun_create_info -- assert: { that: lun_create_info.status_code==201 } diff --git a/netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml deleted file mode 100644 index c4442f61..00000000 --- a/netapp_ontap-formula/ansible/tasks/delete_lun_restit.yml +++ /dev/null @@ -1,16 +0,0 @@ ---- -- name: Delete LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - http_port: '{{ ontap_port }}' - use_rest: always - validate_certs: false - api: 'storage/luns' - method: DELETE - query: - uuid: '{{ ontap_lun_uuid }}' - register: lun_delete_info - -- assert: { that: lun_delete_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml b/netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml deleted file mode 100644 index 57a2685f..00000000 --- a/netapp_ontap-formula/ansible/tasks/delete_lun_restit_by_path.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Delete LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - http_port: '{{ ontap_port }}' - use_rest: always - validate_certs: false - api: 'storage/luns' - method: DELETE - query: - name: '/vol/{{ ontap_volume }}/{{ ontap_lun_name }}' - register: lun_delete_info - -- debug: var=lun_delete_info -- assert: { that: lun_delete_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml b/netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml deleted file mode 100644 index caab4469..00000000 --- a/netapp_ontap-formula/ansible/tasks/get_lun_by_name_restit.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Get LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - http_port: '{{ ontap_port }}' - use_rest: always - validate_certs: false - wait_for_completion: true - api: 'storage/luns' - query: - comment: '{{ ontap_lun_comment }}' - fields: 'space,status.mapped' - register: lun_info - -- assert: { that: lun_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml b/netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml deleted file mode 100644 index a0ae00a2..00000000 --- a/netapp_ontap-formula/ansible/tasks/get_lun_mapping_restit.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -- name: Map LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - http_port: '{{ ontap_port }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - use_rest: always - validate_certs: false - api: protocols/san/lun-maps - query: - igroup: '{{ ontap_igroup }}' - lun: '{{ ontap_lun_path }}' - register: lun_map_out - -- debug: var=lun_map_out -- assert: { that: lun_map_out.status_code==200 } - diff --git a/netapp_ontap-formula/ansible/tasks/get_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/get_lun_restit.yml deleted file mode 100644 index 30e4efb3..00000000 --- a/netapp_ontap-formula/ansible/tasks/get_lun_restit.yml +++ /dev/null @@ -1,17 +0,0 @@ ---- -- name: Get LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - http_port: '{{ ontap_port }}' - use_rest: always - validate_certs: false - wait_for_completion: true - api: 'storage/luns' - query: - uuid: '{{ ontap_lun_uuid }}' - fields: 'space,status.mapped' - register: lun_info - -- assert: { that: lun_info.status_code==200 } diff --git a/netapp_ontap-formula/ansible/tasks/get_luns_restit.yml b/netapp_ontap-formula/ansible/tasks/get_luns_restit.yml deleted file mode 100644 index 3a21f4b5..00000000 --- a/netapp_ontap-formula/ansible/tasks/get_luns_restit.yml +++ /dev/null @@ -1,36 +0,0 @@ ---- -- name: Get LUNs - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - http_port: '{{ ontap_port }}' - use_rest: always - validate_certs: false - wait_for_completion: true - api: storage/luns - query: - fields: comment - register: lun_info -- debug: var=lun_info -- assert: { that: lun_info.status_code==200 } - -- name: Store LUN dump - local_action: - module: copy - content: '{{ lun_info }}' - dest: /dev/shm/luns.out - -# I'm sure there's a native way to do this... -- name: Get highest LUN - ansible.builtin.shell: - cmd: "grep -Po 'lun\\K(\\d+)' /dev/shm/luns.out | sort -nu | tail -n1" - register: lun_grep - -- name: Define LUN number - set_fact: - lun_id: '{{ lun_grep.stdout | int + 1 }}' - -- name: Dump gathered variables - ansible.builtin.debug: - msg: 'LUN ID defined as {{ lun_id }}' diff --git a/netapp_ontap-formula/ansible/tasks/map_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/map_lun_restit.yml deleted file mode 100644 index 28a5d69e..00000000 --- a/netapp_ontap-formula/ansible/tasks/map_lun_restit.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -- name: Map LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - http_port: '{{ ontap_port }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - use_rest: always - validate_certs: false - api: protocols/san/lun-maps - method: POST - body: - svm: - name: '{{ ontap_vserver }}' - igroup: - name: '{{ ontap_igroup }}' - lun: - name: '{{ ontap_lun_path }}' - logical_unit_number: '{{ ontap_lun_id }}' - register: lun_map_out - -- debug: var=lun_map_out -- assert: { that: lun_map_out.status_code==201 } - diff --git a/netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml deleted file mode 100644 index 8931da3f..00000000 --- a/netapp_ontap-formula/ansible/tasks/patch_lun_restit.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -- name: Patch LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - http_port: '{{ ontap_port }}' - use_rest: always - validate_certs: false - api: storage/luns - method: PATCH - query: - name: '{{ ontap_lun_path }}' - body: - space: - size: '{{ ontap_size }}' - register: lun_patch_info - -- debug: var=lun_patch_info -- assert: { that: lun_patch_info.status_code==200 } - -- local_action: - module: copy - content: '{{ lun_patch_info }}' - dest: /tmp/lun_patch.out diff --git a/netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml b/netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml deleted file mode 100644 index 41dbd6c5..00000000 --- a/netapp_ontap-formula/ansible/tasks/unmap_lun_restit.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -- name: Map LUN - netapp.ontap.na_ontap_restit: - hostname: '{{ ontap_host }}' - http_port: '{{ ontap_port }}' - cert_filepath: '{{ ontap_crt }}' - key_filepath: '{{ ontap_key }}' - use_rest: always - validate_certs: false - api: protocols/san/lun-maps - method: DELETE - query: - igroup: '{{ ontap_igroup }}' - lun: '{{ ontap_lun_path }}' - register: lun_unmap_out - -- debug: var=lun_unmap_out -- assert: { that: lun_unmap_out.status_code==200 } - From 777242e8e3ed77a004996162e998b34759bc2ffc Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 23:18:10 +0200 Subject: [PATCH 22/34] netapp_ontap: clean up lun_present Drop old mapping logic (LUN's are now to be mapped using "lun_mapped" instead), use paths instead of comments for validation. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_states/ontap.py | 73 +++------------------------ 1 file changed, 6 insertions(+), 67 deletions(-) diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index ab2dcb31..c413c3c5 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -40,15 +40,9 @@ def _parse_size(size): number, unit = [string.strip() for string in size.split()] return int(float(number)*units[unit]) -def lun_present(name, comment, size, volume, vserver, lunid=None, igroup=None): +def lun_present(name, comment, size, volume, vserver): path = f'/vol/{volume}/{name}' ret = {'name': path, 'result': False, 'changes': {}, 'comment': ''} - size_ok = False - map_ok = False - if not None in [lunid, igroup]: - do_map = True - else: - do_map = False def _size(details, human=False): size = details.get('space', {}).get('size') @@ -56,40 +50,22 @@ def _size(details, human=False): return _humansize(size) return size - # FIXME drop mapping logic from lun_present in favor of lun_mapped - def _map(name, lunid, volume, vserver, igroup): - ok = False - map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) - if map_out.get('result', False) and map_out.get('status') == 201: - comment = f'Mapped LUN to ID {lunid}' - ok = True - # consider another get_lun to validate .. given the queries being expensive in time, it should be combined with the resize validation - else: - comment = 'LUN mapping failed' - return comment, ok - - query = __salt__['ontap.get_lun']() - #luns = query[0] - luns = query - next_free = __salt__['ontap.get_next_free']('wilde') # drop this + luns = __salt__['ontap.get_lun']() for lun in luns: lun_path = lun.get('name') lun_comment = lun.get('comment') lun_uuid = lun.get('uuid') - if lun_comment == comment or lun_path == path: + if lun_path == path: log.debug(f'netapp_ontap: found existing LUN {name}') if lun_uuid is None: log.error(f'netapp_ontap: found LUN with no UUID') lun_details = __salt__['ontap.get_lun'](uuid=lun_uuid, human=False) lun_size = _size(lun_details[0], True) # lun_size_human = needed? - lun_mapping = __salt__['ontap.get_lun_mapped'](lun_result=lun_details) - lun_mapped = lun_mapping.get(name) - # lun_id = needed? if lun_size == size: comment_size = f'Size {size} matches' - size_ok = True + ret['result'] = True elif lun_size != size: if __opts__['test']: comment_size = f'Would resize LUN to {size}' @@ -100,38 +76,14 @@ def _map(name, lunid, volume, vserver, igroup): comment_size = f'LUN from {lun_size} to {size}' if lun2_size != lun_size and lun2_size == size: comment_size = f'Sucessfully resized {comment_size}' - size_ok = True + ret['result'] = True elif lun2_size == lun_size: comment_size = f'Failed to resize {comment_size}, it is still {lun2_size}' else: comment_size = f'Unexpected outcome while resizing {comment_size}' - if not do_map: - comment_mapping = None - map_ok = True - else: - if lun_mapped: - comment_mapping = 'Already mapped' - map_ok = True - else: - map_out = _map(name, lunid, volume, vserver, igroup) - comment_mapping = map_out[0] - map_ok = map_out[1] - - #map_out = __salt__['ontap.map_lun'](name, lunid, volume, vserver, igroup) - #if map_out.get('result', False) and map_out.get('status') == 201: - # comment_mapping = f'Mapped LUN to ID {lunid}' - # map_ok = True - # # consider another get_lun to validate .. given the queries being expensive in time, it should be combined with the resize validation - #else: - # comment_mapping = 'LUN mapping failed' - comment_base = 'LUN is already present' - if size_ok and map_ok: - ret['result'] = True retcomment = f'{comment_base}; {comment_size}' - if comment_mapping is not None: - retcomment = f'{retcomment}, {comment_mapping}' if __opts__['test']: ret['result'] = None ret['comment'] = retcomment @@ -143,11 +95,7 @@ def _map(name, lunid, volume, vserver, igroup): return ret __salt__['ontap.provision_lun'](name, size, volume, vserver, comment) - if do_map: - map_out = _map(name, lunid, volume, vserver, igroup) - comment_mapping = map_out[0] - map_ok = map_out[1] - lun2_details = __salt__['ontap.get_lun'](comment)[0] + lun2_details = __salt__['ontap.get_lun'](path=path)[0] lun2_size = _size(lun2_details) # FIXME changes dict @@ -167,15 +115,6 @@ def _map(name, lunid, volume, vserver, igroup): comment = f'LUN {comment_path} {comment_size}.' - if do_map: - if map_ok: - ret['result'] = True - comment_mapping = f'mapped to ID {lunid}' - else: - ret['result'] = False - comment_mapping = f'mapping to ID {lunid} failed' - comment = f'{comment} LUN {comment_mapping}.' - ret['comment'] = comment return ret From 29db36c9c6206710614a8f98ecc55445e851dcfe Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 25 Jun 2023 23:21:13 +0200 Subject: [PATCH 23/34] netapp_ontap: remove leftover Ansible references Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 6 +++--- netapp_ontap-formula/_states/ontap.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index c722f063..d21ad513 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -1,5 +1,5 @@ """ -Salt execution module for maging ONTAP based NetApp storage systems using Ansible +Salt execution module for maging ONTAP based NetApp storage systems Copyright (C) 2023 SUSE LLC This program is free software: you can redistribute it and/or modify @@ -48,7 +48,7 @@ def _result(result): Used for DELETE/PATCH/POST output, not for GET result = the output to parse """ - log.debug(f'ontap_ansible: parsing result: {result}') + log.debug(f'netapp_ontap: parsing result: {result}') error = result.is_err status = result.http_response.status_code @@ -71,7 +71,7 @@ def _result(result): resmap.update(res) return resmap - log.warning('ontap_ansible: dumping unknown result') + log.warning('netapp_ontap: dumping unknown result') return result def _strip(resource, inners=[]): diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index c413c3c5..c41328b3 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -1,5 +1,5 @@ """ -Salt state module for managing LUNs using the ONTAP Ansible collection +Salt state module for maging ONTAP based NetApp storage systems Copyright (C) 2023 SUSE LLC This program is free software: you can redistribute it and/or modify From d758a54fbc96dba0a54279d64fd3537a41cacb76 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Tue, 27 Jun 2023 01:12:49 +0200 Subject: [PATCH 24/34] netapp_ontap: correct test=True results Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_states/ontap.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index c41328b3..beecee64 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -84,8 +84,6 @@ def _size(details, human=False): comment_base = 'LUN is already present' retcomment = f'{comment_base}; {comment_size}' - if __opts__['test']: - ret['result'] = None ret['comment'] = retcomment return ret @@ -131,6 +129,7 @@ def lun_mapped(name, lunid, volume, vserver, igroup): if not mapping_out or igroup != current_igroup or vserver != current_vserver: if __opts__['test']: comment = f'Would map ID {lunid}{comment_details}' + ret['result'] = None elif mapping_out and igroup == current_igroup or vserver == current_svm: comment = f'Already mapped{comment_details}' ret['result'] = True @@ -139,8 +138,6 @@ def lun_mapped(name, lunid, volume, vserver, igroup): comment = 'Something weird happened' if __opts__['test'] or ret['result'] is True: - if __opts__['test']: - ret['result'] = None ret['comment'] = comment return ret @@ -161,7 +158,6 @@ def lun_unmapped(name, volume, igroup): if __opts__['test']: result = __salt__['ontap.get_lun_mapping'](name, volume, igroup) - ret['result'] = None else: result = __salt__['ontap.unmap_lun'](name, volume, igroup) rr = result.get('result', True) @@ -170,6 +166,7 @@ def lun_unmapped(name, volume, igroup): if __opts__['test'] and result: comment = f'Would unmap LUN' + ret['result'] = None elif not result: comment = 'Nothing to unmap' elif rr is True and rs == 200: From c6decf9678ecc7e4db8717cb15fef2a591c1d830 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 03:41:08 +0200 Subject: [PATCH 25/34] netapp_ontap: initialize tests Initial test structure. Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/tests/conftest.py | 6 ++++++ netapp_ontap-formula/tests/lib.py | 4 ++++ netapp_ontap-formula/tests/test_20_lun.py | 16 ++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 netapp_ontap-formula/tests/conftest.py create mode 100644 netapp_ontap-formula/tests/lib.py create mode 100644 netapp_ontap-formula/tests/test_20_lun.py diff --git a/netapp_ontap-formula/tests/conftest.py b/netapp_ontap-formula/tests/conftest.py new file mode 100644 index 00000000..b0ec47a7 --- /dev/null +++ b/netapp_ontap-formula/tests/conftest.py @@ -0,0 +1,6 @@ +import pytest + +@pytest.fixture +def target(): + # FIXME make this configurable + return '10.168.0.97' diff --git a/netapp_ontap-formula/tests/lib.py b/netapp_ontap-formula/tests/lib.py new file mode 100644 index 00000000..cdc1a66e --- /dev/null +++ b/netapp_ontap-formula/tests/lib.py @@ -0,0 +1,4 @@ +import requests + +def api(target, path, params={}): + return requests.get(url=f'https://{target}/api/{path}', params=params, verify=False, auth=requests.auth.HTTPBasicAuth('pytest', 'cats2023')).json() diff --git a/netapp_ontap-formula/tests/test_20_lun.py b/netapp_ontap-formula/tests/test_20_lun.py new file mode 100644 index 00000000..911b1c9c --- /dev/null +++ b/netapp_ontap-formula/tests/test_20_lun.py @@ -0,0 +1,16 @@ +from lib import api + +def test_lun_provision(host, target): + r = host.salt('ontap.provision_lun', ['testlun', '10MB', 'myvol', 'mysvm', 'a lonely lun']) + assert r['result'] + assert r['status'] == 201 + result_rest = api(target, 'storage/luns', {'fields': 'comment,space.size'}) + records = result_rest['records'] + for entry, record in enumerate(records): + if record['name'] == '/vol/myvol/testlun': + myrecord = records[entry] + break + assert myrecord, 'Did not find any record matching the created LUN name' + assert myrecord['name'] == '/vol/myvol/testlun' + assert myrecord['comment'] == 'a lonely lun' + assert myrecord['space']['size'] == 10485760 From 40d92a110350dfbf0aacea6de3d8bd136244c59f Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 03:54:29 +0200 Subject: [PATCH 26/34] netapp_ontap: add README to tests Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/tests/README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 netapp_ontap-formula/tests/README.md diff --git a/netapp_ontap-formula/tests/README.md b/netapp_ontap-formula/tests/README.md new file mode 100644 index 00000000..c0f6591a --- /dev/null +++ b/netapp_ontap-formula/tests/README.md @@ -0,0 +1,18 @@ +# NetApp ONTAP formula test suite + +These tests are intended to be run against a Salt master with access to an ONTAP simulator. +The workstation the test suite is run from needs HTTPS access to the same simulator in order to validate the results. +Configure the simulator with a `pytest` user and role in the correct SVM: + +``` +::> security login rest-role create -vserver mysvm -role pytest -api /api/storage -access readonly +Warning: ... + +::> security login create -user-or-group-name pytest -application http -authentication-method password -role pytest -vserver mysvm +Please enter a password for user 'pytest': cats2023 +Please enter it again: cats2023 +``` + +Invocation: + +`pytest -v -rx --hosts user@master --ssh-config /dev/null netapp_ontap-formula/tests/` From 190e90fc4d422a6ccccfd493e5442cc66ef4d2a0 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 19:14:53 +0200 Subject: [PATCH 27/34] netapp_ontap: expand tests - use shared LUN fixture - add coverage for deletion and update operations Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/tests/conftest.py | 31 ++++++++++++ netapp_ontap-formula/tests/test_20_lun.py | 62 +++++++++++++++++++++-- 2 files changed, 90 insertions(+), 3 deletions(-) diff --git a/netapp_ontap-formula/tests/conftest.py b/netapp_ontap-formula/tests/conftest.py index b0ec47a7..327c47ef 100644 --- a/netapp_ontap-formula/tests/conftest.py +++ b/netapp_ontap-formula/tests/conftest.py @@ -1,6 +1,37 @@ +from lib import api import pytest @pytest.fixture def target(): # FIXME make this configurable return '10.168.0.97' + +@pytest.fixture +def lun(host, target): + r = host.salt('ontap.provision_lun', ['testlun', '10MB', 'myvol', 'mysvm', 'a lonely lun']) + assert r['result'] + assert r['status'] == 201 + r_prequery = api(target, 'storage/luns') + records_pre = r_prequery['records'] + for entry, record in enumerate(records_pre): + if record['name'] == '/vol/myvol/testlun': + myrecord = records_pre[entry] + assert myrecord, 'Did not find any record matching the created LUN fixture name' + myuuid = myrecord['uuid'] + assert any(record['uuid'] == myuuid for record in records_pre), 'LUN fixture does not show in records' + + yield myrecord + + r_midquery = api(target, 'storage/luns') + records_mid = r_midquery['records'] + for entry, record in enumerate(records_mid): + if record['name'] == '/vol/myvol/testlun': + myuuid = records_mid[entry]['uuid'] + # probably better to delete using a direct API call here in case there is a problem with delete_lun_uuid + r = host.salt('ontap.delete_lun_uuid', myuuid) + assert r['result'] + assert r['status'] == 200 + + r_finquery = api(target, 'storage/luns') + records_fin = r_finquery['records'] + assert not any(record['uuid'] == myuuid for record in records_fin), 'Failed to clean up LUN fixture' diff --git a/netapp_ontap-formula/tests/test_20_lun.py b/netapp_ontap-formula/tests/test_20_lun.py index 911b1c9c..f0b0517f 100644 --- a/netapp_ontap-formula/tests/test_20_lun.py +++ b/netapp_ontap-formula/tests/test_20_lun.py @@ -1,16 +1,72 @@ from lib import api +import json +import pytest + +@pytest.mark.dependency(name='provision') def test_lun_provision(host, target): - r = host.salt('ontap.provision_lun', ['testlun', '10MB', 'myvol', 'mysvm', 'a lonely lun']) + r = host.salt('ontap.provision_lun', ['provisiontestlun', '10MB', 'myvol', 'mysvm', 'a provisioned lun']) assert r['result'] assert r['status'] == 201 result_rest = api(target, 'storage/luns', {'fields': 'comment,space.size'}) records = result_rest['records'] + for entry, record in enumerate(records): + if record['name'] == '/vol/myvol/provisiontestlun': + myrecord = records[entry] + break + assert myrecord, 'Did not find any record matching the created LUN name' + assert myrecord['name'] == '/vol/myvol/provisiontestlun' + assert myrecord['comment'] == 'a provisioned lun' + assert myrecord['space']['size'] == 10485760 + + +@pytest.mark.dependency(depends=['provision']) +def test_lun_delete_name(host, target): + r = host.salt('ontap.delete_lun_name', ['provisiontestlun', 'myvol']) + assert r['result'] + assert r['status'] == 200 + result_rest = api(target, 'storage/luns', {'fields': 'comment,space.size'}) + records = result_rest['records'] + assert not '/vol/myvol/provisiontestlun' in records + + +def test_lun_delete_uuid(host, target, lun): + myuuid = lun['uuid'] + r = host.salt('ontap.delete_lun_uuid', myuuid) + assert r['result'] + assert r['status'] == 200 + result_rest = api(target, 'storage/luns') + records = result_rest['records'] + assert not any(record['uuid'] == myuuid for record in records) + + +def test_lun_resize(host, target, lun): + myuuid = lun['uuid'] + r = host.salt('ontap.update_lun', [myuuid, '20MB']) + assert r['result'] + assert r['status'] == 200 + result_rest = api(target, 'storage/luns', {'fields': 'space.size'}) + records = result_rest['records'] + for entry, record in enumerate(records): + if record['name'] == '/vol/myvol/testlun': + myrecord = records[entry] + break + assert myrecord, 'Did not find any record matching the created LUN name' + assert myrecord['space']['size'] == 20971520 + + +def test_lun_resize_reduce_fail(host, target, lun): + myuuid = lun['uuid'] + r = host.run_expect([1], f'salt-call --out=json ontap.update_lun {myuuid} 5MB') + rout = json.loads(r.stdout)['local'] + assert not rout['result'] + assert rout['status'] == 400 + assert rout['message'] == 'Reducing the size of a LUN might cause permanent data loss or corruption and is not supported by the LUN REST API. The ONTAP command "lun resize" can be used to reduce the LUN size.' + result_rest = api(target, 'storage/luns', {'fields': 'space.size'}) + records = result_rest['records'] for entry, record in enumerate(records): if record['name'] == '/vol/myvol/testlun': myrecord = records[entry] break assert myrecord, 'Did not find any record matching the created LUN name' - assert myrecord['name'] == '/vol/myvol/testlun' - assert myrecord['comment'] == 'a lonely lun' assert myrecord['space']['size'] == 10485760 From 77b648b9d8efbd9d63885a60468aa5083076c239 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 20:52:52 +0200 Subject: [PATCH 28/34] netapp_ontap: add LUN mapping test coverage Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/tests/conftest.py | 28 ++++++- netapp_ontap-formula/tests/test_20_lun.py | 90 +++++++++++++++++++++++ 2 files changed, 115 insertions(+), 3 deletions(-) diff --git a/netapp_ontap-formula/tests/conftest.py b/netapp_ontap-formula/tests/conftest.py index 327c47ef..448ee2d2 100644 --- a/netapp_ontap-formula/tests/conftest.py +++ b/netapp_ontap-formula/tests/conftest.py @@ -6,12 +6,13 @@ def target(): # FIXME make this configurable return '10.168.0.97' + @pytest.fixture def lun(host, target): r = host.salt('ontap.provision_lun', ['testlun', '10MB', 'myvol', 'mysvm', 'a lonely lun']) assert r['result'] assert r['status'] == 201 - r_prequery = api(target, 'storage/luns') + r_prequery = api(target, 'storage/luns', {'fields': 'comment,space.size,svm.name'}) records_pre = r_prequery['records'] for entry, record in enumerate(records_pre): if record['name'] == '/vol/myvol/testlun': @@ -22,11 +23,13 @@ def lun(host, target): yield myrecord - r_midquery = api(target, 'storage/luns') + r_midquery = api(target, 'storage/luns', {'fields': 'status.mapped'}) records_mid = r_midquery['records'] for entry, record in enumerate(records_mid): if record['name'] == '/vol/myvol/testlun': - myuuid = records_mid[entry]['uuid'] + myuuid = record['uuid'] + if record['status']['mapped']: + r = host.salt('ontap.unmap_lun', ['testlun', 'myvol', 'mysvm']) # probably better to delete using a direct API call here in case there is a problem with delete_lun_uuid r = host.salt('ontap.delete_lun_uuid', myuuid) assert r['result'] @@ -35,3 +38,22 @@ def lun(host, target): r_finquery = api(target, 'storage/luns') records_fin = r_finquery['records'] assert not any(record['uuid'] == myuuid for record in records_fin), 'Failed to clean up LUN fixture' + + +@pytest.fixture +def lun_mapped(host, target, lun): + myuuid = lun['uuid'] + mypath = lun['name'] + myvolume, myname = mypath.split('/')[2:] + mysvm = lun['svm']['name'] + r = host.salt('ontap.map_lun', [myname, '900', myvolume, mysvm, 'myigroup']) + assert r['result'] + assert r['status'] == 201 + r_prequery = api(target, 'storage/luns', {'fields': 'comment,space.size,svm.name'}) + records_pre = r_prequery['records'] + for entry, record in enumerate(records_pre): + if record['name'] == '/vol/myvol/testlun': + myrecord = records_pre[entry] + assert myrecord, 'Did not find any record matching the created LUN fixture name' + + yield myrecord diff --git a/netapp_ontap-formula/tests/test_20_lun.py b/netapp_ontap-formula/tests/test_20_lun.py index f0b0517f..a8c6cd27 100644 --- a/netapp_ontap-formula/tests/test_20_lun.py +++ b/netapp_ontap-formula/tests/test_20_lun.py @@ -70,3 +70,93 @@ def test_lun_resize_reduce_fail(host, target, lun): break assert myrecord, 'Did not find any record matching the created LUN name' assert myrecord['space']['size'] == 10485760 + + +@pytest.mark.parametrize("human", [True, False]) +def test_get_lun(host, target, lun, human): + mycomment = lun['comment'] + myuuid = lun['uuid'] + myname = lun['name'] + r = host.salt('ontap.get_lun', ['a lonely lun', f'human={human}']) + r0 = r[0] + assert len(r0.keys()) == 5 + assert r0['comment'] == mycomment + assert r0['name'] == myname + if human: + expsize = '10MB' + elif not human: + expsize = 10485760 + assert r0['space']['size'] == expsize + assert r0['status']['mapped'] is False + assert r0['uuid'] == myuuid + + +def test_map_lun(host, target, lun): + myuuid = lun['uuid'] + mypath = lun['name'] + myvolume, myname = mypath.split('/')[2:] + mysvm = lun['svm']['name'] + r = host.salt('ontap.map_lun', [myname, '900', myvolume, mysvm, 'myigroup']) + assert r['result'] + assert r['status'] == 201 + result_rest = api(target, 'protocols/san/lun-maps', {'lun.uuid': myuuid}) + records = result_rest['records'] + r0 = records[0] + assert r0['lun']['name'] == mypath + + +def test_get_lun_mapped(host, target, lun_mapped): + lun = lun_mapped + mypath = lun['name'] + mycomment = lun['comment'] + r = host.salt('ontap.get_lun_mapped', mycomment) + assert r[mypath] is True + + +def test_get_lun_mapping(host, target, lun_mapped): + lun = lun_mapped + mysvm = lun['svm']['name'] + mypath = lun['name'] + myvolume, myname = mypath.split('/')[2:] + r = host.salt('ontap.get_lun_mapping', [myname, myvolume, 'myigroup']) + assert r['lun']['name'] == mypath + assert r['svm']['name'] == mysvm + + +@pytest.mark.parametrize("igroup", [None, 'myigroup']) +def test_get_lun_mappings(host, target, lun_mapped, igroup): + lun = lun_mapped + mysvm = lun['svm']['name'] + mypath = lun['name'] + myvolume, myname = mypath.split('/')[2:] + params = [myname, myvolume] + if igroup is not None: + params.append(igroup) + r = host.salt('ontap.get_lun_mappings', params) + assert len(r) == 1 + r0 = r[0] + if igroup is not None: + assert r0['igroup']['name'] == igroup + assert r0['lun']['name'] == mypath + assert r0['svm']['name'] == mysvm + + +def test_unmap_lun(host, target, lun_mapped): + lun = lun_mapped + mypath = lun['name'] + myvolume, myname = mypath.split('/')[2:] + r = host.salt('ontap.unmap_lun', [myname, myvolume, 'myigroup']) + assert r['result'] + assert r['status'] == 200 + + +# to-do: test multiple unmappings +def test_unmap_luns(host, target, lun_mapped): + lun = lun_mapped + mypath = lun['name'] + myvolume, myname = mypath.split('/')[2:] + r = host.salt('ontap.unmap_luns', [myname, myvolume, 'myigroup']) + assert len(r) == 1 + r0 = r[0] + assert r0['result'] + assert r0['status'] == 200 From 325a3fb50d7c6f6ec073675f9bc5ab3fd068722a Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 20:53:29 +0200 Subject: [PATCH 29/34] netapp_ontap: repair mapping igroup filter Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_modules/ontap.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/netapp_ontap-formula/_modules/ontap.py b/netapp_ontap-formula/_modules/ontap.py index d21ad513..fa316ee5 100644 --- a/netapp_ontap-formula/_modules/ontap.py +++ b/netapp_ontap-formula/_modules/ontap.py @@ -273,7 +273,10 @@ def get_lun_mappings(name, volume, igroup=None): log.debug(f'netapp_ontap: found mappings: {mymrs}') for mr in mymrs: mr_stripped = _strip(mr, ['igroup', 'lun', 'svm']) - if mr_stripped['lun']['uuid'] == lun_uuid and mr_stripped['igroup']['uuid'] == igroup_uuid: + if mr_stripped['lun']['uuid'] == lun_uuid: + if igroup is not None and mr_stripped['igroup']['uuid'] != igroup_uuid: + log.debug('netapp_ontap: igroup UUID does not match') + continue log.debug(f'netapp_ontap: elected {mr_stripped}') result.append(mr_stripped) From 5a0d46f1b2b10e3ad419c896d255ff14057344c5 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 22:42:44 +0200 Subject: [PATCH 30/34] netapp_ontap: move module tests to subdirectory Signed-off-by: Georg Pfuetzenreuter --- .../tests/{test_20_lun.py => modules/test_120_lun.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename netapp_ontap-formula/tests/{test_20_lun.py => modules/test_120_lun.py} (100%) diff --git a/netapp_ontap-formula/tests/test_20_lun.py b/netapp_ontap-formula/tests/modules/test_120_lun.py similarity index 100% rename from netapp_ontap-formula/tests/test_20_lun.py rename to netapp_ontap-formula/tests/modules/test_120_lun.py From ea1784fb2aa8e253c60623074d87441a69d52d46 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 22:43:09 +0200 Subject: [PATCH 31/34] netapp_ontap: repair test resize state return and typo Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/_states/ontap.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/netapp_ontap-formula/_states/ontap.py b/netapp_ontap-formula/_states/ontap.py index beecee64..36f97f0d 100644 --- a/netapp_ontap-formula/_states/ontap.py +++ b/netapp_ontap-formula/_states/ontap.py @@ -69,13 +69,14 @@ def _size(details, human=False): elif lun_size != size: if __opts__['test']: comment_size = f'Would resize LUN to {size}' + ret['result'] = None else: __salt__['ontap.update_lun'](lun_uuid, size) lun2_details = __salt__['ontap.get_lun'](uuid=lun_uuid, human=False) lun2_size = _size(lun2_details[0], True) comment_size = f'LUN from {lun_size} to {size}' if lun2_size != lun_size and lun2_size == size: - comment_size = f'Sucessfully resized {comment_size}' + comment_size = f'Successfully resized {comment_size}' ret['result'] = True elif lun2_size == lun_size: comment_size = f'Failed to resize {comment_size}, it is still {lun2_size}' From 9113804e446653c84dabd077c1e35a6616847a71 Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Thu, 29 Jun 2023 22:43:32 +0200 Subject: [PATCH 32/34] netapp_ontap: add test coverage for state modules Signed-off-by: Georg Pfuetzenreuter --- .../tests/states/test_220_lun.py | 103 ++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 netapp_ontap-formula/tests/states/test_220_lun.py diff --git a/netapp_ontap-formula/tests/states/test_220_lun.py b/netapp_ontap-formula/tests/states/test_220_lun.py new file mode 100644 index 00000000..1da979ce --- /dev/null +++ b/netapp_ontap-formula/tests/states/test_220_lun.py @@ -0,0 +1,103 @@ +import pytest + + +params_present = ['ontap.lun_present', 'name=testlun', 'comment="a present lun"', 'size=10MB', 'volume=myvol', 'vserver=mysvm'] +state_present = 'ontap_|-testlun_|-testlun_|-lun_present' +params_mapped = ['ontap.lun_mapped', 'name=testlun', 'lunid=900', 'volume=myvol', 'vserver=mysvm', 'igroup=myigroup'] +state_mapped = 'ontap_|-testlun_|-testlun_|-lun_mapped' +params_unmapped = ['ontap.lun_unmapped', 'name=testlun', 'volume=myvol', 'igroup=myigroup'] +state_unmapped = 'ontap_|-testlun_|-testlun_|-lun_unmapped' + + +@pytest.mark.dependency(name='present') +@pytest.mark.parametrize('test', [True, False]) +def test_lun_present(host, target, test): + r = host.salt('state.single', params_present + [f'test={test}']) + sr = r[state_present] + assert sr['name'] == '/vol/myvol/testlun' + if test: + comment = 'Would provision LUN' + result = None + elif not test: + comment = 'LUN /vol/myvol/testlun created with size 10MB.' + result = True + assert sr['comment'] == comment + assert sr['result'] is result + + +@pytest.mark.dependency(depends=['present'], name='present_already') +@pytest.mark.parametrize('test', [True, False]) +def test_lun_present_already(host, target, test): + r = host.salt('state.single', params_present + [f'test={test}']) + sr = r[state_present] + assert sr['name'] == '/vol/myvol/testlun' + assert sr['comment'] == 'LUN is already present; Size 10MB matches' + assert sr['result'] is True + + +@pytest.mark.dependency(depends=['present_already']) +@pytest.mark.parametrize('test', [True, False]) +def test_lun_present_resize(host, target, test): + r = host.salt('state.single', ['ontap.lun_present', 'name=testlun', 'comment="a present lun"', 'size=20MB', 'volume=myvol', 'vserver=mysvm', f'test={test}']) + sr = r[state_present] + assert sr['name'] == '/vol/myvol/testlun' + if test: + comment = 'LUN is already present; Would resize LUN to 20MB' + result = None + elif not test: + comment = 'LUN is already present; Successfully resized LUN from 10MB to 20MB' + result = True + assert sr['comment'] == comment + assert sr['result'] is result + + +@pytest.mark.dependency(depends=['present_already'], name='mapped') +@pytest.mark.parametrize('test', [True, False]) +def test_lun_mapped(host, target, test): + r = host.salt('state.single', params_mapped + [f'test={test}']) + sr = r[state_mapped] + assert sr['name'] == '/vol/myvol/testlun' + if test: + comment = 'Would map ID 900 to igroup myigroup in SVM mysvm' + result = None + elif not test: + comment = 'Mapped LUN to ID 900 in igroup myigroup' + result = True + assert sr['comment'] == comment + assert sr['result'] is result + + +@pytest.mark.dependency(depends=['mapped'], name='mapped_already') +@pytest.mark.parametrize('test', [True, False]) +def test_lun_mapped_already(host, target, test): + r = host.salt('state.single', params_mapped + [f'test={test}']) + sr = r[state_mapped] + assert sr['name'] == '/vol/myvol/testlun' + assert sr['comment'] == 'Already mapped to igroup myigroup in SVM mysvm' + assert sr['result'] is True + + +@pytest.mark.dependency(depends=['mapped'], name='unmapped') +@pytest.mark.parametrize('test', [True, False]) +def test_lun_unmapped(host, target, test): + r = host.salt('state.single', params_unmapped + [f'test={test}']) + sr = r[state_unmapped] + assert sr['name'] == '/vol/myvol/testlun' + if test: + comment = 'Would unmap LUN' + result = None + elif not test: + comment = 'Unmapped LUN' + result = True + assert sr['comment'] == comment + assert sr['result'] is result + + +@pytest.mark.dependency(depends=['unmapped'], name='unmapped_already') +@pytest.mark.parametrize('test', [True, False]) +def test_lun_unmapped_already(host, target, test): + r = host.salt('state.single', params_unmapped + [f'test={test}']) + sr = r[state_unmapped] + assert sr['name'] == '/vol/myvol/testlun' + assert sr['comment'] == 'Nothing to unmap' + assert sr['result'] is True From 0cbcd50fbe8646189af28d4f4ee02bdb86cfb8bd Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Fri, 30 Jun 2023 19:23:41 +0200 Subject: [PATCH 33/34] netapp_ontap: add simulator bootstrap script Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/bin/init-dotsim.pl | 461 ++++++++++++++++++++++++ 1 file changed, 461 insertions(+) create mode 100644 netapp_ontap-formula/bin/init-dotsim.pl diff --git a/netapp_ontap-formula/bin/init-dotsim.pl b/netapp_ontap-formula/bin/init-dotsim.pl new file mode 100644 index 00000000..f260bd18 --- /dev/null +++ b/netapp_ontap-formula/bin/init-dotsim.pl @@ -0,0 +1,461 @@ +#!/usr/bin/perl +# Experimental script to initialize a NetApp Data ONTAP simulator on a KVM hypervisor +# Copyright (C) 2023 Georg Pfuetzenreuter +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# to-do: +# - clean up, move expect prompts/answers to a hash or similar +# - handle CLI prompts better, sometimes the answers don't align with the prompts +# - read multipath devices from domain XML file + +use v5.26.1; +use Archive::Tar; +use Expect; +use File::pushd; +use File::Temp; +use Getopt::Long; +use Sys::Virt; + +my $source; +my $mpaths; +my $domain; +my $force; +my $only_vm; +my $address; +my $netmask; +my $gateway; +my $passphrase; +my $reset; + +GetOptions + ("source=s" => \$source, "mpaths=s" => \$mpaths, "domain=s", => \$domain, "force" => \$force, "only_vm" => \$only_vm, + "address=s" => \$address, "netmask=s" => \$netmask, "gateway=s" => \$gateway, "passphrase=s" => \$passphrase, "reset" => \$reset + ) + or die "\nFailed to set arguments"; + +if(!$source){die "Please specify the full path to an OVA image to use using --source"} +if(!$mpaths){die "Please specify the multipath devices to write to using --mpaths. Pass them comma separated in the order to use."} +if(!$domain){die "Please specify the Libvirt domain XML file to use using --name"} +if(!$address||!$netmask||!$gateway){die "Please specify --address, --netmask and --gateway"} +if(!$passphrase){die "Please specify an admin passphrase using --passphrase"} + +my @paths; +my @vmdks; +my $vircon = Sys::Virt->new(uri => 'qemu:///system'); +my $attempts = 1; + +sub extract { + print("Extracting ...\n"); + if (! -e $source) { + die "No OVA file at $source, aborting"; + } + my $tar=Archive::Tar->new(); + $tar->read($source); + $tar->extract(); +} + +sub convert { + print("Converting ...\n"); + while (my $vmdk = glob("*.vmdk")) { + system("qemu-img convert -p -fvmdk -Oraw $vmdk $vmdk.raw"); + push @vmdks, "$vmdk.raw"; + } +} + +sub check { + print("Validating target disks ...\n"); + foreach my $target (split(',', $mpaths)) { + my $path = "/dev/disk/by-id/dm-uuid-mpath-$target"; + if (! -b $path) { + print("Disk at $path is not valid.\n"); + next; + } + my $status = system("partx -rgoNR $path >/dev/null 2>&1"); + if ($status != 0 || $force) { + push @paths, $path; + } else { + print("Disk at $path seems to contain an existing file system. Refusing destruction without --force.\n"); + } + } +} + +sub dd { + print("Writing ...\n"); + my $loop = 0; + foreach my $vmdk (@vmdks) { + system("dd if=$vmdk of=@paths[$loop] bs=16M status=progress"); + $loop ++; + } +} + +sub vm { + my $reinit = 0; + my $xml; + my $fh; + open($fh, '<', $domain) or die "Cannot open XML file $domain"; + { local $/; $xml = <$fh> }; + my $domain = $vircon->create_domain($xml); + my $domid = $domain->get_id(); + print("\nStarted domain with ID $domid"); + my $domvp = `virsh vncdisplay $domid`; + print(", VNC host is $domvp"); + my $firstboot = Expect->spawn("virsh console $domid") or die "Unable to spawn console"; + $firstboot->restart_timeout_upon_receive(1); + $firstboot->expect(10, + [ + qr/Hit \[Enter\] to boot immediately, or any other key for command prompt\./, + sub { + my $ex = shift; + $ex->send("j"); + } + ] + ); + $firstboot->expect(10, + [ + qr/VLOADER>/, + sub { + my $ex = shift; + $ex->send("set console=comconsole\n"); + } + ] + ); + $firstboot->expect(10, + [ + qr/VLOADER>/, + sub { + my $ex = shift; + $ex->send("set comconsole_speed=115200\n"); + } + ] + ); + $firstboot->expect(10, + [ + qr/VLOADER>/, + sub { + my $ex = shift; + $ex->send("boot\n"); + } + ], + ); + if ($reset) { + $firstboot->expect(60, + [ + qr/\* Press Ctrl-C for Boot Menu\. \*/, + sub { + my $ex = shift; + $ex->send("\cC"); + exp_continue; + } + ], + [ + qr/Selection \(1-11\)\?/, + sub { + my $ex = shift; + $ex->send("4\n"); + exp_continue; + } + ], + [ + qr/Zero disks, reset config and install a new file system\?:/, + sub { + my $ex = shift; + $ex->send("yes\n"); + exp_continue; + } + ], + [ + qr/This will erase all the data on the disks, are you sure\?:/, + sub { + my $ex = shift; + $ex->send("yes\n"); + exp_continue; + } + ], + [ + qr/Rebooting to finish wipeconfig request\./ + ] + ); + $firstboot->expect(30, + [ + qr/Hit \[Enter\] to boot immediately, or any other key for command prompt\./, + sub { + my $ex = shift; + $ex->send("\n"); + } + ] + ); + } + $firstboot->expect(480, + [ + qr/(?:System initialization has completed successfully\.|Welcome to the cluster setup wizard\.)/, + ], + [ + qr/DUMPCORE: START/, + sub { + print("System is broken, starting from scratch...\n"); + $reinit = 1; + } + ], + [ + timeout => + sub { + print("System did not finish booting in time.\n"); + # what to do now ? + } + ] + ); + if ($reinit) + { + $firstboot->soft_close(); + $domain->destroy(); + if ($attempts == 2) { + die("System is still broken after two attempts, giving up.\n"); + } + if ($only_vm) { + die("Unsetting --only_vm in an attempt to start from scratch!\n"); + $only_vm = 0; + } + $force = 1; + $attempts ++; + run(); + } + $firstboot->expect(10, + [ + qr/Type yes to confirm and continue \{yes\}:/, + sub { + my $ex = shift; + $ex->send("exit\n"); + exp_continue; + } + ], + [ + qr/login:/, + sub { + my $ex = shift; + $ex->send("admin\n"); + exp_continue; + } + ], + [ + qr/::>/, + sub { + my $ex = shift; + $ex->send("network interface del -vserver Cluster -lif clus1\n"); + } + ] + ); + $firstboot->expect(10, + [ + qr/::>/, + sub { + my $ex = shift; + $ex->send("network port modify -node localhost -port e0a -ipspace Default\n"); + } + ] + ); + $firstboot->expect(10, + [ + qr/::>/, + sub { + my $ex = shift; + $ex->send("network interface create -vserver Default -lif mgmt -role node-mgmt -address $address -netmask $netmask -home-node localhost -home-port e0a\n"); + } + ] + ); + $firstboot->expect(10, + [ + qr/::>/, + sub { + my $ex = shift; + $ex->send("network route create -vserver Default -destination 0.0.0.0/0 -gateway $gateway\n"); + } + ] + ); + $firstboot->expect(10, + [ + qr/::>/, + sub { + my $ex = shift; + $ex->send("cluster setup\n"); + exp_continue; + } + ], + [ + qr/Press to page down, for next line, or 'q' to quit\.\.\./, + sub { + my $ex = shift; + $ex->send("q\n"); + exp_continue; + } + ], + [ + qr/Type yes to confirm and continue \{yes\}:/, + sub { + my $ex = shift; + $ex->send("yes\n"); + exp_continue; + } + ], + [ + qr/Enter the node management interface port/, + sub { + my $ex = shift; + $ex->send("e0a\n"); + exp_continue; + } + ], + [ + qr/Enter the node management interface IP address \[10\.168\.0\.96\]:/, + sub { + my $ex = shift; + $ex->send("\n"); + exp_continue; + } + ], + [ + qr/Enter the node management interface netmask \[255\.255\.254\.0\]:/, + sub { + my $ex = shift; + $ex->send("\n"); + exp_continue; + } + ], + [ + qr/Enter the node management interface default gateway \[10\.168\.1\.254\]:/, + sub { + my $ex = shift; + $ex->send("\n"); + exp_continue; + } + ], + [ + qr/Otherwise, press Enter to complete cluster setup using the command line/, + sub { + my $ex = shift; + $ex->send("\n"); + exp_continue; + } + ], + [ + qr/Do you want to create a new cluster or join an existing cluster\? \{create, join\}:/, + sub { + my $ex = shift; + $ex->send("create\n"); + exp_continue; + } + ], + [ + qr/Do you intend for this node to be used as a single node cluster\? \{yes, no\} \[no\]:/, + sub { + my $ex = shift; + $ex->send("yes\n"); + exp_continue; + } + ], + [ + qr/Enter the cluster administrator's \(username "admin"\) password:/, + sub { + my $ex = shift; + $ex->send("$passphrase\n"); + exp_continue; + } + ], + [ + qr/Retype the password:/, + sub { + my $ex = shift; + $ex->send("$passphrase\n"); + exp_continue; + } + ], + [ + qr/Enter the cluster name:/, + sub { + my $ex = shift; + $ex->send("labA\n"); + } + ], + ); + $firstboot->expect(300, + [ + qr/Creating cluster labA/, + sub { + exp_continue; + } + ], + [ + qr/Starting cluster support services/ + ] + ); + $firstboot->expect(10, + [ + qr/Enter an additional license key \[\]:/, + sub { + my $ex = shift; + $ex->send("\n"); + exp_continue; + } + ], + [ + qr/Enter the cluster management interface port/, + sub { + my $ex = shift; + $ex->send("exit\n"); + exp_continue; + } + ], + [ + qr/labA::>/, + sub { + my $ex = shift; + $ex->send("version\n"); + } + ], + ); + $firstboot->soft_close(); + if ($reinit) + { + $domain->destroy(); + if ($attempts == 2) { + die("System is still broken after two attempts, giving up.\n"); + } + if ($only_vm) { + die("Unsetting --only_vm in an attempt to start from scratch!\n"); + $only_vm = 0; + } + $force = 1; + $attempts ++; + run(); + } +} + +sub run { + if (!$only_vm) { + check(); + my $good_paths = @paths; + if ($good_paths < 4) { + die("Not enough writeable disks, aborting.\n") + } + my $outdir = tempd(); + print("Working in temporary directory $outdir\n"); + extract(); + convert(); + dd(); + } + vm(); + print("\nDone, simulator should be reachable at $address.\n"); +} + +run(); From b90888439ee8b6c10562b25fca22bfa4c4106ddc Mon Sep 17 00:00:00 2001 From: Georg Pfuetzenreuter Date: Sun, 2 Jul 2023 07:07:00 +0200 Subject: [PATCH 34/34] netapp_ontap: tidy up simulator bootstrap script Signed-off-by: Georg Pfuetzenreuter --- netapp_ontap-formula/bin/init-dotsim.pl | 668 ++++++++++-------------- 1 file changed, 272 insertions(+), 396 deletions(-) diff --git a/netapp_ontap-formula/bin/init-dotsim.pl b/netapp_ontap-formula/bin/init-dotsim.pl index f260bd18..f078b96d 100644 --- a/netapp_ontap-formula/bin/init-dotsim.pl +++ b/netapp_ontap-formula/bin/init-dotsim.pl @@ -16,21 +16,21 @@ # along with this program. If not, see . # to-do: -# - clean up, move expect prompts/answers to a hash or similar -# - handle CLI prompts better, sometimes the answers don't align with the prompts # - read multipath devices from domain XML file use v5.26.1; use Archive::Tar; use Expect; -use File::pushd; use File::Temp; -use Getopt::Long; +use File::pushd; +use Getopt::Long qw(:config auto_version); use Sys::Virt; +our $VERSION = '0.1'; my $source; my $mpaths; my $domain; +my $cluster; my $force; my $only_vm; my $address; @@ -39,423 +39,299 @@ my $passphrase; my $reset; -GetOptions - ("source=s" => \$source, "mpaths=s" => \$mpaths, "domain=s", => \$domain, "force" => \$force, "only_vm" => \$only_vm, - "address=s" => \$address, "netmask=s" => \$netmask, "gateway=s" => \$gateway, "passphrase=s" => \$passphrase, "reset" => \$reset - ) - or die "\nFailed to set arguments"; +sub usage { + print <<~'EOH' + Please specify all of the following arguments for this script to proceed: + --address, --netmask, --gateway : IP details for the VM to use on the node management interface + --cluster : Name to give the cluster + --domain : Path to an existing Libvirt domain XML file the VM should be started from + --mpaths : Comma separated list of four multipath devices the individual disk images should be written to + --passphrase : Passphrase to set for the default "admin" user + --source : Path to the NetApp provided OVA image (aka tarball) + EOH + ; exit; +} -if(!$source){die "Please specify the full path to an OVA image to use using --source"} -if(!$mpaths){die "Please specify the multipath devices to write to using --mpaths. Pass them comma separated in the order to use."} -if(!$domain){die "Please specify the Libvirt domain XML file to use using --name"} -if(!$address||!$netmask||!$gateway){die "Please specify --address, --netmask and --gateway"} -if(!$passphrase){die "Please specify an admin passphrase using --passphrase"} +GetOptions( + "address=s" => \$address, + "cluster=s" => \$cluster, + "domain=s" => \$domain, + "force" => \$force, + "gateway=s" => \$gateway, + "mpaths=s" => \$mpaths, + "netmask=s" => \$netmask, + "only_vm" => \$only_vm, + "passphrase=s" => \$passphrase, + "reset" => \$reset, + "source=s" => \$source, + "help" => sub { usage(); } +) or exit; +usage() unless ( $address && $netmask && $gateway && $cluster && $domain && $mpaths && $passphrase && $source ); +my $attempts = 1; +my $vircon = Sys::Virt->new( uri => 'qemu:///system' ); my @paths; my @vmdks; -my $vircon = Sys::Virt->new(uri => 'qemu:///system'); -my $attempts = 1; sub extract { - print("Extracting ...\n"); - if (! -e $source) { - die "No OVA file at $source, aborting"; - } - my $tar=Archive::Tar->new(); - $tar->read($source); - $tar->extract(); + print("Extracting ...\n"); + if ( !-e $source ) { + die "No OVA file at $source, aborting"; + } + my $tar = Archive::Tar->new(); + $tar->read($source); + $tar->extract(); } sub convert { - print("Converting ...\n"); - while (my $vmdk = glob("*.vmdk")) { - system("qemu-img convert -p -fvmdk -Oraw $vmdk $vmdk.raw"); - push @vmdks, "$vmdk.raw"; - } + print("Converting ...\n"); + while ( my $vmdk = glob("*.vmdk") ) { + system("qemu-img convert -p -fvmdk -Oraw $vmdk $vmdk.raw"); + push @vmdks, "$vmdk.raw"; + } } sub check { - print("Validating target disks ...\n"); - foreach my $target (split(',', $mpaths)) { - my $path = "/dev/disk/by-id/dm-uuid-mpath-$target"; - if (! -b $path) { - print("Disk at $path is not valid.\n"); - next; - } - my $status = system("partx -rgoNR $path >/dev/null 2>&1"); - if ($status != 0 || $force) { - push @paths, $path; - } else { - print("Disk at $path seems to contain an existing file system. Refusing destruction without --force.\n"); + print("Validating target disks ...\n"); + foreach my $target ( split( ',', $mpaths ) ) { + my $path = "/dev/disk/by-id/dm-uuid-mpath-$target"; + if ( !-b $path ) { + print("Disk at $path is not valid.\n"); + next; + } + my $status = system("partx -rgoNR $path >/dev/null 2>&1"); + if ( $status != 0 || $force ) { + push @paths, $path; + } + else { + print("Disk at $path seems to contain an existing file system. Refusing destruction without --force.\n"); + } } - } } sub dd { - print("Writing ...\n"); - my $loop = 0; - foreach my $vmdk (@vmdks) { - system("dd if=$vmdk of=@paths[$loop] bs=16M status=progress"); - $loop ++; - } -} + print("Writing ...\n"); + my $loop = 0; + foreach my $vmdk (@vmdks) { + system("dd if=$vmdk of=@paths[$loop] bs=16M status=progress"); + $loop++; + } +} sub vm { - my $reinit = 0; - my $xml; - my $fh; - open($fh, '<', $domain) or die "Cannot open XML file $domain"; - { local $/; $xml = <$fh> }; - my $domain = $vircon->create_domain($xml); - my $domid = $domain->get_id(); - print("\nStarted domain with ID $domid"); - my $domvp = `virsh vncdisplay $domid`; - print(", VNC host is $domvp"); - my $firstboot = Expect->spawn("virsh console $domid") or die "Unable to spawn console"; - $firstboot->restart_timeout_upon_receive(1); - $firstboot->expect(10, - [ - qr/Hit \[Enter\] to boot immediately, or any other key for command prompt\./, - sub { - my $ex = shift; - $ex->send("j"); - } - ] - ); - $firstboot->expect(10, - [ - qr/VLOADER>/, - sub { - my $ex = shift; - $ex->send("set console=comconsole\n"); - } - ] - ); - $firstboot->expect(10, - [ - qr/VLOADER>/, - sub { - my $ex = shift; - $ex->send("set comconsole_speed=115200\n"); - } - ] - ); - $firstboot->expect(10, - [ - qr/VLOADER>/, - sub { - my $ex = shift; - $ex->send("boot\n"); - } - ], - ); - if ($reset) { - $firstboot->expect(60, - [ - qr/\* Press Ctrl-C for Boot Menu\. \*/, - sub { - my $ex = shift; - $ex->send("\cC"); - exp_continue; - } - ], - [ - qr/Selection \(1-11\)\?/, - sub { - my $ex = shift; - $ex->send("4\n"); - exp_continue; - } - ], - [ - qr/Zero disks, reset config and install a new file system\?:/, - sub { - my $ex = shift; - $ex->send("yes\n"); - exp_continue; - } - ], - [ - qr/This will erase all the data on the disks, are you sure\?:/, - sub { - my $ex = shift; - $ex->send("yes\n"); - exp_continue; - } - ], - [ - qr/Rebooting to finish wipeconfig request\./ - ] - ); - $firstboot->expect(30, - [ - qr/Hit \[Enter\] to boot immediately, or any other key for command prompt\./, - sub { - my $ex = shift; - $ex->send("\n"); + my $reinit = 0; + my $xml; + my $fh; + + open( $fh, '<', $domain ) or die "Cannot open XML file $domain"; + { local $/; $xml = <$fh> }; + my $domain = $vircon->create_domain($xml); + my $domid = $domain->get_id(); + print("\nStarted domain with ID $domid"); + my $domvp = `virsh vncdisplay $domid`; + print(", VNC host is $domvp"); + my $console = Expect->spawn("virsh console $domid") + or die "Unable to spawn console"; + $console->restart_timeout_upon_receive(1); + + sub handle_boot { + my $prompt = 'VLOADER>'; + my @commands = ( + "set console=comconsole\n", + "set comconsole_speed=115200\n", + "boot\n", + ); + sub boot { + $console->expect(20, + [ + qr/Hit \[Enter\] to boot immediately, or any other key for command prompt\./ + ] + ); + $console->send( $_[0] ); + } + boot('x'); + foreach my $command (@commands) { + $console->expect( 10, $prompt ); + $console->send($command); + } + if ($reset) { + my %dialogue = ( + '* Press Ctrl-C for Boot Menu. *' => "\cC", + 'Selection (1-11)?' => "4\n", + 'Zero disks, reset config and install a new file system?' => "yes\n", + 'This will erase all the data on the disks, are you sure?' => "yes\n", + 'Rebooting to finish wipeconfig request.' => '', + ); + my $prompts = join( '|', map { qr{\Q$_\E} } keys %dialogue ); + $console->expect(60, + -re => $prompts, + sub { + my $ex = shift; + my $matched = $ex->match; + my $answer = delete $dialogue{$matched}; + $ex->send($answer); + exp_continue if keys %dialogue; + } + ); + boot("\n"); + } + $console->expect(480, + [ + qr/(?:System initialization has completed successfully\.|Welcome to the cluster setup wizard\.)/, + ], + [ + qr/DUMPCORE: START/, + sub { + print("System is broken, starting from scratch...\n"); + $reinit = 1; + } + ], + [ + timeout => sub { + print("System did not finish booting in time.\n"); + # what to do now ? + } + ] + ); + } + + sub handle_setup { + $console->expect(10, + [ + qr/Type yes to confirm and continue \{yes\}:/, + sub { + my $ex = shift; + $ex->send("exit\n"); + exp_continue; + } + ], + [ + qr/login:/, + sub { + my $ex = shift; + $ex->send("admin\n"); + exp_continue; + } + ], + ); + my $prompt = '::>'; + my @commands = ( + "network interface del -vserver Cluster -lif clus1\n", + "network port modify -node localhost -port e0a -ipspace Default\n", + "network interface create -vserver Default -lif mgmt -role node-mgmt -address $address -netmask $netmask -home-node localhost -home-port e0a\n", + "network route create -vserver Default -destination 0.0.0.0/0 -gateway $gateway\n", + "cluster setup\n", + ); + foreach my $command (@commands) { + $console->expect( 10, $prompt ); + $console->send($command); } - ] - ); - } - $firstboot->expect(480, - [ - qr/(?:System initialization has completed successfully\.|Welcome to the cluster setup wizard\.)/, - ], - [ - qr/DUMPCORE: START/, - sub { - print("System is broken, starting from scratch...\n"); - $reinit = 1; - } - ], - [ - timeout => - sub { - print("System did not finish booting in time.\n"); - # what to do now ? - } - ] - ); - if ($reinit) - { - $firstboot->soft_close(); - $domain->destroy(); - if ($attempts == 2) { - die("System is still broken after two attempts, giving up.\n"); + my %dialogue = ( + "Press to page down, for next line, or 'q' to quit..." => 'q', + 'Type yes to confirm and continue {yes}:' => 'yes', + 'Enter the node management interface port' => 'e0a', + 'Enter the node management interface IP address [10.168.0.96]:' => '', + 'Enter the node management interface netmask [255.255.254.0]:' => '', + 'Enter the node management interface default gateway [10.168.1.254]:' => '', + 'Otherwise, press Enter to complete cluster setup using the command line' => '', + 'Do you want to create a new cluster or join an existing cluster? {create, join}:' => 'create', + 'Do you intend for this node to be used as a single node cluster? {yes, no} [no]:' => 'yes', + "Enter the cluster administrator's (username \"admin\") password:" => "$passphrase", + 'Retype the password:' => "$passphrase", + 'Enter the cluster name:' => "$cluster", + ); + $console->expect( 10, 'Welcome to the cluster setup wizard.' ); + my $prompts = join( '|', map { qr {\Q$_\E} } keys %dialogue ); + $console->expect(10, + -re => $prompts, + sub { + my $ex = shift; + my $matched = $ex->match; + my $answer = delete $dialogue{$matched}; + if ( $answer ne 'q' ) { $answer = $answer . "\n" } + $ex->send($answer); + sleep(1); + exp_continue if keys %dialogue; + } + ); + $console->expect(300, + [ + qr/Creating cluster $cluster/, + sub { + exp_continue; + } + ], + [ + qr/Starting cluster support services/ + ] + ); + $console->expect(10, + [ + qr/Enter an additional license key \[\]:/, + sub { + my $ex = shift; + $ex->send("\n"); + exp_continue; + } + ], + [ + qr/Enter the cluster management interface port/, + sub { + my $ex = shift; + $ex->send("exit\n"); + exp_continue; + } + ], + [ + qr/${cluster}::>/, + sub { + my $ex = shift; + $ex->send("version\n"); + } + ], + ); } - if ($only_vm) { - die("Unsetting --only_vm in an attempt to start from scratch!\n"); - $only_vm = 0; - } - $force = 1; - $attempts ++; - run(); - } - $firstboot->expect(10, - [ - qr/Type yes to confirm and continue \{yes\}:/, - sub { - my $ex = shift; - $ex->send("exit\n"); - exp_continue; - } - ], - [ - qr/login:/, - sub { - my $ex = shift; - $ex->send("admin\n"); - exp_continue; - } - ], - [ - qr/::>/, - sub { - my $ex = shift; - $ex->send("network interface del -vserver Cluster -lif clus1\n"); - } - ] - ); - $firstboot->expect(10, - [ - qr/::>/, - sub { - my $ex = shift; - $ex->send("network port modify -node localhost -port e0a -ipspace Default\n"); - } - ] - ); - $firstboot->expect(10, - [ - qr/::>/, - sub { - my $ex = shift; - $ex->send("network interface create -vserver Default -lif mgmt -role node-mgmt -address $address -netmask $netmask -home-node localhost -home-port e0a\n"); - } - ] - ); - $firstboot->expect(10, - [ - qr/::>/, - sub { - my $ex = shift; - $ex->send("network route create -vserver Default -destination 0.0.0.0/0 -gateway $gateway\n"); - } - ] - ); - $firstboot->expect(10, - [ - qr/::>/, - sub { - my $ex = shift; - $ex->send("cluster setup\n"); - exp_continue; - } - ], - [ - qr/Press to page down, for next line, or 'q' to quit\.\.\./, - sub { - my $ex = shift; - $ex->send("q\n"); - exp_continue; - } - ], - [ - qr/Type yes to confirm and continue \{yes\}:/, - sub { - my $ex = shift; - $ex->send("yes\n"); - exp_continue; - } - ], - [ - qr/Enter the node management interface port/, - sub { - my $ex = shift; - $ex->send("e0a\n"); - exp_continue; - } - ], - [ - qr/Enter the node management interface IP address \[10\.168\.0\.96\]:/, - sub { - my $ex = shift; - $ex->send("\n"); - exp_continue; - } - ], - [ - qr/Enter the node management interface netmask \[255\.255\.254\.0\]:/, - sub { - my $ex = shift; - $ex->send("\n"); - exp_continue; - } - ], - [ - qr/Enter the node management interface default gateway \[10\.168\.1\.254\]:/, - sub { - my $ex = shift; - $ex->send("\n"); - exp_continue; - } - ], - [ - qr/Otherwise, press Enter to complete cluster setup using the command line/, - sub { - my $ex = shift; - $ex->send("\n"); - exp_continue; - } - ], - [ - qr/Do you want to create a new cluster or join an existing cluster\? \{create, join\}:/, - sub { - my $ex = shift; - $ex->send("create\n"); - exp_continue; - } - ], - [ - qr/Do you intend for this node to be used as a single node cluster\? \{yes, no\} \[no\]:/, - sub { - my $ex = shift; - $ex->send("yes\n"); - exp_continue; - } - ], - [ - qr/Enter the cluster administrator's \(username "admin"\) password:/, - sub { - my $ex = shift; - $ex->send("$passphrase\n"); - exp_continue; - } - ], - [ - qr/Retype the password:/, - sub { - my $ex = shift; - $ex->send("$passphrase\n"); - exp_continue; - } - ], - [ - qr/Enter the cluster name:/, - sub { - my $ex = shift; - $ex->send("labA\n"); - } - ], - ); - $firstboot->expect(300, - [ - qr/Creating cluster labA/, - sub { - exp_continue; - } - ], - [ - qr/Starting cluster support services/ - ] - ); - $firstboot->expect(10, - [ - qr/Enter an additional license key \[\]:/, - sub { - my $ex = shift; - $ex->send("\n"); - exp_continue; - } - ], - [ - qr/Enter the cluster management interface port/, - sub { - my $ex = shift; - $ex->send("exit\n"); - exp_continue; - } - ], - [ - qr/labA::>/, - sub { - my $ex = shift; - $ex->send("version\n"); - } - ], - ); - $firstboot->soft_close(); - if ($reinit) - { - $domain->destroy(); - if ($attempts == 2) { - die("System is still broken after two attempts, giving up.\n"); + + sub handle_reinit { + if ($reinit) { + $console->soft_close(); + $domain->destroy(); + if ( $attempts == 2 ) { + die("System is still broken after two attempts, giving up.\n"); + } + if ($only_vm) { + die("Unsetting --only_vm in an attempt to start from scratch!\n" + ); + $only_vm = 0; + } + $force = 1; + $attempts++; + run(); + } } - if ($only_vm) { - die("Unsetting --only_vm in an attempt to start from scratch!\n"); - $only_vm = 0; - } - $force = 1; - $attempts ++; - run(); - } + + handle_boot(); + handle_reinit(); + handle_setup(); + $console->soft_close(); + handle_reinit(); #why here? } sub run { - if (!$only_vm) { - check(); - my $good_paths = @paths; - if ($good_paths < 4) { - die("Not enough writeable disks, aborting.\n") + if ( !$only_vm ) { + check(); + my $good_paths = @paths; + if ( $good_paths < 4 ) { + die("Not enough writeable disks, aborting.\n"); + } + my $outdir = tempd(); + print("Working in temporary directory $outdir\n"); + extract(); + convert(); + dd(); } - my $outdir = tempd(); - print("Working in temporary directory $outdir\n"); - extract(); - convert(); - dd(); - } - vm(); - print("\nDone, simulator should be reachable at $address.\n"); + vm(); + print("\nDone, simulator should be reachable at $address.\n"); } run();