Skip to content

Commit 2543032

Browse files
mapk-amazonthheinendreambeyondorange
authored
Thheinen/efs accesspoints (#6381)
* Add EFS access point support (aws-parallelcluster#2337) * Rename new setting, Add changelog, Fix style, fix some tests * Added tests for EFS Access Points * Improved unit test for access point * Access point requires TLS https://docs.aws.amazon.com/efs/latest/ug/efs-access-points.html#mount-with-access-point * Updated snapshot to included AccessPointId * Ignored Flake8 too-long-line --------- Signed-off-by: Thomas Heinen <[email protected]> Co-authored-by: Thomas Heinen <[email protected]> Co-authored-by: Ryan Anderson <[email protected]>
1 parent d1af051 commit 2543032

File tree

21 files changed

+345
-9
lines changed

21 files changed

+345
-9
lines changed

‎CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ CHANGELOG
1010
- Add support for ap-southeast-3 region.
1111
- Add security groups to login node network load balancer.
1212
- Add `AllowedIps` configuration for login nodes.
13+
- Add new configuration `SharedStorage/EfsSettings/AccessPointId` to specify an optional EFS access point for a mount
1314
- Allow multiple login node pools.
1415

1516
**BUG FIXES**

‎cli/src/pcluster/aws/efs.py

+11
Original file line numberDiff line numberDiff line change
@@ -80,3 +80,14 @@ def describe_file_system(self, efs_fs_id):
8080
:return: the mount_target_ids
8181
"""
8282
return self._client.describe_file_systems(FileSystemId=efs_fs_id)
83+
84+
@AWSExceptionHandler.handle_client_exception
85+
@Cache.cached
86+
def describe_access_point(self, access_point_id):
87+
"""
88+
Describe access point attributes for the given EFS access point id.
89+
90+
:param efaccess_point_ids_ap_id: EFS access point Id
91+
:return: the access_point details
92+
"""
93+
return self._client.describe_access_points(AccessPointId=access_point_id)

‎cli/src/pcluster/config/cluster_config.py

+9-1
Original file line numberDiff line numberDiff line change
@@ -149,7 +149,7 @@
149149
PlacementGroupCapacityTypeValidator,
150150
PlacementGroupNamingValidator,
151151
)
152-
from pcluster.validators.efs_validators import EfsMountOptionsValidator
152+
from pcluster.validators.efs_validators import EfsAccessPointOptionsValidator, EfsMountOptionsValidator
153153
from pcluster.validators.feature_validators import FeatureRegionValidator
154154
from pcluster.validators.fsx_validators import (
155155
FsxAutoImportValidator,
@@ -371,6 +371,7 @@ def __init__(
371371
deletion_policy: str = None,
372372
encryption_in_transit: bool = None,
373373
iam_authorization: bool = None,
374+
access_point_id: str = None,
374375
):
375376
super().__init__()
376377
self.mount_dir = Resource.init_param(mount_dir)
@@ -387,6 +388,7 @@ def __init__(
387388
)
388389
self.encryption_in_transit = Resource.init_param(encryption_in_transit, default=False)
389390
self.iam_authorization = Resource.init_param(iam_authorization, default=False)
391+
self.access_point_id = Resource.init_param(access_point_id)
390392

391393
def _register_validators(self, context: ValidatorContext = None): # noqa: D102 #pylint: disable=unused-argument
392394
self._register_validator(SharedStorageNameValidator, name=self.name)
@@ -400,6 +402,12 @@ def _register_validators(self, context: ValidatorContext = None): # noqa: D102
400402
iam_authorization=self.iam_authorization,
401403
name=self.name,
402404
)
405+
self._register_validator(
406+
EfsAccessPointOptionsValidator,
407+
access_point_id=self.access_point_id,
408+
file_system_id=self.file_system_id,
409+
encryption_in_transit=self.encryption_in_transit,
410+
)
403411

404412

405413
class BaseSharedFsx(Resource):

‎cli/src/pcluster/schemas/cluster_schema.py

+9-1
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,9 @@ class EfsSettingsSchema(BaseSchema):
321321
deletion_policy = fields.Str(
322322
validate=validate.OneOf(DELETION_POLICIES), metadata={"update_policy": UpdatePolicy.SUPPORTED}
323323
)
324+
access_point_id = fields.Str(
325+
validate=validate.Regexp(r"^fsap-[0-9a-z]{17}$"), metadata={"update_policy": UpdatePolicy.UNSUPPORTED}
326+
)
324327
encryption_in_transit = fields.Bool(metadata={"update_policy": UpdatePolicy.UNSUPPORTED})
325328
iam_authorization = fields.Bool(metadata={"update_policy": UpdatePolicy.UNSUPPORTED})
326329

@@ -331,7 +334,12 @@ def validate_file_system_id_ignored_parameters(self, data, **kwargs):
331334
messages = []
332335
if data.get("file_system_id") is not None:
333336
for key in data:
334-
if key is not None and key not in ["encryption_in_transit", "iam_authorization", "file_system_id"]:
337+
if key is not None and key not in [
338+
"encryption_in_transit",
339+
"iam_authorization",
340+
"file_system_id",
341+
"access_point_id",
342+
]:
335343
messages.append(EFS_MESSAGES["errors"]["ignored_param_with_efs_fs_id"].format(efs_param=key))
336344
if messages:
337345
raise ValidationError(message=messages)

‎cli/src/pcluster/templates/cluster_stack.py

+5
Original file line numberDiff line numberDiff line change
@@ -1108,6 +1108,7 @@ def _add_efs_storage(self, id: str, shared_efs: SharedEfs):
11081108
shared_efs.encryption_in_transit
11091109
)
11101110
self.shared_storage_attributes[SharedStorageType.EFS]["IamAuthorizations"].append(shared_efs.iam_authorization)
1111+
self.shared_storage_attributes[SharedStorageType.EFS]["AccessPointIds"].append(shared_efs.access_point_id)
11111112

11121113
return efs_id
11131114

@@ -1287,6 +1288,10 @@ def _add_head_node(self):
12871288
"efs_iam_authorizations": to_comma_separated_string(
12881289
self.shared_storage_attributes[SharedStorageType.EFS]["IamAuthorizations"], use_lower_case=True
12891290
),
1291+
"efs_access_point_ids": to_comma_separated_string(
1292+
self.shared_storage_attributes[SharedStorageType.EFS]["AccessPointIds"],
1293+
use_lower_case=True,
1294+
),
12901295
"fsx_fs_ids": get_shared_storage_ids_by_type(self.shared_storage_infos, SharedStorageType.FSX),
12911296
"fsx_mount_names": to_comma_separated_string(
12921297
self.shared_storage_attributes[SharedStorageType.FSX]["MountNames"]

‎cli/src/pcluster/templates/login_nodes_stack.py

+4
Original file line numberDiff line numberDiff line change
@@ -240,6 +240,10 @@ def _add_login_nodes_pool_launch_template(self):
240240
self._shared_storage_attributes[SharedStorageType.EFS]["IamAuthorizations"],
241241
use_lower_case=True,
242242
),
243+
"efs_access_point_ids": to_comma_separated_string(
244+
self._shared_storage_attributes[SharedStorageType.EFS]["AccessPointIds"],
245+
use_lower_case=True,
246+
),
243247
"enable_intel_hpc_platform": "true" if self._config.is_intel_hpc_platform_enabled else "false",
244248
"ephemeral_dir": DEFAULT_EPHEMERAL_DIR,
245249
"fsx_fs_ids": get_shared_storage_ids_by_type(self._shared_storage_infos, SharedStorageType.FSX),

‎cli/src/pcluster/templates/queues_stack.py

+4
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,10 @@ def _add_compute_resource_launch_template(
335335
self._shared_storage_attributes[SharedStorageType.EFS]["IamAuthorizations"],
336336
use_lower_case=True,
337337
),
338+
"efs_access_point_ids": to_comma_separated_string(
339+
self._shared_storage_attributes[SharedStorageType.EFS]["AccessPointIds"],
340+
use_lower_case=True,
341+
),
338342
"fsx_fs_ids": get_shared_storage_ids_by_type(self._shared_storage_infos, SharedStorageType.FSX),
339343
"fsx_mount_names": to_comma_separated_string(
340344
self._shared_storage_attributes[SharedStorageType.FSX]["MountNames"]

‎cli/src/pcluster/validators/efs_validators.py

+25
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,28 @@ def _validate(self, encryption_in_transit: bool, iam_authorization: bool, name:
2525
f"Please either disable IAM authorization or enable encryption in-transit for file system {name}",
2626
FailureLevel.ERROR,
2727
)
28+
29+
30+
class EfsAccessPointOptionsValidator(Validator):
31+
"""
32+
EFS Mount Options validator.
33+
34+
IAM Authorization requires Encryption in Transit.
35+
"""
36+
37+
def _validate(self, access_point_id: str, file_system_id: str, encryption_in_transit: bool):
38+
39+
if access_point_id and not file_system_id:
40+
self._add_failure(
41+
"An access point can only be specified when using an existing EFS file system. "
42+
f"Please either remove the access point id {access_point_id} "
43+
"or provide the file system id for the access point",
44+
FailureLevel.ERROR,
45+
)
46+
47+
if access_point_id and not encryption_in_transit:
48+
self._add_failure(
49+
"An access point can only be specified when encryption in transit is enabled. "
50+
f"Please either remove the access point id {access_point_id} or enable encryption in transit.",
51+
FailureLevel.ERROR,
52+
)

‎cli/src/pcluster3_config_converter/pcluster3_config_converter.py

+1
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,7 @@ def convert_efs_settings(self, section_name):
296296
("efs_kms_key_id", "KmsKeyId"),
297297
("provisioned_throughput", "ProvisionedThroughput", "getint"),
298298
("throughput_mode", "ThroughputMode"),
299+
("access_point_id", "AccessPointId"),
299300
]
300301
efs_section, efs_dict, _section_label = self.convert_storage_base(
301302
"efs", efs_label.strip(), additional_items

‎cli/tests/pcluster/templates/test_cluster_stack/test_cluster_config_limits/slurm.full_config.snapshot.yaml

+21
Original file line numberDiff line numberDiff line change
@@ -3888,6 +3888,7 @@ SharedStorage:
38883888
Name: nameebs4
38893889
StorageType: Ebs
38903890
- EfsSettings:
3891+
AccessPointId: null
38913892
DeletionPolicy: Delete
38923893
Encrypted: false
38933894
EncryptionInTransit: false
@@ -3901,6 +3902,7 @@ SharedStorage:
39013902
Name: efs0
39023903
StorageType: Efs
39033904
- EfsSettings:
3905+
AccessPointId: null
39043906
DeletionPolicy: null
39053907
Encrypted: false
39063908
EncryptionInTransit: false
@@ -3914,6 +3916,7 @@ SharedStorage:
39143916
Name: existing-efs10
39153917
StorageType: Efs
39163918
- EfsSettings:
3919+
AccessPointId: null
39173920
DeletionPolicy: null
39183921
Encrypted: false
39193922
EncryptionInTransit: false
@@ -3927,6 +3930,7 @@ SharedStorage:
39273930
Name: existing-efs11
39283931
StorageType: Efs
39293932
- EfsSettings:
3933+
AccessPointId: null
39303934
DeletionPolicy: null
39313935
Encrypted: false
39323936
EncryptionInTransit: false
@@ -3940,6 +3944,7 @@ SharedStorage:
39403944
Name: existing-efs12
39413945
StorageType: Efs
39423946
- EfsSettings:
3947+
AccessPointId: null
39433948
DeletionPolicy: null
39443949
Encrypted: false
39453950
EncryptionInTransit: false
@@ -3953,6 +3958,7 @@ SharedStorage:
39533958
Name: existing-efs13
39543959
StorageType: Efs
39553960
- EfsSettings:
3961+
AccessPointId: null
39563962
DeletionPolicy: null
39573963
Encrypted: false
39583964
EncryptionInTransit: false
@@ -3966,6 +3972,7 @@ SharedStorage:
39663972
Name: existing-efs14
39673973
StorageType: Efs
39683974
- EfsSettings:
3975+
AccessPointId: null
39693976
DeletionPolicy: null
39703977
Encrypted: false
39713978
EncryptionInTransit: false
@@ -3979,6 +3986,7 @@ SharedStorage:
39793986
Name: existing-efs15
39803987
StorageType: Efs
39813988
- EfsSettings:
3989+
AccessPointId: null
39823990
DeletionPolicy: null
39833991
Encrypted: false
39843992
EncryptionInTransit: false
@@ -3992,6 +4000,7 @@ SharedStorage:
39924000
Name: existing-efs16
39934001
StorageType: Efs
39944002
- EfsSettings:
4003+
AccessPointId: null
39954004
DeletionPolicy: null
39964005
Encrypted: false
39974006
EncryptionInTransit: false
@@ -4005,6 +4014,7 @@ SharedStorage:
40054014
Name: existing-efs17
40064015
StorageType: Efs
40074016
- EfsSettings:
4017+
AccessPointId: null
40084018
DeletionPolicy: null
40094019
Encrypted: false
40104020
EncryptionInTransit: false
@@ -4018,6 +4028,7 @@ SharedStorage:
40184028
Name: existing-efs18
40194029
StorageType: Efs
40204030
- EfsSettings:
4031+
AccessPointId: null
40214032
DeletionPolicy: null
40224033
Encrypted: false
40234034
EncryptionInTransit: false
@@ -4031,6 +4042,7 @@ SharedStorage:
40314042
Name: existing-efs19
40324043
StorageType: Efs
40334044
- EfsSettings:
4045+
AccessPointId: null
40344046
DeletionPolicy: null
40354047
Encrypted: false
40364048
EncryptionInTransit: false
@@ -4044,6 +4056,7 @@ SharedStorage:
40444056
Name: existing-efs20
40454057
StorageType: Efs
40464058
- EfsSettings:
4059+
AccessPointId: null
40474060
DeletionPolicy: null
40484061
Encrypted: false
40494062
EncryptionInTransit: false
@@ -4057,6 +4070,7 @@ SharedStorage:
40574070
Name: existing-efs21
40584071
StorageType: Efs
40594072
- EfsSettings:
4073+
AccessPointId: null
40604074
DeletionPolicy: null
40614075
Encrypted: false
40624076
EncryptionInTransit: false
@@ -4070,6 +4084,7 @@ SharedStorage:
40704084
Name: existing-efs22
40714085
StorageType: Efs
40724086
- EfsSettings:
4087+
AccessPointId: null
40734088
DeletionPolicy: null
40744089
Encrypted: false
40754090
EncryptionInTransit: false
@@ -4083,6 +4098,7 @@ SharedStorage:
40834098
Name: existing-efs23
40844099
StorageType: Efs
40854100
- EfsSettings:
4101+
AccessPointId: null
40864102
DeletionPolicy: null
40874103
Encrypted: false
40884104
EncryptionInTransit: false
@@ -4096,6 +4112,7 @@ SharedStorage:
40964112
Name: existing-efs24
40974113
StorageType: Efs
40984114
- EfsSettings:
4115+
AccessPointId: null
40994116
DeletionPolicy: null
41004117
Encrypted: false
41014118
EncryptionInTransit: false
@@ -4109,6 +4126,7 @@ SharedStorage:
41094126
Name: existing-efs25
41104127
StorageType: Efs
41114128
- EfsSettings:
4129+
AccessPointId: null
41124130
DeletionPolicy: null
41134131
Encrypted: false
41144132
EncryptionInTransit: false
@@ -4122,6 +4140,7 @@ SharedStorage:
41224140
Name: existing-efs26
41234141
StorageType: Efs
41244142
- EfsSettings:
4143+
AccessPointId: null
41254144
DeletionPolicy: null
41264145
Encrypted: false
41274146
EncryptionInTransit: false
@@ -4135,6 +4154,7 @@ SharedStorage:
41354154
Name: existing-efs27
41364155
StorageType: Efs
41374156
- EfsSettings:
4157+
AccessPointId: null
41384158
DeletionPolicy: null
41394159
Encrypted: false
41404160
EncryptionInTransit: false
@@ -4148,6 +4168,7 @@ SharedStorage:
41484168
Name: existing-efs28
41494169
StorageType: Efs
41504170
- EfsSettings:
4171+
AccessPointId: null
41514172
DeletionPolicy: null
41524173
Encrypted: false
41534174
EncryptionInTransit: false

‎cli/tests/pcluster/templates/test_cluster_stack/test_head_node_dna_json/head_node_default.dna.json

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"efs_fs_ids": "",
2121
"efs_iam_authorizations": "",
2222
"efs_shared_dirs": "",
23+
"efs_access_point_ids": "",
2324
"enable_intel_hpc_platform": "false",
2425
"ephemeral_dir": "/scratch",
2526
"fsx_dns_names": "",

‎cli/tests/pcluster/templates/test_login_nodes_stack/test_login_nodes_dna_json/dna-1.json

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"efs_fs_ids": "",
2323
"efs_iam_authorizations": "",
2424
"efs_shared_dirs": "",
25+
"efs_access_point_ids": "",
2526
"enable_intel_hpc_platform": "false",
2627
"ephemeral_dir": "/scratch",
2728
"fsx_dns_names": "",

‎cli/tests/pcluster/templates/test_login_nodes_stack/test_login_nodes_dna_json/dna-2.json

+1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
"efs_fs_ids": "",
2323
"efs_iam_authorizations": "",
2424
"efs_shared_dirs": "",
25+
"efs_access_point_ids": "",
2526
"enable_intel_hpc_platform": "false",
2627
"ephemeral_dir": "/scratch",
2728
"fsx_dns_names": "",

‎cli/tests/pcluster/templates/test_queues_stack/test_compute_nodes_dna_json/dna-1.json

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"efs_fs_ids": "",
2121
"efs_iam_authorizations": "",
2222
"efs_shared_dirs": "",
23+
"efs_access_point_ids": "",
2324
"enable_efa": "NONE",
2425
"enable_efa_gdr": "NONE",
2526
"enable_intel_hpc_platform": "false",

‎cli/tests/pcluster/templates/test_queues_stack/test_compute_nodes_dna_json/dna-2.json

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
"efs_fs_ids": "",
2121
"efs_iam_authorizations": "",
2222
"efs_shared_dirs": "",
23+
"efs_access_point_ids": "",
2324
"enable_efa": "NONE",
2425
"enable_efa_gdr": "NONE",
2526
"enable_intel_hpc_platform": "false",

0 commit comments

Comments
 (0)