diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b1b9adfd9..f81d6b50c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,8 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti - Added a Management API endpoint to reset principal credentials, controlled by the `ENABLE_CREDENTIAL_RESET` (default: true) feature flag. +- Added support for S3-compatible storage that does not have STS (use `stsUavailable: true` in catalog storage configuration) + ### Changes * The following APIs will now return the newly-created objects as part of the successful 201 response: createCatalog, createPrincipalRole, createCatalogRole. diff --git a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java index 44f261c4cc..a312995047 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/entity/CatalogEntity.java @@ -155,6 +155,7 @@ private StorageConfigInfo getStorageInfo(Map internalProperties) .setEndpoint(awsConfig.getEndpoint()) .setStsEndpoint(awsConfig.getStsEndpoint()) .setPathStyleAccess(awsConfig.getPathStyleAccess()) + .setStsUnavailable(awsConfig.getStsUnavailable()) .setEndpointInternal(awsConfig.getEndpointInternal()) .build(); } @@ -299,6 +300,7 @@ public Builder setStorageConfigurationInfo( .endpoint(awsConfigModel.getEndpoint()) .stsEndpoint(awsConfigModel.getStsEndpoint()) .pathStyleAccess(awsConfigModel.getPathStyleAccess()) + .stsUnavailable(awsConfigModel.getStsUnavailable()) .endpointInternal(awsConfigModel.getEndpointInternal()) .build(); config = awsConfig; diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java index 3e93ba7b4b..8023f7a607 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsCredentialsStorageIntegration.java @@ -79,43 +79,46 @@ public AccessConfig getSubscopedCreds( int storageCredentialDurationSeconds = realmConfig.getConfig(STORAGE_CREDENTIAL_DURATION_SECONDS); AwsStorageConfigurationInfo storageConfig = config(); - AssumeRoleRequest.Builder request = - AssumeRoleRequest.builder() - .externalId(storageConfig.getExternalId()) - .roleArn(storageConfig.getRoleARN()) - .roleSessionName("PolarisAwsCredentialsStorageIntegration") - .policy( - policyString( - storageConfig.getAwsPartition(), - allowListOperation, - allowedReadLocations, - allowedWriteLocations) - .toJson()) - .durationSeconds(storageCredentialDurationSeconds); - credentialsProvider.ifPresent( - cp -> request.overrideConfiguration(b -> b.credentialsProvider(cp))); - String region = storageConfig.getRegion(); - @SuppressWarnings("resource") - // Note: stsClientProvider returns "thin" clients that do not need closing - StsClient stsClient = - stsClientProvider.stsClient(StsDestination.of(storageConfig.getStsEndpointUri(), region)); - - AssumeRoleResponse response = stsClient.assumeRole(request.build()); AccessConfig.Builder accessConfig = AccessConfig.builder(); - accessConfig.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId()); - accessConfig.put( - StorageAccessProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); - accessConfig.put(StorageAccessProperty.AWS_TOKEN, response.credentials().sessionToken()); - Optional.ofNullable(response.credentials().expiration()) - .ifPresent( - i -> { - accessConfig.put( - StorageAccessProperty.EXPIRATION_TIME, String.valueOf(i.toEpochMilli())); - accessConfig.put( - StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, - String.valueOf(i.toEpochMilli())); - }); + + if (shouldUseSts(storageConfig)) { + AssumeRoleRequest.Builder request = + AssumeRoleRequest.builder() + .externalId(storageConfig.getExternalId()) + .roleArn(storageConfig.getRoleARN()) + .roleSessionName("PolarisAwsCredentialsStorageIntegration") + .policy( + policyString( + storageConfig.getAwsPartition(), + allowListOperation, + allowedReadLocations, + allowedWriteLocations) + .toJson()) + .durationSeconds(storageCredentialDurationSeconds); + credentialsProvider.ifPresent( + cp -> request.overrideConfiguration(b -> b.credentialsProvider(cp))); + + @SuppressWarnings("resource") + // Note: stsClientProvider returns "thin" clients that do not need closing + StsClient stsClient = + stsClientProvider.stsClient(StsDestination.of(storageConfig.getStsEndpointUri(), region)); + + AssumeRoleResponse response = stsClient.assumeRole(request.build()); + accessConfig.put(StorageAccessProperty.AWS_KEY_ID, response.credentials().accessKeyId()); + accessConfig.put( + StorageAccessProperty.AWS_SECRET_KEY, response.credentials().secretAccessKey()); + accessConfig.put(StorageAccessProperty.AWS_TOKEN, response.credentials().sessionToken()); + Optional.ofNullable(response.credentials().expiration()) + .ifPresent( + i -> { + accessConfig.put( + StorageAccessProperty.EXPIRATION_TIME, String.valueOf(i.toEpochMilli())); + accessConfig.put( + StorageAccessProperty.AWS_SESSION_TOKEN_EXPIRES_AT_MS, + String.valueOf(i.toEpochMilli())); + }); + } if (region != null) { accessConfig.put(StorageAccessProperty.CLIENT_REGION, region); @@ -149,6 +152,10 @@ public AccessConfig getSubscopedCreds( return accessConfig.build(); } + private boolean shouldUseSts(AwsStorageConfigurationInfo storageConfig) { + return !Boolean.TRUE.equals(storageConfig.getStsUnavailable()); + } + /** * generate an IamPolicy from the input readLocations and writeLocations, optionally with list * support. Credentials will be scoped to exactly the resources provided. If read and write diff --git a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java index 3a2d706632..b3d7d60790 100644 --- a/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java +++ b/polaris-core/src/main/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfo.java @@ -98,6 +98,12 @@ public URI getInternalEndpointUri() { /** Flag indicating whether path-style bucket access should be forced in S3 clients. */ public abstract @Nullable Boolean getPathStyleAccess(); + /** + * Flag indicating whether STS is available or not. It is modeled in the negative to simplify + * support for unset values ({@code null} being interpreted as {@code false}). + */ + public abstract @Nullable Boolean getStsUnavailable(); + /** Endpoint URI for STS API calls */ @Nullable public abstract String getStsEndpoint(); diff --git a/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java index 0e238775b4..3460dc23f7 100644 --- a/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java +++ b/polaris-core/src/test/java/org/apache/polaris/core/storage/aws/AwsStorageConfigurationInfoTest.java @@ -118,6 +118,14 @@ public void testPathStyleAccess() { assertThat(newBuilder().pathStyleAccess(true).build().getPathStyleAccess()).isTrue(); } + @Test + public void testStsUnavailable() { + assertThat(newBuilder().build().getStsUnavailable()).isNull(); + assertThat(newBuilder().stsUnavailable(null).build().getStsUnavailable()).isNull(); + assertThat(newBuilder().stsUnavailable(false).build().getStsUnavailable()).isFalse(); + assertThat(newBuilder().stsUnavailable(true).build().getStsUnavailable()).isTrue(); + } + @Test public void testRoleArnParsing() { AwsStorageConfigurationInfo awsConfig = diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java index 7fd263e44f..561e76938f 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/RestCatalogMinIOSpecialIT.java @@ -19,12 +19,15 @@ package org.apache.polaris.service.it; import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.iceberg.CatalogProperties.TABLE_DEFAULT_PREFIX; import static org.apache.iceberg.aws.AwsClientProperties.REFRESH_CREDENTIALS_ENDPOINT; import static org.apache.iceberg.aws.s3.S3FileIOProperties.ACCESS_KEY_ID; import static org.apache.iceberg.aws.s3.S3FileIOProperties.ENDPOINT; import static org.apache.iceberg.aws.s3.S3FileIOProperties.SECRET_ACCESS_KEY; import static org.apache.iceberg.types.Types.NestedField.optional; import static org.apache.iceberg.types.Types.NestedField.required; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_KEY_ID; +import static org.apache.polaris.core.storage.StorageAccessProperty.AWS_SECRET_KEY; import static org.apache.polaris.service.catalog.AccessDelegationMode.VENDED_CREDENTIALS; import static org.apache.polaris.service.it.env.PolarisClient.polarisClient; import static org.assertj.core.api.Assertions.assertThat; @@ -165,8 +168,10 @@ private RESTCatalog createCatalog( Optional endpoint, Optional stsEndpoint, boolean pathStyleAccess, - Optional delegationMode) { - return createCatalog(endpoint, stsEndpoint, pathStyleAccess, Optional.empty(), delegationMode); + Optional delegationMode, + boolean stsEnabled) { + return createCatalog( + endpoint, stsEndpoint, pathStyleAccess, Optional.empty(), delegationMode, stsEnabled); } private RESTCatalog createCatalog( @@ -174,11 +179,13 @@ private RESTCatalog createCatalog( Optional stsEndpoint, boolean pathStyleAccess, Optional endpointInternal, - Optional delegationMode) { + Optional delegationMode, + boolean stsEnabled) { AwsStorageConfigInfo.Builder storageConfig = AwsStorageConfigInfo.builder() .setStorageType(StorageConfigInfo.StorageTypeEnum.S3) .setPathStyleAccess(pathStyleAccess) + .setStsUnavailable(!stsEnabled) .setAllowedLocations(List.of(storageBase.toString())); endpoint.ifPresent(storageConfig::setEndpoint); @@ -187,6 +194,12 @@ private RESTCatalog createCatalog( CatalogProperties.Builder catalogProps = CatalogProperties.builder(storageBase.toASCIIString() + "/" + catalogName); + if (!stsEnabled) { + catalogProps.addProperty( + TABLE_DEFAULT_PREFIX + AWS_KEY_ID.getPropertyName(), MINIO_ACCESS_KEY); + catalogProps.addProperty( + TABLE_DEFAULT_PREFIX + AWS_SECRET_KEY.getPropertyName(), MINIO_SECRET_KEY); + } Catalog catalog = PolarisCatalog.builder() .setType(Catalog.TypeEnum.INTERNAL) @@ -227,9 +240,12 @@ public void cleanUp() { } @ParameterizedTest - @ValueSource(booleans = {true, false}) - public void testCreateTable(boolean pathStyle) throws IOException { - LoadTableResponse response = doTestCreateTable(pathStyle, Optional.empty()); + @CsvSource("true, true,") + @CsvSource("false, true,") + @CsvSource("true, false,") + @CsvSource("false, false,") + public void testCreateTable(boolean pathStyle, boolean stsEnabled) throws IOException { + LoadTableResponse response = doTestCreateTable(pathStyle, Optional.empty(), stsEnabled); assertThat(response.config()).doesNotContainKey(SECRET_ACCESS_KEY); assertThat(response.config()).doesNotContainKey(ACCESS_KEY_ID); assertThat(response.config()).doesNotContainKey(REFRESH_CREDENTIALS_ENDPOINT); @@ -239,7 +255,8 @@ public void testCreateTable(boolean pathStyle) throws IOException { @ParameterizedTest @ValueSource(booleans = {true, false}) public void testCreateTableVendedCredentials(boolean pathStyle) throws IOException { - LoadTableResponse response = doTestCreateTable(pathStyle, Optional.of(VENDED_CREDENTIALS)); + LoadTableResponse response = + doTestCreateTable(pathStyle, Optional.of(VENDED_CREDENTIALS), true); assertThat(response.config()) .containsEntry( REFRESH_CREDENTIALS_ENDPOINT, @@ -247,10 +264,10 @@ public void testCreateTableVendedCredentials(boolean pathStyle) throws IOExcepti assertThat(response.credentials()).hasSize(1); } - private LoadTableResponse doTestCreateTable(boolean pathStyle, Optional dm) - throws IOException { + private LoadTableResponse doTestCreateTable( + boolean pathStyle, Optional dm, boolean stsEnabled) throws IOException { try (RESTCatalog restCatalog = - createCatalog(Optional.of(endpoint), Optional.empty(), pathStyle, dm)) { + createCatalog(Optional.of(endpoint), Optional.empty(), pathStyle, dm, stsEnabled)) { LoadTableResponse loadTableResponse = doTestCreateTable(restCatalog, dm); if (pathStyle) { assertThat(loadTableResponse.config()) @@ -268,7 +285,8 @@ public void testInternalEndpoints() throws IOException { Optional.of(endpoint), false, Optional.of(endpoint), - Optional.empty())) { + Optional.empty(), + true)) { StorageConfigInfo storageConfig = managementApi.getCatalog(catalogName).getStorageConfigInfo(); assertThat((AwsStorageConfigInfo) storageConfig) @@ -319,18 +337,22 @@ public LoadTableResponse doTestCreateTable( } @ParameterizedTest - @CsvSource("true,") - @CsvSource("false,") - @CsvSource("true,VENDED_CREDENTIALS") - @CsvSource("false,VENDED_CREDENTIALS") - public void testAppendFiles(boolean pathStyle, AccessDelegationMode delegationMode) + @CsvSource("true, true,") + @CsvSource("false, true,") + @CsvSource("true, false,") + @CsvSource("false, false,") + @CsvSource("true, true, VENDED_CREDENTIALS") + @CsvSource("false, true, VENDED_CREDENTIALS") + public void testAppendFiles( + boolean pathStyle, boolean stsEnabled, AccessDelegationMode delegationMode) throws IOException { try (RESTCatalog restCatalog = createCatalog( Optional.of(endpoint), Optional.of(endpoint), pathStyle, - Optional.ofNullable(delegationMode))) { + Optional.ofNullable(delegationMode), + stsEnabled)) { catalogApi.createNamespace(catalogName, "test-ns"); TableIdentifier id = TableIdentifier.of("test-ns", "t1"); Table table = restCatalog.createTable(id, SCHEMA); @@ -344,7 +366,8 @@ public void testAppendFiles(boolean pathStyle, AccessDelegationMode delegationMo table .locationProvider() .newDataLocation( - String.format("test-file-%s-%s.txt", pathStyle, delegationMode))); + String.format( + "test-file-%s-%s-%s.txt", pathStyle, delegationMode, stsEnabled))); OutputFile f1 = io.newOutputFile(loc.toString()); try (PositionOutputStream os = f1.create()) { os.write("Hello World".getBytes(UTF_8)); diff --git a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java index 54ff3e1cea..d788963254 100644 --- a/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java +++ b/runtime/service/src/main/java/org/apache/polaris/service/admin/PolarisServiceImpl.java @@ -160,6 +160,10 @@ private void validateStorageConfig(StorageConfigInfo storageConfigInfo) { || s3Config.getEndpointInternal() != null) { throw new IllegalArgumentException("Explicitly setting S3 endpoints is not allowed."); } + + if (s3Config.getStsUnavailable() != null) { + throw new IllegalArgumentException("Explicitly disabling STS is not allowed."); + } } } } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java index 5ce6031783..ea034a6748 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/admin/ManagementServiceTest.java @@ -140,6 +140,12 @@ public void testCreateCatalogWithDisallowedS3Endpoints() { assertThatThrownBy(createCatalog::get) .isInstanceOf(IllegalArgumentException.class) .hasMessage("Explicitly setting S3 endpoints is not allowed."); + + storageConfig.setEndpointInternal(null); + storageConfig.setStsUnavailable(false); + assertThatThrownBy(createCatalog::get) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Explicitly disabling STS is not allowed."); } @Test diff --git a/spec/polaris-management-service.yml b/spec/polaris-management-service.yml index 7e93f01ab3..84546bbf40 100644 --- a/spec/polaris-management-service.yml +++ b/spec/polaris-management-service.yml @@ -1119,6 +1119,13 @@ components: endpoint for STS requests made by the Polaris Server (optional). If not set, defaults to 'endpointInternal' (which in turn defaults to `endpoint`). example: "https://sts.example.com:1234" + stsUnavailable: + type: boolean + description: >- + if set to `true`, instructs Polaris Servers to avoid using the STS endpoints when obtaining credentials + for accessing data and metadata files within the related catalog. Setting this property to `true` + effectively disables vending storage credentials to clients. This setting is intended for configuring + catalogs with S3-compatible storage implementations that do not support STS. endpointInternal: type: string description: >-