Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
- Added an OpenTelemetry event listener for emitting Polaris audit events as OpenTelemetry log records.
- Added optional `sessionPolicy` field to `SigV4AuthenticationParameters` for catalog federation. When set, the IAM session policy JSON is attached to the STS AssumeRole request, allowing administrators to restrict vended credentials to only the required AWS services and actions (Principle of Least Privilege).
- Python CLI: added `--scheme` to specify URL scheme
- Python CLI: `catalogs update` now supports `--no-sts` and `--no-kms` to toggle STS/KMS availability on an existing S3 catalog. Previously these were only settable at `catalogs create` time.
- Added opt-in idempotency for `createTable` and `updateTable` in the Iceberg REST catalog. When enabled via `polaris.idempotency.enabled=true` (default `false`), a client-supplied `Idempotency-Key` header is embedded into the table entity and committed in the same transaction as the operation; a retry carrying the same key within the TTL window (`polaris.idempotency.ttl`, default `PT5M`) replays the original success instead of failing — with `AlreadyExists` for `createTable`, or with `CommitFailedException` for `updateTable` when the request's requirements no longer match the already-advanced table. When idempotency is enabled, the reuse window is advertised to clients through the `idempotency-key-lifetime` field of the `GET /v1/config` response.

### Changes
Expand Down
26 changes: 19 additions & 7 deletions client/python/apache_polaris/cli/command/catalogs.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,6 +267,8 @@ def _has_aws_storage_info(self) -> bool:
or self.current_kms_key
or self.allowed_kms_keys
or self.path_style_access
or self.sts_unavailable
or self.kms_unavailable
)

def _has_azure_storage_info(self) -> bool:
Expand All @@ -280,6 +282,14 @@ def _has_azure_storage_info(self) -> bool:
def _has_gcs_storage_info(self) -> bool:
return bool(self.service_account)

@staticmethod
def _require_s3(storage_info: StorageConfigInfo, flag: str) -> None:
# Note: We have to lowercase the returned value because the server enum
# is uppercase but we defined the StorageType enums as lowercase.
storage_type = storage_info.storage_type
if storage_type.lower() != StorageType.S3.value:
raise CliError(f"{flag} requires S3 storage_type, got: {storage_type}")

def _build_storage_config_info(self) -> Optional[StorageConfigInfo]:
config = None
if self.storage_type == StorageType.S3.value:
Expand Down Expand Up @@ -501,15 +511,17 @@ def execute(self, api: PolarisDefaultApi) -> None:
)

if self.region:
# Note: We have to lowercase the returned value because the server enum
# is uppercase but we defined the StorageType enums as lowercase.
storage_type = updated_storage_info.storage_type
if storage_type.lower() != StorageType.S3.value:
raise CliError(
f"--region requires S3 storage_type, got: {storage_type}"
)
self._require_s3(updated_storage_info, "--region")
updated_storage_info.region = self.region

if self.sts_unavailable:
self._require_s3(updated_storage_info, "--no-sts")
updated_storage_info.sts_unavailable = self.sts_unavailable
Comment thread
yalindogusahin marked this conversation as resolved.

if self.kms_unavailable:
self._require_s3(updated_storage_info, "--no-kms")
updated_storage_info.kms_unavailable = self.kms_unavailable

request = UpdateCatalogRequest(
current_entity_version=catalog.entity_version,
properties=catalog.properties.to_dict(),
Expand Down
12 changes: 12 additions & 0 deletions client/python/apache_polaris/cli/options/option_tree.py
Original file line number Diff line number Diff line change
Expand Up @@ -398,6 +398,18 @@ def _catalogs_option() -> Option:
Hints.S3_REGION,
group="AWS S3 Storage Options",
),
Argument(
Arguments.STS_UNAVAILABLE,
bool,
Hints.S3_STS_UNAVAILABLE,
group="AWS S3 Storage Options",
),
Argument(
Arguments.KMS_UNAVAILABLE,
bool,
Hints.S3_KMS_UNAVAILABLE,
group="AWS S3 Storage Options",
),
Argument(
Arguments.SET_PROPERTY,
str,
Expand Down
61 changes: 61 additions & 0 deletions client/python/tests/test_catalogs_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
PolarisCatalog,
CatalogProperties,
AwsStorageConfigInfo,
StorageConfigInfo,
)


Expand Down Expand Up @@ -464,6 +465,66 @@ def test_catalog_name_parsing(self) -> None:
self.mock_execute(mock_client, ["catalogs", "get", "foo"])
mock_client.get_catalog.assert_called_with("foo")

def test_catalog_update_s3_options(self) -> None:
mock_client = self.build_mock_client()
mock_client.get_catalog.return_value = PolarisCatalog(
type="INTERNAL",
name="s3-catalog",
entity_version=1,
properties=CatalogProperties(
default_base_location="s3://bucket/path", additional_properties={}
),
storage_config_info=AwsStorageConfigInfo(
storage_type="S3", allowed_locations=[]
),
)
self.mock_execute(
mock_client,
["catalogs", "update", "s3-catalog", "--no-sts", "--no-kms"],
)
call_args = mock_client.update_catalog.call_args[0][1]
self.assertTrue(call_args.storage_config_info.sts_unavailable)
self.assertTrue(call_args.storage_config_info.kms_unavailable)

# --no-kms alone (without any other AWS storage option) must still be applied
mock_client.get_catalog.return_value = PolarisCatalog(
type="INTERNAL",
name="s3-catalog",
entity_version=1,
properties=CatalogProperties(
default_base_location="s3://bucket/path", additional_properties={}
),
storage_config_info=AwsStorageConfigInfo(
storage_type="S3", allowed_locations=[]
),
)
self.mock_execute(
mock_client,
["catalogs", "update", "s3-catalog", "--no-kms"],
)
call_args = mock_client.update_catalog.call_args[0][1]
self.assertTrue(call_args.storage_config_info.kms_unavailable)

# --no-kms against a non-S3 catalog should raise
mock_client.get_catalog.return_value = PolarisCatalog(
type="INTERNAL",
name="gcs-catalog",
entity_version=1,
properties=CatalogProperties(
default_base_location="gs://bucket/path", additional_properties={}
),
storage_config_info=StorageConfigInfo(
storage_type="GCS", allowed_locations=[]
),
)
self.check_exception(
lambda: self.mock_execute(
mock_client,
["catalogs", "update", "gcs-catalog", "--no-kms"],
),
"--no-kms requires S3 storage_type",
)

def test_catalog_list(self) -> None:
mock_client = self.build_mock_client()
mock_client.list_catalogs.return_values.catalogs = []
Expand Down
4 changes: 4 additions & 0 deletions site/content/in-dev/unreleased/command-line-interface.md
Original file line number Diff line number Diff line change
Expand Up @@ -351,6 +351,8 @@ Command Options:

AWS S3 Storage Options:
--region REGION The region to use when connecting to S3
--no-sts Indicates that Polaris should not use STS (e.g. if STS is not available)
--no-kms Indicates that Polaris should not use KMS (e.g. if KMS is not available)
```

##### Examples
Expand All @@ -359,6 +361,8 @@ AWS S3 Storage Options:
polaris catalogs update --set-property tag=new_value my_catalog

polaris catalogs update --default-base-location s3://new-bucket/my_data my_catalog

polaris catalogs update --no-kms my_catalog
```

### Principals
Expand Down