Skip to content

Commit 4fee6f1

Browse files
authored
unhide parquet runtime configs (#7032)
Signed-off-by: yeya24 <[email protected]>
1 parent 6436235 commit 4fee6f1

File tree

2 files changed

+33
-5
lines changed

2 files changed

+33
-5
lines changed

docs/configuration/config-file-reference.md

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4126,6 +4126,22 @@ The `limits_config` configures default and per-tenant limits imposed by Cortex s
41264126
# zones are not available.
41274127
[query_partial_data: <boolean> | default = false]
41284128

4129+
# The maximum number of rows that can be fetched when querying parquet storage.
4130+
# Each row maps to a series in a parquet file. This limit applies before
4131+
# materializing chunks. 0 to disable.
4132+
# CLI flag: -querier.parquet-queryable.max-fetched-row-count
4133+
[parquet_max_fetched_row_count: <int> | default = 0]
4134+
4135+
# The maximum number of bytes that can be used to fetch chunk column pages when
4136+
# querying parquet storage. 0 to disable.
4137+
# CLI flag: -querier.parquet-queryable.max-fetched-chunk-bytes
4138+
[parquet_max_fetched_chunk_bytes: <int> | default = 0]
4139+
4140+
# The maximum number of bytes that can be used to fetch all column pages when
4141+
# querying parquet storage. 0 to disable.
4142+
# CLI flag: -querier.parquet-queryable.max-fetched-data-bytes
4143+
[parquet_max_fetched_data_bytes: <int> | default = 0]
4144+
41294145
# Maximum number of outstanding requests per tenant per request queue (either
41304146
# query frontend or query scheduler); requests beyond this error with HTTP 429.
41314147
# CLI flag: -frontend.max-outstanding-requests-per-tenant
@@ -4224,6 +4240,18 @@ query_rejection:
42244240
# CLI flag: -compactor.partition-series-count
42254241
[compactor_partition_series_count: <int> | default = 0]
42264242

4243+
# If set, enables the Parquet converter to create the parquet files.
4244+
# CLI flag: -parquet-converter.enabled
4245+
[parquet_converter_enabled: <boolean> | default = false]
4246+
4247+
# The default tenant's shard size when the shuffle-sharding strategy is used by
4248+
# the parquet converter. When this setting is specified in the per-tenant
4249+
# overrides, a value of 0 disables shuffle sharding for the tenant. If the value
4250+
# is < 1 and > 0 the shard size will be a percentage of the total parquet
4251+
# converters.
4252+
# CLI flag: -parquet-converter.tenant-shard-size
4253+
[parquet_converter_tenant_shard_size: <float> | default = 0]
4254+
42274255
# S3 server-side encryption type. Required to enable server-side encryption
42284256
# overrides for a specific tenant. If not set, the default S3 client settings
42294257
# are used.

pkg/util/validation/limits.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -189,9 +189,9 @@ type Limits struct {
189189
QueryPartialData bool `yaml:"query_partial_data" json:"query_partial_data" doc:"nocli|description=Enable to allow queries to be evaluated with data from a single zone, if other zones are not available.|default=false"`
190190

191191
// Parquet Queryable enforced limits.
192-
ParquetMaxFetchedRowCount int `yaml:"parquet_max_fetched_row_count" json:"parquet_max_fetched_row_count" doc:"hidden"`
193-
ParquetMaxFetchedChunkBytes int `yaml:"parquet_max_fetched_chunk_bytes" json:"parquet_max_fetched_chunk_bytes" doc:"hidden"`
194-
ParquetMaxFetchedDataBytes int `yaml:"parquet_max_fetched_data_bytes" json:"parquet_max_fetched_data_bytes" doc:"hidden"`
192+
ParquetMaxFetchedRowCount int `yaml:"parquet_max_fetched_row_count" json:"parquet_max_fetched_row_count"`
193+
ParquetMaxFetchedChunkBytes int `yaml:"parquet_max_fetched_chunk_bytes" json:"parquet_max_fetched_chunk_bytes"`
194+
ParquetMaxFetchedDataBytes int `yaml:"parquet_max_fetched_data_bytes" json:"parquet_max_fetched_data_bytes"`
195195

196196
// Query Frontend / Scheduler enforced limits.
197197
MaxOutstandingPerTenant int `yaml:"max_outstanding_requests_per_tenant" json:"max_outstanding_requests_per_tenant"`
@@ -220,8 +220,8 @@ type Limits struct {
220220
CompactorPartitionSeriesCount int64 `yaml:"compactor_partition_series_count" json:"compactor_partition_series_count"`
221221

222222
// Parquet converter
223-
ParquetConverterEnabled bool `yaml:"parquet_converter_enabled" json:"parquet_converter_enabled" doc:"hidden"`
224-
ParquetConverterTenantShardSize float64 `yaml:"parquet_converter_tenant_shard_size" json:"parquet_converter_tenant_shard_size" doc:"hidden"`
223+
ParquetConverterEnabled bool `yaml:"parquet_converter_enabled" json:"parquet_converter_enabled"`
224+
ParquetConverterTenantShardSize float64 `yaml:"parquet_converter_tenant_shard_size" json:"parquet_converter_tenant_shard_size"`
225225

226226
// This config doesn't have a CLI flag registered here because they're registered in
227227
// their own original config struct.

0 commit comments

Comments
 (0)