Skip to content

Commit 173d6a5

Browse files
ianton-ruianton-ru
committed
Make fields in object storage configuration private
1 parent 319e424 commit 173d6a5

21 files changed

+119
-99
lines changed

‎src/Storages/ObjectStorage/Azure/Configuration.cpp

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -149,9 +149,9 @@ void StorageAzureConfiguration::fromNamedCollection(const NamedCollection & coll
149149
if (collection.has("account_key"))
150150
account_key = collection.get<String>("account_key");
151151

152-
structure = collection.getOrDefault<String>("structure", "auto");
153-
format = collection.getOrDefault<String>("format", format);
154-
compression_method = collection.getOrDefault<String>("compression_method", collection.getOrDefault<String>("compression", "auto"));
152+
setStructure(collection.getOrDefault<String>("structure", "auto"));
153+
setFormat(collection.getOrDefault<String>("format", getFormat()));
154+
setCompressionMethod(collection.getOrDefault<String>("compression_method", collection.getOrDefault<String>("compression", "auto")));
155155

156156
blobs_paths = {blob_path};
157157
connection_params = getConnectionParams(connection_url, container_name, account_name, account_key, context);
@@ -187,12 +187,12 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
187187
auto fourth_arg = checkAndGetLiteralArgument<String>(engine_args[3], "format/account_name");
188188
if (is_format_arg(fourth_arg))
189189
{
190-
format = fourth_arg;
190+
setFormat(fourth_arg);
191191
}
192192
else
193193
{
194194
if (with_structure)
195-
structure = fourth_arg;
195+
setStructure(fourth_arg);
196196
else
197197
throw Exception(
198198
ErrorCodes::BAD_ARGUMENTS,
@@ -204,8 +204,8 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
204204
auto fourth_arg = checkAndGetLiteralArgument<String>(engine_args[3], "format/account_name");
205205
if (is_format_arg(fourth_arg))
206206
{
207-
format = fourth_arg;
208-
compression_method = checkAndGetLiteralArgument<String>(engine_args[4], "compression");
207+
setFormat(fourth_arg);
208+
setCompressionMethod(checkAndGetLiteralArgument<String>(engine_args[4], "compression"));
209209
}
210210
else
211211
{
@@ -220,9 +220,9 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
220220
{
221221
if (with_structure)
222222
{
223-
format = fourth_arg;
224-
compression_method = checkAndGetLiteralArgument<String>(engine_args[4], "compression");
225-
structure = checkAndGetLiteralArgument<String>(engine_args[5], "structure");
223+
setFormat(fourth_arg);
224+
setCompressionMethod(checkAndGetLiteralArgument<String>(engine_args[4], "compression"));
225+
setStructure(checkAndGetLiteralArgument<String>(engine_args[5], "structure"));
226226
}
227227
else
228228
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Format and compression must be last arguments");
@@ -234,12 +234,12 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
234234
auto sixth_arg = checkAndGetLiteralArgument<String>(engine_args[5], "format/structure");
235235
if (is_format_arg(sixth_arg))
236236
{
237-
format = sixth_arg;
237+
setFormat(sixth_arg);
238238
}
239239
else
240240
{
241241
if (with_structure)
242-
structure = sixth_arg;
242+
setStructure(sixth_arg);
243243
else
244244
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown format {}", sixth_arg);
245245
}
@@ -258,8 +258,8 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
258258
auto sixth_arg = checkAndGetLiteralArgument<String>(engine_args[5], "format/account_name");
259259
if (!is_format_arg(sixth_arg))
260260
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown format {}", sixth_arg);
261-
format = sixth_arg;
262-
compression_method = checkAndGetLiteralArgument<String>(engine_args[6], "compression");
261+
setFormat(sixth_arg);
262+
setCompressionMethod(checkAndGetLiteralArgument<String>(engine_args[6], "compression"));
263263
}
264264
else if (with_structure && engine_args.size() == 8)
265265
{
@@ -269,9 +269,9 @@ void StorageAzureConfiguration::fromAST(ASTs & engine_args, ContextPtr context,
269269
auto sixth_arg = checkAndGetLiteralArgument<String>(engine_args[5], "format");
270270
if (!is_format_arg(sixth_arg))
271271
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown format {}", sixth_arg);
272-
format = sixth_arg;
273-
compression_method = checkAndGetLiteralArgument<String>(engine_args[6], "compression");
274-
structure = checkAndGetLiteralArgument<String>(engine_args[7], "structure");
272+
setFormat(sixth_arg);
273+
setCompressionMethod (checkAndGetLiteralArgument<String>(engine_args[6], "compression"));
274+
setStructure(checkAndGetLiteralArgument<String>(engine_args[7], "structure"));
275275
}
276276

277277
blobs_paths = {blob_path};

‎src/Storages/ObjectStorage/DataLakes/DataLakeConfiguration.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,14 @@ class StorageIcebergConfiguration : public StorageObjectStorage::Configuration,
356356
return getImpl().createArgsWithAccessData();
357357
}
358358

359+
const String & getFormat() const override { return getImpl().getFormat(); }
360+
const String & getCompressionMethod() const override { return getImpl().getCompressionMethod(); }
361+
const String & getStructure() const override { return getImpl().getStructure(); }
362+
363+
void setFormat(const String & format_) override { getImpl().setFormat(format_); }
364+
void setCompressionMethod(const String & compression_method_) override { getImpl().setCompressionMethod(compression_method_); }
365+
void setStructure(const String & structure_) override { getImpl().setStructure(structure_); }
366+
359367
protected:
360368
void fromNamedCollection(const NamedCollection & collection, ContextPtr context) override
361369
{ return getImpl().fromNamedCollection(collection, context); }

‎src/Storages/ObjectStorage/DataLakes/HudiMetadata.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ Strings HudiMetadata::getDataFilesImpl() const
4444
{
4545
auto configuration_ptr = configuration.lock();
4646
auto log = getLogger("HudiMetadata");
47-
const auto keys = listFiles(*object_storage, *configuration_ptr, "", Poco::toLower(configuration_ptr->format));
47+
const auto keys = listFiles(*object_storage, *configuration_ptr, "", Poco::toLower(configuration_ptr->getFormat()));
4848

4949
using Partition = std::string;
5050
using FileID = std::string;

‎src/Storages/ObjectStorage/HDFS/Configuration.cpp

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -111,23 +111,23 @@ void StorageHDFSConfiguration::fromAST(ASTs & args, ContextPtr context, bool wit
111111

112112
if (args.size() > 1)
113113
{
114-
format = checkAndGetLiteralArgument<String>(args[1], "format_name");
114+
setFormat(checkAndGetLiteralArgument<String>(args[1], "format_name"));
115115
}
116116

117117
if (with_structure)
118118
{
119119
if (args.size() > 2)
120120
{
121-
structure = checkAndGetLiteralArgument<String>(args[2], "structure");
121+
setStructure(checkAndGetLiteralArgument<String>(args[2], "structure"));
122122
}
123123
if (args.size() > 3)
124124
{
125-
compression_method = checkAndGetLiteralArgument<String>(args[3], "compression_method");
125+
setCompressionMethod(checkAndGetLiteralArgument<String>(args[3], "compression_method"));
126126
}
127127
}
128128
else if (args.size() > 2)
129129
{
130-
compression_method = checkAndGetLiteralArgument<String>(args[2], "compression_method");
130+
setCompressionMethod(checkAndGetLiteralArgument<String>(args[2], "compression_method"));
131131
}
132132

133133
setURL(url_str);
@@ -143,10 +143,10 @@ void StorageHDFSConfiguration::fromNamedCollection(const NamedCollection & colle
143143
else
144144
url_str = collection.get<String>("url");
145145

146-
format = collection.getOrDefault<String>("format", "auto");
147-
compression_method = collection.getOrDefault<String>("compression_method",
148-
collection.getOrDefault<String>("compression", "auto"));
149-
structure = collection.getOrDefault<String>("structure", "auto");
146+
setFormat(collection.getOrDefault<String>("format", "auto"));
147+
setCompressionMethod(collection.getOrDefault<String>("compression_method",
148+
collection.getOrDefault<String>("compression", "auto")));
149+
setStructure(collection.getOrDefault<String>("structure", "auto"));
150150

151151
setURL(url_str);
152152
}

‎src/Storages/ObjectStorage/Local/Configuration.cpp

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
2424
void StorageLocalConfiguration::fromNamedCollection(const NamedCollection & collection, ContextPtr)
2525
{
2626
path = collection.get<String>("path");
27-
format = collection.getOrDefault<String>("format", "auto");
28-
compression_method = collection.getOrDefault<String>("compression_method", collection.getOrDefault<String>("compression", "auto"));
29-
structure = collection.getOrDefault<String>("structure", "auto");
27+
setFormat(collection.getOrDefault<String>("format", "auto"));
28+
setCompressionMethod(collection.getOrDefault<String>("compression_method", collection.getOrDefault<String>("compression", "auto")));
29+
setStructure(collection.getOrDefault<String>("structure", "auto"));
3030
paths = {path};
3131
}
3232

@@ -46,23 +46,23 @@ void StorageLocalConfiguration::fromAST(ASTs & args, ContextPtr context, bool wi
4646

4747
if (args.size() > 1)
4848
{
49-
format = checkAndGetLiteralArgument<String>(args[1], "format_name");
49+
setFormat(checkAndGetLiteralArgument<String>(args[1], "format_name"));
5050
}
5151

5252
if (with_structure)
5353
{
5454
if (args.size() > 2)
5555
{
56-
structure = checkAndGetLiteralArgument<String>(args[2], "structure");
56+
setStructure(checkAndGetLiteralArgument<String>(args[2], "structure"));
5757
}
5858
if (args.size() > 3)
5959
{
60-
compression_method = checkAndGetLiteralArgument<String>(args[3], "compression_method");
60+
setCompressionMethod(checkAndGetLiteralArgument<String>(args[3], "compression_method"));
6161
}
6262
}
6363
else if (args.size() > 2)
6464
{
65-
compression_method = checkAndGetLiteralArgument<String>(args[2], "compression_method");
65+
setCompressionMethod(checkAndGetLiteralArgument<String>(args[2], "compression_method"));
6666
}
6767
paths = {path};
6868
}

‎src/Storages/ObjectStorage/ReadBufferIterator.cpp

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@ ReadBufferIterator::ReadBufferIterator(
3838
, read_keys(read_keys_)
3939
, prev_read_keys_size(read_keys_.size())
4040
{
41-
if (configuration->format != "auto")
42-
format = configuration->format;
41+
if (configuration->getFormat() != "auto")
42+
format = configuration->getFormat();
4343
}
4444

4545
SchemaCache::Key ReadBufferIterator::getKeyForSchemaCache(const ObjectInfo & object_info, const String & format_name) const
@@ -148,7 +148,7 @@ std::unique_ptr<ReadBuffer> ReadBufferIterator::recreateLastReadBuffer()
148148

149149
auto impl = StorageObjectStorageSource::createReadBuffer(*current_object_info, object_storage, context, getLogger("ReadBufferIterator"));
150150

151-
const auto compression_method = chooseCompressionMethod(current_object_info->getFileName(), configuration->compression_method);
151+
const auto compression_method = chooseCompressionMethod(current_object_info->getFileName(), configuration->getCompressionMethod());
152152
const auto zstd_window = static_cast<int>(context->getSettingsRef()[Setting::zstd_window_log_max]);
153153

154154
return wrapReadBufferWithCompressionMethod(std::move(impl), compression_method, zstd_window);
@@ -266,13 +266,13 @@ ReadBufferIterator::Data ReadBufferIterator::next()
266266
using ObjectInfoInArchive = StorageObjectStorageSource::ArchiveIterator::ObjectInfoInArchive;
267267
if (const auto * object_info_in_archive = dynamic_cast<const ObjectInfoInArchive *>(current_object_info.get()))
268268
{
269-
compression_method = chooseCompressionMethod(filename, configuration->compression_method);
269+
compression_method = chooseCompressionMethod(filename, configuration->getCompressionMethod());
270270
const auto & archive_reader = object_info_in_archive->archive_reader;
271271
read_buf = archive_reader->readFile(object_info_in_archive->path_in_archive, /*throw_on_not_found=*/true);
272272
}
273273
else
274274
{
275-
compression_method = chooseCompressionMethod(filename, configuration->compression_method);
275+
compression_method = chooseCompressionMethod(filename, configuration->getCompressionMethod());
276276
read_buf = StorageObjectStorageSource::createReadBuffer(*current_object_info, object_storage, getContext(), getLogger("ReadBufferIterator"));
277277
}
278278

‎src/Storages/ObjectStorage/S3/Configuration.cpp

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -186,9 +186,9 @@ void StorageS3Configuration::fromNamedCollection(const NamedCollection & collect
186186
auth_settings[S3AuthSetting::expiration_window_seconds] = collection.getOrDefault<UInt64>("expiration_window_seconds", S3::DEFAULT_EXPIRATION_WINDOW_SECONDS);
187187
auth_settings[S3AuthSetting::session_token] = collection.getOrDefault<String>("session_token", "");
188188

189-
format = collection.getOrDefault<String>("format", format);
190-
compression_method = collection.getOrDefault<String>("compression_method", collection.getOrDefault<String>("compression", "auto"));
191-
structure = collection.getOrDefault<String>("structure", "auto");
189+
setFormat(collection.getOrDefault<String>("format", getFormat()));
190+
setCompressionMethod(collection.getOrDefault<String>("compression_method", collection.getOrDefault<String>("compression", "auto")));
191+
setStructure(collection.getOrDefault<String>("structure", "auto"));
192192

193193
request_settings = S3::S3RequestSettings(collection, settings, /* validate_settings */true);
194194

@@ -433,14 +433,14 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
433433
/// Set format to configuration only of it's not 'auto',
434434
/// because we can have default format set in configuration.
435435
if (format_ != "auto")
436-
format = format_;
436+
setFormat(format_);
437437
}
438438

439439
if (engine_args_to_idx.contains("structure"))
440-
structure = checkAndGetLiteralArgument<String>(args[engine_args_to_idx["structure"]], "structure");
440+
setStructure(checkAndGetLiteralArgument<String>(args[engine_args_to_idx["structure"]], "structure"));
441441

442442
if (engine_args_to_idx.contains("compression_method"))
443-
compression_method = checkAndGetLiteralArgument<String>(args[engine_args_to_idx["compression_method"]], "compression_method");
443+
setCompressionMethod(checkAndGetLiteralArgument<String>(args[engine_args_to_idx["compression_method"]], "compression_method"));
444444

445445
if (engine_args_to_idx.contains("access_key_id"))
446446
auth_settings[S3AuthSetting::access_key_id] = checkAndGetLiteralArgument<String>(args[engine_args_to_idx["access_key_id"]], "access_key_id");
@@ -683,10 +683,10 @@ ASTPtr StorageS3Configuration::createArgsWithAccessData() const
683683
arguments->children.push_back(std::make_shared<ASTLiteral>(auth_settings[S3AuthSetting::secret_access_key].value));
684684
if (!auth_settings[S3AuthSetting::session_token].value.empty())
685685
arguments->children.push_back(std::make_shared<ASTLiteral>(auth_settings[S3AuthSetting::session_token].value));
686-
if (format != "auto")
687-
arguments->children.push_back(std::make_shared<ASTLiteral>(format));
688-
if (!compression_method.empty())
689-
arguments->children.push_back(std::make_shared<ASTLiteral>(compression_method));
686+
if (getFormat() != "auto")
687+
arguments->children.push_back(std::make_shared<ASTLiteral>(getFormat()));
688+
if (!getCompressionMethod().empty())
689+
arguments->children.push_back(std::make_shared<ASTLiteral>(getCompressionMethod()));
690690

691691
if (!auth_settings[S3AuthSetting::role_arn].value.empty())
692692
{

‎src/Storages/ObjectStorage/StorageObjectStorage.cpp

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ StorageObjectStorage::StorageObjectStorage(
9999
, distributed_processing(distributed_processing_)
100100
, log(getLogger(fmt::format("Storage{}({})", configuration->getEngineName(), table_id_.getFullTableName())))
101101
{
102-
bool do_lazy_init = lazy_init && !columns_.empty() && !configuration->format.empty();
102+
bool do_lazy_init = lazy_init && !columns_.empty() && !configuration->getFormat().empty();
103103
update_configuration_on_read = !is_table_function_ || do_lazy_init;
104104
bool failed_init = false;
105105
auto do_init = [&]()
@@ -115,7 +115,7 @@ StorageObjectStorage::StorageObjectStorage(
115115
{
116116
// If we don't have format or schema yet, we can't ignore failed configuration update,
117117
// because relevant configuration is crucial for format and schema inference
118-
if (mode <= LoadingStrictnessLevel::CREATE || columns_.empty() || (configuration->format == "auto"))
118+
if (mode <= LoadingStrictnessLevel::CREATE || columns_.empty() || (configuration->getFormat() == "auto"))
119119
{
120120
throw;
121121
}
@@ -132,7 +132,7 @@ StorageObjectStorage::StorageObjectStorage(
132132

133133
std::string sample_path;
134134
ColumnsDescription columns{columns_};
135-
resolveSchemaAndFormat(columns, configuration->format, object_storage, configuration, format_settings, sample_path, context);
135+
resolveSchemaAndFormat(columns, object_storage, configuration, format_settings, sample_path, context);
136136
configuration->check(context);
137137

138138
StorageInMemoryMetadata metadata;
@@ -172,17 +172,17 @@ String StorageObjectStorage::getName() const
172172

173173
bool StorageObjectStorage::prefersLargeBlocks() const
174174
{
175-
return FormatFactory::instance().checkIfOutputFormatPrefersLargeBlocks(configuration->format);
175+
return FormatFactory::instance().checkIfOutputFormatPrefersLargeBlocks(configuration->getFormat());
176176
}
177177

178178
bool StorageObjectStorage::parallelizeOutputAfterReading(ContextPtr context) const
179179
{
180-
return FormatFactory::instance().checkParallelizeOutputAfterReading(configuration->format, context);
180+
return FormatFactory::instance().checkParallelizeOutputAfterReading(configuration->getFormat(), context);
181181
}
182182

183183
bool StorageObjectStorage::supportsSubsetOfColumns(const ContextPtr & context) const
184184
{
185-
return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(configuration->format, context, format_settings);
185+
return FormatFactory::instance().checkIfFormatSupportsSubsetOfColumns(configuration->getFormat(), context, format_settings);
186186
}
187187

188188
void StorageObjectStorage::Configuration::update(ObjectStoragePtr object_storage_ptr, ContextPtr context)
@@ -529,7 +529,7 @@ ColumnsDescription StorageObjectStorage::resolveSchemaFromData(
529529

530530
ObjectInfos read_keys;
531531
auto iterator = createReadBufferIterator(object_storage, configuration, format_settings, read_keys, context);
532-
auto schema = readSchemaFromFormat(configuration->format, format_settings, *iterator, context);
532+
auto schema = readSchemaFromFormat(configuration->getFormat(), format_settings, *iterator, context);
533533
sample_path = iterator->getLastFilePath();
534534
return schema;
535535
}
@@ -550,7 +550,7 @@ std::string StorageObjectStorage::resolveFormatFromData(
550550

551551
std::pair<ColumnsDescription, std::string> StorageObjectStorage::resolveSchemaAndFormatFromData(
552552
const ObjectStoragePtr & object_storage,
553-
const ConfigurationPtr & configuration,
553+
ConfigurationPtr & configuration,
554554
const std::optional<FormatSettings> & format_settings,
555555
std::string & sample_path,
556556
const ContextPtr & context)
@@ -559,13 +559,13 @@ std::pair<ColumnsDescription, std::string> StorageObjectStorage::resolveSchemaAn
559559
auto iterator = createReadBufferIterator(object_storage, configuration, format_settings, read_keys, context);
560560
auto [columns, format] = detectFormatAndReadSchema(format_settings, *iterator, context);
561561
sample_path = iterator->getLastFilePath();
562-
configuration->format = format;
562+
configuration->setFormat(format);
563563
return std::pair(columns, format);
564564
}
565565

566566
void StorageObjectStorage::addInferredEngineArgsToCreateQuery(ASTs & args, const ContextPtr & context) const
567567
{
568-
configuration->addStructureAndFormatToArgsIfNeeded(args, "", configuration->format, context, /*with_structure=*/false);
568+
configuration->addStructureAndFormatToArgsIfNeeded(args, "", configuration->getFormat(), context, /*with_structure=*/false);
569569
}
570570

571571
SchemaCache & StorageObjectStorage::getSchemaCache(const ContextPtr & context, const std::string & storage_type_name)

0 commit comments

Comments
 (0)