Skip to content

Commit e346693

Browse files
raulcdraulcd
committed
GH-46369: [C++] Add key-value pairs to the FileSystemFactory interface
1 parent 3589dc2 commit e346693

5 files changed

Lines changed: 111 additions & 17 deletions

File tree

‎cpp/src/arrow/filesystem/filesystem.cc‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -893,18 +893,18 @@ Status LoadFileSystemFactories(const char* libpath) {
893893

894894
namespace {
895895

896-
Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
897-
const std::string& uri_string,
898-
const io::IOContext& io_context,
899-
std::string* out_path) {
896+
Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(
897+
const Uri& uri, const std::string& uri_string,
898+
const std::vector<std::pair<std::string, std::any>>& options,
899+
const io::IOContext& io_context, std::string* out_path) {
900900
const auto scheme = uri.scheme();
901901

902902
{
903903
ARROW_ASSIGN_OR_RAISE(
904904
auto* factory,
905905
FileSystemFactoryRegistry::GetInstance()->FactoryForScheme(scheme));
906906
if (factory != nullptr) {
907-
return factory->function(uri, io_context, out_path);
907+
return factory->function(uri, options, io_context, out_path);
908908
}
909909
}
910910

@@ -962,14 +962,28 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(const Uri& uri,
962962

963963
Result<std::shared_ptr<FileSystem>> FileSystemFromUri(const std::string& uri_string,
964964
std::string* out_path) {
965-
return FileSystemFromUri(uri_string, io::default_io_context(), out_path);
965+
return FileSystemFromUri(uri_string, /*options=*/{}, io::default_io_context(),
966+
out_path);
967+
}
968+
969+
Result<std::shared_ptr<FileSystem>> FileSystemFromUri(
970+
const std::string& uri_string,
971+
const std::vector<std::pair<std::string, std::any>>& options, std::string* out_path) {
972+
return FileSystemFromUri(uri_string, options, io::default_io_context(), out_path);
966973
}
967974

968975
Result<std::shared_ptr<FileSystem>> FileSystemFromUri(const std::string& uri_string,
969976
const io::IOContext& io_context,
970977
std::string* out_path) {
978+
return FileSystemFromUri(uri_string, /*options=*/{}, io_context, out_path);
979+
}
980+
981+
Result<std::shared_ptr<FileSystem>> FileSystemFromUri(
982+
const std::string& uri_string,
983+
const std::vector<std::pair<std::string, std::any>>& options,
984+
const io::IOContext& io_context, std::string* out_path) {
971985
ARROW_ASSIGN_OR_RAISE(auto fsuri, ParseFileSystemUri(uri_string));
972-
return FileSystemFromUriReal(fsuri, uri_string, io_context, out_path);
986+
return FileSystemFromUriReal(fsuri, uri_string, options, io_context, out_path);
973987
}
974988

975989
Result<std::shared_ptr<FileSystem>> FileSystemFromUriOrPath(const std::string& uri_string,

‎cpp/src/arrow/filesystem/filesystem.h‎

Lines changed: 67 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
#pragma once
1919

20+
#include <any>
2021
#include <chrono>
2122
#include <cstdint>
2223
#include <functional>
@@ -28,6 +29,7 @@
2829

2930
#include "arrow/filesystem/type_fwd.h"
3031
#include "arrow/io/interfaces.h"
32+
#include "arrow/result.h"
3133
#include "arrow/type_fwd.h"
3234
#include "arrow/util/compare.h"
3335
#include "arrow/util/macros.h"
@@ -359,11 +361,34 @@ class ARROW_EXPORT FileSystem
359361

360362
struct FileSystemFactory {
361363
std::function<Result<std::shared_ptr<FileSystem>>(
362-
const Uri& uri, const io::IOContext& io_context, std::string* out_path)>
364+
const Uri& uri, const std::vector<std::pair<std::string, std::any>>& options,
365+
const io::IOContext& io_context, std::string* out_path)>
363366
function;
364367
std::string_view file;
365368
int line;
366369

370+
/// Construct from an options-aware factory function.
371+
FileSystemFactory(std::function<Result<std::shared_ptr<FileSystem>>(
372+
const Uri&, const std::vector<std::pair<std::string, std::any>>&,
373+
const io::IOContext&, std::string*)>
374+
fn,
375+
std::string_view file, int line)
376+
: function(std::move(fn)), file(file), line(line) {}
377+
378+
/// Construct from a non-options aware factory function maintaing source compatibility
379+
/// with existing factories.
380+
FileSystemFactory(std::function<Result<std::shared_ptr<FileSystem>>(
381+
const Uri&, const io::IOContext&, std::string*)>
382+
fn,
383+
std::string_view file, int line)
384+
: function([fn = std::move(fn)](
385+
const Uri& uri,
386+
const std::vector<std::pair<std::string, std::any>>& /*ignored*/,
387+
const io::IOContext& ctx,
388+
std::string* out_path) { return fn(uri, ctx, out_path); }),
389+
file(file),
390+
line(line) {}
391+
367392
bool operator==(const FileSystemFactory& other) const {
368393
// In the case where libarrow is linked statically both to the executable and to a
369394
// dynamically loaded filesystem implementation library, the library contains a
@@ -547,6 +572,26 @@ ARROW_EXPORT
547572
Result<std::shared_ptr<FileSystem>> FileSystemFromUri(const std::string& uri,
548573
std::string* out_path = NULLPTR);
549574

575+
/// \brief Create a new FileSystem by URI with extended backend-specific filesystem
576+
/// options
577+
///
578+
/// Recognized schemes are "file", "mock", "hdfs", "viewfs", "s3",
579+
/// "gs" and "gcs".
580+
///
581+
/// Support for other schemes can be added using RegisterFileSystemFactory.
582+
///
583+
/// \param[in] uri the URI to give access to
584+
/// \param[in] options a list of backend-specific filesystem options
585+
/// Each option is a (name, value) pair.
586+
/// The expected type is specific to the backend and
587+
/// option name.
588+
/// \param[out] out_path (optional) Path inside the filesystem.
589+
/// \return out_fs FileSystem instance.
590+
ARROW_EXPORT
591+
Result<std::shared_ptr<FileSystem>> FileSystemFromUri(
592+
const std::string& uri, const std::vector<std::pair<std::string, std::any>>& options,
593+
std::string* out_path = NULLPTR);
594+
550595
/// \brief Create a new FileSystem by URI with a custom IO context
551596
///
552597
/// Recognized schemes are "file", "mock", "hdfs", "viewfs", "s3",
@@ -563,6 +608,27 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUri(const std::string& uri,
563608
const io::IOContext& io_context,
564609
std::string* out_path = NULLPTR);
565610

611+
/// \brief Create a new FileSystem by URI with a custom IO context with backend-specific
612+
/// filesystem options
613+
///
614+
/// Recognized schemes are "file", "mock", "hdfs", "viewfs", "s3",
615+
/// "gs" and "gcs".
616+
///
617+
/// Support for other schemes can be added using RegisterFileSystemFactory.
618+
///
619+
/// \param[in] uri a URI-based path, ex: file:///some/local/path
620+
/// \param[in] options a list of backend-specific filesystem options
621+
/// Each option is a (name, value) pair.
622+
/// The expected type is specific to the backend and
623+
/// option name.
624+
/// \param[in] io_context an IOContext which will be associated with the filesystem
625+
/// \param[out] out_path (optional) Path inside the filesystem.
626+
/// \return out_fs FileSystem instance.
627+
ARROW_EXPORT
628+
Result<std::shared_ptr<FileSystem>> FileSystemFromUri(
629+
const std::string& uri, const std::vector<std::pair<std::string, std::any>>& options,
630+
const io::IOContext& io_context, std::string* out_path = NULLPTR);
631+
566632
/// \brief Create a new FileSystem by URI
567633
///
568634
/// Support for other schemes can be added using RegisterFileSystemFactory.

‎cpp/src/arrow/filesystem/localfs_test.cc‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -171,10 +171,21 @@ TEST(FileSystemFromUri, RuntimeRegisteredFactory) {
171171
}
172172

173173
FileSystemRegistrar kSegfaultFileSystemModule[]{
174-
ARROW_REGISTER_FILESYSTEM("segfault", nullptr, {}),
175-
ARROW_REGISTER_FILESYSTEM("segfault", nullptr, {}),
176-
ARROW_REGISTER_FILESYSTEM("segfault", nullptr, {}),
177-
};
174+
ARROW_REGISTER_FILESYSTEM(
175+
"segfault",
176+
std::function<Result<std::shared_ptr<FileSystem>>(
177+
const Uri&, const io::IOContext&, std::string*)>(nullptr),
178+
{}),
179+
ARROW_REGISTER_FILESYSTEM(
180+
"segfault",
181+
std::function<Result<std::shared_ptr<FileSystem>>(
182+
const Uri&, const io::IOContext&, std::string*)>(nullptr),
183+
{}),
184+
ARROW_REGISTER_FILESYSTEM(
185+
"segfault",
186+
std::function<Result<std::shared_ptr<FileSystem>>(
187+
const Uri&, const io::IOContext&, std::string*)>(nullptr),
188+
{})};
178189
TEST(FileSystemFromUri, LinkedRegisteredFactoryNameCollision) {
179190
// Since multiple registrars are defined in this translation unit which all
180191
// register factories for the 'segfault' scheme, using that scheme in FileSystemFromUri

‎cpp/src/arrow/filesystem/s3fs.cc‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3603,11 +3603,13 @@ Result<std::string> ResolveS3BucketRegion(const std::string& bucket) {
36033603

36043604
auto kS3FileSystemModule = ARROW_REGISTER_FILESYSTEM(
36053605
"s3",
3606-
[](const arrow::util::Uri& uri, const io::IOContext& io_context,
3606+
[](const arrow::util::Uri& uri,
3607+
const std::vector<std::pair<std::string, std::any>>& options,
3608+
const io::IOContext& io_context,
36073609
std::string* out_path) -> Result<std::shared_ptr<fs::FileSystem>> {
36083610
RETURN_NOT_OK(EnsureS3Initialized());
3609-
ARROW_ASSIGN_OR_RAISE(auto options, S3Options::FromUri(uri, out_path));
3610-
return S3FileSystem::Make(options, io_context);
3611+
ARROW_ASSIGN_OR_RAISE(auto s3_options, S3Options::FromUri(uri, out_path));
3612+
return S3FileSystem::Make(s3_options, io_context);
36113613
},
36123614
[] { DCHECK_OK(EnsureS3Finalized()); });
36133615

‎cpp/src/arrow/testing/examplefs.cc‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,13 @@ namespace arrow::fs {
2626

2727
auto kExampleFileSystemModule = ARROW_REGISTER_FILESYSTEM(
2828
"example",
29-
[](const Uri& uri, const io::IOContext& io_context,
29+
[](const Uri& uri, const std::vector<std::pair<std::string, std::any>>& options,
30+
const io::IOContext& io_context,
3031
std::string* out_path) -> Result<std::shared_ptr<FileSystem>> {
3132
constexpr std::string_view kScheme = "example";
3233
EXPECT_EQ(uri.scheme(), kScheme);
3334
auto local_uri = "file" + uri.ToString().substr(kScheme.size());
34-
return FileSystemFromUri(local_uri, io_context, out_path);
35+
return FileSystemFromUri(local_uri, options, io_context, out_path);
3536
},
3637
{});
3738

0 commit comments

Comments
 (0)