diff --git a/.gitignore b/.gitignore index 1ec31d2..feb7e02 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ github_* CMakeLists.txt.user compile_commands.json +/build/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 4394994..c807e7e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,6 +13,7 @@ option(USE_EXTERNAL_GRPC "Use external GRPC (set GRPC_INSTALL_DIR env variable a option(HAVE_TESTS "Enable tests" ON) option(HAVE_PYTHON_BINDINGS "Enable python bindings" ON) option(HAVE_JAVA_BINDINGS "Enable Java bindings (SET JAVA_HOME env variable accordingly)." ON) +option(HAVE_RUST_BINDINGS "Enable Rust bindings" ON) option(HAVE_RDMA_SUPPORT "Supports RDMA" OFF) option(HAVE_DEFAULT_BUCKET "Creates a default (default) bucket." ON) option(HAVE_PROMETHEUS_HISTOGRAM_BUCKETS "Enable Prometheus Histogram Buckets." OFF) @@ -312,7 +313,7 @@ add_custom_target(build-all metadataserver test_metadataserver pygeds - + geds_rs geds_java geds_jar ) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 77d348b..8aea55c 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -26,3 +26,7 @@ endif() if(HAVE_JAVA_BINDINGS) add_subdirectory(java) endif() + +if(HAVE_RUST_BINDINGS) + add_subdirectory(rust) +endif() diff --git a/src/rust/CMakeLists.txt b/src/rust/CMakeLists.txt new file mode 100644 index 0000000..366627f --- /dev/null +++ b/src/rust/CMakeLists.txt @@ -0,0 +1,124 @@ +# +# Copyright 2022- IBM Inc. All rights reserved +# SPDX-License-Identifier: Apache-2.0 +# + +if (NOT HAVE_RUST_BINDINGS) + message(error "This module requires the Rust bindings flag.") +else () + message(STATUS "Installing Rust library.") +endif () + +# Install Rust library files +install(FILES build.rs Cargo.toml Cargo.lock + COMPONENT geds + DESTINATION rust/ + COMPONENT geds) + +install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/rust/src" DESTINATION rust/ COMPONENT geds) + +# GEDS libgeds headers +install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/libgeds/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" +) + +# GEDS utility headers +install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/utility/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" +) + +# GEDS common headers +install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/common/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" +) + +# GEDS protos headers +install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/protos/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" +) + +# GEDS s3 headers +install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/s3/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" +) + +# GEDS statistics headers +install(DIRECTORY "${CMAKE_SOURCE_DIR}/src/statistics/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" +) + +# GEDS gens headers +install(DIRECTORY "${CMAKE_BINARY_DIR}/gens/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.inc" +) + +# grpc headers +install(DIRECTORY "${CMAKE_BINARY_DIR}/_deps/grpc-src/include/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.inc" +) + +# abseil headers +install(DIRECTORY "${CMAKE_BINARY_DIR}/_deps/grpc-src/third_party/abseil-cpp/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.inc" +) + +# protobuf headers +install(DIRECTORY "${CMAKE_BINARY_DIR}/_deps/grpc-src/third_party/protobuf/src/" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.h" + PATTERN "*.inc" +) + +# boost headers +install(DIRECTORY "${BOOST_ROOT}/include/boost" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.ipp" + PATTERN "*.h" +) + +# AWS S3 SDK headers +install(DIRECTORY "${AWSSDK_ROOT}/include" + COMPONENT geds + DESTINATION "rust/include" + FILES_MATCHING + PATTERN "*.hpp" + PATTERN "*.ipp" + PATTERN "*.h" + PATTERN "*.inc" + PATTERN "*.inl" +) diff --git a/src/rust/Cargo.lock b/src/rust/Cargo.lock new file mode 100644 index 0000000..eca8c4f --- /dev/null +++ b/src/rust/Cargo.lock @@ -0,0 +1,233 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cc" +version = "1.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945" +dependencies = [ + "shlex", +] + +[[package]] +name = "codespan-reporting" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e" +dependencies = [ + "termcolor", + "unicode-width", +] + +[[package]] +name = "cxx" +version = "1.0.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbdc8cca144dce1c4981b5c9ab748761619979e515c3d53b5df385c677d1d007" +dependencies = [ + "cc", + "cxxbridge-flags", + "cxxbridge-macro", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5764c3142ab44fcf857101d12c0ddf09c34499900557c764f5ad0597159d1fc" +dependencies = [ + "cc", + "codespan-reporting", + "once_cell", + "proc-macro2", + "quote", + "scratch", + "syn", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d422aff542b4fa28c2ce8e5cc202d42dbf24702345c1fba3087b2d3f8a1b90ff" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.129" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1719100f31492cd6adeeab9a0f46cdbc846e615fdb66d7b398aa46ec7fdd06f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "geds_rs" +version = "1.0.0" +dependencies = [ + "cxx", + "cxx-build", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d240c6f7e1ba3a28b0249f774e6a9dd0175054b52dfbb61b16eb8505c3785c9" +dependencies = [ + "cc", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "proc-macro2" +version = "1.0.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "scratch" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3cf7c11c38cb994f3d40e8a8cde3bbd1f72a435e4c49e85d6553d8312306152" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "unicode-ident" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" diff --git a/src/rust/Cargo.toml b/src/rust/Cargo.toml new file mode 100644 index 0000000..3e6da5e --- /dev/null +++ b/src/rust/Cargo.toml @@ -0,0 +1,16 @@ +# +# Copyright 2022- IBM Inc. All rights reserved +# SPDX-License-Identifier: Apache-2.0 +# + +[package] +name = "geds_rs" +authors = ["Pablo Gimeno Sarroca "] +version = "1.0.0" +edition = "2021" + +[dependencies] +cxx = "=1.0.129" + +[build-dependencies] +cxx-build = "=1.0.129" \ No newline at end of file diff --git a/src/rust/README.md b/src/rust/README.md new file mode 100644 index 0000000..0236780 --- /dev/null +++ b/src/rust/README.md @@ -0,0 +1,14 @@ +# geds-rs: Rust bindings for GEDS + +This project contains a Rust crate that implements a Rust API for GEDS' C++ implementation. +It utilizes [CXX](https://cxx.rs) to generate a safe C ABI for Rust/C++ interoperability. + +To use in a Rust project, add the ```geds-rs``` crate as a dependency in your Cargo.toml file, specifying the path to the install +directory of ```geds-rs``` (under ```${GEDS_INSTALL_DIR}/rust```): + +```toml +[dependencies] +geds-rs = { path = "path/to/geds-rs" } +``` + +This crate requires that ```libgeds.so``` is installed and available in one of the system's library paths. ```geds-rs``` is currently supported in Ubuntu 22.04 and 20.04. diff --git a/src/rust/build.rs b/src/rust/build.rs new file mode 100644 index 0000000..8a8889d --- /dev/null +++ b/src/rust/build.rs @@ -0,0 +1,21 @@ +// +// Copyright 2022- IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 +// + +fn main() { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + + cxx_build::bridge("src/lib.rs") + .file(format!("{manifest_dir}/src/GEDSFileWrapper.cpp")) + .file(format!("{manifest_dir}/src/GEDSWrapper.cpp")) + .include(format!("{manifest_dir}/src")) + .include(format!("{manifest_dir}/include")) + .std("c++20") + .compile("geds_rs"); + + println!("cargo:rerun-if-changed={manifest_dir}/src/lib.rs"); + println!("cargo:rerun-if-changed={manifest_dir}/src/GEDSWrapper.cpp"); + println!("cargo:rerun-if-changed={manifest_dir}/src/GEDSFileWrapper.cpp"); + println!("cargo:rustc-link-lib=dylib=geds"); +} diff --git a/src/rust/src/GEDSFileWrapper.cpp b/src/rust/src/GEDSFileWrapper.cpp new file mode 100644 index 0000000..bce1765 --- /dev/null +++ b/src/rust/src/GEDSFileWrapper.cpp @@ -0,0 +1,74 @@ +/** +* Copyright 2022- IBM Inc. All rights reserved +* SPDX-License-Identifier: Apache-2.0 +*/ + +#include "GEDSFileWrapper.h" + +#include "GEDSFile.h" +#include "Logging.h" +#include "geds_rs/src/lib.rs.h" + +namespace geds_rs { + GEDSFileWrapper::GEDSFileWrapper(const GEDSFile &filePtr) { + gedsFile = std::make_unique(filePtr); + } + + size_t GEDSFileWrapper::size() const { + return gedsFile->size(); + } + + rust::String GEDSFileWrapper::identifier() const { + return rust::String(gedsFile->identifier()); + } + + bool GEDSFileWrapper::is_writeable() const { + return gedsFile->isWriteable(); + } + + rust::String GEDSFileWrapper::metadata() const { + return rust::String(gedsFile->metadata().value()); + } + + template + std::vector toStdVector(rust::Vec v) { + std::vector stdv; + + return stdv; + } + + shared::Status GEDSFileWrapper::seal() const { + const auto status = gedsFile->seal(); + return {.message = rust::string(status.ToString()), .ok = status.ok()}; + } + + shared::Status GEDSFileWrapper::truncate(const size_t size) const { + const auto status = gedsFile->truncate(size); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSFileWrapper::set_metadata(const rust::Str metadata, const bool seal) const { + auto cppMetadata = std::string(metadata); + const auto status = gedsFile->setMetadata(cppMetadata, seal); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::StatusOrUsize GEDSFileWrapper::read(rust::Vec &buffer, const size_t position, + const size_t length) const { + // TODO: can we get rid of the extra copy? Working with buffer.data() as in write() does not work + std::vector stdv; + const auto status = gedsFile->read(stdv, position, length); + std::copy(stdv.begin(), stdv.end(), std::back_inserter(buffer)); + + return shared::StatusOrUsize{ + .status = {.message = status.status().ToString(), .ok = status.ok()}, + .value = status.value_or(0) + }; + } + + shared::Status GEDSFileWrapper::write(const rust::Vec &buffer, const size_t position, + const size_t length) const { + const auto status = gedsFile->write(buffer.data(), position, length); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } +} diff --git a/src/rust/src/GEDSFileWrapper.h b/src/rust/src/GEDSFileWrapper.h new file mode 100644 index 0000000..fefb650 --- /dev/null +++ b/src/rust/src/GEDSFileWrapper.h @@ -0,0 +1,46 @@ +/** +* Copyright 2022- IBM Inc. All rights reserved +* SPDX-License-Identifier: Apache-2.0 +*/ + +#ifndef GEDSFILEWRAPPER_H +#define GEDSFILEWRAPPER_H + +#include +#include "rust/cxx.h" + +class GEDSFile; + +namespace shared { + struct Status; + struct StatusOrUsize; +} + +namespace geds_rs { + class GEDSFileWrapper { + std::unique_ptr gedsFile; + + public: + explicit GEDSFileWrapper(const GEDSFile &filePtr); + + [[nodiscard]] size_t size() const; + + [[nodiscard]] rust::String identifier() const; + + [[nodiscard]] bool is_writeable() const; + + [[nodiscard]] rust::String metadata() const; + + [[nodiscard]] shared::Status seal() const; + + [[nodiscard]] shared::Status truncate(size_t size) const; + + [[nodiscard]] shared::Status set_metadata(const rust::Str metadata, bool seal) const; + + [[nodiscard]] shared::StatusOrUsize read(rust::Vec &buffer, size_t position, size_t length) const; + + [[nodiscard]] shared::Status write(const rust::Vec &buffer, size_t position, size_t length) const; + }; +} + +#endif //GEDSFILEWRAPPER_H diff --git a/src/rust/src/GEDSWrapper.cpp b/src/rust/src/GEDSWrapper.cpp new file mode 100644 index 0000000..7016680 --- /dev/null +++ b/src/rust/src/GEDSWrapper.cpp @@ -0,0 +1,193 @@ +/** +* Copyright 2022- IBM Inc. All rights reserved +* SPDX-License-Identifier: Apache-2.0 +*/ + +#include "GEDSWrapper.h" +#include "GEDS.h" +#include "PubSub.h" +#include "GEDSConfig.h" +#include "geds_rs/src/lib.rs.h" +#include "geds.grpc.pb.h" + +namespace geds_rs { + std::unique_ptr new_wrapper(const shared::GEDSConfig &sharedConfig) { + return std::make_unique(sharedConfig); + } + + GEDSWrapper::GEDSWrapper(const shared::GEDSConfig &sharedConfig) { + auto config = GEDSConfig(std::string(sharedConfig.metadata_service_address)); + config.listenAddress = std::string(sharedConfig.listen_address); + config.port = sharedConfig.port; + config.portHttpServer = sharedConfig.port_http_server; + config.localStoragePath = std::string(sharedConfig.local_storage_path); + config.cacheBlockSize = sharedConfig.cache_block_size; + config.cache_objects_from_s3 = sharedConfig.cache_objects_from_s3; + config.available_local_storage = sharedConfig.available_local_storage; + config.available_local_memory = sharedConfig.available_local_memory; + config.force_relocation_when_stopping = sharedConfig.force_relocation_when_stopping; + config.pubSubEnabled = sharedConfig.pub_sub_enabled; + + if (sharedConfig.hostname == "null") { + config.hostname = std::nullopt; + } + gedsPtr = GEDS::factory(config); + } + + shared::Status GEDSWrapper::start() const { + const auto status = gedsPtr->start(); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::stop() const { + const auto status = gedsPtr->stop(); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::StatusOrGEDSFileWrapper GEDSWrapper::create(const rust::Str bucket, const rust::Str key, + const bool overwrite) const { + auto status = gedsPtr->create(std::string(bucket), std::string(key), overwrite); + std::shared_ptr value; + if (status.ok()) { + value = std::make_shared(*status); + } + return { + .status = {.message = status.status().ToString(), .ok = status.ok()}, + .value = value + }; + } + + shared::Status GEDSWrapper::create_bucket(const rust::Str bucket) const { + const auto status = gedsPtr->createBucket(std::string(bucket)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::mkdirs(const rust::Str bucket, const rust::Str path) const { + const auto status = gedsPtr->mkdirs(std::string(bucket), std::string(path)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + rust::Vec to_rust_vec(const std::vector &stdv) { + rust::Vec rustv; + + for (const auto &[key, size, is_directory]: stdv) { + rustv.emplace_back(shared::GEDSFileStatus{.key = key, .size = size, .is_directory = is_directory}); + } + + return rustv; + } + + shared::StatusOrVecGEDSFileStatus + GEDSWrapper::list(const rust::Str bucket, const rust::Str key) const { + auto status = gedsPtr->list(std::string(bucket), std::string(key)); + rust::Vec rustVec; + if (status.ok()) { + rustVec = to_rust_vec(*status); + } + return {.status = {.message = status.status().ToString(), .ok = status.ok()}, .value = rustVec}; + } + + shared::StatusOrVecGEDSFileStatus GEDSWrapper::list_folder(const rust::Str bucket, + const rust::Str prefix) const { + auto status = gedsPtr->listAsFolder(std::string(bucket), std::string(prefix)); + rust::Vec rustVec; + if (status.ok()) { + rustVec = to_rust_vec(*status); + } + return {.status = {.message = status.status().ToString(), .ok = status.ok()}, .value = rustVec}; + } + + shared::StatusOrGEDSFileStatus GEDSWrapper::status(const rust::Str bucket, const rust::Str key) const { + const auto status = gedsPtr->status(std::string(bucket), std::string(key)); + shared::GEDSFileStatus fileStatus; + if (status.ok()) { + fileStatus.is_directory = status->isDirectory; + fileStatus.key = status->key; + fileStatus.size = status->size; + } + return {.status = {.message = status.status().ToString(), .ok = status.ok()}, .value = fileStatus}; + } + + auto GEDSWrapper::open(const rust::Str bucket, const rust::Str key) const -> shared::StatusOrGEDSFileWrapper { + const auto status = gedsPtr->open(std::string(bucket), std::string(key)); + std::shared_ptr value; + if (status.ok()) { + value = std::make_shared(*status); + } + return { + .status = {.message = status.status().ToString(), .ok = status.ok()}, + .value = value + }; + } + + shared::Status GEDSWrapper::delete_object(const rust::Str bucket, const rust::Str key) const { + const auto status = gedsPtr->deleteObject(std::string(bucket), std::string(key)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::delete_object_prefix(const rust::Str bucket, const rust::Str prefix) const { + const auto status = gedsPtr->deleteObjectPrefix(std::string(bucket), std::string(prefix)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::rename(const rust::Str src_bucket, const rust::Str src_key, + const rust::Str dest_bucket, const rust::Str dest_key) const { + const auto status = gedsPtr->rename(std::string(src_bucket), std::string(src_key), std::string(dest_bucket), + std::string(dest_key)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::rename_prefix(const rust::Str src_bucket, const rust::Str src_prefix, + const rust::Str dest_bucket, const rust::Str dest_prefix) const { + const auto status = gedsPtr->renamePrefix(std::string(src_bucket), std::string(src_prefix), + std::string(dest_bucket), std::string(dest_prefix)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::copy(const rust::Str src_bucket, const rust::Str src_key, + const rust::Str dest_bucket, const rust::Str dest_key) const { + const auto status = gedsPtr->copy(std::string(src_bucket), std::string(src_key), std::string(dest_bucket), + std::string(dest_key)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::copy_prefix(const rust::Str src_bucket, const rust::Str src_prefix, + const rust::Str dest_bucket, const rust::Str dest_prefix) const { + const auto status = gedsPtr->copyPrefix(std::string(src_bucket), std::string(src_prefix), + std::string(dest_bucket), std::string(dest_prefix)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + rust::String GEDSWrapper::local_path(const rust::Str bucket, const rust::Str key) const { + const auto path = gedsPtr->getLocalPath(std::string(bucket), std::string(key)); + return {path}; + } + + shared::Status GEDSWrapper::register_object_store_config(const rust::Str bucket, + const rust::Str endpoint_url, + const rust::Str access_key, + const rust::Str secret_key) const { + const auto status = gedsPtr->registerObjectStoreConfig(std::string(bucket), std::string(endpoint_url), + std::string(access_key), std::string(secret_key)); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + shared::Status GEDSWrapper::sync_object_store_configs() const { + const auto status = gedsPtr->syncObjectStoreConfigs(); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } + + void GEDSWrapper::relocate(const bool force) const { + gedsPtr->relocate(force); + } + + shared::Status GEDSWrapper::subscribe(const rust::Str bucket, const rust::Str key, const int &subscription_type) const { + geds::rpc::SubscriptionType type = static_cast(subscription_type); + geds::SubscriptionEvent *event = new geds::SubscriptionEvent(); + event->bucket = std::string(bucket); + event->key = std::string(key); + event->subscriptionType = type; + const auto status = gedsPtr->subscribe(*event); + return shared::Status{.message = status.ToString(), .ok = status.ok()}; + } +} diff --git a/src/rust/src/GEDSWrapper.h b/src/rust/src/GEDSWrapper.h new file mode 100644 index 0000000..a739b71 --- /dev/null +++ b/src/rust/src/GEDSWrapper.h @@ -0,0 +1,86 @@ +/** +* Copyright 2022- IBM Inc. All rights reserved +* SPDX-License-Identifier: Apache-2.0 +*/ + +#ifndef GEDSWRAPPER_H +#define GEDSWRAPPER_H + +#include +#include "rust/cxx.h" + +class GEDSFile; +class GEDS; + +namespace shared { + struct GEDSConfig; + struct GEDSFileStatus; + struct Status; + struct StatusOrGEDSFileWrapper; + struct StatusOrGEDSFileStatus; + struct StatusOrVecGEDSFileStatus; +} + +namespace geds_rs { + class GEDSWrapper { + protected: + std::shared_ptr gedsPtr; + + public: + explicit GEDSWrapper(const shared::GEDSConfig &); + + [[nodiscard]] shared::Status start() const; + + [[nodiscard]] shared::Status stop() const; + + [[nodiscard]] shared::StatusOrGEDSFileWrapper create(const rust::Str bucket, const rust::Str key, + bool overwrite) const; + + [[nodiscard]] shared::Status create_bucket(const rust::Str bucket) const; + + [[nodiscard]] shared::Status mkdirs(const rust::Str bucket, const rust::Str path) const; + + [[nodiscard]] shared::StatusOrVecGEDSFileStatus list(const rust::Str bucket, const rust::Str key) const; + + [[nodiscard]] shared::StatusOrVecGEDSFileStatus list_folder(const rust::Str bucket, + const rust::Str prefix) const; + + [[nodiscard]] shared::StatusOrGEDSFileStatus status(const rust::Str bucket, const rust::Str key) const; + + [[nodiscard]] shared::StatusOrGEDSFileWrapper open(const rust::Str bucket, const rust::Str key) const; + + [[nodiscard]] shared::Status delete_object(const rust::Str bucket, const rust::Str key) const; + + [[nodiscard]] shared::Status delete_object_prefix(const rust::Str bucket, const rust::Str prefix) const; + + [[nodiscard]] shared::Status rename(const rust::Str src_bucket, const rust::Str src_key, + const rust::Str dest_bucket, const rust::Str dest_key) const; + + [[nodiscard]] shared::Status rename_prefix(const rust::Str src_bucket, const rust::Str src_prefix, + const rust::Str dest_bucket, const rust::Str dest_prefix) const; + + [[nodiscard]] shared::Status copy(const rust::Str src_bucket, const rust::Str src_key, + const rust::Str dest_bucket, const rust::Str dest_key) const; + + [[nodiscard]] shared::Status copy_prefix(const rust::Str src_bucket, const rust::Str src_prefix, + const rust::Str dest_bucket, const rust::Str dest_prefix) const; + + [[nodiscard]] rust::String local_path(const rust::Str bucket, const rust::Str key) const; + + [[nodiscard]] shared::Status register_object_store_config(const rust::Str bucket, + const rust::Str endpoint_url, + const rust::Str access_key, + const rust::Str secret_key) const; + + [[nodiscard]] shared::Status sync_object_store_configs() const; + + void relocate(bool force) const; + + [[nodiscard]] shared::Status subscribe(const rust::Str bucket, const rust::Str key, const int &subscription_type) const; + + }; + + std::unique_ptr new_wrapper(const shared::GEDSConfig &sharedConfig); +} + +#endif //GEDSWRAPPER_H diff --git a/src/rust/src/lib.rs b/src/rust/src/lib.rs new file mode 100644 index 0000000..cae51b7 --- /dev/null +++ b/src/rust/src/lib.rs @@ -0,0 +1,582 @@ +// +// Copyright 2022- IBM Inc. All rights reserved +// SPDX-License-Identifier: Apache-2.0 +// + +/// Rust API for [GEDS](https://github.com/IBM/GEDS) +use cxx::{SharedPtr, UniquePtr}; + +#[cxx::bridge(namespace = geds_rs)] +pub mod ffi { + unsafe extern "C++" { + include!("GEDSFileWrapper.h"); + type GEDSFileWrapper; + fn seal(&self) -> Status; + fn truncate(&self, size: usize) -> Status; + fn set_metadata(&self, metadata: &str, seal: bool) -> Status; + fn read(&self, buffer: &mut Vec, position: usize, length: usize) -> StatusOrUsize; + fn write(&self, buffer: &Vec, position: usize, length: usize) -> Status; + + // GEDSFile properties + fn size(&self) -> usize; + fn is_writeable(&self) -> bool; + fn identifier(&self) -> String; + fn metadata(&self) -> String; + } + + unsafe extern "C++" { + include!("GEDSWrapper.h"); + type GEDSWrapper; + fn start(&self) -> Status; + fn stop(&self) -> Status; + fn create(&self, bucket: &str, key: &str, overwrite: bool) -> StatusOrGEDSFileWrapper; + fn create_bucket(&self, bucket: &str) -> Status; + fn mkdirs(&self, bucket: &str, path: &str) -> Status; + fn list(&self, bucket: &str, key: &str) -> StatusOrVecGEDSFileStatus; + fn list_folder(&self, bucket: &str, prefix: &str) -> StatusOrVecGEDSFileStatus; + fn status(&self, bucket: &str, key: &str) -> StatusOrGEDSFileStatus; + fn open(&self, bucket: &str, key: &str) -> StatusOrGEDSFileWrapper; + fn delete_object(&self, bucket: &str, key: &str) -> Status; + fn delete_object_prefix(&self, bucket: &str, prefix: &str) -> Status; + fn subscribe(&self, bucket: &str, key: &str, subscription_type: &i32) -> Status; + fn rename( + &self, + src_bucket: &str, + src_key: &str, + dest_bucket: &str, + dest_key: &str, + ) -> Status; + fn rename_prefix( + &self, + src_bucket: &str, + src_prefix: &str, + dest_bucket: &str, + dest_prefix: &str, + ) -> Status; + fn copy( + &self, + src_bucket: &str, + src_key: &str, + dest_bucket: &str, + dest_key: &str, + ) -> Status; + fn copy_prefix( + &self, + src_bucket: &str, + src_prefix: &str, + dest_bucket: &str, + dest_prefix: &str, + ) -> Status; + fn local_path(&self, bucket: &str, key: &str) -> String; + fn register_object_store_config( + &self, + bucket: &str, + endpoint_url: &str, + access_key: &str, + secret_key: &str, + ) -> Status; + fn sync_object_store_configs(&self) -> Status; + fn relocate(&self, force: bool); + + pub fn new_wrapper(config: &GEDSConfig) -> UniquePtr; + } + + #[namespace = "shared"] + pub struct GEDSConfig { + pub metadata_service_address: String, + pub listen_address: String, + pub hostname: String, + pub port: u16, + pub port_http_server: u16, + pub local_storage_path: String, + pub cache_block_size: usize, + pub cache_objects_from_s3: bool, + pub available_local_storage: usize, + pub available_local_memory: usize, + pub force_relocation_when_stopping: bool, + pub pub_sub_enabled: bool, + } + + #[namespace = "shared"] + pub struct GEDSFileStatus { + key: String, + size: usize, + is_directory: bool, + } + + #[namespace = "shared"] + pub struct Status { + message: String, + ok: bool, + } + + #[namespace = "shared"] + pub struct StatusOrGEDSFileWrapper { + status: Status, + value: SharedPtr, + } + + #[namespace = "shared"] + pub struct StatusOrGEDSFileStatus { + status: Status, + value: GEDSFileStatus, + } + + #[namespace = "shared"] + pub struct StatusOrVecGEDSFileStatus { + status: Status, + value: Vec, + } + + #[namespace = "shared"] + pub struct StatusOrUsize { + status: Status, + value: usize, + } +} + +// +pub struct GEDS { + geds_ptr: UniquePtr, +} + +impl GEDS { + /// Start GEDS. + pub fn start(&self) -> Result<(), String> { + let status = self.geds_ptr.start(); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Stop GEDS. + pub fn stop(&self) -> Result<(), String> { + let status = self.geds_ptr.stop(); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Create object located at bucket/key. + /// The object is registered with the metadata service once the file is sealed. + pub fn create(&self, bucket: &str, key: &str, overwrite: bool) -> Result { + let status_or_result = self.geds_ptr.create(bucket, key, overwrite); + if status_or_result.status.ok { + let file = GEDSFile { + file_ptr: status_or_result.value, + }; + Ok(file) + } else { + Err(status_or_result.status.message) + } + } + + /// Register a bucket with the metadata server. + pub fn create_bucket(&self, bucket: &str) -> Result<(), String> { + let status = self.geds_ptr.create_bucket(bucket); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Recursively create directory using directory markers. + pub fn mkdirs(&self, bucket: &str, path: &str) -> Result<(), String> { + let status = self.geds_ptr.mkdirs(bucket, path); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// List objects in bucket where the key starts with 'prefix'. + pub fn list(&self, bucket: &str, prefix: &str) -> Result, String> { + let status_or_result = self.geds_ptr.list(bucket, prefix); + if status_or_result.status.ok { + Ok(status_or_result.value) + } else { + Err(status_or_result.status.message) + } + } + + /// List objects in `bucket` where the key starts with `prefix` and the postfix does not contain `delimiter`. + // - If the delimiter set to `0` will list all keys starting with prefix. + // - If the delimiter is set to a value != 0, then the delimiter will be used as a folder + // separator. Keys ending with "/_$folder$" will be used as directory markers (where '/' is used + // as a delimiter). + pub fn list_folder( + &self, + bucket: &str, + prefix: &str, + ) -> Result, String> { + let status_or_result = self.geds_ptr.list_folder(bucket, prefix); + if status_or_result.status.ok { + Ok(status_or_result.value) + } else { + Err(status_or_result.status.message) + } + } + + /// Get status of `key` in `bucket` + pub fn status(&self, bucket: &str, key: &str) -> Result { + let status_or_result = self.geds_ptr.status(bucket, key); + if status_or_result.status.ok { + Ok(status_or_result.value) + } else { + Err(status_or_result.status.message) + } + } + + /// Open object located at `bucket`/`key`. + pub fn open(&self, bucket: &str, key: &str) -> Result { + let status_or_result = self.geds_ptr.open(bucket, key); + if status_or_result.status.ok { + let file = GEDSFile { + file_ptr: status_or_result.value, + }; + Ok(file) + } else { + Err(status_or_result.status.message) + } + } + + /// Delete object in `bucket` with `key`. + pub fn delete_object(&self, bucket: &str, key: &str) -> Result<(), String> { + let status = self.geds_ptr.delete_object(bucket, key); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Delete objects in `'bucket`' with keys starting with `prefix`. + pub fn delete_object_prefix(&self, bucket: &str, prefix: &str) -> Result<(), String> { + let status = self.geds_ptr.delete_object_prefix(bucket, prefix); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Rename an object. + pub fn rename( + &self, + src_bucket: &str, + src_key: &str, + dest_bucket: &str, + dest_key: &str, + ) -> Result<(), String> { + let status = self + .geds_ptr + .rename(src_bucket, src_key, dest_bucket, dest_key); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Rename a prefix recursively. + pub fn rename_prefix( + &self, + src_bucket: &str, + src_prefix: &str, + dest_bucket: &str, + dest_prefix: &str, + ) -> Result<(), String> { + let status = self + .geds_ptr + .rename_prefix(src_bucket, src_prefix, dest_bucket, dest_prefix); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Copy an object. + pub fn copy( + &self, + src_bucket: &str, + src_key: &str, + dest_bucket: &str, + dest_key: &str, + ) -> Result<(), String> { + let status = self + .geds_ptr + .copy(src_bucket, src_key, dest_bucket, dest_key); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Copy a file or a folder structure. + pub fn copy_prefix( + &self, + src_bucket: &str, + src_prefix: &str, + dest_bucket: &str, + dest_prefix: &str, + ) -> Result<(), String> { + let status = self + .geds_ptr + .copy_prefix(src_bucket, src_prefix, dest_bucket, dest_prefix); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Compute the path to the files stored in `_pathPrefix` folder. + pub fn local_path(&self, bucket: &str, key: &str) -> String { + self.geds_ptr.local_path(bucket, key) + } + + /// Register an object store configuration with GEDS. + pub fn register_object_store_config( + &self, + bucket: &str, + endpoint_url: &str, + access_key: &str, + secret_key: &str, + ) -> Result<(), String> { + let status = self.geds_ptr.register_object_store_config( + bucket, + endpoint_url, + access_key, + secret_key, + ); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Sync object store configs. + pub fn sync_object_store_configs(&self) -> Result<(), String> { + let status = self.geds_ptr.sync_object_store_configs(); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Relocate objects to S3. + pub fn relocate(&self, force: bool) { + self.geds_ptr.relocate(force); + } + + // NO_SUBSCRIPTION = 0, + // BUCKET = 1, + // OBJECT = 2, + // PREFIX = 3, + pub fn subscribe( + &self, + bucket: &str, + key: &str, + subscription_type: &i32, + ) -> Result<(), String> { + let status = self.geds_ptr.subscribe(bucket, key, subscription_type); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + /// Create new GEDS instance. + pub fn new(config: &ffi::GEDSConfig) -> GEDS { + let wrapper = ffi::new_wrapper(&config); + GEDS { geds_ptr: wrapper } + } + + /// Create a `geds_rs` + pub fn get_default_config() -> ffi::GEDSConfig { + ffi::GEDSConfig { + metadata_service_address: "localhost:4381".to_owned(), + listen_address: "0.0.0.0".to_owned(), + hostname: "null".to_owned(), + port: 4382, + port_http_server: 4380, + local_storage_path: "/tmp/GEDS_XXXXXX".to_owned(), + cache_block_size: 32 * 1024 * 1024, + cache_objects_from_s3: false, + available_local_storage: 100 * 1024 * 1024 * 1024, + available_local_memory: 16 * 1024 * 1024 * 1024, + force_relocation_when_stopping: false, + pub_sub_enabled: false, + } + } +} + +unsafe impl Send for GEDS {} +unsafe impl Sync for GEDS {} + +/// GEDS File abstraction. Exposes a file buffer. +/// +/// - Implements Sparse-File semantics +/// - Sparse File for caching. +/// - Use GEDSFile everywhere to allow unseal. +pub struct GEDSFile { + file_ptr: SharedPtr, +} + +impl GEDSFile { + pub fn seal(&self) -> Result<(), String> { + let status = self.file_ptr.seal(); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + pub fn truncate(&self, size: usize) -> Result<(), String> { + let status = self.file_ptr.truncate(size); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + pub fn set_metadata(&self, metadata: &str, seal: bool) -> Result<(), String> { + let status = self.file_ptr.set_metadata(metadata, seal); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + pub fn read( + &self, + buffer: &mut Vec, + position: usize, + length: usize, + ) -> Result { + let status_or_result = self.file_ptr.read(buffer, position, length); + if status_or_result.status.ok { + Ok(status_or_result.value) + } else { + Err(status_or_result.status.message) + } + } + + pub fn write(&self, buffer: &Vec, position: usize, length: usize) -> Result<(), String> { + let status = self.file_ptr.write(buffer, position, length); + if status.ok { + Ok(()) + } else { + Err(status.message) + } + } + + pub fn size(&self) -> usize { + self.file_ptr.size() + } + pub fn is_writeable(&self) -> bool { + self.file_ptr.is_writeable() + } + pub fn identifier(&self) -> String { + self.file_ptr.identifier() + } + pub fn metadata(&self) -> String { + self.file_ptr.metadata() + } +} + +unsafe impl Send for GEDSFile {} +unsafe impl Sync for GEDSFile {} + +#[cfg(test)] +mod tests { + use crate::GEDS; + + #[test] + fn test_file() { + let geds = create_geds(); + + // GEDS::start + let start_result = geds.start(); + assert!(start_result.is_ok(), "{}", start_result.err().unwrap()); + + // GEDS::create_bucket + let create_bucket_result = geds.create_bucket("test-bucket"); + assert!( + create_bucket_result.is_ok(), + "{}", + create_bucket_result.as_ref().unwrap_err() + ); + + // GEDS::create + let create_result = geds.create("test-bucket", "test-file", true); + assert!(create_result.is_ok(), "{}", create_result.err().unwrap()); + + // GEDSFile::write + let file = create_result.unwrap(); + let write_vec: Vec = "Hello world!".as_bytes().to_vec(); + let write_result = file.write(&write_vec, 0, 12); + assert!(write_result.is_ok(), "{}", write_result.err().unwrap()); + + // GEDSFile::read + let read_vec: &mut Vec = &mut Vec::with_capacity(12); + let read_result = file.read(read_vec, 0, 12); + assert!( + read_result.is_ok() && *read_result.as_ref().unwrap() == 12, + "{}", + read_result.as_ref().unwrap_err() + ); + + assert_eq!(*read_vec, write_vec); + + // GEDSFile::set_metadata + let set_metadata_result = file.set_metadata("test_metadata", false); + assert!( + set_metadata_result.is_ok(), + "{}", + set_metadata_result.err().unwrap() + ); + + // GEDSFile::truncate + assert!(file.truncate(5).is_ok()); + + // GEDSFile::seal + assert!(file.seal().is_ok()); + + // GEDSFile properties + let size = file.size(); + let is_writeable = file.is_writeable(); + let identifier = file.identifier(); + let metadata = file.metadata(); + assert_eq!(size, 5); + assert_eq!(is_writeable, true); // why? + println!("File identifier is {identifier}"); + assert_eq!(metadata, "test_metadata"); + + // GEDS::delete_object + assert!(geds.delete_object("test-bucket", "test-file").is_ok()); + + // GEDS::stop + let stop_result = geds.stop(); + assert!(stop_result.is_ok()); + } + + fn create_geds() -> GEDS { + let config = GEDS::get_default_config(); + GEDS::new(&config) + } +}