From 0a20ee8399357f18499ef7f7b67ce6c8273ce0e4 Mon Sep 17 00:00:00 2001 From: Andreas Joachim Peters Date: Wed, 22 May 2024 10:27:28 +0200 Subject: [PATCH] S3: add documentation to all classes/functions S3: refactor defines with pragma once S3: add license headers to the whole cocde --- src/XrdS3/XrdS3.cc | 83 ++++++++- src/XrdS3/XrdS3.hh | 41 ++++- src/XrdS3/XrdS3Action.hh | 29 ++- src/XrdS3/XrdS3Api.cc | 176 +++++++++++++++++- src/XrdS3/XrdS3Api.hh | 39 +++- src/XrdS3/XrdS3Auth.cc | 9 +- src/XrdS3/XrdS3Auth.hh | 35 +++- src/XrdS3/XrdS3Crypt.cc | 41 ++++- src/XrdS3/XrdS3Crypt.hh | 33 +++- src/XrdS3/XrdS3ErrorResponse.hh | 36 +++- src/XrdS3/XrdS3ObjectStore.cc | 308 +++++++++++++++++++++++++++++++- src/XrdS3/XrdS3ObjectStore.hh | 26 ++- src/XrdS3/XrdS3Req.cc | 26 ++- src/XrdS3/XrdS3Req.hh | 30 +++- src/XrdS3/XrdS3Response.cc | 6 +- src/XrdS3/XrdS3Response.hh | 31 +++- src/XrdS3/XrdS3Router.cc | 85 ++++++++- src/XrdS3/XrdS3Router.hh | 41 ++++- src/XrdS3/XrdS3Utils.cc | 136 +++++++++++++- src/XrdS3/XrdS3Utils.hh | 32 +++- src/XrdS3/XrdS3Xml.cc | 38 +++- src/XrdS3/XrdS3Xml.hh | 29 ++- 22 files changed, 1219 insertions(+), 91 deletions(-) diff --git a/src/XrdS3/XrdS3.cc b/src/XrdS3/XrdS3.cc index be4d23c7204..3106d6305b2 100644 --- a/src/XrdS3/XrdS3.cc +++ b/src/XrdS3/XrdS3.cc @@ -1,24 +1,65 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +//------------------------------------------------------------------------------ #include "XrdS3.hh" - +//------------------------------------------------------------------------------ #include - #include #include - +//------------------------------------------------------------------------------ #include "XrdHttp/XrdHttpExtHandler.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucString.hh" #include "XrdS3ErrorResponse.hh" #include "XrdVersion.hh" +//------------------------------------------------------------------------------ + +//------------------------------------------------------------------------------ +//! XRootD S3 plug-in implementation +//------------------------------------------------------------------------------ namespace S3 { XrdVERSIONINFO(XrdHttpGetExtHandler, HttpS3); +//------------------------------------------------------------------------------ +//! This is the default handler for requests that are not handled by the +//! router. It returns a 404 error. +//------------------------------------------------------------------------------ int NotFoundHandler(XrdS3Req &req) { return req.S3ErrorResponse(S3Error::NoSuchAccessPoint); } +//------------------------------------------------------------------------------ +//! S3Handler constructor. +//! +//! \param log The logger to use. +//! \param config The configuration string. +//! \param myEnv The environment to use. +//! +//------------------------------------------------------------------------------ S3Handler::S3Handler(XrdSysError *log, const char *config, XrdOucEnv *myEnv) : mLog(log->logger(), "S3_"), mApi(), mRouter(&mLog, NotFoundHandler) { if (!ParseConfig(config, *myEnv)) { @@ -35,6 +76,14 @@ S3Handler::S3Handler(XrdSysError *log, const char *config, XrdOucEnv *myEnv) mLog.Say("Finished configuring S3 Handler"); } +//------------------------------------------------------------------------------ +//! S3Handler ConfigureRouter. +//! +//! \param log The logger to use. +//! \param config The configuration string. +//! \param myEnv The environment to use. +//! +//------------------------------------------------------------------------------ void S3Handler::ConfigureRouter() { #define HANDLER(f) #f, [this](XrdS3Req &req) { return mApi.f##Handler(req); } // The router needs to be initialized in the right order, with the most @@ -458,6 +507,15 @@ void S3Handler::ConfigureRouter() { #undef HANDLER } +//------------------------------------------------------------------------------ +//! Parse the configuration file and populate the environment +//! +//! @param config The configuration file to parse +//! @param env The environment to populate +//! +//! @return true if the configuration file was parsed successfully +//! +//------------------------------------------------------------------------------ bool S3Handler::ParseConfig(const char *config, XrdOucEnv &env) { XrdOucStream Config(&mLog, getenv("XRDINSTANCE"), &env, "=====> "); @@ -515,6 +573,15 @@ bool S3Handler::ParseConfig(const char *config, XrdOucEnv &env) { S3Handler::~S3Handler() = default; +//------------------------------------------------------------------------------ +//! Match the request against the handler +//! +//! @param verb The HTTP verb +//! @param path The HTTP path +//! +//! @return true if the request matches the handler +//! +//------------------------------------------------------------------------------ bool S3Handler::MatchesPath(const char *verb, const char *path) { // match all paths for now, as we do not have access to request headers here. return true; @@ -532,6 +599,16 @@ int S3Handler::ProcessReq(XrdHttpExtReq &req) { extern "C" { +//------------------------------------------------------------------------------ +//! Get external handler instance +//! +//! @param log The system log +//! @param config The configuration file +//! @param parms The parameters +//! @param myEnv The environment +//! +//! @return external handler instance +//------------------------------------------------------------------------------ XrdHttpExtHandler *XrdHttpGetExtHandler(XrdSysError *log, const char *config, const char *parms, XrdOucEnv *myEnv) { return new S3Handler(log, config, myEnv); diff --git a/src/XrdS3/XrdS3.hh b/src/XrdS3/XrdS3.hh index 55750a446eb..b08e2e9bf66 100644 --- a/src/XrdS3/XrdS3.hh +++ b/src/XrdS3/XrdS3.hh @@ -1,10 +1,32 @@ - -#ifndef XROOTD_XRDS3_HH -#define XROOTD_XRDS3_HH - +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ +#pragma once + +//------------------------------------------------------------------------------ #include #include - +//------------------------------------------------------------------------------ #include "XrdHttp/XrdHttpChecksumHandler.hh" #include "XrdHttp/XrdHttpExtHandler.hh" #include "XrdPosix/XrdPosixXrootd.hh" @@ -14,9 +36,16 @@ #include "XrdS3Router.hh" #include "XrdS3Utils.hh" #include "XrdSys/XrdSysError.hh" +//------------------------------------------------------------------------------ + namespace S3 { + +//------------------------------------------------------------------------------ +//! \brief S3Handler is a class that implements the XRootD HTTP extension +//! handler for S3. +//------------------------------------------------------------------------------ class S3Handler : public XrdHttpExtHandler { public: S3Handler(XrdSysError *log, const char *config, XrdOucEnv *myEnv); @@ -52,5 +81,3 @@ class S3Handler : public XrdHttpExtHandler { }; } // namespace S3 - -#endif diff --git a/src/XrdS3/XrdS3Action.hh b/src/XrdS3/XrdS3Action.hh index 792a299883b..09b1558761e 100644 --- a/src/XrdS3/XrdS3Action.hh +++ b/src/XrdS3/XrdS3Action.hh @@ -1,12 +1,34 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on //. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDSACTION_HH -#define XROOTD_XRDSACTION_HH +#pragma once namespace S3 { +//------------------------------------------------------------------------------ +//! Action to perform on S3 +//------------------------------------------------------------------------------ enum class Action { Unknown, AbortMultipartUpload, @@ -108,4 +130,3 @@ enum class Action { } // namespace S -#endif // XROOTD_XRDSACTION_HH diff --git a/src/XrdS3/XrdS3Api.cc b/src/XrdS3/XrdS3Api.cc index a9acc5ddc53..4ce0ce51da1 100644 --- a/src/XrdS3/XrdS3Api.cc +++ b/src/XrdS3/XrdS3Api.cc @@ -1,18 +1,41 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/16/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ #include "XrdS3Api.hh" - +//------------------------------------------------------------------------------ #include - +//------------------------------------------------------------------------------ #include "XrdCks/XrdCksCalcmd5.hh" #include "XrdS3Auth.hh" #include "XrdS3ErrorResponse.hh" #include "XrdS3Response.hh" +//------------------------------------------------------------------------------ namespace S3 { +//------------------------------------------------------------------------------ #define VALIDATE_REQUEST(action) \ auto [err, bucket] = \ auth.ValidateRequest(req, action, req.bucket, req.object); \ @@ -20,12 +43,20 @@ namespace S3 { return req.S3ErrorResponse(err); \ } +//------------------------------------------------------------------------------ #define RET_ON_ERROR(action) \ err = action; \ if (err != S3Error::None) { \ return req.S3ErrorResponse(err); \ } +//------------------------------------------------------------------------------ +//! \brief Parse the XML body of a CreateBucket request +//! \param body The body of the request +//! \param length The length of the body +//! \param location The location of the bucket +//! \return true if the body is valid, false otherwise +//------------------------------------------------------------------------------ bool ParseCreateBucketBody(char* body, int length, std::string& location) { tinyxml2::XMLDocument doc; @@ -58,6 +89,12 @@ bool ParseCreateBucketBody(char* body, int length, std::string& location) { return true; } +//------------------------------------------------------------------------------ +//! \brief Parse the XML body of a DeleteBucket request +//! \param body The body of the request +//! \param length The length of the body +//! \return true if the body is valid, false otherwise +//------------------------------------------------------------------------------ int S3Api::CreateBucketHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::CreateBucket) @@ -95,6 +132,11 @@ int S3Api::CreateBucketHandler(XrdS3Req& req) { return req.S3Response(200, headers, ""); } +//------------------------------------------------------------------------------ +//! ListBucketsHandler +//! \param req +//! \return +//------------------------------------------------------------------------------ int S3Api::ListBucketsHandler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::ListBuckets) @@ -103,6 +145,11 @@ int S3Api::ListBucketsHandler(S3::XrdS3Req& req) { return ListBucketsResponse(req, req.id, req.id, buckets); } +//------------------------------------------------------------------------------ +//! HeadBucketHandler +//! \param req +//! \return +//------------------------------------------------------------------------------ int S3Api::HeadBucketHandler(S3::XrdS3Req& req) { auto [err, bucket] = auth.ValidateRequest(req, Action::HeadBucket, req.bucket, req.object); @@ -114,6 +161,11 @@ int S3Api::HeadBucketHandler(S3::XrdS3Req& req) { return req.Ok(); } +//------------------------------------------------------------------------------ +//! DeleteBucketHandler +//! \param req +//! \return +//------------------------------------------------------------------------------ int S3Api::DeleteBucketHandler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::DeleteBucket) @@ -122,6 +174,11 @@ int S3Api::DeleteBucketHandler(S3::XrdS3Req& req) { return req.S3Response(204); } +//------------------------------------------------------------------------------ +//! DeleteObjectHandler +//! \param req +//! \return +//------------------------------------------------------------------------------ int S3Api::DeleteObjectHandler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::DeleteObject) @@ -130,6 +187,13 @@ int S3Api::DeleteObjectHandler(S3::XrdS3Req& req) { return req.S3Response(204); } +//------------------------------------------------------------------------------ +//! ValidatePreconditions +//! \param etag +//! \param last_modified +//! \param headers +//! \return +//------------------------------------------------------------------------------ S3Error ValidatePreconditions(const std::string& etag, time_t last_modified, const Headers& headers) { // See https://datatracker.ietf.org/doc/html/rfc7232#section-6 for @@ -172,6 +236,11 @@ S3Error ValidatePreconditions(const std::string& etag, time_t last_modified, return S3Error::None; } +//------------------------------------------------------------------------------ +//! GetObjectHandler +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::GetObjectHandler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::GetObject) @@ -275,6 +344,15 @@ int S3Api::GetObjectHandler(S3::XrdS3Req& req) { } } +//------------------------------------------------------------------------------ +//! ParseCommonQueryParams +//! \param query_params +//! \param delimiter +//! \param encode_values +//! \param max_keys +//! \param prefix +//! \return S3Error +//------------------------------------------------------------------------------ S3Error ParseCommonQueryParams( const std::map& query_params, char& delimiter, bool& encode_values, int& max_keys, std::string& prefix) { @@ -318,6 +396,11 @@ S3Error ParseCommonQueryParams( return S3Error::None; } +//------------------------------------------------------------------------------ +//! ListObjectVersionsHandler - List object versions +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::ListObjectVersionsHandler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::ListObjectVersions) @@ -349,6 +432,11 @@ int S3Api::ListObjectVersionsHandler(S3::XrdS3Req& req) { #define PUT_LIMIT 5000000000 +//------------------------------------------------------------------------------ +//! CopyObjectHandler +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::CopyObjectHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::CopyObject) @@ -405,6 +493,11 @@ int S3Api::CopyObjectHandler(XrdS3Req& req) { return req.ChunkResp(nullptr, 0); } +//------------------------------------------------------------------------------ +//! PutObjectHandler - Put object +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::PutObjectHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::PutObject) @@ -451,6 +544,11 @@ int S3Api::PutObjectHandler(XrdS3Req& req) { return req.S3Response(200, headers, ""); } +//------------------------------------------------------------------------------ +//! HeadObjectHandler - Head object +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::HeadObjectHandler(XrdS3Req& req) { auto [err, bucket] = auth.ValidateRequest(req, Action::HeadObject, req.bucket, req.object); @@ -481,6 +579,14 @@ struct DeleteObjectsQuery { std::vector objects; }; +//------------------------------------------------------------------------------ +//! ParseDeleteObjectsBody - Parse the body of a delete objects request and +//! populate the query +//! \param body +//! \param length +//! \param query +//! \return bool +//------------------------------------------------------------------------------ bool ParseDeleteObjectsBody(char* body, int length, DeleteObjectsQuery& query) { tinyxml2::XMLDocument doc; @@ -530,6 +636,11 @@ bool ParseDeleteObjectsBody(char* body, int length, DeleteObjectsQuery& query) { return true; } +//------------------------------------------------------------------------------ +//! DeleteObjectHandler - Delete object +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::DeleteObjectsHandler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::DeleteObjects) @@ -569,6 +680,11 @@ int S3Api::DeleteObjectsHandler(S3::XrdS3Req& req) { return DeleteObjectsResponse(req, query.quiet, deleted, error); } +//------------------------------------------------------------------------------ +//! ListObjectsV2Handler - List objects +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::ListObjectsV2Handler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::ListObjectsV2) @@ -610,6 +726,11 @@ int S3Api::ListObjectsV2Handler(S3::XrdS3Req& req) { encode_values, objectinfo); } +//------------------------------------------------------------------------------ +//! ListObjectVersionsHandler - List object versions +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::ListObjectsHandler(S3::XrdS3Req& req) { VALIDATE_REQUEST(Action::ListObjects) @@ -631,6 +752,12 @@ int S3Api::ListObjectsHandler(S3::XrdS3Req& req) { return ListObjectsResponse(req, req.bucket, prefix, delimiter, marker, max_keys, encode_values, objectinfo); } + +//------------------------------------------------------------------------------ +//! CreateMultipartUploadHandler - Create multipart upload request +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::CreateMultipartUploadHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::CreateMultipartUpload) @@ -643,6 +770,11 @@ int S3Api::CreateMultipartUploadHandler(XrdS3Req& req) { return CreateMultipartUploadResponse(req, upload_id); } +//------------------------------------------------------------------------------ +//! ListMultipartUploadsHandler - List multipart uploads +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::ListMultipartUploadsHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::ListMultipartUploads) @@ -651,6 +783,11 @@ int S3Api::ListMultipartUploadsHandler(XrdS3Req& req) { return ListMultipartUploadResponse(req, multipart_uploads); } +//------------------------------------------------------------------------------ +//! AbortMultipartUploadHandler - Abort multipart upload +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::AbortMultipartUploadHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::AbortMultipartUpload) @@ -663,6 +800,11 @@ int S3Api::AbortMultipartUploadHandler(XrdS3Req& req) { return req.S3Response(204); } +//------------------------------------------------------------------------------ +//! ListPartsHandler - List parts +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::ListPartsHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::ListParts) @@ -678,6 +820,12 @@ int S3Api::ListPartsHandler(XrdS3Req& req) { return ListPartsResponse(req, upload_id, parts); } + +//------------------------------------------------------------------------------ +//! UploadPartHandler - Upload part +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::UploadPartHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::UploadPart) @@ -726,6 +874,13 @@ int S3Api::UploadPartHandler(XrdS3Req& req) { return req.S3Response(200, headers, ""); } +//------------------------------------------------------------------------------ +//! ParseCompleteMultipartUploadBody - Parse complete multipart upload body +//! \param body +//! \param length +//! \param query +//! \return bool +//------------------------------------------------------------------------------ bool ParseCompleteMultipartUploadBody( char* body, int length, std::vector& query) { tinyxml2::XMLDocument doc; @@ -778,6 +933,11 @@ bool ParseCompleteMultipartUploadBody( return true; } +//------------------------------------------------------------------------------ +//! CompleteMultipartUploadHandler - Complete multipart upload +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::CompleteMultipartUploadHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::CompleteMultipartUpload) @@ -817,12 +977,22 @@ int S3Api::CompleteMultipartUploadHandler(XrdS3Req& req) { return CompleteMultipartUploadResponse(req); } +//------------------------------------------------------------------------------ +//! GetBucketAclHandler - Get bucket acl +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::GetBucketAclHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::GetBucketAcl) return GetAclResponse(req, bucket); } +//------------------------------------------------------------------------------ +//! GetObjectAclHandler - Get object acl +//! \param req +//! \return int +//------------------------------------------------------------------------------ int S3Api::GetObjectAclHandler(XrdS3Req& req) { VALIDATE_REQUEST(Action::GetObjectAcl) diff --git a/src/XrdS3/XrdS3Api.hh b/src/XrdS3/XrdS3Api.hh index 558ebb96ca5..a0ebe09aaad 100644 --- a/src/XrdS3/XrdS3Api.hh +++ b/src/XrdS3/XrdS3Api.hh @@ -1,10 +1,33 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/16/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3API_HH -#define XROOTD_XRDS3API_HH - +#pragma once +#include +#include +#include +#include +#include #include #include "XrdS3Auth.hh" @@ -13,6 +36,13 @@ namespace S3 { +//------------------------------------------------------------------------------ +//! \brief S3 API class +//! +//! This class is the main entry point to the S3 API. +//! It is responsible for parsing the request and calling the appropriate +//! handler. +//------------------------------------------------------------------------------ class S3Api { public: S3Api() = default; @@ -286,4 +316,3 @@ class S3Api { } // namespace S3 -#endif // XROOTD_XRDS3API_HH diff --git a/src/XrdS3/XrdS3Auth.cc b/src/XrdS3/XrdS3Auth.cc index 293c40761eb..0ad93abd129 100644 --- a/src/XrdS3/XrdS3Auth.cc +++ b/src/XrdS3/XrdS3Auth.cc @@ -2,18 +2,19 @@ // Created by segransm on 11/9/23. // +//------------------------------------------------------------------------------ #include "XrdS3Auth.hh" - +//------------------------------------------------------------------------------ #include #include - #include #include - +//------------------------------------------------------------------------------ #include "XrdOuc/XrdOucStream.hh" #include "XrdOuc/XrdOucTUtils.hh" #include "XrdOuc/XrdOucUtils.hh" #include "XrdPosix/XrdPosixExtern.hh" +//------------------------------------------------------------------------------ namespace S3 { @@ -412,7 +413,7 @@ S3Auth::S3Auth(const std::filesystem::path &path, std::string region, std::string access_key_id = entry->d_name; auto filepath = keystore / access_key_id; - + auto user_id = S3Utils::GetXattr(filepath, "user"); if (user_id.empty()) { continue; diff --git a/src/XrdS3/XrdS3Auth.hh b/src/XrdS3/XrdS3Auth.hh index d17f09205c8..de3f48ed697 100644 --- a/src/XrdS3/XrdS3Auth.hh +++ b/src/XrdS3/XrdS3Auth.hh @@ -1,17 +1,44 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/9/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3AUTH_HH -#define XROOTD_XRDS3AUTH_HH +#pragma once +//------------------------------------------------------------------------------ #include #include +#include +#include +//------------------------------------------------------------------------------ +#include "XrdS3ErrorResponse.hh" #include "XrdS3Action.hh" #include "XrdS3Crypt.hh" #include "XrdS3Req.hh" +//------------------------------------------------------------------------------ +// S3Auth +//------------------------------------------------------------------------------ namespace S3 { enum class AuthType { @@ -121,4 +148,4 @@ class S3Auth { } // namespace S3 -#endif // XROOTD_XRDS3AUTH_HH + diff --git a/src/XrdS3/XrdS3Crypt.cc b/src/XrdS3/XrdS3Crypt.cc index b3bfd28b9e2..dc1f17323ee 100644 --- a/src/XrdS3/XrdS3Crypt.cc +++ b/src/XrdS3/XrdS3Crypt.cc @@ -2,16 +2,20 @@ // Created by segransm on 11/3/23. // +//------------------------------------------------------------------------------ #include "XrdS3Crypt.hh" - +//------------------------------------------------------------------------------ #include - #include #include - +//------------------------------------------------------------------------------ namespace S3 { #if OPENSSL_VERSION_NUMBER < 0x30000000L + +//------------------------------------------------------------------------------ +//! \brief SHA256 implementation using OpenSSL +//------------------------------------------------------------------------------ S3Crypt::S3SHA256::S3SHA256() { md = (EVP_MD*)EVP_sha256(); if (md == nullptr) { @@ -34,9 +38,18 @@ S3Crypt::S3SHA256::~S3SHA256() { EVP_cleanup(); } +//------------------------------------------------------------------------------ +//! \brief Initialize the digest +//! @param ctx The digest context +//! @param md The digest object +//! @param digest The digest buffer +//------------------------------------------------------------------------------ void S3Crypt::S3SHA256::Init() { EVP_DigestInit_ex(ctx, nullptr, nullptr); } #else - + +//------------------------------------------------------------------------------ +//! \brief SHA256 implementation using OpenSSL +//------------------------------------------------------------------------------ S3Crypt::S3SHA256::S3SHA256() { md = EVP_MD_fetch(nullptr, "SHA256", nullptr); if (md == nullptr) { @@ -56,19 +69,37 @@ S3Crypt::S3SHA256::S3SHA256() { } } +//------------------------------------------------------------------------------ +//! \brief destructor +//------------------------------------------------------------------------------ S3Crypt::S3SHA256::~S3SHA256() { EVP_MD_CTX_free(ctx); EVP_MD_free(md); } +//------------------------------------------------------------------------------ +//! \brief Initialize the digest +//! @param ctx The digest context +//! @param md The digest object +//! @param digest The digest buffer +//------------------------------------------------------------------------------ void S3Crypt::S3SHA256::Init() { EVP_DigestInit_ex2(ctx, nullptr, nullptr); } #endif - +//------------------------------------------------------------------------------ +//! \brief Update the digest +//! @param src The source buffer +//! @param size The size of the buffer +//------------------------------------------------------------------------------ void S3Crypt::S3SHA256::Update(const char *src, size_t size) { EVP_DigestUpdate(ctx, src, size); } + +//------------------------------------------------------------------------------ +//! \brief Finish the digest and return the digest +//! @return The digest +//------------------------------------------------------------------------------ sha256_digest S3Crypt::S3SHA256::Finish() { unsigned int outl; if (!EVP_DigestFinal_ex(ctx, digest.data(), &outl)) { diff --git a/src/XrdS3/XrdS3Crypt.hh b/src/XrdS3/XrdS3Crypt.hh index 5351f467b00..db173e95f95 100644 --- a/src/XrdS3/XrdS3Crypt.hh +++ b/src/XrdS3/XrdS3Crypt.hh @@ -1,9 +1,28 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/3/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3CRYPT_HH -#define XROOTD_XRDS3CRYPT_HH +#pragma once #include #include @@ -21,6 +40,9 @@ namespace S3 { using sha256_digest = std::array; +//------------------------------------------------------------------------------ +//! S3Crypt - S3 encryption/decryption +//------------------------------------------------------------------------------ class S3Crypt { public: S3Crypt() = default; @@ -47,6 +69,9 @@ class S3Crypt { return digest; } +//------------------------------------------------------------------------------ +//! S3SHA256 - SHA256 hash +//------------------------------------------------------------------------------ class S3SHA256 { public: S3SHA256(); @@ -117,4 +142,4 @@ class S3Crypt { } // namespace S3 -#endif // XROOTD_XRDS3CRYPT_HH + diff --git a/src/XrdS3/XrdS3ErrorResponse.hh b/src/XrdS3/XrdS3ErrorResponse.hh index 4ca73866ba0..2944ddbf32e 100644 --- a/src/XrdS3/XrdS3ErrorResponse.hh +++ b/src/XrdS3/XrdS3ErrorResponse.hh @@ -1,9 +1,28 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/14/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3ERRORRESPONSE_HH -#define XROOTD_XRDS3ERRORRESPONSE_HH +#pragma once #include #include @@ -11,12 +30,18 @@ namespace S3 { +//------------------------------------------------------------------------------ +//! \brief S3 error code +//------------------------------------------------------------------------------ struct S3ErrorCode { std::string code; std::string description; int httpCode; }; +//------------------------------------------------------------------------------ +//! \brief S3 error enums +//------------------------------------------------------------------------------ enum class S3Error { None, AccessControlListNotSupported, @@ -129,6 +154,9 @@ enum class S3Error { ObjectExistInObjectPath, }; +//------------------------------------------------------------------------------ +//! \brief S3 error codes and descriptions +//------------------------------------------------------------------------------ const std::map S3ErrorMap = { {S3Error::NotImplemented, {.code = "NotImplemented", @@ -251,5 +279,3 @@ const std::map S3ErrorMap = { }; } // namespace S3 - -#endif // XROOTD_XRDS3ERRORRESPONSE_HH diff --git a/src/XrdS3/XrdS3ObjectStore.cc b/src/XrdS3/XrdS3ObjectStore.cc index 46d9a8d66c6..cab5d576198 100644 --- a/src/XrdS3/XrdS3ObjectStore.cc +++ b/src/XrdS3/XrdS3ObjectStore.cc @@ -1,28 +1,54 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/17/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ #include "XrdS3ObjectStore.hh" - +//------------------------------------------------------------------------------ #include #include #include #include - -#include +//------------------------------------------------------------------------------ #include #include #include #include #include - +//------------------------------------------------------------------------------ +#include #include "XrdCks/XrdCksCalcmd5.hh" #include "XrdPosix/XrdPosixExtern.hh" #include "XrdS3Auth.hh" #include "XrdS3Req.hh" +//------------------------------------------------------------------------------ namespace S3 { - +//------------------------------------------------------------------------------ +//! \brief S3ObjectStore Constructor +//! \param config Path to the configuration file +//! \param mtpu Path to the MTPU directory +//------------------------------------------------------------------------------ S3ObjectStore::S3ObjectStore(const std::string &config, const std::string &mtpu) : config_path(config), mtpu_path(mtpu) { user_map = config_path / "users"; @@ -31,6 +57,11 @@ S3ObjectStore::S3ObjectStore(const std::string &config, const std::string &mtpu) XrdPosix_Mkdir(mtpu_path.c_str(), S_IRWXU | S_IRWXG); } +//------------------------------------------------------------------------------ +//! \brief ValidateBucketName +//! \param name Bucket name to validate +//! \return true if the name is valid, false otherwise +//------------------------------------------------------------------------------ bool S3ObjectStore::ValidateBucketName(const std::string &name) { if (name.size() < 3 || name.size() > 63) { return false; @@ -45,11 +76,22 @@ bool S3ObjectStore::ValidateBucketName(const std::string &name) { }); } +//------------------------------------------------------------------------------ +//! \brief GetUserDefaultBucketPath Get the default bucket path for a user +//! \param user_id User ID +//! \return Default bucket path for the user +//------------------------------------------------------------------------------ std::string S3ObjectStore::GetUserDefaultBucketPath( const std::string &user_id) const { return S3Utils::GetXattr(user_map / user_id, "new_bucket_path"); } +//------------------------------------------------------------------------------ +//! \brief SetMetadata Set metadata for a file or directory +//! \param object Object path +//! \param metadata Metadata to set +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::SetMetadata( const std::string &object, const std::map &metadata) { @@ -61,6 +103,11 @@ S3Error S3ObjectStore::SetMetadata( return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief GetPartsNumber Get the parts number for a file +//! \param path Object path +//! \return Parts number for the file +//------------------------------------------------------------------------------ std::vector S3ObjectStore::GetPartsNumber( const std::string &path) { auto p = S3Utils::GetXattr(path, "parts"); @@ -75,7 +122,13 @@ std::vector S3ObjectStore::GetPartsNumber( return res; } -// todo: + +//------------------------------------------------------------------------------ +//! SetPartsNumber Set the parts number for a file +//! \param path Object path +//! \param parts Parts number to set +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::SetPartsNumbers(const std::string &path, std::vector &parts) { auto p = S3Utils::stringJoin(',', parts); @@ -86,6 +139,12 @@ S3Error S3ObjectStore::SetPartsNumbers(const std::string &path, } // TODO: We need a mutex here as multiple threads can write to the same file +//------------------------------------------------------------------------------ +//! \brief AddPartAttr Add a part attribute for a file +//! \param object Object path +//! \param part_number Part number +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::AddPartAttr(const std::string &object, size_t part_number) { auto parts = GetPartsNumber(object); @@ -98,6 +157,13 @@ S3Error S3ObjectStore::AddPartAttr(const std::string &object, return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief CreateBucket Create a bucket +//! \param auth Authentication object +//! \param bucket Bucket to create +//! \param _location Location to create the bucket in +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::CreateBucket(S3Auth &auth, S3Auth::Bucket bucket, const std::string &_location) { if (!ValidateBucketName(bucket.name)) { @@ -145,6 +211,11 @@ S3Error S3ObjectStore::CreateBucket(S3Auth &auth, S3Auth::Bucket bucket, return S3Error::None; } +//------------------------------------------------------------------------------ +//! BaseDir Split a path into a base directory and a file name +//! \param p Path to split +//! \return Pair of base directory and file name +//------------------------------------------------------------------------------ std::pair BaseDir(std::string p) { std::string basedir; auto pos = p.rfind('/'); @@ -156,6 +227,12 @@ std::pair BaseDir(std::string p) { return {basedir, p}; } +//------------------------------------------------------------------------------ +//! DeleteBucket - Delete a bucket and all its contents +//! \param auth Authentication object +//! \param bucket Bucket to delete +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::DeleteBucket(S3Auth &auth, const S3Auth::Bucket &bucket) { if (!S3Utils::IsDirEmpty(bucket.path)) { @@ -192,6 +269,9 @@ S3Error S3ObjectStore::DeleteBucket(S3Auth &auth, return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief Object destructor - close the file descriptor if open +//------------------------------------------------------------------------------ S3ObjectStore::Object::~Object() { if (fd != 0) { XrdPosix_Close(fd); @@ -201,6 +281,11 @@ S3ObjectStore::Object::~Object() { // TODO: Replace with the real XrdPosix_Listxattr once implemented. #define XrdPosix_Listxattr listxattr +//------------------------------------------------------------------------------ +//! \brief Object init +//! \param p Path to the object +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::Object::Init(const std::filesystem::path &p) { struct stat buf; @@ -234,6 +319,12 @@ S3Error S3ObjectStore::Object::Init(const std::filesystem::path &p) { return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief Read a file +//! \param length Number of bytes to read +//! \param ptr Pointer to the buffer +//! \return Number of bytes read +//------------------------------------------------------------------------------ ssize_t S3ObjectStore::Object::Read(size_t length, char **ptr) { if (!init) { return 0; @@ -253,6 +344,12 @@ ssize_t S3ObjectStore::Object::Read(size_t length, char **ptr) { return ret; } +//------------------------------------------------------------------------------ +//! \brief lseek a file +//! \param offset Offset +//! \param whence Whence +//! \return Offset +//------------------------------------------------------------------------------ off_t S3ObjectStore::Object::Lseek(off_t offset, int whence) { if (!init) { return -1; @@ -266,11 +363,24 @@ off_t S3ObjectStore::Object::Lseek(off_t offset, int whence) { return XrdPosix_Lseek(fd, offset, whence); } +//------------------------------------------------------------------------------ +//! \brief GetObject - Get an object from the store +//! \param bucket Bucket name +//! \param object Object name +//! \param obj Object to fill +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::GetObject(const S3Auth::Bucket &bucket, const std::string &object, Object &obj) { return obj.Init(bucket.path / object); } +//------------------------------------------------------------------------------ +//! \brief DeleteObject - Delete an object from the store +//! \param bucket Bucket name +//! \param object Object name +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::DeleteObject(const S3Auth::Bucket &bucket, const std::string &key) { std::string base, obj; @@ -288,6 +398,12 @@ S3Error S3ObjectStore::DeleteObject(const S3Auth::Bucket &bucket, return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief ListBuckets - List all buckets in the store for a user +//! \param id User ID +//! \param buckets Buckets to fill +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ std::vector S3ObjectStore::ListBuckets( const std::string &id) const { std::vector buckets; @@ -309,6 +425,13 @@ std::vector S3ObjectStore::ListBuckets( // TODO: At the moment object versioning is not supported, this returns a // hardcoded object version. +//------------------------------------------------------------------------------ +//! \brief ListObjectVersions - List all versions of an object in the store +//! \param bucket Bucket name +//! \param object Object name +//! \param versions Versions to fill +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ ListObjectsInfo S3ObjectStore::ListObjectVersions( const S3Auth::Bucket &bucket, const std::string &prefix, const std::string &key_marker, const std::string &version_id_marker, @@ -328,6 +451,15 @@ ListObjectsInfo S3ObjectStore::ListObjectVersions( true, f); } +//------------------------------------------------------------------------------ +//! \brief CopyObject - Copy an object from the store +//! \param bucket Bucket name +//! \param key Object name +//! \param source_obj Object to copy +//! \param reqheaders Request headers +//! \param headers Response headers +//! \return S3Error::None if successful, S3Error::InternalError otherwise +//------------------------------------------------------------------------------ S3Error S3ObjectStore::CopyObject(const S3Auth::Bucket &bucket, const std::string &key, Object &source_obj, const Headers &reqheaders, Headers &headers) { @@ -418,6 +550,16 @@ S3Error S3ObjectStore::CopyObject(const S3Auth::Bucket &bucket, //! - There must be no gap in the part numbers (this will be checked when //! completing the multipart upload)\n //! - There cannot be another upload in progress of the same part.\n + +//------------------------------------------------------------------------------ +//! \brief KeepOptimize - Optimize the file by keeping only the parts that are needed +//! \param upload_path - The path to the file to optimize +//! \param part_number - The part number of the part to optimize +//! \param size - The size of the part to optimize +//! \param tmp_path - The path to the temporary file +//! \param part_size - The size of the part +//! \return true if the file can be optimized, false otherwise +//------------------------------------------------------------------------------ bool S3ObjectStore::KeepOptimize(const std::filesystem::path &upload_path, size_t part_number, unsigned long size, const std::string &tmp_path, @@ -474,6 +616,16 @@ bool S3ObjectStore::KeepOptimize(const std::filesystem::path &upload_path, return true; } +//------------------------------------------------------------------------------ +//! \brief ReadBufferAt - Read a buffer from the request +//! \param req - The request to read from +//! \param md5XS - The md5 checksum +//! \param sha256XS - The sha256 checksum +//! \param fd - The file descriptor to write to +//! \param length - The length of the buffer to read +//! \return S3Error::None if successful, S3Error::IncompleteBody otherwise +//------------------------------------------------------------------------------ + S3Error ReadBufferAt(XrdS3Req &req, XrdCksCalcmd5 &md5XS, S3Crypt::S3SHA256 &sha256XS, int fd, unsigned long length) { @@ -502,6 +654,15 @@ S3Error ReadBufferAt(XrdS3Req &req, XrdCksCalcmd5 &md5XS, return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief ReadBufferIntoFile - Read a buffer from the request into a file +//! \param req - The request to read from +//! \param md5XS - The md5 checksum +//! \param sha256XS - The sha256 checksum +//! \param fd - The file descriptor to write to +//! \param length - The length of the buffer to read +//! \return S3Error::None if successful, S3Error::IncompleteBody otherwise +//------------------------------------------------------------------------------ std::pair ReadBufferIntoFile(XrdS3Req &req, XrdCksCalcmd5 &md5XS, S3Crypt::S3SHA256 &sha256XS, @@ -543,6 +704,9 @@ std::pair ReadBufferIntoFile(XrdS3Req &req, #undef PUT_LIMIT } +//------------------------------------------------------------------------------ +//! \breif FileUploadResult - Result of uploading a file +//------------------------------------------------------------------------------ struct FileUploadResult { S3Error result; sha256_digest sha256; @@ -550,6 +714,14 @@ struct FileUploadResult { size_t size; }; +//------------------------------------------------------------------------------ +//! \brief FileUploader - Upload a file +//! \param req - The request to read from +//! \param chunked - Whether the file is chunked +//! \param size - The size of the file +//! \param path - The path to the file +//! \return The result of the upload +//------------------------------------------------------------------------------ FileUploadResult FileUploader(XrdS3Req &req, bool chunked, size_t size, std::filesystem::path &path) { auto fd = XrdPosix_Open(path.c_str(), O_CREAT | O_EXCL | O_WRONLY, @@ -592,6 +764,16 @@ FileUploadResult FileUploader(XrdS3Req &req, bool chunked, size_t size, return FileUploadResult{error, sha256, md5hex, final_size}; } +//------------------------------------------------------------------------------ +//! \brief UploadPartOptimized - Upload a part of a file +//! \param req - The request to read from +//! \param tmp_path - The path to the temporary file +//! \param part_size - The size of the part +//! \param part_number - The part number +//! \param size - The size of the file +//! \param headers - The headers to set +//! \return The result of the upload +//------------------------------------------------------------------------------ S3Error S3ObjectStore::UploadPartOptimized(XrdS3Req &req, const std::string &tmp_path, size_t part_size, size_t part_number, @@ -646,6 +828,16 @@ S3Error S3ObjectStore::UploadPartOptimized(XrdS3Req &req, // TODO: Needs a mutex for some operations // TODO: Can be optimized by keeping in memory part information data instead of // using xattr + +//------------------------------------------------------------------------------ +//! \brief UploadPart - Upload a part of a file +//! \param req - The request to read from +//! \param part_number - The part number +//! \param size - The size of the file +//! \param chunked - Whether the file is chunked +//! \param headers - The headers to set +//! \return The result of the upload +//------------------------------------------------------------------------------ S3Error S3ObjectStore::UploadPart(XrdS3Req &req, const std::string &upload_id, size_t part_number, unsigned long size, bool chunked, Headers &headers) { @@ -705,6 +897,15 @@ S3Error S3ObjectStore::UploadPart(XrdS3Req &req, const std::string &upload_id, return error; } +//------------------------------------------------------------------------------ +//! \brief PutObject - Put an object +//! \param req - The request to read from +//! \param bucket - The bucket to put the object in +//! \param size - The size of the file +//! \param chunked - Whether the file is chunked +//! \param headers - The headers to set +//! \return The result of the upload +//------------------------------------------------------------------------------ S3Error S3ObjectStore::PutObject(XrdS3Req &req, const S3Auth::Bucket &bucket, unsigned long size, bool chunked, Headers &headers) { @@ -774,6 +975,12 @@ S3Error S3ObjectStore::PutObject(XrdS3Req &req, const S3Auth::Bucket &bucket, return error; } +//------------------------------------------------------------------------------ +//! \brief DeleteObjects - Delete multiple objects +//! \param bucket - The bucket to delete the objects in +//! \param objects - The objects to delete +//! \return The result of the delete +//------------------------------------------------------------------------------ std::tuple, std::vector> S3ObjectStore::DeleteObjects(const S3Auth::Bucket &bucket, const std::vector &objects) { @@ -791,6 +998,17 @@ S3ObjectStore::DeleteObjects(const S3Auth::Bucket &bucket, return {deleted, error}; } +//------------------------------------------------------------------------------ +//! \brief ListObjectsV2 - List objects in a bucket +//! \param bucket - The bucket to list the objects in +//! \param prefix - The prefix to filter the objects by +//! \param continuation_token - The continuation token to use for pagination +//! \param delimiter - The delimiter to use for grouping objects +//! \param max_keys - The maximum number of keys to return +//! \param fetch_owner - Whether to fetch the owner of the objects +//! \param start_after - The key to start after +//! \return The result of the list +//------------------------------------------------------------------------------ ListObjectsInfo S3ObjectStore::ListObjectsV2( const S3Auth::Bucket &bucket, const std::string &prefix, const std::string &continuation_token, const char delimiter, int max_keys, @@ -817,6 +1035,15 @@ ListObjectsInfo S3ObjectStore::ListObjectsV2( max_keys, false, f); } +//------------------------------------------------------------------------------ +//! \brief ListObjects - List objects in a bucket +//! \param bucket - The bucket to list the objects in +//! \param prefix - The prefix to filter the objects by +//! \param marker - The marker to use for pagination +//! \param delimiter - The delimiter to use for grouping objects +//! \param max_keys - The maximum number of keys to return +//! \return The result of the list +//------------------------------------------------------------------------------ ListObjectsInfo S3ObjectStore::ListObjects(const S3Auth::Bucket &bucket, const std::string &prefix, const std::string &marker, @@ -840,6 +1067,17 @@ ListObjectsInfo S3ObjectStore::ListObjects(const S3Auth::Bucket &bucket, // custom function. #define XrdPosix_Scandir scandir +//------------------------------------------------------------------------------ +//! \brief ListObjectsCommon - Common logic for listing objects +//! \param bucket - The bucket to list the objects in +//! \param prefix - The prefix to filter the objects by +//! \param marker - The marker to use for pagination +//! \param delimiter - The delimiter to use for grouping objects +//! \param max_keys - The maximum number of keys to return +//! \param get_versions - Whether to get versions of the objects +//! \param f - The function to call for each object +//! \return The result of the list +//------------------------------------------------------------------------------ ListObjectsInfo S3ObjectStore::ListObjectsCommon( const S3Auth::Bucket &bucket, std::string prefix, const std::string &marker, char delimiter, int max_keys, bool get_versions, @@ -957,6 +1195,12 @@ ListObjectsInfo S3ObjectStore::ListObjectsCommon( return list; } +//------------------------------------------------------------------------------ +//! \brief CreateMultipartUpload - Create a multipart upload +//! \param bucket - The bucket to create the upload in +//! \param key - The key to create the upload for +//! \return The result of the upload +//------------------------------------------------------------------------------ std::pair S3ObjectStore::CreateMultipartUpload( const S3Auth::Bucket &bucket, const std::string &key) { // TODO: Metadata uploaded with the create multipart upload operation is not @@ -1000,6 +1244,11 @@ std::pair S3ObjectStore::CreateMultipartUpload( return {upload_id, S3Error::None}; } +//------------------------------------------------------------------------------ +//! \brief ListMultipartUploads - List all multipart uploads for a bucket +//! \param bucket - bucket name +//! \return vector of multipart upload info +//------------------------------------------------------------------------------ std::vector S3ObjectStore::ListMultipartUploads(const std::string &bucket) { auto upload_path = mtpu_path / bucket; @@ -1020,6 +1269,13 @@ S3ObjectStore::ListMultipartUploads(const std::string &bucket) { return uploads; } +//------------------------------------------------------------------------------ +//! \brief AbortMultipartUpload - Abort a multipart upload +//! \param bucket - The bucket to abort the upload in +//! \param key - The key to abort the upload for +//! \param upload_id - The upload id to abort +//! \return The result of the abort +//------------------------------------------------------------------------------ S3Error S3ObjectStore::AbortMultipartUpload(const S3Auth::Bucket &bucket, const std::string &key, const std::string &upload_id) { @@ -1036,6 +1292,13 @@ S3Error S3ObjectStore::AbortMultipartUpload(const S3Auth::Bucket &bucket, return err; }; +//------------------------------------------------------------------------------ +//! \brief DeleteMultipartUpload - Delete a multipart upload +//! \param bucket - The bucket to delete the upload in +//! \param key - The key to delete the upload for +//! \param upload_id - The upload id to delete +//! \return The result of the delete +//------------------------------------------------------------------------------ S3Error S3ObjectStore::DeleteMultipartUpload(const S3Auth::Bucket &bucket, const std::string &key, const std::string &upload_id) { @@ -1061,6 +1324,12 @@ S3Error S3ObjectStore::DeleteMultipartUpload(const S3Auth::Bucket &bucket, return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief ValidateMultipartUpload - Validate a multipart upload +//! \param upload_path - The path to the upload +//! \param key - The key to validate the upload for +//! \return The result of the validation +//------------------------------------------------------------------------------ S3Error S3ObjectStore::ValidateMultipartUpload(const std::string &upload_path, const std::string &key) { struct stat buf; @@ -1077,6 +1346,13 @@ S3Error S3ObjectStore::ValidateMultipartUpload(const std::string &upload_path, return S3Error::None; } +//------------------------------------------------------------------------------ +//! \brief ListParts - List all parts for a multipart upload +//! \param bucket - The bucket to list the parts in +//! \param key - The key to list the parts for +//! \param upload_id - The upload id to list the parts for +//! \return The result of the list +//------------------------------------------------------------------------------ std::pair> S3ObjectStore::ListParts(const std::string &bucket, const std::string &key, const std::string &upload_id) { @@ -1113,6 +1389,15 @@ S3ObjectStore::ListParts(const std::string &bucket, const std::string &key, return {S3Error::None, parts}; } +//------------------------------------------------------------------------------ +//! \brief Complete an optimized multipart upload +//! +//! \param final_path Path to the final file +//! \param tmp_path Path to the temporary directory +//! \param parts Vector of parts to upload +//! +//! \return True if the upload was successful, false otherwise +//------------------------------------------------------------------------------ bool S3ObjectStore::CompleteOptimizedMultipartUpload( const std::filesystem::path &final_path, const std::filesystem::path &tmp_path, const std::vector &parts) { @@ -1137,6 +1422,15 @@ bool S3ObjectStore::CompleteOptimizedMultipartUpload( return true; } +//------------------------------------------------------------------------------ +//! \brief CompleteMultipartUpload - Complete a multipart upload +//! \param req - The request object +//! \param bucket - The bucket to complete the upload in +//! \param key - The key to complete the upload for +//! \param upload_id - The upload id to complete +//! \param parts - The parts to complete the upload with +//! \return The result of the complete +//------------------------------------------------------------------------------ S3Error S3ObjectStore::CompleteMultipartUpload( XrdS3Req &req, const S3Auth::Bucket &bucket, const std::string &key, const std::string &upload_id, const std::vector &parts) { diff --git a/src/XrdS3/XrdS3ObjectStore.hh b/src/XrdS3/XrdS3ObjectStore.hh index be4953a9e43..d0cd502c0ae 100644 --- a/src/XrdS3/XrdS3ObjectStore.hh +++ b/src/XrdS3/XrdS3ObjectStore.hh @@ -1,9 +1,28 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/17/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3OBJECTSTORE_HH -#define XROOTD_XRDS3OBJECTSTORE_HH +#pragma once #include #include @@ -225,4 +244,3 @@ class S3ObjectStore { } // namespace S3 -#endif // XROOTD_XRDS3OBJECTSTORE_HH diff --git a/src/XrdS3/XrdS3Req.cc b/src/XrdS3/XrdS3Req.cc index 474a43a5d1e..9da8bf3dce8 100644 --- a/src/XrdS3/XrdS3Req.cc +++ b/src/XrdS3/XrdS3Req.cc @@ -1,15 +1,37 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/9/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ #include "XrdS3Req.hh" - +//------------------------------------------------------------------------------ #include "XrdOfs/XrdOfs.hh" #include "XrdOuc/XrdOucString.hh" #include "XrdOuc/XrdOucTUtils.hh" #include "XrdS3.hh" #include "XrdS3Auth.hh" #include "XrdS3Xml.hh" +//------------------------------------------------------------------------------ namespace S3 { diff --git a/src/XrdS3/XrdS3Req.hh b/src/XrdS3/XrdS3Req.hh index 2fc7387a8ba..ab0b0199d9e 100644 --- a/src/XrdS3/XrdS3Req.hh +++ b/src/XrdS3/XrdS3Req.hh @@ -1,16 +1,37 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/9/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3REQ_HH -#define XROOTD_XRDS3REQ_HH +#pragma once +//------------------------------------------------------------------------------ #include - +//------------------------------------------------------------------------------ #include "XrdHttp/XrdHttpExtHandler.hh" #include "XrdS3Crypt.hh" #include "XrdS3ErrorResponse.hh" #include "XrdS3Utils.hh" +//------------------------------------------------------------------------------ namespace S3 { @@ -97,4 +118,3 @@ using HandlerFunc = std::function; } // namespace S3 -#endif // XROOTD_XRDS3REQ_HH diff --git a/src/XrdS3/XrdS3Response.cc b/src/XrdS3/XrdS3Response.cc index 6088f0eba18..afb6bf2caab 100644 --- a/src/XrdS3/XrdS3Response.cc +++ b/src/XrdS3/XrdS3Response.cc @@ -2,12 +2,14 @@ // Created by segransm on 11/16/23. // +//------------------------------------------------------------------------------ #include "XrdS3Response.hh" - +//------------------------------------------------------------------------------ #include - +//------------------------------------------------------------------------------ #include "XrdS3ObjectStore.hh" #include "XrdS3Xml.hh" +//------------------------------------------------------------------------------ namespace S3 { diff --git a/src/XrdS3/XrdS3Response.hh b/src/XrdS3/XrdS3Response.hh index ef58bad95fb..277dd2077e7 100644 --- a/src/XrdS3/XrdS3Response.hh +++ b/src/XrdS3/XrdS3Response.hh @@ -1,15 +1,36 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/16/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3RESPONSE_HH -#define XROOTD_XRDS3RESPONSE_HH +#pragma once +//------------------------------------------------------------------------------ #include #include - +//------------------------------------------------------------------------------ #include "XrdS3ObjectStore.hh" #include "XrdS3Req.hh" +//------------------------------------------------------------------------------ namespace S3 { @@ -54,5 +75,3 @@ int CompleteMultipartUploadResponse(XrdS3Req& req); int GetAclResponse(XrdS3Req& req, const S3Auth::Bucket& bucket); } // namespace S3 - -#endif // XROOTD_XRDS3RESPONSE_HH diff --git a/src/XrdS3/XrdS3Router.cc b/src/XrdS3/XrdS3Router.cc index 68711eeb996..dc3d5af3a91 100644 --- a/src/XrdS3/XrdS3Router.cc +++ b/src/XrdS3/XrdS3Router.cc @@ -1,22 +1,60 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/9/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ #include "XrdS3Router.hh" +//------------------------------------------------------------------------------ namespace S3 { +//------------------------------------------------------------------------------ +//! \brief match the method and the request +//! \param m the method +//! \param req the request +//! \return s3route +//------------------------------------------------------------------------------ S3Route &S3Route::Method(const HttpMethod &m) { matchers.emplace_back( [m](const XrdS3Req &req) { return MatchMethod(m, req); }); return *this; } +//------------------------------------------------------------------------------ +//! \brief match the path +//! \param p the path +//! \return s3route +//------------------------------------------------------------------------------ S3Route &S3Route::Path(const PathMatch &p) { matchers.emplace_back([p](const XrdS3Req &req) { return MatchPath(p, req); }); return *this; } +//------------------------------------------------------------------------------ +//! \brief match the bucket +//! \param b the bucket +//! \return s3route +//------------------------------------------------------------------------------ S3Route &S3Route::Queries( const std::vector> &q) { matchers.emplace_back( @@ -24,6 +62,11 @@ S3Route &S3Route::Queries( return *this; } +//------------------------------------------------------------------------------ +//! \brief match headers +//! \param h the headers +//! \return s3route +//------------------------------------------------------------------------------ S3Route &S3Route::Headers( const std::vector> &h) { matchers.emplace_back( @@ -31,19 +74,44 @@ S3Route &S3Route::Headers( return *this; } +//------------------------------------------------------------------------------ +//! \brief match request +//! \param req the request +//! \return true if the request matches the route +//------------------------------------------------------------------------------ bool S3Route::Match(const XrdS3Req &req) const { return std::all_of(matchers.begin(), matchers.end(), [req](const auto &matcher) { return matcher(req); }); } +//------------------------------------------------------------------------------ +//! \brief return handler +//! \return handler +//------------------------------------------------------------------------------ const HandlerFunc &S3Route::Handler() const { return handler; }; +//------------------------------------------------------------------------------ +//! \brief return name +//! \return name +//------------------------------------------------------------------------------ const std::string &S3Route::GetName() const { return name; }; +//------------------------------------------------------------------------------ +//! \brief match the method +//! \param method the method +//! \param req the request +//! \return true if the method matches the request +//------------------------------------------------------------------------------ bool S3Route::MatchMethod(const HttpMethod &method, const XrdS3Req &req) { return method == req.method; } +//------------------------------------------------------------------------------ +//! \brief match the path +//! \param path the path +//! \param req the request +//! \return true if the path matches the request +//------------------------------------------------------------------------------ bool S3Route::MatchPath(const PathMatch &path, const XrdS3Req &req) { // Path will always start with '/' switch (path) { @@ -57,6 +125,12 @@ bool S3Route::MatchPath(const PathMatch &path, const XrdS3Req &req) { return false; } +//------------------------------------------------------------------------------ +//! \brief match requirements against map +//! \param required the requirements +//! \param map the map +//! \return true if the requirements match the map +//------------------------------------------------------------------------------ bool S3Route::MatchMap( const std::vector> &required, const std::map &map) { @@ -71,11 +145,20 @@ bool S3Route::MatchMap( }); } +//------------------------------------------------------------------------------ +//! \brief add route to router +//! \param route the route +//------------------------------------------------------------------------------ void S3Router::AddRoute(S3Route &route) { mLog.Say("Registered route:", route.GetName().c_str()); routes.push_back(route); } +//------------------------------------------------------------------------------ +//! \brief process request +//! \param req the request +//! \return the status code +//------------------------------------------------------------------------------ int S3Router::ProcessReq(XrdS3Req &req) { for (const auto &route : routes) { if (route.Match(req)) { diff --git a/src/XrdS3/XrdS3Router.hh b/src/XrdS3/XrdS3Router.hh index a5921f2512a..4a1b466a30f 100644 --- a/src/XrdS3/XrdS3Router.hh +++ b/src/XrdS3/XrdS3Router.hh @@ -1,25 +1,51 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/9/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3ROUTER_HH -#define XROOTD_XRDS3ROUTER_HH +#pragma once +//------------------------------------------------------------------------------ #include #include #include #include #include #include - +//------------------------------------------------------------------------------ #include "XrdS3Action.hh" #include "XrdS3Req.hh" #include "XrdSys/XrdSysError.hh" +//------------------------------------------------------------------------------ namespace S3 { enum class PathMatch { MatchObject, MatchBucket, MatchNoBucket }; +//------------------------------------------------------------------------------ +//! S3Route is a class that represents a route in the S3 server. +//! It contains a handler function and a list of matchers. +//! The matchers are used to match the request to the route. +//------------------------------------------------------------------------------ class S3Route { public: explicit S3Route(HandlerFunc fn) : handler(std::move(fn)){}; @@ -58,6 +84,11 @@ class S3Route { const std::string name; }; +//------------------------------------------------------------------------------ +//! S3Router is a class that represents a router in the S3 server. +//! It contains a list of routes and a not found handler. +//! The router is used to match the request to the route. +//------------------------------------------------------------------------------ class S3Router { public: explicit S3Router(XrdSysError *log, HandlerFunc fn) @@ -78,4 +109,4 @@ class S3Router { } // namespace S3 -#endif // XROOTD_XRDS3ROUTER_HH + diff --git a/src/XrdS3/XrdS3Utils.cc b/src/XrdS3/XrdS3Utils.cc index 24f012df61d..406deb70970 100644 --- a/src/XrdS3/XrdS3Utils.cc +++ b/src/XrdS3/XrdS3Utils.cc @@ -1,19 +1,46 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/3/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ #include "XrdS3Utils.hh" - +//------------------------------------------------------------------------------ #include - #include #include #include - +//------------------------------------------------------------------------------ #include "XrdPosix/XrdPosixExtern.hh" +//------------------------------------------------------------------------------ namespace S3 { +//------------------------------------------------------------------------------ +//! Encode a string according to RFC 3986 +//! +//! @param str The string to encode +//! @return The encoded string +//------------------------------------------------------------------------------ std::string S3Utils::UriEncode(const std::bitset<256> &encoder, const std::string &str) { std::ostringstream res; @@ -30,6 +57,12 @@ std::string S3Utils::UriEncode(const std::bitset<256> &encoder, return res.str(); } +//------------------------------------------------------------------------------ +//! Decode a string according to RFC 3986 +//! +//! @param str The string to decode +//! @return The decoded string +//------------------------------------------------------------------------------ std::string S3Utils::UriDecode(const std::string &str) { size_t i = 0; std::string res; @@ -53,6 +86,9 @@ std::string S3Utils::UriDecode(const std::string &str) { return res; } +//------------------------------------------------------------------------------ +//! Constructor +//------------------------------------------------------------------------------ S3Utils::S3Utils() { for (int i = 0; i < 256; ++i) { mEncoder[i] = isalnum(i) || i == '-' || i == '.' || i == '_' || i == '~'; @@ -69,14 +105,32 @@ S3Utils::S3Utils() { // AWS uses a different uri encoding for objects during canonical signature // calculation. + +//------------------------------------------------------------------------------ +//! Encode a string according to the AWS URI encoding rules +//! +//! @param str The string to encode +//! @return The encoded string +//------------------------------------------------------------------------------ std::string S3Utils::UriEncode(const std::string &str) { return UriEncode(mEncoder, str); } +//------------------------------------------------------------------------------ +//! Encode a string for objects according to the AWS URI encoding rules +//! +//! @param str The string to encode +//! @return The encoded string +//------------------------------------------------------------------------------ std::string S3Utils::ObjectUriEncode(const std::string &str) { return UriEncode(mObjectEncoder, str); } +//------------------------------------------------------------------------------ +//! trim all whitespace from a string +//! +//! @param str The string to trim +//------------------------------------------------------------------------------ void S3Utils::TrimAll(std::string &str) { // Trim leading non-letters size_t n = 0; @@ -110,11 +164,26 @@ void S3Utils::TrimAll(std::string &str) { } } +//------------------------------------------------------------------------------ +//! Check if a map has a key +//! +//! @param map The map to check +//! @param key The key to check for +//! @return True if the map has the key, false otherwise +//------------------------------------------------------------------------------ bool S3Utils::MapHasKey(const std::map &map, const std::string &key) { return (map.find(key) != map.end()); } +//------------------------------------------------------------------------------ +//! Check if a map has a key and a value +//! +//! @param map The map to check +//! @param key The key to check for +//! @param val The value to check for +//! @return True if the map has the key and the value, false otherwise +//------------------------------------------------------------------------------ bool S3Utils::MapHasEntry(const std::map &map, const std::string &key, const std::string &val) { auto it = map.find(key); @@ -122,6 +191,14 @@ bool S3Utils::MapHasEntry(const std::map &map, return (it != map.end() && it->second == val); } +//------------------------------------------------------------------------------ +//! Check if a map has a key and a value that starts with a given value +//! +//! @param map The map to check +//! @param key The key to check for +//! @param val The value to check for +//! @return True if the map has the key and the value, false otherwise +//------------------------------------------------------------------------------ bool S3Utils::MapEntryStartsWith(const std::map &map, const std::string &key, const std::string &val) { @@ -130,6 +207,12 @@ bool S3Utils::MapEntryStartsWith(const std::map &map, return (it != map.end() && it->second.substr(0, val.length()) == val); } +//------------------------------------------------------------------------------ +//! string timestamp to iso8016 format +//! +//! @param t The timestamp to convert +//! @return The converted timestamp +//------------------------------------------------------------------------------ std::string S3Utils::timestampToIso8016(const std::string &t) { try { return timestampToIso8016(std::stol(t)); @@ -138,11 +221,23 @@ std::string S3Utils::timestampToIso8016(const std::string &t) { } } +//------------------------------------------------------------------------------ +//! timestamp to iso8016 format +//! +//! @param t The timestamp to convert +//! @return The converted timestamp +//------------------------------------------------------------------------------ std::string S3Utils::timestampToIso8016(const time_t &t) { struct tm *date = gmtime(&t); return timestampToIso8016(date); } +//------------------------------------------------------------------------------ +//! timestamp to iso8016 format +//! +//! @param t The timestamp to convert +//! @return The converted timestamp +//------------------------------------------------------------------------------ std::string S3Utils::timestampToIso8016(const struct tm *t) { char date_iso8601[17]{}; if (!t || strftime(date_iso8601, 17, "%Y%m%dT%H%M%SZ", t) != 16) { @@ -151,6 +246,13 @@ std::string S3Utils::timestampToIso8016(const struct tm *t) { return date_iso8601; } +//------------------------------------------------------------------------------ +//! make a path +//! +//! @param path The path to create +//! @param mode The mode to use +//! @return 0 on success, an error code otherwise +//------------------------------------------------------------------------------ int S3Utils::makePath(char *path, mode_t mode) { char *next_path = path + 1; struct stat buf; @@ -180,6 +282,12 @@ int S3Utils::makePath(char *path, mode_t mode) { return 0; } +//------------------------------------------------------------------------------ +//! delete a path +//! +//! @param path The path to delete +//! @param stop The path to stop at +//------------------------------------------------------------------------------ void S3Utils::RmPath(std::filesystem::path path, const std::filesystem::path &stop) { while (path != stop && !XrdPosix_Rmdir(path.c_str())) { @@ -187,6 +295,13 @@ void S3Utils::RmPath(std::filesystem::path path, } } +//------------------------------------------------------------------------------ +//! get an xattr +//! +//! @param path The path to get the xattr from +//! @param key The key to get +//! @return The xattr value +//------------------------------------------------------------------------------ std::string S3Utils::GetXattr(const std::filesystem::path &path, const std::string &key) { std::vector res; @@ -218,6 +333,12 @@ int S3Utils::SetXattr(const std::filesystem::path &path, const std::string &key, #undef XrdPosix_Setxattr +//------------------------------------------------------------------------------ +//! check if a directory is empty +//! +//! @param path The path to check +//! @return True if the directory is empty, false otherwise +//------------------------------------------------------------------------------ bool S3Utils::IsDirEmpty(const std::filesystem::path &path) { auto dir = XrdPosix_Opendir(path.c_str()); @@ -236,6 +357,13 @@ bool S3Utils::IsDirEmpty(const std::filesystem::path &path) { return n <= 2; } +//------------------------------------------------------------------------------ +//! iterate over a directory +//! +//! @param path The path to iterate over +//! @param f The function to call for each entry +//! @return 0 on success, 1 on error +//------------------------------------------------------------------------------ int S3Utils::DirIterator(const std::filesystem::path &path, const std::function &f) { auto dir = XrdPosix_Opendir(path.c_str()); diff --git a/src/XrdS3/XrdS3Utils.hh b/src/XrdS3/XrdS3Utils.hh index 59b06afba79..452020e2fd3 100644 --- a/src/XrdS3/XrdS3Utils.hh +++ b/src/XrdS3/XrdS3Utils.hh @@ -1,12 +1,31 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/3/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3UTILS_HH -#define XROOTD_XRDS3UTILS_HH +#pragma once +//------------------------------------------------------------------------------ #include - #include #include #include @@ -15,9 +34,13 @@ #include #include #include +//------------------------------------------------------------------------------ namespace S3 { +//------------------------------------------------------------------------------ +//! S3 Utility functions handling encoding and timestamps etc. +//------------------------------------------------------------------------------ class S3Utils { public: S3Utils(); @@ -129,4 +152,3 @@ class S3Utils { } // namespace S3 -#endif // XROOTD_XRDS3UTILS_HH diff --git a/src/XrdS3/XrdS3Xml.cc b/src/XrdS3/XrdS3Xml.cc index fa9e3e88e7c..a5780c8dacc 100644 --- a/src/XrdS3/XrdS3Xml.cc +++ b/src/XrdS3/XrdS3Xml.cc @@ -1,21 +1,57 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/13/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ +//------------------------------------------------------------------------------ #include "XrdS3Xml.hh" +//------------------------------------------------------------------------------ namespace S3 { +//------------------------------------------------------------------------------ +// S3Xml +//------------------------------------------------------------------------------ S3Xml::S3Xml() : tinyxml2::XMLPrinter(nullptr, true) { PushDeclaration(R"(xml version="1.0" encoding="UTF-8")"); } +//------------------------------------------------------------------------------ +//! \brief Open an element with the given name +//! \param[in] elem The name of the element to open +//------------------------------------------------------------------------------ void S3Xml::OpenElement(const char *elem) { tinyxml2::XMLPrinter::OpenElement(elem, true); } +//------------------------------------------------------------------------------ +//! \brief Close the current element +//------------------------------------------------------------------------------ void S3Xml::CloseElement() { tinyxml2::XMLPrinter::CloseElement(true); } +//------------------------------------------------------------------------------ +//! \brief Add a text element to the current element +//! \param[in] value The text to add +//------------------------------------------------------------------------------ void S3Xml::AddElement(const char *key, const std::string &value) { OpenElement(key); if (!value.empty()) { diff --git a/src/XrdS3/XrdS3Xml.hh b/src/XrdS3/XrdS3Xml.hh index 0364d431547..d2d461820b5 100644 --- a/src/XrdS3/XrdS3Xml.hh +++ b/src/XrdS3/XrdS3Xml.hh @@ -1,13 +1,33 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2024 by European Organization for Nuclear Research (CERN) +// Author: Mano Segransan / CERN EOS Project +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. // -// Created by segransm on 11/13/23. +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. // +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ -#ifndef XROOTD_XRDS3XML_HH -#define XROOTD_XRDS3XML_HH +#pragma once +//------------------------------------------------------------------------------ #include - #include +//------------------------------------------------------------------------------ namespace S3 { @@ -31,4 +51,3 @@ class S3Xml : public tinyxml2::XMLPrinter { } // namespace S3 -#endif // XROOTD_XRDS3XML_HH