XrdS3: support 'virtual' directory objects tagged by content-type from clients

Author: apeters1971

src/XrdS3/XrdS3Api.cc

@@ -1,3 +1,4 @@
+
 //------------------------------------------------------------------------------
 // Copyright (c) 2024 by European Organization for Nuclear Research (CERN)
 // Author: Mano Segransan / CERN EOS Project <[email protected]>
@@ -444,18 +445,17 @@ int S3Api::ListObjectVersionsHandler(S3::XrdS3Req& req) {
 //------------------------------------------------------------------------------
 int S3Api::CopyObjectHandler(XrdS3Req& req) {
   VALIDATE_REQUEST(Action::CopyObject)
-
   auto source =
       req.ctx->utils.UriDecode(req.lowercase_headers["x-amz-copy-source"]);
-  auto pos = source.find('/');
+  auto pos = source.find('/',1);
   if (pos == std::string::npos) {
     return req.S3ErrorResponse(S3Error::InvalidArgument);
   }
   auto bucket_src = source.substr(0, pos);
   auto object_src = source.substr(pos + 1);
-
   auto [error, source_bucket] =
       auth.ValidateRequest(req, Action::GetObject, bucket_src, object_src);
+
   if (error != S3Error::None) {
     return req.S3ErrorResponse(error);
   }
@@ -467,9 +467,10 @@ int S3Api::CopyObjectHandler(XrdS3Req& req) {
   S3ObjectStore::Object obj;
   RET_ON_ERROR(objectStore.GetObject(source_bucket, object_src, obj))
 
-  if (!S3Utils::MapHasKey(obj.GetAttributes(), "etag")) {
-    return req.S3ErrorResponse(S3Error::InternalError);
-  }
+  // TODO: fix me, we disable this for the time being
+    //  if (!S3Utils::MapHasKey(obj.GetAttributes(), "etag")) {
+    //    return req.S3ErrorResponse(S3Error::InternalError);
+    //  }
 
   //  std::string etag = headers["etag"];
   //  time_t last_modified = std::stoul(headers["last-modified"]);
@@ -486,6 +487,7 @@ int S3Api::CopyObjectHandler(XrdS3Req& req) {
   Headers hd = {{"Content-Type", "application/xml"}};
   req.StartChunkedResp(200, hd);
 
+  std::cerr << "copyObject" << std::endl;
   std::map<std::string, std::string> headers;
   err = objectStore.CopyObject(bucket, req.object, obj, req.lowercase_headers,
                                headers);
@@ -546,7 +548,8 @@ int S3Api::PutObjectHandler(XrdS3Req& req) {
   }
 
   std::map<std::string, std::string> headers;
-  RET_ON_ERROR(objectStore.PutObject(req, bucket, length, chunked, headers))
+
+  RET_ON_ERROR(objectStore.PutObject(req, bucket, length, chunked, req.lowercase_headers))
 
   return req.S3Response(200, headers, "");
 }
@@ -577,7 +580,13 @@ int S3Api::HeadObjectHandler(XrdS3Req& req) {
 
   // content length is added in S3Response
   headers["Last-Modified"] = S3Utils::timestampToRFC7231(last_modified);
-  headers["Accept-Ranges"] = "\"bytes\"";
+  headers["Accept-Ranges"] = "bytes";
+
+  // add directory flag , if the object has a / in the end
+  if (obj.Directory()) {
+    headers["Content-Type"] = "application/x-directory";
+  }
+
   return req.S3Response(200, headers, nullptr, content_length);
 }
 

src/XrdS3/XrdS3Auth.cc

@@ -310,7 +310,6 @@ std::pair<S3Error, S3Auth::Bucket> S3Auth::ValidateRequest(
   if (err != S3Error::None) {
     return {err, {}};
   }
-
   return AuthorizeRequest(req, action, bucket, object);
 }
 

src/XrdS3/XrdS3ObjectStore.cc

@@ -355,13 +355,15 @@ S3Error S3ObjectStore::Object::Init(const std::filesystem::path &p, uid_t uid,
 
   // Do the backend operations with the users filesystem id
   ScopedFsId scope(uid, gid, id);
-  if (XrdPosix_Stat(p.c_str(), &buf) || S_ISDIR(buf.st_mode)) {
+  if (XrdPosix_Stat(p.c_str(), &buf)) {
     S3::S3Handler::Logger()->Log(S3::ERROR, "ObjectStore::Object::Init",
                                  "no such object - object-path=%s owner(%u:%u)",
                                  p.c_str(), uid, gid);
     return S3Error::NoSuchKey;
   }
 
+  // Flag directories
+  this->directory = S_ISDIR(buf.st_mode);
   std::vector<std::string> attrnames;
   std::vector<char> attrlist;
   auto attrlen = XrdPosix_Listxattr(p.c_str(), nullptr, 0);
@@ -464,16 +466,21 @@ S3Error S3ObjectStore::GetObject(const S3Auth::Bucket &bucket,
 S3Error S3ObjectStore::DeleteObject(const S3Auth::Bucket &bucket,
                                     const std::string &key) {
   std::string base, obj;
-
+  ScopedFsId scope(bucket.owner.uid, bucket.owner.gid, bucket.owner.id);
   auto full_path = bucket.path / key;
   S3::S3Handler::Logger()->Log(S3::DEBUG, "ObjectStore::DeleteObject",
                                "object-path=%s", full_path.c_str());
 
   if (XrdPosix_Unlink(full_path.c_str())) {
-    S3::S3Handler::Logger()->Log(S3::ERROR, "ObjectStore::DeleteObject",
-                                 "failed to delete object-path=%s",
-                                 full_path.c_str());
-    return S3Error::NoSuchKey;
+    if (errno == EISDIR) {
+      if (XrdPosix_Rmdir(full_path.c_str())) {
+	// TODO: error handling
+	S3::S3Handler::Logger()->Log(S3::ERROR, "ObjectStore::DeleteObject",
+				     "failed to delete object-path=%s",
+				     full_path.c_str());
+	return S3Error::NoSuchKey;
+      }
+    }
   }
 
   do {
@@ -564,7 +571,7 @@ S3Error S3ObjectStore::CopyObject(const S3Auth::Bucket &bucket,
   auto final_path = bucket.path / key;
 
   struct stat buf;
-  if (!XrdPosix_Stat(final_path.c_str(), &buf) && S_ISDIR(buf.st_mode)) {
+  if (!XrdPosix_Stat(final_path.c_str(), &buf)) {
     S3::S3Handler::Logger()->Log(
         S3::ERROR, "ObjectStore::CopyObject",
         "target:%s is directory => bucket:%s key:%s src=:%s",
@@ -576,6 +583,11 @@ S3Error S3ObjectStore::CopyObject(const S3Auth::Bucket &bucket,
       bucket.path / ("." + key + "." + std::to_string(std::time(nullptr)) +
                      std::to_string(std::rand()));
 
+  if (reqheaders.count("x-amz-metadata-directive") && reqheaders.at("x-amz-metadata-directive") == "REPLACE") {
+    // do the meta-data replacement
+    return S3Error::None;
+  }
+
   auto err = S3Utils::makePath((char *)final_path.parent_path().c_str(),
                                S_IRWXU | S_IRWXG);
 
@@ -1149,6 +1161,26 @@ S3Error S3ObjectStore::PutObject(XrdS3Req &req, const S3Auth::Bucket &bucket,
     return S3Error::ObjectExistAsDir;
   }
 
+  // special treatment for 'non-standard' directory creation
+  auto it = headers.find("content-type");
+  if (it != headers.end() && it->second == "application/x-directory") {
+    // this is a 'virtual 'directory creation request
+    auto err = S3Utils::makePath((char *)final_path.c_str(),
+				 S_IRWXU | S_IRGRP);
+    if (err == ENOTDIR) {
+      S3::S3Handler::Logger()->Log(S3::ERROR, "ObjectStore::PutObject",
+				   "object exists in object path -  path:%s",
+				   final_path.c_str());
+      return S3Error::ObjectExistInObjectPath;
+    } else if (err != 0) {
+      S3::S3Handler::Logger()->Log(S3::ERROR, "ObjectStore::PutObject",
+				   "internal error makeing parent : path:%s",
+				   final_path.c_str());
+      return S3Error::InternalError;
+    }
+    return S3Error::None;
+  }
+
   auto tmp_path =
       final_path.parent_path() /
       ("." + final_path.filename().string() + "." +
@@ -1375,6 +1407,7 @@ ListObjectsInfo S3ObjectStore::ListObjectsCommon(
   std::vector<S3Utils::BasicPath> ent;
   std::deque<S3Utils::BasicPath> entries;
 
+  ScopedFsId scope(bucket.owner.uid, bucket.owner.gid, bucket.owner.id);
   int n;
   // get full listing
   if ((n = S3Utils::ScanDir((fullpath/basedir), basedir, ent)) <= 0) {

src/XrdS3/XrdS3ObjectStore.hh

@@ -106,6 +106,7 @@ class S3ObjectStore {
     ssize_t Read(size_t length, char **data);
     off_t Lseek(off_t offset, int whence);
     std::string Name() const { return name; }
+    bool Directory() const { return directory; }
 
     const std::map<std::string, std::string> &GetAttributes() const {
       return attributes;
@@ -123,6 +124,7 @@ class S3ObjectStore {
     uid_t uid;
     gid_t gid;
     std::string id;
+    bool directory;
     std::map<std::string, std::string> attributes{};
   };