Skip to content

Commit 14bb81a

Browse files
authored
Buffered io nonstriperbuffer (#43)
* Add capability for buffer io raw to use striperless reads * Add capability for buffer io raw to use striperless reads * Add a maybe striper for reading in ceph posix * Use striperless reads when bypassing the buffer
1 parent 6a168c6 commit 14bb81a

File tree

7 files changed

+45
-10
lines changed

7 files changed

+45
-10
lines changed

src/XrdCeph/XrdCephBuffers/CephIOAdapterRaw.cc

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@ using namespace XrdCephBuffer;
1111
using myclock = std::chrono::steady_clock;
1212
//using myseconds = std::chrono::duration<float,
1313

14-
CephIOAdapterRaw::CephIOAdapterRaw(IXrdCephBufferData * bufferdata, int fd) :
15-
m_bufferdata(bufferdata),m_fd(fd) {
14+
CephIOAdapterRaw::CephIOAdapterRaw(IXrdCephBufferData * bufferdata, int fd,
15+
bool useStriperlessReads) :
16+
m_bufferdata(bufferdata),m_fd(fd),
17+
m_useStriperlessReads(useStriperlessReads) {
1618
}
1719

1820
CephIOAdapterRaw::~CephIOAdapterRaw() {
@@ -30,7 +32,9 @@ CephIOAdapterRaw::~CephIOAdapterRaw() {
3032
<< " write_MBs:" << write_speed
3133
<< " nread:" << m_stats_read_req << " bytesread:" << m_stats_read_bytes << " read_s:"
3234
<< m_stats_read_timer * 1e-3 << " readmax_s:" << m_stats_read_longest * 1e-3
33-
<< " read_MBs:" << read_speed );
35+
<< " read_MBs:" << read_speed
36+
<< " striperlessRead: " << m_useStriperlessReads
37+
);
3438

3539
}
3640

@@ -60,10 +64,11 @@ ssize_t CephIOAdapterRaw::read(off64_t offset, size_t count) {
6064
if (!buf) {
6165
return -EINVAL;
6266
}
67+
ssize_t rc {0};
6368

6469
// no check is made whether the buffer has sufficient capacity
6570
auto start = std::chrono::steady_clock::now();
66-
ssize_t rc = ceph_posix_pread(m_fd,buf,count,offset);
71+
rc = ceph_posix_maybestriper_pread(m_fd,buf,count,offset, m_useStriperlessReads);
6772
auto end = std::chrono::steady_clock::now();
6873
//auto elapsed = end-start;
6974
auto int_ms = std::chrono::duration_cast<std::chrono::milliseconds>(end-start);

src/XrdCeph/XrdCephBuffers/CephIOAdapterRaw.hh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ namespace XrdCephBuffer {
2727
*/
2828
class CephIOAdapterRaw: public virtual ICephIOAdapter {
2929
public:
30-
CephIOAdapterRaw(IXrdCephBufferData * bufferdata, int fd);
30+
CephIOAdapterRaw(IXrdCephBufferData * bufferdata, int fd,
31+
bool useStriperlessReads);
3132
virtual ~CephIOAdapterRaw();
3233

3334
/**
@@ -57,6 +58,7 @@ class CephIOAdapterRaw: public virtual ICephIOAdapter {
5758
private:
5859
IXrdCephBufferData * m_bufferdata; //!< no ownership of pointer (consider shared ptrs, etc)
5960
int m_fd;
61+
bool m_useStriperlessReads {true}; //!< use the striperless read code
6062

6163
// timer and counter info
6264
std::atomic< long> m_stats_read_timer{0}, m_stats_write_timer{0};

src/XrdCeph/XrdCephBuffers/XrdCephBufferAlgSimple.cc

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ using namespace XrdCephBuffer;
1818

1919

2020
XrdCephBufferAlgSimple::XrdCephBufferAlgSimple(std::unique_ptr<IXrdCephBufferData> buffer,
21-
std::unique_ptr<ICephIOAdapter> cephio, int fd ):
22-
m_bufferdata(std::move(buffer)), m_cephio(std::move(cephio)), m_fd(fd){
21+
std::unique_ptr<ICephIOAdapter> cephio, int fd,
22+
bool useStriperlessReads):
23+
m_bufferdata(std::move(buffer)), m_cephio(std::move(cephio)), m_fd(fd),
24+
m_useStriperlessReads(useStriperlessReads) {
2325

2426
}
2527

@@ -111,7 +113,8 @@ ssize_t XrdCephBufferAlgSimple::read(volatile void *buf, off_t offset, size_t
111113
m_bufferdata->invalidate();
112114
m_bufferLength =0; // ensure cached data is set to zero length
113115
// #FIXME JW: const_cast is probably a bit poor.
114-
ssize_t rc = ceph_posix_pread(m_fd, const_cast<void*>(buf), blen, offset);
116+
117+
ssize_t rc = ceph_posix_maybestriper_pread (m_fd, const_cast<void*>(buf), blen, offset, m_useStriperlessReads);
115118
if (rc > 0) {
116119
m_stats_bytes_fromceph += rc;
117120
m_stats_bytes_toclient += rc;

src/XrdCeph/XrdCephBuffers/XrdCephBufferAlgSimple.hh

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ namespace XrdCephBuffer {
2626

2727
class XrdCephBufferAlgSimple : public virtual IXrdCephBufferAlg {
2828
public:
29-
XrdCephBufferAlgSimple(std::unique_ptr<IXrdCephBufferData> buffer, std::unique_ptr<ICephIOAdapter> cephio, int fd );
29+
XrdCephBufferAlgSimple(std::unique_ptr<IXrdCephBufferData> buffer, std::unique_ptr<ICephIOAdapter> cephio, int fd,
30+
bool useStriperlessReads = true );
3031
virtual ~XrdCephBufferAlgSimple();
3132

3233
virtual ssize_t read_aio (XrdSfsAio *aoip) override;
@@ -49,6 +50,7 @@ class XrdCephBufferAlgSimple : public virtual IXrdCephBufferAlg {
4950
std::unique_ptr<IXrdCephBufferData> m_bufferdata; //! this algorithm takes ownership of the buffer, and will delete it on destruction
5051
std::unique_ptr<ICephIOAdapter> m_cephio ; // no ownership is taken here
5152
int m_fd = -1;
53+
bool m_useStriperlessReads {true};
5254

5355
off_t m_bufferStartingOffset = 0;
5456
size_t m_bufferLength = 0;

src/XrdCeph/XrdCephOssBufferedFile.cc

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -327,7 +327,8 @@ std::unique_ptr<XrdCephBuffer::IXrdCephBufferAlg> XrdCephOssBufferedFile::create
327327
if (m_bufferIOmode == "aio") {
328328
cephio = std::unique_ptr<ICephIOAdapter>(new CephIOAdapterAIORaw(cephbuffer.get(),m_fd));
329329
} else if (m_bufferIOmode == "io") {
330-
cephio = std::unique_ptr<ICephIOAdapter>(new CephIOAdapterRaw(cephbuffer.get(),m_fd));
330+
cephio = std::unique_ptr<ICephIOAdapter>(new CephIOAdapterRaw(cephbuffer.get(),m_fd,
331+
!m_cephoss->m_useDefaultPreadAlg));
331332
} else {
332333
BUFLOG("XrdCephOssBufferedFile: buffer mode needs to be one of aio|io " );
333334
m_xrdOssDF->Close();

src/XrdCeph/XrdCephPosix.cc

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1089,6 +1089,26 @@ ssize_t ceph_posix_pread(int fd, void *buf, size_t count, off64_t offset) {
10891089
}
10901090
}
10911091

1092+
ssize_t ceph_posix_maybestriper_pread(int fd, void *buf, size_t count, off64_t offset, bool allowStriper) {
1093+
ssize_t rc {0};
1094+
if (!allowStriper) {
1095+
rc = ceph_posix_pread(fd,buf,count,offset);
1096+
return rc;
1097+
}
1098+
rc = ceph_posix_nonstriper_pread(fd, buf, count,offset);
1099+
if (-ENOENT == rc || -ENOTSUP == rc) {
1100+
//This might be a sparse file or nbstripes > 1, so let's try striper read
1101+
rc = ceph_posix_pread(fd, buf, count,offset);
1102+
if (rc >= 0) {
1103+
char err_str[100]; //99 symbols should be enough for the short message
1104+
snprintf(err_str, 100, "WARNING! The file (fd %d) seem to be sparse, this is not expected", fd);
1105+
logwrapper(err_str);
1106+
}
1107+
}
1108+
return rc;
1109+
}
1110+
1111+
10921112
static void ceph_aio_read_complete(rados_completion_t c, void *arg) {
10931113
AioArgs *awa = reinterpret_cast<AioArgs*>(arg);
10941114
size_t rc = rados_aio_get_return_value(c);

src/XrdCeph/XrdCephPosix.hh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,8 @@ ssize_t ceph_striper_readv(int fd, XrdOucIOVec *readV, int n);
6969
ssize_t ceph_posix_read(int fd, void *buf, size_t count);
7070
ssize_t ceph_posix_nonstriper_pread(int fd, void *buf, size_t count, off64_t offset);
7171
ssize_t ceph_posix_pread(int fd, void *buf, size_t count, off64_t offset);
72+
ssize_t ceph_posix_maybestriper_pread(int fd, void *buf, size_t count, off64_t offset, bool allowStriper=true);
73+
7274
ssize_t ceph_aio_read(int fd, XrdSfsAio *aiop, AioCB *cb);
7375
int ceph_posix_fstat(int fd, struct stat *buf);
7476
int ceph_posix_stat(XrdOucEnv* env, const char *pathname, struct stat *buf);

0 commit comments

Comments
 (0)