Skip to content

Commit

Permalink
avoid open remote data source for fully cached files
Browse files Browse the repository at this point in the history
  • Loading branch information
wyang007 committed Sep 11, 2018
1 parent 587fc46 commit 042f711
Show file tree
Hide file tree
Showing 7 changed files with 207 additions and 27 deletions.
11 changes: 7 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,14 @@ endif

FLAGS=-D_REENTRANT -D_THREAD_SAFE -Wno-deprecated -std=c++0x

HEADERS=rucioGetMetaLink.hh pfn2cache.hh
SOURCES=XrdOucName2NameDCP4RUCIO.cc XrdOssStatInfoDCP.cc rucioGetMetaLink.cc pfn2cache.cc
OBJECTS=XrdOucName2NameDCP4RUCIO.o XrdOssStatInfoDCP.o rucioGetMetaLink.o pfn2cache.o
HEADERS=rucioGetMetaLink.hh pfn2cache.hh checkPFCcinfo.hh
SOURCES=XrdOucName2NameDCP4RUCIO.cc XrdOssStatInfoDCP.cc rucioGetMetaLink.cc pfn2cache.cc checkPFCcinfo.cc
OBJECTS=XrdOucName2NameDCP4RUCIO.o XrdOssStatInfoDCP.o rucioGetMetaLink.o pfn2cache.o checkPFCcinfo.o

DEBUG=-g

XrdName2NameDCP4RUCIO.so: $(OBJECTS) Makefile
g++ ${DEBUG} -shared -fPIC -o $@ $(OBJECTS) -ldl -lssl -lcurl -lXrdCl -lstdc++
g++ ${DEBUG} -shared -fPIC -o $@ $(OBJECTS) -ldl -lssl -lcurl -lXrdCl -lXrdFileCache-4 -lstdc++

XrdOucName2NameDCP4RUCIO.o: XrdOucName2NameDCP4RUCIO.cc ${HEADERS} Makefile
g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I ${XRD_LIB} -c -o $@ $<
Expand All @@ -34,6 +34,9 @@ rucioGetMetaLink.o: rucioGetMetaLink.cc ${HEADERS} Makefile
pfn2cache.o: pfn2cache.cc ${HEADERS} Makefile
g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I ${XRD_LIB} -c -o $@ $<

checkPFCcinfo.o: checkPFCcinfo.cc ${HEADERS} Makefile
g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I /afs/slac/package/xrootd/githead/xrootd/src -c -o $@ $<

clean:
rm -vf *.{o,so}

Expand Down
13 changes: 8 additions & 5 deletions XrdOucName2NameDCP4RUCIO.cc
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ XrdVERSIONINFO(XrdOucgetName2Name, "N2N-DCP4RUCIO");

#include "rucioGetMetaLink.hh"
#include "pfn2cache.hh"
#include "checkPFCcinfo.hh"
#include "XrdOuc/XrdOucEnv.hh"
#include "XrdOuc/XrdOucName2Name.hh"
#include "XrdSys/XrdSysPlatform.hh"
Expand All @@ -25,7 +26,7 @@ class XrdOucName2NameDiskCacheProxy4Rucio : public XrdOucName2Name
virtual int lfn2rfn(const char* lfn, char* buff, int blen);
virtual int pfn2lfn(const char* lfn, char* buff, int blen);

XrdOucName2NameDiskCacheProxy4Rucio(XrdSysError *erp, const char* parms);
XrdOucName2NameDiskCacheProxy4Rucio(XrdSysError *erp, const char* confg, const char* parms);
virtual ~XrdOucName2NameDiskCacheProxy4Rucio() {};

friend XrdOucName2Name *XrdOucgetName2Name(XrdOucgetName2NameArgs);
Expand All @@ -35,7 +36,9 @@ class XrdOucName2NameDiskCacheProxy4Rucio : public XrdOucName2Name
bool isCmsd;
};

XrdOucName2NameDiskCacheProxy4Rucio::XrdOucName2NameDiskCacheProxy4Rucio(XrdSysError* erp, const char* parms)
XrdOucName2NameDiskCacheProxy4Rucio::XrdOucName2NameDiskCacheProxy4Rucio(XrdSysError* erp,
const char* confg,
const char* parms)
{
std::string myProg;
std::string opts, message, key, value;
Expand Down Expand Up @@ -81,7 +84,7 @@ XrdOucName2NameDiskCacheProxy4Rucio::XrdOucName2NameDiskCacheProxy4Rucio(XrdSysE
if (x == 1) value += *it;
}
}
rucioGetMetaLinkInit(localMetaLinkRootDir);
rucioGetMetaLinkInit(localMetaLinkRootDir, checkPFCcinfoInit(confg));
}

int XrdOucName2NameDiskCacheProxy4Rucio::lfn2pfn(const char* lfn, char* buff, int blen)
Expand All @@ -105,7 +108,7 @@ int XrdOucName2NameDiskCacheProxy4Rucio::lfn2pfn(const char* lfn, char* buff, in
// see comments in pfn2cache()

if (myLfn.find("/root:/") == 0)
myPfn = makeMetaLink(lfn); // Assume the client know the data source...
myPfn = makeMetaLink(eDest, myName, lfn); // Assume the client know the data source...
else
{
i = myLfn.rfind("/atlas/rucio");
Expand Down Expand Up @@ -156,7 +159,7 @@ XrdOucName2Name *XrdOucgetName2Name(XrdOucgetName2NameArgs)

if (inst) return (XrdOucName2Name *)inst;

inst = new XrdOucName2NameDiskCacheProxy4Rucio(eDest, parms);
inst = new XrdOucName2NameDiskCacheProxy4Rucio(eDest, confg, parms);
if (!inst) return NULL;

return (XrdOucName2Name *)inst;
Expand Down
91 changes: 91 additions & 0 deletions checkPFCcinfo.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// to compile g++ -o x.exe x.cc -I/afs/slac/package/xrootd/githead/xrootd/src -lXrdFileCache-4 -lXrdXrootd-4
//
//
using namespace std;

#include <stdio.h>
#include <string>
#include <errno.h>
#include <iostream>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include "XrdFileCache/XrdFileCachePrint.hh"
#include "XrdOuc/XrdOucEnv.hh"
#include "XrdOuc/XrdOucStream.hh"
#include "XrdOuc/XrdOucArgs.hh"
#include "XrdSys/XrdSysTrace.hh"
#include "XrdOfs/XrdOfsConfigPI.hh"
#include "XrdSys/XrdSysLogger.hh"
#include "XrdFileCache/XrdFileCacheInfo.hh"
#include "XrdOss/XrdOssApi.hh"

//______________________________________________________________________________

std::string LocalRoot;
XrdOss *oss;

std::string checkPFCcinfoInit(const char* confg)
{
XrdOucEnv myEnv;
XrdSysLogger log;
XrdSysError err(&log);

XrdOucStream Config(&err, NULL, &myEnv, "=====> ");

// suppress oss init messages
int efs = open("/dev/null",O_RDWR, 0);
XrdSysLogger ossLog(efs);
XrdSysError ossErr(&ossLog, "print");
XrdOfsConfigPI *ofsCfg = XrdOfsConfigPI::New(0,&Config,&ossErr);
bool ossSucc = ofsCfg->Load(XrdOfsConfigPI::theOssLib);
if (ossSucc)
ofsCfg->Plugin(oss);
else
oss = NULL;

int fd = open(confg, O_RDONLY, 0);
Config.Attach(fd);
int n = Config.FDNum();
char *var;
while((var = Config.GetFirstWord()))
{
if (! strncmp(var,"oss.localroot", strlen("oss.localroot")))
{
LocalRoot = Config.GetWord();
return LocalRoot;
}
}
return NULL;
}

bool checkPFCcinfoIsComplete(std::string cinfofile)
{
bool rc;
XrdOucEnv myEnv;

if (oss == NULL) return false;

XrdOssFile* fh = (XrdOssFile*) oss->newFile("nobody");
fh->Open(cinfofile.c_str(), O_RDONLY, 0600, myEnv);

struct stat stBuf;
if (fh->Fstat(&stBuf)) return false;

if (fh)
{
XrdSysTrace tr(""); tr.What = 2;
XrdFileCache::Info cfi(&tr);

if (! cfi.Read(fh))
rc = false;
else
rc = cfi.IsComplete();
fh->Close();
}
else
rc = false;
delete(fh);
return rc;
}
2 changes: 2 additions & 0 deletions checkPFCcinfo.hh
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
std::string checkPFCcinfoInit(const char* confg);
bool checkPFCcinfoIsComplete(std::string cinfofile);
11 changes: 9 additions & 2 deletions pfn2cache.cc
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,22 @@ std::string pfn2cache(const std::string localMetaLinkRootDir, const char* pfn)
char md5string[MD5_DIGEST_LENGTH*2+1];

myPfn = pfn;
// if myPfn starts with "localMetaLinkRootDir" and ends with ".meta4", this function is called after lfn2pfn
// and we need to remove the starting "localMetaLinkRootDir" and tailing ".meta4"
// if myPfn starts with "localMetaLinkRootDir" and ends with ".meta4" or ".metalink", this function is
// called after lfn2pfn and we need to remove the starting "localMetaLinkRootDir" and tailing ".meta4"
if (myPfn.find(localMetaLinkRootDir) == 0 &&
myPfn.rfind(".meta4") == (myPfn.length() - 6))
{
myPfn.replace(0, localMetaLinkRootDir.length(), "");
myPfn.replace(myPfn.rfind(".meta4"), 6, "");
}

if (myPfn.find(localMetaLinkRootDir) == 0 &&
myPfn.rfind(".metalink") == (myPfn.length() - 9))
{
myPfn.replace(0, localMetaLinkRootDir.length(), "");
myPfn.replace(myPfn.rfind(".metalink"), 9, "");
}

i = myPfn.rfind("/rucio/");
// if pfn doesn't have "/rucio/", then rucioDID = pfn
// buff = pfn
Expand Down
102 changes: 88 additions & 14 deletions rucioGetMetaLink.cc
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ using namespace std;
#include <openssl/md5.h>
#include <string>
#include <thread>
#include "checkPFCcinfo.hh"
#include "pfn2cache.hh"
#include "XrdCl/XrdClURL.hh"
#include "XrdSys/XrdSysError.hh"

Expand All @@ -40,6 +42,7 @@ struct rucioMetaLink
};

std::string localMetaLinkRootDir;
std::string ossLocalRoot;

void cleaner()
{
Expand All @@ -56,9 +59,10 @@ void cleaner()

static int Xcache4RUCIO_DBG = 0;

void rucioGetMetaLinkInit(const std::string dir)
void rucioGetMetaLinkInit(const std::string dir, const std::string osslocalroot)
{
localMetaLinkRootDir = dir;
ossLocalRoot = osslocalroot;

std::thread cleanning(cleaner);
cleanning.detach();
Expand Down Expand Up @@ -109,7 +113,11 @@ int mkdir_p(const std::string dir)

// Both makeMetaLink() and getMetaLink() should return metaLinkFile even if we can't create or download
// a metalink. A file-not-found error will eventually be gerenated.
std::string makeMetaLink(const std::string pfn)

// Use .meta4 for files that have not been fully cached.
// Use .metalink for files that have been completely cached. In this case, a .metalink contains only one
// data source at file://localhost//... so that we don't need to Open() with a remote data source.
std::string makeMetaLink(XrdSysError* eDest, const std::string myName, const std::string pfn)
{
std::string metaLinkDir, metaLinkFile, myPfn, tmp;
size_t i;
Expand All @@ -129,11 +137,43 @@ std::string makeMetaLink(const std::string pfn)
metaLinkFile = metaLinkFile.replace(0, 7, ""); // remote "root://"
metaLinkFile = metaLinkFile.replace(0, metaLinkFile.find("/")+2, ""); // remote loginid@hostnaem:port//
}
metaLinkFile = localMetaLinkRootDir + "/" + metaLinkFile + ".meta4";

std::string cinfofile = "/" + pfn2cache("", metaLinkFile.c_str()) + ".cinfo";
metaLinkFile = localMetaLinkRootDir + "/" + metaLinkFile + ".metalink";

metaLinkDir = metaLinkFile;
i = metaLinkDir.rfind("/");
metaLinkDir.replace(i, metaLinkDir.length() - i+1, "");
if (mkdir_p(metaLinkDir)) return metaLinkFile;
if (mkdir_p(metaLinkDir))
{
eDest->Say((myName + ": Fail to create metalink dir " + metaLinkDir).c_str());
return metaLinkFile;
}

struct stat statBuf;
if (! stat(metaLinkFile.c_str(), &statBuf))
return metaLinkFile;

if (checkPFCcinfoIsComplete(cinfofile))
{
FILE *fd = fopen(metaLinkFile.c_str(), "w");
if (fd != NULL)
{
tmp = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
tmp += "<metalink xmlns=\"urn:ietf:params:xml:ns:metalink\">\n";
tmp += " <file name=\"x\">\n";
tmp += " <url location=\"LOCAL\" priority=\"1\">file://localhost/" + ossLocalRoot
+ cinfofile.substr(0, cinfofile.rfind(".cinfo"))+ "</url>\n";
tmp += " </file>\n";
tmp += "</metalink>\n";

fprintf(fd, "%s", tmp.c_str());
fclose(fd);
}
return metaLinkFile;
}
else
metaLinkFile = metaLinkFile.replace(metaLinkFile.rfind(".metalink"), 9, ".meta4");

FILE *fd = fopen(metaLinkFile.c_str(), "w");
if (fd != NULL)
Expand Down Expand Up @@ -181,14 +221,52 @@ std::string getMetaLink(XrdSysError* eDest, const std::string myName, const std:
rucioDID = scope + ":" + file;
MD5_Update(&c, rucioDID.c_str(), rucioDID.length());
MD5_Final(md5digest, &c);
for(i = 0; i < MD5_DIGEST_LENGTH; ++i)
// for(i = 0; i < MD5_DIGEST_LENGTH; ++i)
for(i = 0; i < 2; ++i)
sprintf(&md5string[i*2], "%02x", (unsigned int)md5digest[i]);
md5string[MD5_DIGEST_LENGTH*2+1] = '\0';
// md5string[MD5_DIGEST_LENGTH*2+1] = '\0';
md5string[2*2+1] = '\0';
tmp = md5string;

std::string cinfofile;

cinfofile = "/atlas/rucio/" + slashScope + "/"
+ tmp.substr(0, 2) + "/" + tmp.substr(2, 2) + "/"
+ file + ".cinfo";

metaLinkDir = localMetaLinkRootDir + "/atlas/rucio/" + slashScope + "/"
+ tmp.substr(0, 2) + "/" + tmp.substr(2, 2);
metaLinkFile = metaLinkDir + "/" + file + ".meta4";
if (mkdir_p(metaLinkDir))
{
eDest->Say((myName + ": Fail to create metalink dir " + metaLinkDir).c_str());
return metaLinkFile;
}
metaLinkFile = metaLinkDir + "/" + file + ".metalink";

if (! stat(metaLinkFile.c_str(), &statBuf))
return metaLinkFile;

if (checkPFCcinfoIsComplete(cinfofile))
{
FILE *fd = fopen(metaLinkFile.c_str(), "w");
if (fd != NULL)
{
tmp = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n";
tmp += "<metalink xmlns=\"urn:ietf:params:xml:ns:metalink\">\n";
tmp += " <file name=\"x\">\n";
tmp += " <url location=\"LOCAL\" priority=\"1\">file://localhost/" + ossLocalRoot
+ cinfofile.substr(0, cinfofile.rfind(".cinfo"))+ "</url>\n";
tmp += " </file>\n";
tmp += "</metalink>\n";

fprintf(fd, "%s", tmp.c_str());
fclose(fd);
}
return metaLinkFile;
}
else
metaLinkFile = metaLinkFile.replace(metaLinkFile.rfind(".metalink"), 9, ".meta4");
// metaLinkFile = metaLinkDir + "/" + file + ".meta4";

time_t t_now = time(NULL);
if (stat(metaLinkFile.c_str(), &statBuf) == 0 &&
Expand All @@ -197,11 +275,6 @@ std::string getMetaLink(XrdSysError* eDest, const std::string myName, const std:
return metaLinkFile;
}

if (mkdir_p(metaLinkDir))
{
eDest->Say((myName + ": Fail to create metalink dir " + metaLinkDir).c_str());
return metaLinkFile;
}
rucioMetaLinkURL = rucioServerUrl + scope + "/" + file + rucioServerCgi;

// -f prevent an output to be created if DID doesn't exist
Expand Down Expand Up @@ -237,8 +310,9 @@ std::string getMetaLink(XrdSysError* eDest, const std::string myName, const std:
// check for errors
if(res == CURLE_OK)
{
if (! strncmp(chunk.data, "HTTP/1.1 200 OK", 15) ||
! strncmp(chunk.data + chunk.size - 11, "</metalink>", 11)) // simple sanity check
if (! strncmp(chunk.data, "HTTP/1.1 200 OK", 15) &&
(! strncmp(chunk.data + chunk.size - 11, "</metalink>", 11) || // simple sanity check
! strncmp(chunk.data + chunk.size - 12, "</metalink>", 11))) // maybe a "\n" at the end?
{
FILE *fd = fopen(metaLinkFile.c_str(), "w");
if (fd != NULL)
Expand Down
4 changes: 2 additions & 2 deletions rucioGetMetaLink.hh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,6 @@

#include "XrdSys/XrdSysError.hh"

void rucioGetMetaLinkInit(const std::string);
void rucioGetMetaLinkInit(const std::string, const std::string);
string getMetaLink(XrdSysError*, const std::string, const std::string);
string makeMetaLink(const std::string);
string makeMetaLink(XrdSysError*, const std::string, const std::string);

0 comments on commit 042f711

Please sign in to comment.