From 4a231b73d3db9971545eade453016f8c5a531948 Mon Sep 17 00:00:00 2001 From: Wei Yang Date: Sat, 1 Jun 2019 01:20:12 -0700 Subject: [PATCH] use the new XrdPosixCache interface to query cache file info --- Makefile | 15 +++++++----- XrdOucName2NameDCP4RUCIO.cc | 3 +-- cacheQuery.cc | 48 +++++++++++++++++++++++++++++++++++++ cacheQuery.hh | 5 ++++ rucioGetMetaLink.cc | 27 +++++++++------------ rucioGetMetaLink.hh | 2 +- 6 files changed, 75 insertions(+), 25 deletions(-) create mode 100644 cacheQuery.cc create mode 100644 cacheQuery.hh diff --git a/Makefile b/Makefile index 17c58e0..d77159f 100644 --- a/Makefile +++ b/Makefile @@ -13,14 +13,14 @@ endif FLAGS=-D_REENTRANT -D_THREAD_SAFE -Wno-deprecated -std=c++0x -HEADERS=rucioGetMetaLink.hh pfn2cache.hh checkPFCcinfo.hh -SOURCES=XrdOucName2NameDCP4RUCIO.cc XrdOssStatInfoDCP.cc rucioGetMetaLink.cc pfn2cache.cc checkPFCcinfo.cc -OBJECTS=XrdOucName2NameDCP4RUCIO.o XrdOssStatInfoDCP.o rucioGetMetaLink.o pfn2cache.o checkPFCcinfo.o +HEADERS=rucioGetMetaLink.hh pfn2cache.hh cacheQuery.hh +SOURCES=XrdOucName2NameDCP4RUCIO.cc XrdOssStatInfoDCP.cc rucioGetMetaLink.cc pfn2cache.cc cacheQuery.cc +OBJECTS=XrdOucName2NameDCP4RUCIO.o XrdOssStatInfoDCP.o rucioGetMetaLink.o pfn2cache.o cacheQuery.o DEBUG=-g XrdName2NameDCP4RUCIO.so: $(OBJECTS) Makefile - g++ ${DEBUG} -shared -fPIC -o $@ $(OBJECTS) -L${XRD_LIB} -L${XRD_LIB}/XrdCl -ldl -lssl -lcurl -lXrdCl -lXrdFileCache-4 -lstdc++ + g++ ${DEBUG} -shared -fPIC -o $@ $(OBJECTS) -L${XRD_LIB} -L${XRD_LIB}/XrdCl -ldl -lssl -lcurl -lXrdCl -lXrdFileCache-4 -lXrdPosix -lstdc++ XrdOucName2NameDCP4RUCIO.o: XrdOucName2NameDCP4RUCIO.cc ${HEADERS} Makefile g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I ${XRD_LIB} -c -o $@ $< @@ -34,8 +34,11 @@ rucioGetMetaLink.o: rucioGetMetaLink.cc ${HEADERS} Makefile pfn2cache.o: pfn2cache.cc ${HEADERS} Makefile g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I ${XRD_LIB} -c -o $@ $< -checkPFCcinfo.o: checkPFCcinfo.cc ${HEADERS} Makefile - g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I /afs/slac/package/xrootd/githead/xrootd/src -c -o $@ $< +cacheQuery.o: cacheQuery.cc ${HEADERS} Makefile + g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I ${XRD_LIB} -c -o $@ $< + +#checkPFCcinfo.o: checkPFCcinfo.cc ${HEADERS} Makefile +# g++ ${DEBUG} ${FLAGS} -fPIC -I ${XRD_INC} -I ${XRD_LIB} -c -o $@ $< clean: rm -vf *.{o,so} diff --git a/XrdOucName2NameDCP4RUCIO.cc b/XrdOucName2NameDCP4RUCIO.cc index 6a9f75e..ad081f2 100644 --- a/XrdOucName2NameDCP4RUCIO.cc +++ b/XrdOucName2NameDCP4RUCIO.cc @@ -13,7 +13,6 @@ XrdVERSIONINFO(XrdOucgetName2Name, "N2N-Xcache4RUCIO"); #include "rucioGetMetaLink.hh" #include "pfn2cache.hh" -#include "checkPFCcinfo.hh" #include "XrdOuc/XrdOucEnv.hh" #include "XrdOuc/XrdOucName2Name.hh" #include "XrdSys/XrdSysPlatform.hh" @@ -99,7 +98,7 @@ XrdOucName2NameDiskCacheProxy4Rucio::XrdOucName2NameDiskCacheProxy4Rucio(XrdSysE message = myName + " Init: RUCIO metalink server = " + rucioServer; eDest->Say(message.c_str()); - rucioGetMetaLinkInit(localMetaLinkRootDir, gLFNprefix, rucioServer, checkPFCcinfoInit(confg)); + rucioGetMetaLinkInit(localMetaLinkRootDir, gLFNprefix, rucioServer); } int XrdOucName2NameDiskCacheProxy4Rucio::lfn2pfn(const char* lfn, char* buff, int blen) diff --git a/cacheQuery.cc b/cacheQuery.cc new file mode 100644 index 0000000..ce1ecc1 --- /dev/null +++ b/cacheQuery.cc @@ -0,0 +1,48 @@ +/* + * Author: Wei Yang (SLAC National Accelerator Laboratory / Stanford University, 2019) + */ + +using namespace std; + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "XrdVersion.hh" +#include "XrdOuc/XrdOucCacheCM.hh" +#include "XrdPosix/XrdPosixCache.hh" + +//______________________________________________________________________________ + + +XrdPosixCache *myCache; + +extern "C" { +XrdOucCacheCMInit_t XrdOucCacheCMInit(XrdPosixCache &Cache, + XrdSysLogger *Logger, + const char *Config, + const char *Parms, + XrdOucEnv *envP) +{ + myCache = &Cache; +} +}; +XrdVERSIONINFO(XrdOucCacheCMInit,CacheCM-4-RUCIO); + +int cachedFileIsComplete(std::string url, std::string *localfile) +{ + int rc; + char pfn[1024]; + char *myurl = strdup(url.c_str()); + rc = myCache->CacheQuery(myurl, 0); + myCache->CachePath(myurl, pfn, 1024); + free(myurl); + *localfile = pfn; + return rc; +} + diff --git a/cacheQuery.hh b/cacheQuery.hh new file mode 100644 index 0000000..ee8239c --- /dev/null +++ b/cacheQuery.hh @@ -0,0 +1,5 @@ +/* + * Author: Wei Yang (SLAC National Accelerator Laboratory / Stanford University, 2019) + */ + +int cachedFileIsComplete(std::string url, std::string *localpath); diff --git a/rucioGetMetaLink.cc b/rucioGetMetaLink.cc index b4fac32..b1dbc8c 100644 --- a/rucioGetMetaLink.cc +++ b/rucioGetMetaLink.cc @@ -16,7 +16,7 @@ using namespace std; #include #include #include -#include "checkPFCcinfo.hh" +#include "cacheQuery.hh" #include "pfn2cache.hh" #include "XrdCl/XrdClURL.hh" #include "XrdSys/XrdSysError.hh" @@ -36,7 +36,6 @@ std::string rucioServerCgi = "/metalink?schemes=root&sort=geoip"; #define MetaLinkLifeTsec MetaLinkLifeTmin*60 static std::string localMetaLinkRootDir; -static std::string ossLocalRoot; static std::string gLFNprefix; struct rucioMetaLink @@ -60,12 +59,11 @@ void cleaner() static int Xcache4RUCIO_DBG = 0; -void rucioGetMetaLinkInit(const std::string dir, const std::string glfnprefix, const std::string rucioserver, const std::string osslocalroot) +void rucioGetMetaLinkInit(const std::string dir, const std::string glfnprefix, const std::string rucioserver) { localMetaLinkRootDir = dir; gLFNprefix = glfnprefix; rucioServerUrl = "https://" + rucioserver + "/redirect/"; - ossLocalRoot = osslocalroot; std::thread cleanning(cleaner); cleanning.detach(); @@ -141,7 +139,7 @@ std::string makeMetaLink(XrdSysError* eDest, const std::string myName, const std metaLinkFile = metaLinkFile.replace(0, metaLinkFile.find("/"), ""); // remove loginid@hostnaem:port/ if (metaLinkFile.substr(0, 1) == "/") metaLinkFile.replace(0, 1, ""); // remove the next leading / - std::string cinfofile = "/" + pfn2cache("", gLFNprefix, metaLinkFile.c_str()) + ".cinfo"; + std::string cachefile = pfn2cache("", gLFNprefix, metaLinkFile.c_str()); metaLinkFile = localMetaLinkRootDir + "/" + metaLinkFile + ".metalink"; metaLinkDir = metaLinkFile; @@ -157,7 +155,8 @@ std::string makeMetaLink(XrdSysError* eDest, const std::string myName, const std if (! stat(metaLinkFile.c_str(), &statBuf)) return metaLinkFile; - if (checkPFCcinfoIsComplete(cinfofile)) + std::string localfile; + if (cachedFileIsComplete(cachefile, &localfile) == 1) { FILE *fd = fopen(metaLinkFile.c_str(), "w"); if (fd != NULL) @@ -165,8 +164,7 @@ std::string makeMetaLink(XrdSysError* eDest, const std::string myName, const std tmp = "\n"; tmp += "\n"; tmp += " \n"; - tmp += " file://localhost/" + ossLocalRoot - + cinfofile.substr(0, cinfofile.rfind(".cinfo"))+ "\n"; + tmp += " file://localhost/" + localfile + "\n"; tmp += " \n"; tmp += "\n"; @@ -231,11 +229,8 @@ std::string getMetaLink(XrdSysError* eDest, const std::string myName, const std: md5string[2*2+1] = '\0'; tmp = md5string; - std::string cinfofile; - - cinfofile = gLFNprefix + "/" + slashScope + "/" - + tmp.substr(0, 2) + "/" + tmp.substr(2, 2) + "/" - + file + ".cinfo"; + std::string cachefile = gLFNprefix + "/" + slashScope + "/" + + tmp.substr(0, 2) + "/" + tmp.substr(2, 2) + "/" + file; metaLinkDir = localMetaLinkRootDir + gLFNprefix + "/" + slashScope + "/" + tmp.substr(0, 2) + "/" + tmp.substr(2, 2); @@ -249,7 +244,8 @@ std::string getMetaLink(XrdSysError* eDest, const std::string myName, const std: if (! stat(metaLinkFile.c_str(), &statBuf)) return metaLinkFile; - if (checkPFCcinfoIsComplete(cinfofile)) + std::string localfile; + if (cachedFileIsComplete(cachefile, &localfile) == 1) { FILE *fd = fopen(metaLinkFile.c_str(), "w"); if (fd != NULL) @@ -257,8 +253,7 @@ std::string getMetaLink(XrdSysError* eDest, const std::string myName, const std: tmp = "\n"; tmp += "\n"; tmp += " \n"; - tmp += " file://localhost/" + ossLocalRoot - + cinfofile.substr(0, cinfofile.rfind(".cinfo"))+ "\n"; + tmp += " file://localhost/" + localfile + "\n"; tmp += " \n"; tmp += "\n"; diff --git a/rucioGetMetaLink.hh b/rucioGetMetaLink.hh index 6f8db86..2c57da8 100644 --- a/rucioGetMetaLink.hh +++ b/rucioGetMetaLink.hh @@ -4,6 +4,6 @@ #include "XrdSys/XrdSysError.hh" -void rucioGetMetaLinkInit(const std::string, const std::string, const std::string, const std::string); +void rucioGetMetaLinkInit(const std::string, const std::string, const std::string); string getMetaLink(XrdSysError*, const std::string, const std::string); string makeMetaLink(XrdSysError*, const std::string, const std::string);