XrdApps:JCache : add python and C++ cache cleaner execs - add journal…

… cache implementation to cache simple reads - add option to enable journal and readv caching on demand

src/XrdApps/XrdClJCachePlugin/CacheCleaner.py

@@ -0,0 +1,90 @@
+#/usr/bin/pythyon3 
+##------------------------------------------------------------------------------
+## Copyright (c) 2024 by European Organization for Nuclear Research (CERN)
+## Author: Andreas-Joachim Peters <[email protected]>
+##------------------------------------------------------------------------------
+## This file is part of the XRootD software suite.
+##
+## XRootD is free software: you can redistribute it and/or modify
+## it under the terms of the GNU Lesser General Public License as published by
+## the Free Software Foundation, either version 3 of the License, or
+## (at your option) any later version.
+##
+## XRootD is distributed in the hope that it will be useful,
+## but WITHOUT ANY WARRANTY; without even the implied warranty of
+## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+## GNU General Public License for more details.
+##
+## You should have received a copy of the GNU Lesser General Public License
+## along with XRootD.  If not, see <http://www.gnu.org/licenses/>.
+##
+## In applying this licence, CERN does not waive the privileges and immunities
+## granted to it by virtue of its status as an Intergovernmental Organization
+## or submit itself to any jurisdiction.
+
+import os
+import time
+import argparse
+
+def get_directory_size(directory):
+    """Calculate the total size of all files in the directory subtree."""
+    total_size = 0
+    for dirpath, dirnames, filenames in os.walk(directory):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            if os.path.isfile(fp):
+                total_size += os.path.getsize(fp)
+    return total_size
+
+def get_files_by_access_time(directory):
+    """Get a list of files sorted by their access time (oldest first)."""
+    file_list = []
+    for dirpath, dirnames, filenames in os.walk(directory):
+        for f in filenames:
+            fp = os.path.join(dirpath, f)
+            if os.path.isfile(fp):
+                access_time = os.path.getatime(fp)
+                file_list.append((access_time, fp))
+    file_list.sort()  # Sort by access time (oldest first)
+    return file_list
+
+def clean_directory(directory, high_watermark, low_watermark):
+    """Clean the directory by deleting files until the size is below the low watermark."""
+    current_size = get_directory_size(directory)
+    if current_size <= high_watermark:
+        print("Directory size is within the limit. No action needed.")
+        return
+
+    files = get_files_by_access_time(directory)
+
+    for access_time, file_path in files:
+        if current_size <= low_watermark:
+            break
+        file_size = os.path.getsize(file_path)
+        try:
+            os.remove(file_path)
+            current_size -= file_size
+            print(f"Deleted: {file_path} (Size: {file_size} bytes)")
+        except Exception as e:
+            print(f"Error deleting {file_path}: {e}")
+
+def main():
+    parser = argparse.ArgumentParser(description="Directory size monitor and cleaner.")
+    parser.add_argument("directory", type=str, help="Directory to monitor and clean.")
+    parser.add_argument("highwatermark", type=int, help="High watermark in bytes.")
+    parser.add_argument("lowwatermark", type=int, help="Low watermark in bytes.")
+    parser.add_argument("interval", type=int, help="Interval time in seconds between checks.")
+
+    args = parser.parse_args()
+
+    directory = args.directory
+    high_watermark = args.highwatermark
+    low_watermark = args.lowwatermark
+    interval = args.interval
+
+    while True:
+        clean_directory(directory, high_watermark, low_watermark)
+        time.sleep(interval)
+
+if __name__ == "__main__":
+    main()

src/XrdApps/XrdClJCachePlugin/XrdClCacheCleaner.cc

@@ -0,0 +1,129 @@
+//------------------------------------------------------------------------------
+// Copyright (c) 2024 by European Organization for Nuclear Research (CERN)
+// Author: Andreas-Joachim Peters <[email protected]>
+//------------------------------------------------------------------------------
+// This file is part of the XRootD software suite.
+//
+// XRootD is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// XRootD is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with XRootD.  If not, see <http://www.gnu.org/licenses/>.
+//
+// In applying this licence, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+#include <iostream>
+#include <filesystem>
+#include <chrono>
+#include <thread>
+#include <vector>
+#include <algorithm>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <ctime>
+#include <iomanip>
+
+namespace fs = std::filesystem;
+
+void printCurrentTime() {
+    auto now = std::chrono::system_clock::now();
+    auto now_ns = std::chrono::time_point_cast<std::chrono::nanoseconds>(now);
+    auto now_ns_since_epoch = now_ns.time_since_epoch();
+    auto seconds = std::chrono::duration_cast<std::chrono::seconds>(now_ns_since_epoch);
+    auto nanoseconds = std::chrono::duration_cast<std::chrono::nanoseconds>(now_ns_since_epoch - seconds);
+
+    auto now_t = std::chrono::system_clock::to_time_t(now);
+    struct tm* tm = std::localtime(&now_t);
+
+    int year = tm->tm_year - 100; // tm_year represents years since 1900
+
+    std::cout << std::setfill('0') << std::setw(2) << year << std::setfill('0') << std::setw(4) << (tm->tm_mon + 1) * 100 + tm->tm_mday << " ";
+    std::cout << std::setw(2) << std::setfill('0') << tm->tm_hour << ":" << std::setw(2) << tm->tm_min << ":" << std::setw(2) << tm->tm_sec << " ";
+    std::cout << "time=" << seconds.count() << "." << std::setw(9) << std::setfill('0') << nanoseconds.count() << " ";
+}
+
+time_t getLastAccessTime(const fs::path& filePath) {
+    struct stat fileInfo;
+    if (stat(filePath.c_str(), &fileInfo) != 0) {
+        return -1; // Error occurred
+    }
+    return fileInfo.st_atime;
+}
+
+long long getDirectorySize(const fs::path& directory) {
+    long long totalSize = 0;
+    for (const auto& entry : fs::recursive_directory_iterator(directory)) {
+        if (fs::is_regular_file(entry)) {
+            totalSize += fs::file_size(entry);
+        }
+    }
+    return totalSize;
+}
+
+std::vector<std::pair<long long, fs::path>> getFilesByAccessTime(const fs::path& directory) {
+    std::vector<std::pair<long long, fs::path>> fileList;
+    for (const auto& entry : fs::recursive_directory_iterator(directory)) {
+        if (fs::is_regular_file(entry)) {
+            auto accessTime = getLastAccessTime(entry.path());
+            fileList.emplace_back(accessTime, entry.path());
+        }
+    }
+    std::sort(fileList.begin(), fileList.end());
+    return fileList;
+}
+
+void cleanDirectory(const fs::path& directory, long long highWatermark, long long lowWatermark) {
+    long long currentSize = getDirectorySize(directory);
+    if (currentSize <= highWatermark) {
+        printCurrentTime();
+        std::cout << "Directory size is within the limit. No action needed." << std::endl;
+        return;
+    }
+
+    auto files = getFilesByAccessTime(directory);
+
+    for (const auto& [accessTime, filePath] : files) {
+        if (currentSize <= lowWatermark) {
+            break;
+        }
+        long long fileSize = fs::file_size(filePath);
+        try {
+            fs::remove(filePath);
+            currentSize -= fileSize;
+            printCurrentTime();
+            std::cout << "Deleted: " << filePath << " (Size: " << fileSize << " bytes)" << std::endl;
+        } catch (const std::exception& e) {
+            printCurrentTime();
+            std::cerr << "Error deleting " << filePath << ": " << e.what() << std::endl;
+        }
+    }
+}
+
+int main(int argc, char* argv[]) {
+    if (argc != 5) {
+        std::cerr << "Usage: " << argv[0] << " <directory> <highwatermark> <lowwatermark> <interval>" << std::endl;
+        return 1;
+    }
+
+    fs::path directory = argv[1];
+    long long highWatermark = std::stoll(argv[2]);
+    long long lowWatermark = std::stoll(argv[3]);
+    int interval = std::stoi(argv[4]);
+
+    while (true) {
+        cleanDirectory(directory, highWatermark, lowWatermark);
+        std::this_thread::sleep_for(std::chrono::seconds(interval));
+    }
+
+    return 0;
+}