feat: Store the cached index entry mtime in files

Use file attributes to store the cached index entry metadata
persistently on the filesystem.

Try to recreate the volatile metadata cache entry from
the index cache file attributes.

This speeds up the volatile metadata cache regeneration
after restarts.

Author: ravenexp

src/file_cache.rs

@@ -1,6 +1,7 @@
 //! Index entry and crate file cache helpers
 
-use std::fs::{create_dir_all, read, write};
+use std::fs::{create_dir_all, metadata, read, write, File};
+use std::io::Write;
 use std::path::Path;
 
 use log::error;
@@ -35,11 +36,41 @@ pub fn cache_store_index_entry(dir: &Path, entry: &IndexEntry, data: &[u8]) {
         return;
     }
 
-    write(entry_file_path, data)
-        .unwrap_or_else(|e| error!("cache: failed to write index entry file: {e}"));
+    let mut file = match File::create(entry_file_path) {
+        Ok(f) => f,
+        Err(e) => {
+            error!("cache: failed to create index entry file: {e}");
+            return;
+        }
+    };
+
+    if let Err(e) = file.write_all(data) {
+        error!("cache: failed to write index entry data: {e}");
+        return;
+    }
+
+    // Set the cache file mtime according to the Last-Modified HTTP metadata.
+    if let Some(mtime) = entry.mtime() {
+        file.set_modified(mtime)
+            .unwrap_or_else(|e| error!("cache: failed to set index entry file mtime: {e}"));
+    }
 }
 
 /// Fetches the cached index entry file from the local filesystem, if present.
 pub fn cache_fetch_index_entry(dir: &Path, entry: &IndexEntry) -> Option<Vec<u8>> {
     read(dir.join(entry.to_file_path())).ok()
 }
+
+/// Tries to recreate the missing index entry metadata from the cache file metadata.
+pub fn cache_try_find_index_entry(dir: &Path, name: &str) -> Option<IndexEntry> {
+    let mut entry = IndexEntry::new(name);
+
+    let mtime = metadata(dir.join(entry.to_file_path()))
+        .ok()?
+        .modified()
+        .ok()?;
+
+    entry.set_mtime(mtime);
+
+    Some(entry)
+}

src/index_entry.rs

@@ -79,15 +79,23 @@ impl IndexEntry {
     }
 
     /// Gets the HTTP entity tag metadata.
+    #[must_use]
     pub fn etag(&self) -> Option<&str> {
         self.etag.as_deref()
     }
 
     /// Gets the HTTP Last-Modified metadata.
+    #[must_use]
     pub fn last_modified(&self) -> Option<String> {
         self.mtime.map(fmt_http_date)
     }
 
+    /// Gets the file modification time metadata.
+    #[must_use]
+    pub fn mtime(&self) -> Option<SystemTime> {
+        self.mtime
+    }
+
     /// Sets the HTTP entity tag metadata.
     pub fn set_etag(&mut self, etag: &str) {
         self.etag = Some(etag.to_owned());
@@ -98,6 +106,11 @@ impl IndexEntry {
         self.mtime = parse_http_date(last_modified).ok();
     }
 
+    /// Sets the file modification time metadata.
+    pub fn set_mtime(&mut self, mtime: SystemTime) {
+        self.mtime = Some(mtime);
+    }
+
     /// Updates the last upstream server access time metadata.
     pub fn set_last_updated(&mut self) {
         self.atime = Some(Instant::now());

src/main.rs

@@ -47,6 +47,7 @@ use crate::config_json::{gen_config_json_file, is_config_json_url};
 use crate::crate_info::CrateInfo;
 use crate::file_cache::{
     cache_fetch_crate, cache_fetch_index_entry, cache_store_crate, cache_store_index_entry,
+    cache_try_find_index_entry,
 };
 use crate::index_entry::IndexEntry;
 use crate::metadata_cache::{
@@ -447,12 +448,12 @@ fn handle_index_request(request: Request, index_url: &str, config: &ProxyConfig)
     for header in request.headers() {
         if header.field.equiv("If-None-Match") {
             let etag = header.value.as_str();
-            debug!("proxy: checking known index entry {index_entry} with ETag {etag}");
+            debug!("proxy: checking known index entry {index_entry} with ETag: {etag}");
             index_entry.set_etag(etag);
         }
         if header.field.equiv("If-Modified-Since") {
             let last_modified = header.value.as_str();
-            debug!("proxy: checking known index entry {index_entry} with Last-Modified {last_modified}");
+            debug!("proxy: checking known index entry {index_entry} with Last-Modified: {last_modified}");
             index_entry.set_last_modified(last_modified);
         }
     }
@@ -482,8 +483,19 @@ fn handle_index_request(request: Request, index_url: &str, config: &ProxyConfig)
         }
     }
 
+    // Try to recreate the index entry metadata from the cached file mtime.
+    let mtimed_entry = cache_try_find_index_entry(&config.index_dir, index_entry.name());
+
+    if let Some(entry) = &mtimed_entry {
+        let last_modified = entry.last_modified().unwrap();
+
+        info!(
+            "proxy: recreated index cache metadata for {entry} with Last-Modified: {last_modified}"
+        );
+    }
+
     // Fall back to forwarding the request to the upstream registry.
-    forward_index_request(request, index_entry, None, config.clone());
+    forward_index_request(request, index_entry, mtimed_entry, config.clone());
 }
 
 /// Processes one HTTP GET request.