Add ReadDir(Files)Nth API call, which allows skipping files (useful for resuming iteration)

Author: sosthene-nitrokey

src/api.rs

@@ -41,6 +41,7 @@ generate_enums! {
     Hash: 12
     // TODO: add ReadDir{First,Next}, not loading data, if needed for efficiency
     ReadDirFilesFirst: 13
+    ReadDirFilesNth: 31
     ReadDirFilesNext: 14
     ReadFile: 15
     Metadata: 26
@@ -63,6 +64,7 @@ generate_enums! {
 
     // // CreateDir,    <-- implied by WriteFile
     ReadDirFirst: 31 //      <-- gets Option<FileType> to restrict to just dir/file DirEntries,
+    ReadDirNth: 31
     ReadDirNext: 32 //      <-- gets Option<FileType> to restrict to just dir/file DirEntries,
     //                   // returns simplified Metadata
     // // ReadDirFilesFirst: 23 // <-- returns contents
@@ -229,13 +231,24 @@ pub mod request {
           - dir: PathBuf
           - user_attribute: Option<UserAttribute>
 
+        ReadDirFilesNth:
+          - location: Location
+          - dir: PathBuf
+          - start_at: usize
+          - user_attribute: Option<UserAttribute>
+
         ReadDirFilesNext:
 
         ReadDirFirst:
           - location: Location
           - dir: PathBuf
           - not_before_filename: Option<PathBuf>
 
+        ReadDirNth:
+          - location: Location
+          - dir: PathBuf
+          - start_at: usize
+
         ReadDirNext:
 
         ReadFile:
@@ -419,12 +432,18 @@ pub mod reply {
         ReadDirFilesFirst:
           - data: Option<Message>
 
+        ReadDirFilesNth:
+          - data: Option<Message>
+
         ReadDirFilesNext:
           - data: Option<Message>
 
         ReadDirFirst:
           - entry: Option<DirEntry>
 
+        ReadDirNth:
+          - data: Option<Message>
+
         ReadDirNext:
           - entry: Option<DirEntry>
 

src/service.rs

@@ -353,6 +353,20 @@ impl<P: Platform> ServiceResources<P> {
                 };
                 Ok(Reply::ReadDirFirst(reply::ReadDirFirst { entry: maybe_entry } ))
             }
+            Request::ReadDirNth(request) => {
+                let maybe_entry = match filestore.read_dir_nth(&request.dir, request.location, request.start_at)? {
+                    Some((entry, read_dir_state)) => {
+                        ctx.read_dir_state = Some(read_dir_state);
+                        Some(entry)
+                    }
+                    None => {
+                        ctx.read_dir_state = None;
+                        None
+
+                    }
+                };
+                Ok(Reply::ReadDirFirst(reply::ReadDirFirst { entry: maybe_entry } ))
+            }
 
             Request::ReadDirNext(_request) => {
                 // ensure next call has nothing to work with, unless we store state again
@@ -391,6 +405,20 @@ impl<P: Platform> ServiceResources<P> {
                 Ok(Reply::ReadDirFilesFirst(reply::ReadDirFilesFirst { data: maybe_data } ))
             }
 
+            Request::ReadDirFilesNth(request) => {
+                let maybe_data = match filestore.read_dir_files_nth(&request.dir, request.location, request.start_at, request.user_attribute.clone())? {
+                    Some((data, state)) => {
+                        ctx.read_dir_files_state = Some(state);
+                        data
+                    }
+                    None => {
+                        ctx.read_dir_files_state = None;
+                        None
+                    }
+                };
+                Ok(Reply::ReadDirFilesFirst(reply::ReadDirFilesFirst { data: maybe_data } ))
+            }
+
             Request::ReadDirFilesNext(_request) => {
                 let read_dir_files_state = ctx.read_dir_files_state.take();
 

src/store/filestore.rs

@@ -89,6 +89,19 @@ pub trait Filestore {
         filename: PathBuf,
     ) -> Result<Option<PathBuf>>;
 
+    /// Iterate over entries of a directory (both file and directory entries).
+    ///
+    /// This function is modeled after `std::fs::read_dir`, within the limitations of our setup.
+    ///
+    /// In case an entry was found, the returned option also contains state, so the expected
+    /// call to `read_dir_next` can resume operation.
+    fn read_dir_nth(
+        &mut self,
+        dir: &PathBuf,
+        location: Location,
+        start_at: usize,
+    ) -> Result<Option<(DirEntry, ReadDirState)>>;
+
     /// Iterate over entries of a directory (both file and directory entries).
     ///
     /// This function is modeled after `std::fs::read_dir`, within the limitations of our setup.
@@ -125,6 +138,21 @@ pub trait Filestore {
         user_attribute: Option<UserAttribute>,
     ) -> Result<Option<(Option<Message>, ReadDirFilesState)>>;
 
+    /// Iterate over contents of files inside a directory.
+    ///
+    /// This has no equivalent in `std::fs`, it is an optimization to avoid duplicate
+    /// calls and a more complicated state machine (interspersing read_dir_first/next calls
+    /// with some sort of "fetch data").
+    ///
+    /// Additionally, files may optionally be filtered via attributes.
+    fn read_dir_files_nth(
+        &mut self,
+        clients_dir: &PathBuf,
+        location: Location,
+        start_at: usize,
+        user_attribute: Option<UserAttribute>,
+    ) -> Result<Option<(Option<Message>, ReadDirFilesState)>>;
+
     /// Continuation of `read_dir_files_first`.
     fn read_dir_files_next(
         &mut self,
@@ -139,6 +167,7 @@ impl<S: Store> ClientFilestore<S> {
         clients_dir: &PathBuf,
         location: Location,
         not_before: Option<&PathBuf>,
+        start_at: usize,
         fs: &'static Fs<F>,
     ) -> Result<Option<(DirEntry, ReadDirState)>> {
         let dir = self.actual_path(clients_dir)?;
@@ -148,7 +177,7 @@ impl<S: Store> ClientFilestore<S> {
                 // this is an iterator with Item = (usize, Result<DirEntry>)
                 (&mut it)
                     // skip over `.` and `..`
-                    .skip(2)
+                    .skip(2 + start_at)
                     // todo: try ?-ing out of this (the API matches std::fs, where read/write errors
                     // can occur during operation)
                     //
@@ -232,6 +261,7 @@ impl<S: Store> ClientFilestore<S> {
         clients_dir: &PathBuf,
         location: Location,
         user_attribute: Option<UserAttribute>,
+        start_at: usize,
         fs: &'static Fs<F>,
     ) -> Result<Option<(Option<Message>, ReadDirFilesState)>> {
         let dir = self.actual_path(clients_dir)?;
@@ -247,8 +277,7 @@ impl<S: Store> ClientFilestore<S> {
                     .map(Result::unwrap)
                     // skip over directories (including `.` and `..`)
                     .filter(|entry| entry.file_type().is_file())
-                    // take first entry that meets requirements
-                    .find(|entry| {
+                    .filter(|entry| {
                         if let Some(user_attribute) = user_attribute.as_ref() {
                             let mut path = dir.clone();
                             path.push(entry.file_name());
@@ -257,14 +286,16 @@ impl<S: Store> ClientFilestore<S> {
                                 .unwrap();
 
                             if let Some(attribute) = attribute {
-                                user_attribute == attribute.data()
+                                user_attribute != attribute.data()
                             } else {
-                                false
+                                true
                             }
                         } else {
                             true
                         }
                     })
+                    // take nth entry that meets requirements
+                    .nth(start_at)
                     // if there is an entry, construct the state that needs storing out of it,
                     // and return the file's contents.
                     // the client, and return both the entry and the state
@@ -403,13 +434,32 @@ impl<S: Store> Filestore for ClientFilestore<S> {
     ) -> Result<Option<(DirEntry, ReadDirState)>> {
         match location {
             Location::Internal => {
-                self.read_dir_first_impl(clients_dir, location, not_before, self.store.ifs())
+                self.read_dir_first_impl(clients_dir, location, not_before, 0, self.store.ifs())
+            }
+            Location::External => {
+                self.read_dir_first_impl(clients_dir, location, not_before, 0, self.store.efs())
+            }
+            Location::Volatile => {
+                self.read_dir_first_impl(clients_dir, location, not_before, 0, self.store.vfs())
+            }
+        }
+    }
+
+    fn read_dir_nth(
+        &mut self,
+        clients_dir: &PathBuf,
+        location: Location,
+        start_at: usize,
+    ) -> Result<Option<(DirEntry, ReadDirState)>> {
+        match location {
+            Location::Internal => {
+                self.read_dir_first_impl(clients_dir, location, None, start_at, self.store.ifs())
             }
             Location::External => {
-                self.read_dir_first_impl(clients_dir, location, not_before, self.store.efs())
+                self.read_dir_first_impl(clients_dir, location, None, start_at, self.store.efs())
             }
             Location::Volatile => {
-                self.read_dir_first_impl(clients_dir, location, not_before, self.store.vfs())
+                self.read_dir_first_impl(clients_dir, location, None, start_at, self.store.vfs())
             }
         }
     }
@@ -433,18 +483,53 @@ impl<S: Store> Filestore for ClientFilestore<S> {
                 clients_dir,
                 location,
                 user_attribute,
+                0,
+                self.store.ifs(),
+            ),
+            Location::External => self.read_dir_files_first_impl(
+                clients_dir,
+                location,
+                user_attribute,
+                0,
+                self.store.efs(),
+            ),
+            Location::Volatile => self.read_dir_files_first_impl(
+                clients_dir,
+                location,
+                user_attribute,
+                0,
+                self.store.vfs(),
+            ),
+        }
+    }
+
+    fn read_dir_files_nth(
+        &mut self,
+        clients_dir: &PathBuf,
+        location: Location,
+        start_at: usize,
+        user_attribute: Option<UserAttribute>,
+    ) -> Result<Option<(Option<Message>, ReadDirFilesState)>> {
+        match location {
+            Location::Internal => self.read_dir_files_first_impl(
+                clients_dir,
+                location,
+                user_attribute,
+                start_at,
                 self.store.ifs(),
             ),
             Location::External => self.read_dir_files_first_impl(
                 clients_dir,
                 location,
                 user_attribute,
+                start_at,
                 self.store.efs(),
             ),
             Location::Volatile => self.read_dir_files_first_impl(
                 clients_dir,
                 location,
                 user_attribute,
+                start_at,
                 self.store.vfs(),
             ),
         }