diff --git a/src/uu/sed/src/fast_io.rs b/src/uu/sed/src/fast_io.rs index 514f493..b70e453 100644 --- a/src/uu/sed/src/fast_io.rs +++ b/src/uu/sed/src/fast_io.rs @@ -1,7 +1,7 @@ // Zero-copy line-based I/O // // Abstractions that allow file lines to be processed and output -// in mmapped memory space. By coallescing output requests an +// in mmapped memory space. By coalescing output requests an // efficient write(2) system call can be issued for them, bypassing // the copy required for output through BufWriter. // Search for "main" to see a usage example. @@ -26,29 +26,34 @@ use std::io::{self, BufRead, BufReader, BufWriter, Read, Write}; #[cfg(not(unix))] use std::marker::PhantomData; +#[cfg(unix)] +use std::os::fd::RawFd; + #[cfg(unix)] use std::os::unix::io::AsRawFd; -use std::path::PathBuf; use std::str; -#[cfg(unix)] -use uucore::libc::{c_void, write}; - +use std::path::PathBuf; use uucore::error::UError; #[cfg(unix)] use uucore::error::USimpleError; +#[cfg(unix)] +use uucore::libc; + // Define two cursors for iterating over lines: // - MmapLineCursor based on mmap(2), -// - ReadLineCursorbased on BufReader. +// - ReadLineCursor based on BufReader. /// Cursor for zero-copy iteration over mmap’d file. #[cfg(unix)] pub struct MmapLineCursor<'a> { - data: &'a [u8], - pos: usize, + _file: File, // Mmapped file; keeps file open for fast copy + fast_copy: FastCopy, // Data for fast file copy I/O + data: &'a [u8], // Mmapped data + pos: usize, // Position within the data } #[cfg(unix)] @@ -61,8 +66,13 @@ pub struct NextMmapLine<'a> { #[cfg(unix)] impl<'a> MmapLineCursor<'a> { - fn new(data: &'a [u8]) -> Self { - Self { data, pos: 0 } + fn new(file: File, data: &'a [u8]) -> Self { + Self { + fast_copy: FastCopy::new(&file), + _file: file, + data, + pos: 0, + } } /// Return the next line, if available, or None. @@ -254,18 +264,17 @@ impl<'a> IOChunk<'a> { match &self.content { IOChunkContent::Owned { .. } => Ok(()), // already owned #[cfg(unix)] - IOChunkContent::MmapInput { content, full_span } => { - match std::str::from_utf8(content) { - Ok(valid_str) => { - let has_newline = full_span.last().copied() == Some(b'\n'); - self.content = - IOChunkContent::new_owned(valid_str.to_string(), has_newline); - self.utf8_verified.set(true); - Ok(()) - } - Err(e) => Err(USimpleError::new(2, e.to_string())), + IOChunkContent::MmapInput { + content, full_span, .. + } => match std::str::from_utf8(content) { + Ok(valid_str) => { + let has_newline = full_span.last().copied() == Some(b'\n'); + self.content = IOChunkContent::new_owned(valid_str.to_string(), has_newline); + self.utf8_verified.set(true); + Ok(()) } - } + Err(e) => Err(USimpleError::new(2, e.to_string())), + }, } } @@ -286,13 +295,15 @@ impl<'a> IOChunk<'a> { } /// Data to be written to a file. It can come from the mmapped -/// memory space, in which case it is tracked to allow coallescing +/// memory space, in which case it is tracked to allow coalescing /// and bypassing BufWriter, or it can be other data from the process's /// memory space. #[derive(Debug, PartialEq, Eq)] enum IOChunkContent<'a> { #[cfg(unix)] MmapInput { + fast_copy: FastCopy, // Data for fast file copy I/O + base: *const u8, // Mmap start address content: &'a [u8], // Line without newline full_span: &'a [u8], // Line including original newline, if any }, @@ -343,24 +354,74 @@ impl IOChunkContent<'_> { } } +/// Information required for performing I/O using fast file copy +/// operations, such as copy_file_range(2). +#[cfg(unix)] +#[derive(Debug, PartialEq, Eq, Clone)] +struct FastCopy { + fd: i32, // Raw file descriptor + is_regular: bool, // True if this is a regular file + block_size: usize, // Filesystem block size +} + +#[cfg(unix)] +impl FastCopy { + /// Construct with an object on which as_raw_fd() can be called. + pub fn new(f: &T) -> Self { + let fd = f.as_raw_fd(); + + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + + let ret = unsafe { libc::fstat(fd, &mut st) }; + if ret == -1 { + // All fstat errors are programmer rather user faults + // so panic is appropriate. + let err = std::io::Error::last_os_error(); + panic!("fstat failed on fd {}: {}", fd, err); + } + + let ftype = st.st_mode & libc::S_IFMT; + + Self { + fd, + is_regular: ftype == libc::S_IFREG, + block_size: st.st_blksize as usize, + } + } +} + +#[cfg(test)] +#[cfg(unix)] +impl Default for FastCopy { + fn default() -> Self { + FastCopy { + fd: -1, + is_regular: false, + block_size: 0, + } + } +} + /// Unified reader that uses mmap when possible, falls back to buffered reading. -pub enum LineReader { +pub enum LineReader<'a> { #[cfg(unix)] MmapInput { mapped_file: Mmap, // A handle that can derive the mapped file slice - cursor: MmapLineCursor<'static>, + cursor: MmapLineCursor<'a>, }, ReadInput(ReadLineCursor), + #[cfg(not(unix))] + _Phantom(std::marker::PhantomData<&'a ()>), } /// Return a LineReader that uses the ReadInput method fot the specified file. -fn line_reader_read_input(file: File) -> io::Result { +fn line_reader_read_input(file: File) -> io::Result> { let boxed: Box = Box::new(file); let reader = BufReader::new(boxed); Ok(LineReader::ReadInput(ReadLineCursor::new(reader))) } -impl LineReader { +impl<'a> LineReader<'a> { /// Open the specified file for line input. // Use "-" to read from the standard input. pub fn open(path: &PathBuf) -> io::Result { @@ -381,7 +442,7 @@ impl LineReader { let slice: &'static [u8] = unsafe { std::slice::from_raw_parts(mapped_file.as_ptr(), mapped_file.len()) }; - let cursor = MmapLineCursor::new(slice); + let cursor = MmapLineCursor::new(file, slice); Ok(LineReader::MmapInput { mapped_file, cursor, @@ -411,14 +472,21 @@ impl LineReader { match self { #[cfg(unix)] LineReader::MmapInput { cursor, .. } => { + // Obtain fields to prevent borrowing issues. + let fast_copy = cursor.fast_copy.clone(); + let base = cursor.data.as_ptr(); if let Some(NextMmapLine { content, full_span, is_last_line, }) = cursor.get_line()? { - let chunk = - IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let chunk = IOChunk::from_content(IOChunkContent::MmapInput { + fast_copy, + base, + content, + full_span, + }); Ok(Some((chunk, is_last_line))) } else { @@ -435,6 +503,9 @@ impl LineReader { Ok(None) } } + + #[cfg(not(unix))] + LineReader::_Phantom(_) => unreachable!("_Phantom should never be constructed"), } } } @@ -450,27 +521,35 @@ pub trait OutputWrite: Write {} #[cfg(not(unix))] impl OutputWrite for T {} +/// An output data chunk from the mmapped file +/// Data elements allow output to be performed through write(2) +/// or through copy_file_range(2). +#[cfg(unix)] +#[derive(Clone)] +struct MmapOutput { + in_fast_copy: FastCopy, // Data for fast file copy I/O + base_ptr: *const u8, // Base of the entire mmapped region + out_ptr: *const u8, // Start of the output data chunk + len: usize, // Output data chunk size +} + /// Abstraction for outputting data, potentially from the mmapped file -/// Outputs from mmapped data are coallesced and written via a write(2) -/// system call without any copying if worthwhile. +/// Outputs from mmapped data are coalesced and written via the Linux +/// copy_file_range(2) system call without any copying, if possible +/// and worthwhile. As a fallback write(2) is used, which requires +/// the OS to copy data from the mmapped region to the output file +/// page cache. /// All other output is buffered and writen via BufWriter. pub struct OutputBuffer { out: BufWriter>, // Where to write #[cfg(unix)] - mmap_ptr: Option<(*const u8, usize)>, // Start and len of chunk to write + fast_copy: FastCopy, // Data for fast file copy ops + #[cfg(unix)] + max_pending_write: usize, // Max bytes to keep before flushing + #[cfg(unix)] + mmap_chunk: Option, // Chunk to write #[cfg(test)] - writes_issued: usize, // Number of issued write(2) calls -} - -/// Wrapper that issues the write(2) system call -#[cfg(unix)] -fn write_syscall(fd: i32, ptr: *const u8, len: usize) -> io::Result<()> { - let ret = unsafe { write(fd, ptr as *const c_void, len) }; - if ret < 0 { - Err(std::io::Error::last_os_error()) - } else { - Ok(()) - } + low_level_flushes: usize, // Number of system call flushes } /// Threshold to use buffered writes for output @@ -482,21 +561,45 @@ fn write_syscall(fd: i32, ptr: *const u8, len: usize) -> io::Result<()> { #[cfg(unix)] const MIN_DIRECT_WRITE: usize = 4 * 1024; -/// The maximum size of a pending write buffer -// Once more than 64k accumulate, issue a write to allow the OS -// and downstream pipes to handle the output processing in parallel -// with our processing. +/// Maximum size of a pending write buffer for files +// Once more than the specified bytes accumulate, issue a write or +// a copy_file_range(2). This is kept high to reduce the number of +// system calls and (where supported) file extents. #[cfg(unix)] -const MAX_PENDING_WRITE: usize = 64 * 1024; +const MAX_PENDING_WRITE_FILE: usize = 1024 * 1024; + +/// Maximum size of a pending write buffer for non-files (likely pipes) +// Once more than the specified bytes accumulate, issue a write. +// This is set to the common size of Linux pipe buffer to maximize +// throughput and liveness across the pipeline. +#[cfg(unix)] +const MAX_PENDING_WRITE_NON_FILE: usize = 64 * 1024; impl OutputBuffer { + #[cfg(not(unix))] pub fn new(w: Box) -> Self { Self { out: BufWriter::new(w), - #[cfg(unix)] - mmap_ptr: None, #[cfg(test)] - writes_issued: 0, + low_level_flushes: 0, + } + } + + #[cfg(unix)] + pub fn new(w: Box) -> Self { + let fast_copy = FastCopy::new(&*w); + let max_pending_write = if fast_copy.is_regular { + MAX_PENDING_WRITE_FILE + } else { + MAX_PENDING_WRITE_NON_FILE + }; + Self { + out: BufWriter::new(w), + fast_copy, + max_pending_write, + mmap_chunk: None, + #[cfg(test)] + low_level_flushes: 0, } } @@ -510,8 +613,11 @@ impl OutputBuffer { /// Copy the specified file to the output. pub fn copy_file(&mut self, path: &PathBuf) -> io::Result<()> { + // Flush mmap writes, if any. #[cfg(unix)] - self.flush_mmap()?; // Flush mmap writes, if any. + { + self.flush_mmap(WriteRange::Complete)?; + } let file = match File::open(path) { Ok(f) => f, @@ -539,26 +645,63 @@ impl Write for OutputBuffer { } } +#[cfg(unix)] +#[derive(Debug, PartialEq)] +enum WriteRange { + Complete, // Write all specified data. + Blocks, // Finish write on a block boundary (to help alignment). + None, // No writing is needed. +} + #[cfg(unix)] impl OutputBuffer { /// Schedule the specified output chunk for eventual output - pub fn write_chunk(&mut self, chunk: &IOChunk) -> io::Result<()> { - match &chunk.content { - IOChunkContent::MmapInput { full_span, .. } => { - let ptr = full_span.as_ptr(); - let len = full_span.len(); - - if let Some((p, l)) = self.mmap_ptr { - // Coalesce if adjacent - if unsafe { p.add(l) } == ptr && l < MAX_PENDING_WRITE { - self.mmap_ptr = Some((p, l + len)); - return Ok(()); + pub fn write_chunk(&mut self, new_chunk: &IOChunk) -> io::Result<()> { + match &new_chunk.content { + IOChunkContent::MmapInput { + full_span, + fast_copy, + base, + .. + } => { + let new_ptr = full_span.as_ptr(); + let new_len = full_span.len(); + + // Set whether a flush is needed and whether the + // mmap_chunk needs to be reset to the new input. + // This avoids calling mmap_chunk (which borrows self) + // when old_chunk is already borrowed. + let (flush_action, reset) = if let Some(old_chunk) = self.mmap_chunk.as_mut() { + // Coalesce if adjacent. + if unsafe { old_chunk.out_ptr.add(old_chunk.len) } == new_ptr { + // Coalesce. + old_chunk.len += new_len; + if old_chunk.len > self.max_pending_write { + // Too much data; flush some full blocks. + (WriteRange::Blocks, false) + } else { + (WriteRange::None, false) + } } else { - self.flush_mmap()?; // not contiguous + // Not contiguous + (WriteRange::Complete, true) } + } else { + // No chunk yet; start a new one. + (WriteRange::None, true) + }; + + if flush_action != WriteRange::None { + self.flush_mmap(flush_action)?; + } + if reset { + self.mmap_chunk = Some(MmapOutput { + in_fast_copy: fast_copy.clone(), + base_ptr: *base, + out_ptr: new_ptr, + len: new_len, + }); } - self.mmap_ptr = Some((ptr, len)); - Ok(()) } IOChunkContent::Owned { @@ -566,41 +709,62 @@ impl OutputBuffer { has_newline, .. } => { - self.flush_mmap()?; + self.flush_mmap(WriteRange::Complete)?; self.out.write_all(content.as_bytes())?; if *has_newline { self.out.write_all(b"\n")?; } - Ok(()) } } + Ok(()) } - // Flush any pending mmap data + /// Flush any pending mmap data. + /// Cover specifies whether the written data shall terminate + /// on a block boundary. + /// This is only respected when copy_file_range is employed. + /// Return the number of bytes written. #[cfg(unix)] - fn flush_mmap(&mut self) -> io::Result<()> { - if let Some((ptr, len)) = self.mmap_ptr.take() { - if len < MIN_DIRECT_WRITE { + fn flush_mmap(&mut self, cover: WriteRange) -> io::Result<()> { + if let Some(chunk) = self.mmap_chunk.as_mut() { + let written = if chunk.len < MIN_DIRECT_WRITE { // SAFELY treat as &[u8] and write to buffered writer - let slice = unsafe { std::slice::from_raw_parts(ptr, len) }; - return self.out.write_all(slice); + let slice = unsafe { std::slice::from_raw_parts(chunk.out_ptr, chunk.len) }; + self.out.write_all(slice)?; + slice.len() } else { // Large enough: write directly using zero-copy - let fd = self.out.get_ref().as_raw_fd(); self.out.flush()?; // sync any buffered data #[cfg(test)] { - self.writes_issued += 1; + self.low_level_flushes += 1; } - return write_syscall(fd, ptr, len); - } + if chunk.in_fast_copy.is_regular && self.fast_copy.is_regular { + portable_copy_file_range( + chunk.out_ptr, + chunk.in_fast_copy.fd, + // Input file offset + unsafe { chunk.out_ptr.offset_from(chunk.base_ptr) } as i64, + self.fast_copy.fd, + chunk.len, + // Alignment block size: the largest of the two + chunk.in_fast_copy.block_size.max(self.fast_copy.block_size), + cover, + )? + } else { + reliable_write(self.fast_copy.fd, chunk.out_ptr, chunk.len)? + } + }; + + chunk.len -= written; + unsafe { chunk.out_ptr = chunk.out_ptr.add(written) }; } Ok(()) } /// Flush everything: pending mmap and buffered data. pub fn flush(&mut self) -> io::Result<()> { - self.flush_mmap()?; // flush mmap if any + self.flush_mmap(WriteRange::Complete)?; // flush mmap if any self.out.flush() // then flush buffered data } } @@ -630,6 +794,165 @@ impl OutputBuffer { } } +/// Wrapper that issues the write(2) system calls via write_all +// This takes care of partial writes and EAGAIN, EWOULDBLOCK, EINTR. +// Return the number of bytes written. +#[cfg(unix)] +fn reliable_write(fd: i32, ptr: *const u8, len: usize) -> std::io::Result { + // A thin Write-compatible wrapper around a raw file descriptor + // This allows us to issue and utilize the write_all implementatin. + struct FdWriter(RawFd); + + impl Write for FdWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let ret = + unsafe { libc::write(self.0, buf.as_ptr() as *const libc::c_void, buf.len()) }; + if ret < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(ret as usize) + } + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + let mut writer = FdWriter(fd); + let buf: &[u8] = unsafe { std::slice::from_raw_parts(ptr, len) }; + writer.write_all(buf)?; + Ok(len) +} + +/// Copy efficiently len data from the input to the output file. +/// Fall back to write(2) if the platform doesn't support copy_file_range(2). +/// Return the number of bytes written. +#[cfg(unix)] +#[allow(unused_variables)] +fn portable_copy_file_range( + in_ptr: *const u8, + in_fd: i32, + in_off: libc::off_t, + out_fd: i32, + len: usize, + block_size: usize, + cover: WriteRange, +) -> std::io::Result { + if len == 0 { + return Ok(0); + } + + #[cfg(all(target_os = "linux", target_env = "gnu"))] + { + aligned_copy_file_range(in_ptr, in_fd, in_off, out_fd, len, block_size, cover) + } + #[cfg(not(all(target_os = "linux", target_env = "gnu")))] + { + reliable_write(out_fd, in_ptr, len) + } +} + +/// Copy efficiently len data from the input to the output file. +/// Handle partial copies and fall back to write(2) if the +/// file system or options don't support copy_file_range(2). +/// Return the number of bytes written. +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn reliable_copy_file_range( + in_ptr: *const u8, + in_fd: i32, + mut in_off: libc::off_t, + out_fd: i32, + len: usize, +) -> std::io::Result { + let mut pending = len; + while pending > 0 { + let in_off_ptr: *mut i64 = &mut in_off; + let ret = unsafe { + libc::copy_file_range( + in_fd, + in_off_ptr, + out_fd, + std::ptr::null_mut(), // Use and update output offset + pending, + 0, + ) + }; + if ret < 0 { + let err = io::Error::last_os_error(); + return match err.raw_os_error() { + Some(libc::ENOSYS) | Some(libc::EOPNOTSUPP) | Some(libc::EXDEV) => { + // Fallback to write(2). + reliable_write(out_fd, in_ptr, pending) + } + _ => Err(err), + }; + } else if ret == 0 { + // EOF reached + break; + } else { + pending -= ret as usize; + } + } + Ok(len) +} + +/// Copy efficiently len data from the input to the output file. +/// Try to call copy_file_range(2) on block-aligned data, so +/// as to help the filesystem maintain cross-file extents. +/// Return the number of bytes written. +#[cfg(all(target_os = "linux", target_env = "gnu"))] +fn aligned_copy_file_range( + mut in_ptr: *const u8, + in_fd: i32, + mut in_off: libc::off_t, + out_fd: i32, + len: usize, + block_size: usize, + cover: WriteRange, +) -> std::io::Result { + // 1. Get current output offset. + let res = unsafe { libc::lseek(out_fd, 0, libc::SEEK_CUR) as i64 }; + if res < 0 { + return Err(std::io::Error::last_os_error()); + } + let out_off = res as usize; + let mut pending = len; + + // Obtain head alignment. + // Bytes to end of block: + let remainder = in_off as usize % block_size; + // Bytes to write (block_size becomes 0): + let head_align = (block_size - remainder) % block_size; + + if !(out_off + head_align).is_multiple_of(block_size) { + // No hope of alignment, so just copy everything. + return reliable_copy_file_range(in_ptr, in_fd, in_off, out_fd, pending); + } + + if head_align > 0 { + // Align the two files on a block boundary. + let head_len = head_align.min(pending); + reliable_write(out_fd, in_ptr, head_len)?; + in_ptr = unsafe { in_ptr.add(head_len) }; + in_off += head_len as i64; + pending -= head_len; + } + + // Copy aligned blocks. + let aligned_len = pending - (pending % block_size); + let _ = reliable_copy_file_range(in_ptr, in_fd, in_off, out_fd, aligned_len); + pending -= aligned_len; + + // Copy tail if needed. + if pending > 0 && cover == WriteRange::Complete { + in_ptr = unsafe { in_ptr.add(aligned_len) }; + pending -= reliable_write(out_fd, in_ptr, pending)?; + } + + Ok(len - pending) +} + // Usage example (never compiled) #[cfg(any())] pub fn main() -> io::Result<()> { @@ -654,9 +977,11 @@ mod tests { use std::fs; #[cfg(unix)] use std::fs::File; - #[cfg(unix)] + #[cfg(all(target_os = "linux", target_env = "gnu"))] use std::io::{self, Write}; + use std::io::{Seek, SeekFrom}; use tempfile::NamedTempFile; + use tempfile::tempfile; /// Helper: produce a 4k-byte Vec of `'.'`s ending in `'\n'`. #[cfg(unix)] @@ -667,6 +992,19 @@ mod tests { buf } + #[cfg(unix)] + pub fn new_content_mmap_input<'a>( + content: &'a [u8], + full_span: &'a [u8], + ) -> IOChunkContent<'a> { + IOChunkContent::MmapInput { + fast_copy: FastCopy::default(), + base: std::ptr::null(), + content, + full_span, + } + } + #[test] fn test_owned_line_output() -> io::Result<()> { let tmp = NamedTempFile::new()?; @@ -676,7 +1014,7 @@ mod tests { out.write_str("foo\n")?; out.write_str("bar\n")?; out.flush()?; - assert_eq!(out.writes_issued, 0); + assert_eq!(out.low_level_flushes, 0); } // File closes here as it leaves the scope let contents = fs::read(tmp.path())?; @@ -715,7 +1053,7 @@ mod tests { } out.flush()?; - assert_eq!(out.writes_issued, 0); + assert_eq!(out.low_level_flushes, 0); let written = fs::read(&output_path)?; assert_eq!(written.as_slice(), mmap_data); @@ -761,7 +1099,7 @@ mod tests { out.flush()?; // Since all writes are small (<4K), we expect zero zero copy syscalls - assert_eq!(out.writes_issued, 0); + assert_eq!(out.low_level_flushes, 0); // Read both files back and compare let expected = { @@ -806,7 +1144,7 @@ mod tests { assert_eq!(nline, 3); out.flush()?; - assert_eq!(out.writes_issued, 1); + assert_eq!(out.low_level_flushes, 1); // Verify that files match: let expected = fs::read(&input_path)?; @@ -845,7 +1183,7 @@ mod tests { assert_eq!(nline, 4); out.flush()?; - assert_eq!(out.writes_issued, 1); + assert_eq!(out.low_level_flushes, 1); // Verify that files match: let expected = fs::read(&input_path)?; @@ -880,7 +1218,7 @@ mod tests { assert_eq!(nline, 3); out.flush()?; - assert_eq!(out.writes_issued, 0); + assert_eq!(out.low_level_flushes, 0); // Verify that files match: let expected = fs::read(&input_path)?; @@ -915,7 +1253,7 @@ mod tests { assert_eq!(nline, 3); out.flush()?; - assert_eq!(out.writes_issued, 0); + assert_eq!(out.low_level_flushes, 0); // Verify that files match: let expected = fs::read(&input_path)?; @@ -931,8 +1269,9 @@ mod tests { let mut input = NamedTempFile::new()?; write!(input, "first line\nsecond line\n")?; let dot_line = make_dot_line_4k(); - // Write 64k + 16k to ensure one flush when writing - for _i in 0..20 { + // Create more than MAX... to ensure one flush when writing + let nline_in_file = MAX_PENDING_WRITE_FILE / dot_line.len() + 4; + for _i in 0..nline_in_file { input.write_all(&dot_line)?; } input.flush()?; @@ -946,21 +1285,25 @@ mod tests { let output_path = output.path().to_path_buf(); let out_file = File::create(&output_path)?; - // Wrap it in your OutputBuffer and run the loop: + // Wrap it in OutputBuffer and run the loop: let mut out = OutputBuffer::new(Box::new(out_file)); - let mut nline = 0; + let mut nline_written = 0; while let Some((chunk, _last_line)) = reader.get_line()? { out.write_chunk(&chunk)?; - nline += 1; + nline_written += 1; } - assert_eq!(nline, 22); + assert_eq!(nline_written, nline_in_file + 2); out.flush()?; - assert_eq!(out.writes_issued, 2); + assert_eq!(out.low_level_flushes, 2); // Verify that files match: let expected = fs::read(&input_path)?; let actual = fs::read(&output_path)?; + + // Fast fail with a readable message, without displaying large output + assert_eq!(actual.len(), expected.len()); + assert_eq!(actual, expected); Ok(()) } @@ -1126,7 +1469,7 @@ mod tests { fn test_mmap_newline_terminated() { let content = b"line"; let full_span = b"line\n"; - let chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); assert!(chunk.is_newline_terminated()); } @@ -1135,7 +1478,7 @@ mod tests { fn test_mmap_not_newline_terminated() { let content = b"line"; let full_span = b"line"; - let chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); assert!(!chunk.is_newline_terminated()); } @@ -1144,7 +1487,7 @@ mod tests { fn test_mmap_empty() { let content = b""; let full_span = b""; - let chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); assert!(!chunk.is_newline_terminated()); } @@ -1178,7 +1521,7 @@ mod tests { let content = b"mmap string"; let full_span = b"mmap string\n"; - let mut chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); let result = chunk.ensure_owned(); assert!(result.is_ok()); @@ -1202,7 +1545,7 @@ mod tests { let content = b"no newline"; let full_span = b"no newline"; - let mut chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); let result = chunk.ensure_owned(); assert!(result.is_ok()); @@ -1226,7 +1569,7 @@ mod tests { let content = b"bad\xFFutf8"; let full_span = b"bad\xFFutf8\n"; - let mut chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); let result = chunk.ensure_owned(); assert!(result.is_err()); @@ -1255,7 +1598,7 @@ mod tests { fn test_fields_mut_on_mmap_input_valid_utf8() { let content = b"foo"; let full_span = b"foo\n"; - let mut chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); { let (s, _) = chunk.fields_mut().unwrap(); @@ -1270,7 +1613,7 @@ mod tests { fn test_fields_mut_on_utf8_multibyte() { let content = "Ζωντανά!".as_bytes(); let full_span = "Ζωντανά!\n".as_bytes(); - let mut chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); let (s, _) = chunk.fields_mut().unwrap(); s.push_str(" Δεδομένα"); @@ -1283,10 +1626,298 @@ mod tests { fn test_fields_mut_invalid_utf8() { let content = b"abc\xFF"; // invalid UTF-8 let full_span = b"abc\xFF\n"; - let mut chunk = IOChunk::from_content(IOChunkContent::MmapInput { content, full_span }); + let mut chunk = IOChunk::from_content(new_content_mmap_input(content, full_span)); let result = chunk.fields_mut(); assert!(result.is_err()); assert!(format!("{}", result.unwrap_err()).contains("invalid utf-8")); } + + #[cfg(unix)] + #[test] + fn fastcopy_regular_file() { + let mut file = tempfile().expect("create temp file"); + writeln!(file, "hello").unwrap(); + + let fc = FastCopy::new(&file); + + assert!(fc.is_regular, "expected regular file"); + assert!(fc.block_size > 0, "block size should be > 0"); + } + + #[cfg(unix)] + #[test] + fn fastcopy_non_regular_devnull() { + let file = File::open("/dev/null").expect("open /dev/null"); + + let fc = FastCopy::new(&file); + + assert!( + !fc.is_regular, + "expected /dev/null to be reported as non-regular" + ); + } + + #[cfg(unix)] + #[test] + fn test_reliable_write_to_file() { + // Create an anonymous temporary file. + let mut file = tempfile().expect("failed to create tempfile"); + + // The data to write + let data = b"hello reliable_write!"; + let fd = file.as_raw_fd(); + + // Call our function + let written = reliable_write(fd, data.as_ptr(), data.len()).expect("reliable_write failed"); + + assert_eq!(written, data.len()); + + // Rewind file and read back contents + file.rewind().unwrap(); + let mut buf = Vec::new(); + file.read_to_end(&mut buf).unwrap(); + + assert_eq!(&buf, data); + } + + #[cfg(unix)] + #[test] + fn test_reliable_write_handles_partial_write() { + use std::thread; + use std::time::Duration; + + // Create a pipe + let mut fds = [0; 2]; + assert_eq!(unsafe { libc::pipe(fds.as_mut_ptr()) }, 0); + let read_fd = fds[0]; + let write_fd = fds[1]; + + // Spawn a reader that drains slowly + thread::spawn(move || { + let mut buf = [0u8; 1024]; + loop { + let n = unsafe { libc::read(read_fd, buf.as_mut_ptr() as *mut _, buf.len()) }; + if n <= 0 { + break; + } + thread::sleep(Duration::from_millis(10)); + } + }); + + // Prepare data bigger than pipe capacity to force partial writes + let big = vec![b'x'; 200_000]; + + // This will require multiple kernel writes, exercising write_all + let n = reliable_write(write_fd, big.as_ptr(), big.len()).expect("reliable_write failed"); + + assert_eq!(n, big.len()); + + unsafe { libc::close(write_fd) }; + } + + #[test] + #[cfg(all(target_os = "linux", target_env = "gnu"))] + fn test_aligned_copy_cover_includes_tail() { + let mut infile = tempfile().unwrap(); + let mut outfile = tempfile().unwrap(); + + // Input data = 10 bytes + let data = b"abcdefghij"; + infile.write_all(data).unwrap(); + infile.rewind().unwrap(); + + let in_fd = infile.as_raw_fd(); + let out_fd = outfile.as_raw_fd(); + let in_ptr = data.as_ptr(); + + // Copy with block size 4, cover = Complete + let copied = aligned_copy_file_range( + in_ptr, + in_fd, + 0, + out_fd, + data.len(), + 4, + WriteRange::Complete, + ) + .unwrap(); + + assert_eq!(copied, data.len()); + + // Verify outfile contents = full input data + outfile.rewind().unwrap(); + let mut buf = Vec::new(); + outfile.read_to_end(&mut buf).unwrap(); + assert_eq!(&buf, data); + } + + #[test] + #[cfg(all(target_os = "linux", target_env = "gnu"))] + fn test_aligned_copy_blocks_skips_tail() { + let mut infile = tempfile().unwrap(); + let mut outfile = tempfile().unwrap(); + + // Input data = 10 bytes + let data = b"abcdefghij"; + infile.write_all(data).unwrap(); + infile.rewind().unwrap(); + + let in_fd = infile.as_raw_fd(); + let out_fd = outfile.as_raw_fd(); + let in_ptr = data.as_ptr(); + + // Copy with block size 4, cover = Blocks + let copied = + aligned_copy_file_range(in_ptr, in_fd, 0, out_fd, data.len(), 4, WriteRange::Blocks) + .unwrap(); + + // Only full 8 bytes (2 blocks of 4) should be copied + assert_eq!(copied, 8); + + // Verify outfile contents = only the first 8 bytes + outfile.rewind().unwrap(); + let mut buf = Vec::new(); + outfile.read_to_end(&mut buf).unwrap(); + assert_eq!(&buf, b"abcdefgh"); + } + + /////////////////////////////// + // Unit tests for write_chunk() + /////////////////////////////// + + fn new_for_test() -> (OutputBuffer, std::fs::File) { + let file = tempfile().unwrap(); + let buf = OutputBuffer { + out: BufWriter::new(Box::new(file.try_clone().unwrap())), + #[cfg(unix)] + fast_copy: FastCopy { + fd: -1, + is_regular: false, + block_size: 2, + }, + #[cfg(unix)] + max_pending_write: 8, + #[cfg(unix)] + mmap_chunk: None, + low_level_flushes: 0, + }; + (buf, file) + } + + #[cfg(unix)] + fn make_mmap_chunk(bytes: &'static [u8]) -> IOChunk<'static> { + IOChunk { + utf8_verified: Cell::new(true), + content: IOChunkContent::MmapInput { + fast_copy: FastCopy { + fd: -1, + is_regular: false, + block_size: 1, + }, + base: bytes.as_ptr(), + content: bytes, + full_span: bytes, + }, + } + } + + fn make_owned_chunk(s: &str, has_nl: bool) -> IOChunk<'_> { + IOChunk { + utf8_verified: Cell::new(true), + content: IOChunkContent::Owned { + content: s.to_string(), + has_newline: has_nl, + #[cfg(not(unix))] + _phantom: std::marker::PhantomData, + }, + } + } + + #[cfg(unix)] + #[test] + fn mmap_new_chunk_single() { + let (mut outbuf, _file) = new_for_test(); // OutputBuffer + + let c1 = make_mmap_chunk(b"abc"); + + outbuf.write_chunk(&c1).unwrap(); + + assert_eq!(outbuf.mmap_chunk.as_ref().unwrap().len, 3); + } + + #[cfg(unix)] + #[test] + fn mmap_new_chunk_and_coalesce() { + let (mut outbuf, _file) = new_for_test(); // OutputBuffer + + let backing = b"abcdefgh"; // contiguous buffer + let c1 = make_mmap_chunk(&backing[0..4]); // "abcd" + let c2 = make_mmap_chunk(&backing[4..8]); // "efgh" + + outbuf.write_chunk(&c1).unwrap(); + outbuf.write_chunk(&c2).unwrap(); + + assert_eq!(outbuf.mmap_chunk.as_ref().unwrap().len, 8); + } + + #[test] + #[cfg(unix)] + fn mmap_not_contiguous_triggers_flush() { + let (mut buf, _file) = new_for_test(); + let backing = b"abcdefghi"; + let c1 = make_mmap_chunk(&backing[0..4]); // "abcd" + // Guaranteed non-coalescable. Surprisingly, on macOS + // passing two strings resulted in coalescible data. + let c2 = make_mmap_chunk(&backing[5..9]); // "fghi" + + buf.write_chunk(&c1).unwrap(); + assert_eq!(buf.mmap_chunk.as_ref().unwrap().len, 4); + buf.write_chunk(&c2).unwrap(); + // No coalescing + assert_eq!(buf.mmap_chunk.as_ref().unwrap().len, 4); + } + + #[test] + #[cfg(unix)] + fn mmap_coalesce_and_flush_blocks() { + let (mut buf, _file) = new_for_test(); + buf.max_pending_write = 4; + let backing = b"abcdefgh"; // contiguous buffer + let c1 = make_mmap_chunk(&backing[0..5]); // "abcde" + let c2 = make_mmap_chunk(&backing[5..8]); // "fgh" + + buf.write_chunk(&c1).unwrap(); + buf.write_chunk(&c2).unwrap(); + // After a flush + assert_eq!(buf.mmap_chunk.as_ref().unwrap().len, 0); + } + + #[test] + fn owned_without_newline() { + let (mut buf, mut file) = new_for_test(); + let chunk = make_owned_chunk("hello", false); + buf.write_chunk(&chunk).unwrap(); + + buf.out.flush().unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + let mut out = String::new(); + file.read_to_string(&mut out).unwrap(); + + assert_eq!(out, "hello"); + } + + #[test] + fn owned_with_newline() { + let (mut buf, mut file) = new_for_test(); + let chunk = make_owned_chunk("world", true); + buf.write_chunk(&chunk).unwrap(); + + buf.out.flush().unwrap(); + file.seek(SeekFrom::Start(0)).unwrap(); + let mut out = String::new(); + file.read_to_string(&mut out).unwrap(); + + assert_eq!(out, "world\n"); + } } diff --git a/tests/by-util/test_sed.rs b/tests/by-util/test_sed.rs index 433cd70..b8ec7d0 100644 --- a/tests/by-util/test_sed.rs +++ b/tests/by-util/test_sed.rs @@ -93,6 +93,21 @@ fn test_no_script_file() { } } +/// Test correct functioning of copy_file_range. +#[test] +fn test_multiple_input_files() { + new_ucmd!() + .args(&[ + "-e", + "", + "input/dots-64k.txt", + "input/no-new-line.txt", + "input/dots-64k.txt", + ]) + .succeeds() + .stdout_is_fixture("output/multiple_input_files"); +} + #[test] fn test_delete_stdin() { for fixture in INPUT_FILES { @@ -114,6 +129,20 @@ fn test_delete_file() { } } +#[test] +#[cfg(unix)] +fn test_special_file() { + for fixture in INPUT_FILES { + // To test path avoiding copy_file_range + let devnull = fs::File::create("/dev/null").unwrap(); + + new_ucmd!() + .args(&["-e", "", fixture]) + .set_stdout(devnull) + .succeeds(); + } +} + /// Create a new test function to verify an execution for specified output. macro_rules! check_output { ($name:ident, $args:expr) => { diff --git a/tests/fixtures/sed/output/multiple_input_files b/tests/fixtures/sed/output/multiple_input_files new file mode 100644 index 0000000..2529dcc --- /dev/null +++ b/tests/fixtures/sed/output/multiple_input_files @@ -0,0 +1,32 @@ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +Hello................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ +................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................................ diff --git a/util/benchmark.sh b/util/benchmark.sh index a3b6232..17d405c 100755 --- a/util/benchmark.sh +++ b/util/benchmark.sh @@ -22,7 +22,7 @@ SCRIPTS=tests/fixtures/sed/script # Run hyperfine with the specified name and command and collect the results. bench_run() { - if hyperfine --command-name "$1" --warmup 2 --export-csv out.csv "$2" ; then + if hyperfine --command-name "$1" --warmup 2 --export-csv out.csv --output ./out.txt "$2" ; then # Output the results sans-heading. sed 1d out.csv >>"$OUT" else @@ -30,7 +30,7 @@ bench_run() echo "$1,,,,,,," >>"$OUT" fi - rm out.csv + rm out.csv out.txt } # Shared heading