Allow recording the raw HTTP/1 headers sent and received

This allows feeding them into a WARC file
(https://en.wikipedia.org/wiki/WARC_(file_format)). The rest of the
request and response is already available as it's either the explicitly
set body or the received response body.

This includes the final `\r\n\r\n` between the headers and the response
(so that it can be distinguished from bare `\n\n`).

Needed for a similar request in reqwest:
seanmonstar/reqwest#1229.

Author: dis-archive

src/client/conn/http1.rs

@@ -112,6 +112,8 @@ pub struct Builder {
     h1_parser_config: ParserConfig,
     h1_writev: Option<bool>,
     h1_title_case_headers: bool,
+    h1_record_raw_request_headers: bool,
+    h1_record_raw_response_headers: bool,
     h1_preserve_header_case: bool,
     h1_max_headers: Option<usize>,
     #[cfg(feature = "ffi")]
@@ -312,6 +314,8 @@ impl Builder {
             h1_read_buf_exact_size: None,
             h1_parser_config: Default::default(),
             h1_title_case_headers: false,
+            h1_record_raw_request_headers: false,
+            h1_record_raw_response_headers: false,
             h1_preserve_header_case: false,
             h1_max_headers: None,
             #[cfg(feature = "ffi")]
@@ -428,6 +432,22 @@ impl Builder {
         self
     }
 
+    /// Set whether to record the raw headers sent.
+    ///
+    /// Default is false.
+    pub fn record_raw_request_headers(&mut self, enabled: bool) -> &mut Builder {
+        self.h1_record_raw_request_headers = enabled;
+        self
+    }
+
+    /// Set whether to record the raw headers received.
+    ///
+    /// Default is false.
+    pub fn record_raw_response_headers(&mut self, enabled: bool) -> &mut Builder {
+        self.h1_record_raw_response_headers = enabled;
+        self
+    }
+
     /// Set whether to support preserving original header cases.
     ///
     /// Currently, this will record the original cases received, and store them
@@ -539,6 +559,12 @@ impl Builder {
             if opts.h1_title_case_headers {
                 conn.set_title_case_headers();
             }
+            if opts.h1_record_raw_request_headers {
+                conn.set_record_raw_request_headers();
+            }
+            if opts.h1_record_raw_response_headers {
+                conn.set_record_raw_response_headers();
+            }
             if opts.h1_preserve_header_case {
                 conn.set_preserve_header_case();
             }

src/ext/mod.rs

@@ -86,6 +86,38 @@ impl fmt::Debug for Protocol {
     }
 }
 
+/// Raw request headers as sent over the TLS or TCP connection.
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+#[derive(Clone, Debug)]
+pub struct RawRequestHeaders(Bytes);
+
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+impl RawRequestHeaders {
+    /// Returns the raw bytes sent for the header of the request.
+    pub fn as_bytes(&self) -> &[u8] {
+        &self.0
+    }
+    pub(crate) fn from(bytes: Bytes) -> Self {
+        Self(bytes)
+    }
+}
+
+/// Raw response headers as sent over the TLS or TCP connection.
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+#[derive(Clone, Debug)]
+pub struct RawResponseHeaders(Bytes);
+
+#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
+impl RawResponseHeaders {
+    /// Returns the raw bytes received for the header of the response.
+    pub fn as_bytes(&self) -> &[u8] {
+        &self.0
+    }
+    pub(crate) fn from(bytes: Bytes) -> Self {
+        Self(bytes)
+    }
+}
+
 /// A map from header names to their original casing as received in an HTTP message.
 ///
 /// If an HTTP/1 response `res` is parsed on a connection whose option

src/proto/h1/conn.rs

@@ -68,6 +68,9 @@ where
                 date_header: true,
                 #[cfg(feature = "server")]
                 timer: Time::Empty,
+                raw_request_headers: None,
+                record_raw_request_headers: false,
+                record_raw_response_headers: false,
                 preserve_header_case: false,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: false,
@@ -123,6 +126,14 @@ where
         self.state.title_case_headers = true;
     }
 
+    pub(crate) fn set_record_raw_request_headers(&mut self) {
+        self.state.record_raw_request_headers = true;
+    }
+
+    pub(crate) fn set_record_raw_response_headers(&mut self) {
+        self.state.record_raw_response_headers = true;
+    }
+
     pub(crate) fn set_preserve_header_case(&mut self) {
         self.state.preserve_header_case = true;
     }
@@ -241,6 +252,8 @@ where
                 req_method: &mut self.state.method,
                 h1_parser_config: self.state.h1_parser_config.clone(),
                 h1_max_headers: self.state.h1_max_headers,
+                raw_request_headers: self.state.raw_request_headers.as_ref(),
+                record_raw_headers: self.state.record_raw_response_headers,
                 preserve_header_case: self.state.preserve_header_case,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: self.state.preserve_header_order,
@@ -617,6 +630,7 @@ where
         self.enforce_version(&mut head);
 
         let buf = self.io.headers_buf();
+        let headers_start = buf.len();
         match super::role::encode_headers::<T>(
             Encode {
                 head: &mut head,
@@ -633,6 +647,11 @@ where
             Ok(encoder) => {
                 debug_assert!(self.state.cached_headers.is_none());
                 debug_assert!(head.headers.is_empty());
+                if self.state.record_raw_request_headers {
+                    self.state.raw_request_headers = Some(crate::ext::RawRequestHeaders::from(Bytes::copy_from_slice(&buf[headers_start..])));
+                } else {
+                    self.state.raw_request_headers = None;
+                }
                 self.state.cached_headers = Some(head.headers);
 
                 #[cfg(feature = "client")]
@@ -934,6 +953,9 @@ struct State {
     date_header: bool,
     #[cfg(feature = "server")]
     timer: Time,
+    raw_request_headers: Option<crate::ext::RawRequestHeaders>,
+    record_raw_request_headers: bool,
+    record_raw_response_headers: bool,
     preserve_header_case: bool,
     #[cfg(feature = "ffi")]
     preserve_header_order: bool,

src/proto/h1/io.rs

@@ -184,6 +184,8 @@ where
                     req_method: parse_ctx.req_method,
                     h1_parser_config: parse_ctx.h1_parser_config.clone(),
                     h1_max_headers: parse_ctx.h1_max_headers,
+                    raw_request_headers: parse_ctx.raw_request_headers,
+                    record_raw_headers: parse_ctx.record_raw_headers,
                     preserve_header_case: parse_ctx.preserve_header_case,
                     #[cfg(feature = "ffi")]
                     preserve_header_order: parse_ctx.preserve_header_order,
@@ -706,6 +708,8 @@ mod tests {
                 req_method: &mut None,
                 h1_parser_config: Default::default(),
                 h1_max_headers: None,
+                raw_request_headers: None,
+                record_raw_headers: false,
                 preserve_header_case: false,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: false,

src/proto/h1/mod.rs

@@ -73,6 +73,8 @@ pub(crate) struct ParseContext<'a> {
     req_method: &'a mut Option<Method>,
     h1_parser_config: ParserConfig,
     h1_max_headers: Option<usize>,
+    record_raw_headers: bool,
+    raw_request_headers: Option<&'a crate::ext::RawRequestHeaders>,
     preserve_header_case: bool,
     #[cfg(feature = "ffi")]
     preserve_header_order: bool,

src/proto/h1/role.rs

@@ -18,6 +18,7 @@ use crate::body::DecodedLength;
 use crate::common::date;
 use crate::error::Parse;
 use crate::ext::HeaderCaseMap;
+use crate::ext::RawResponseHeaders;
 #[cfg(feature = "ffi")]
 use crate::ext::OriginalHeaderOrder;
 use crate::headers;
@@ -1056,19 +1057,32 @@ impl Http1Transaction for Client {
             };
 
             let mut slice = buf.split_to(len);
+            let raw_headers;
 
-            if ctx
+            let slice = if ctx
                 .h1_parser_config
                 .obsolete_multiline_headers_in_responses_are_allowed()
             {
+                raw_headers = if ctx.record_raw_headers {
+                    Some(RawResponseHeaders::from(slice.clone().freeze()))
+                } else {
+                    None
+                };
                 for header in &mut headers_indices[..headers_len] {
                     // SAFETY: array is valid up to `headers_len`
                     let header = unsafe { header.assume_init_mut() };
                     Client::obs_fold_line(&mut slice, header);
                 }
-            }
-
-            let slice = slice.freeze();
+                slice.freeze()
+            } else {
+                let slice = slice.freeze();
+                raw_headers = if ctx.record_raw_headers {
+                    Some(RawResponseHeaders::from(slice.clone()))
+                } else {
+                    None
+                };
+                slice
+            };
 
             let mut headers = ctx.cached_headers.take().unwrap_or_default();
 
@@ -1119,6 +1133,14 @@ impl Http1Transaction for Client {
 
             let mut extensions = http::Extensions::default();
 
+            if let Some(raw_request_headers) = ctx.raw_request_headers {
+                extensions.insert(raw_request_headers.clone());
+            }
+
+            if let Some(raw_headers) = raw_headers {
+                extensions.insert(raw_headers);
+            }
+
             if let Some(header_case_map) = header_case_map {
                 extensions.insert(header_case_map);
             }
@@ -1656,6 +1678,8 @@ mod tests {
                 req_method: &mut method,
                 h1_parser_config: Default::default(),
                 h1_max_headers: None,
+                raw_request_headers: None,
+                record_raw_headers: false,
                 preserve_header_case: false,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: false,
@@ -1684,6 +1708,8 @@ mod tests {
             req_method: &mut Some(crate::Method::GET),
             h1_parser_config: Default::default(),
             h1_max_headers: None,
+            raw_request_headers: None,
+            record_raw_headers: false,
             preserve_header_case: false,
             #[cfg(feature = "ffi")]
             preserve_header_order: false,
@@ -1708,6 +1734,8 @@ mod tests {
             req_method: &mut None,
             h1_parser_config: Default::default(),
             h1_max_headers: None,
+            raw_request_headers: None,
+            record_raw_headers: false,
             preserve_header_case: false,
             #[cfg(feature = "ffi")]
             preserve_header_order: false,
@@ -1729,6 +1757,8 @@ mod tests {
             req_method: &mut Some(crate::Method::GET),
             h1_parser_config: Default::default(),
             h1_max_headers: None,
+            raw_request_headers: None,
+            record_raw_headers: false,
             preserve_header_case: false,
             #[cfg(feature = "ffi")]
             preserve_header_order: false,
@@ -1752,6 +1782,8 @@ mod tests {
             req_method: &mut Some(crate::Method::GET),
             h1_parser_config: Default::default(),
             h1_max_headers: None,
+            raw_request_headers: None,
+            record_raw_headers: false,
             preserve_header_case: false,
             #[cfg(feature = "ffi")]
             preserve_header_order: false,
@@ -1779,6 +1811,8 @@ mod tests {
             req_method: &mut Some(crate::Method::GET),
             h1_parser_config,
             h1_max_headers: None,
+            raw_request_headers: None,
+            record_raw_headers: false,
             preserve_header_case: false,
             #[cfg(feature = "ffi")]
             preserve_header_order: false,
@@ -1803,6 +1837,8 @@ mod tests {
             req_method: &mut Some(crate::Method::GET),
             h1_parser_config: Default::default(),
             h1_max_headers: None,
+            raw_request_headers: None,
+            record_raw_headers: false,
             preserve_header_case: false,
             #[cfg(feature = "ffi")]
             preserve_header_order: false,
@@ -1823,6 +1859,8 @@ mod tests {
             req_method: &mut None,
             h1_parser_config: Default::default(),
             h1_max_headers: None,
+            raw_request_headers: None,
+            record_raw_headers: false,
             preserve_header_case: true,
             #[cfg(feature = "ffi")]
             preserve_header_order: false,
@@ -1862,6 +1900,8 @@ mod tests {
                     req_method: &mut None,
                     h1_parser_config: Default::default(),
                     h1_max_headers: None,
+                    raw_request_headers: None,
+                    record_raw_headers: false,
                     preserve_header_case: false,
                     #[cfg(feature = "ffi")]
                     preserve_header_order: false,
@@ -1883,6 +1923,8 @@ mod tests {
                     req_method: &mut None,
                     h1_parser_config: Default::default(),
                     h1_max_headers: None,
+                    raw_request_headers: None,
+                    record_raw_headers: false,
                     preserve_header_case: false,
                     #[cfg(feature = "ffi")]
                     preserve_header_order: false,
@@ -2113,6 +2155,8 @@ mod tests {
                     req_method: &mut Some(Method::GET),
                     h1_parser_config: Default::default(),
                     h1_max_headers: None,
+                    raw_request_headers: None,
+                    record_raw_headers: false,
                     preserve_header_case: false,
                     #[cfg(feature = "ffi")]
                     preserve_header_order: false,
@@ -2134,6 +2178,8 @@ mod tests {
                     req_method: &mut Some(m),
                     h1_parser_config: Default::default(),
                     h1_max_headers: None,
+                    raw_request_headers: None,
+                    record_raw_headers: false,
                     preserve_header_case: false,
                     #[cfg(feature = "ffi")]
                     preserve_header_order: false,
@@ -2155,6 +2201,8 @@ mod tests {
                     req_method: &mut Some(Method::GET),
                     h1_parser_config: Default::default(),
                     h1_max_headers: None,
+                    raw_request_headers: None,
+                    record_raw_headers: false,
                     preserve_header_case: false,
                     #[cfg(feature = "ffi")]
                     preserve_header_order: false,
@@ -2725,6 +2773,8 @@ mod tests {
                 req_method: &mut Some(Method::GET),
                 h1_parser_config: Default::default(),
                 h1_max_headers: None,
+                raw_request_headers: None,
+                record_raw_headers: false,
                 preserve_header_case: false,
                 #[cfg(feature = "ffi")]
                 preserve_header_order: false,
@@ -2769,6 +2819,8 @@ mod tests {
                         req_method: &mut None,
                         h1_parser_config: Default::default(),
                         h1_max_headers: max_headers,
+                        raw_request_headers: None,
+                        record_raw_headers: false,
                         preserve_header_case: false,
                         #[cfg(feature = "ffi")]
                         preserve_header_order: false,
@@ -2793,6 +2845,8 @@ mod tests {
                         req_method: &mut None,
                         h1_parser_config: Default::default(),
                         h1_max_headers: max_headers,
+                        raw_request_headers: None,
+                        record_raw_headers: false,
                         preserve_header_case: false,
                         #[cfg(feature = "ffi")]
                         preserve_header_order: false,