Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/chars.rs
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ where
// did not contain a valid rust style float. This is expected to be
// used only to parse f32s or f64s and may panic otherwise.
#[inline(always)]
fn parse_f<T: Tokens<Item = char>, F, B, E>(t: &mut T) -> Option<F>
fn parse_f<T, F, B, E>(t: &mut T) -> Option<F>
where
T: Tokens<Item = char>,
F: core::str::FromStr<Err = E>,
Expand Down
27 changes: 5 additions & 22 deletions src/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -122,28 +122,11 @@ impl<'a> Tokens for StrTokens<'a> {
type Location = StrTokensLocation;

fn next(&mut self) -> Option<Self::Item> {
if self.cursor == self.str.len() {
return None;
}

// Cursor should always start at a valid char boundary.
// So, we just find the next char boundary and return the
// char between those two.
let mut next_char_boundary = self.cursor + 1;
while !self.str.is_char_boundary(next_char_boundary) {
next_char_boundary += 1;
}

// We have to go to &str and then char. Unchecked because we know
// that we are on a valid boundary. There's probably a quicker way..
// To check that bounds detection works even on exotic characters, there's a test included
// at the end of the file.
let next_char = unsafe { self.str.get_unchecked(self.cursor..next_char_boundary) }
.chars()
.next()
.unwrap();

self.cursor = next_char_boundary;
// Cursor should always start at a valid char boundary so this will never panic.
let str = &self.str[self.cursor..];
let mut chars = str.chars();
let next_char = chars.next()?;
self.cursor += str.len() - chars.as_str().len();
Some(next_char)
}

Expand Down