Skip to content

Commit

Permalink
Remove malloc in CRINEX header parsing (#233)
Browse files Browse the repository at this point in the history 
* remove memory allocation in Crinex header parsing
* move observation methods to separate module
* run clippy
* remove unused macro

---------

Signed-off-by: Guillaume W. Bres <[email protected]>
  • Loading branch information
@gwbres
gwbres authored Apr 17, 2024
1 parent bdf3ebb commit b79e25b
Show file tree
Hide file tree
Showing 8 changed files with 207 additions and 172 deletions.
4 changes: 2 additions & 2 deletions rinex/src/doris/mod.rs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
use std::collections::HashMap;
use std::str::FromStr;

use thiserror::Error;

use crate::{
domes::{Domes, Error as DomesParsingError, TrackingPoint as DomesTrackingPoint},
domes::Error as DomesParsingError,
observable::Observable,
prelude::{Duration, Epoch},
};
Expand Down
34 changes: 17 additions & 17 deletions rinex/src/doris/record.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ pub(crate) fn parse_epoch(
> {
let mut obs_idx = 0usize;
let mut epoch = Epoch::default();
let mut flag = EpochFlag::default();
let flag = EpochFlag::default();
let mut station = Option::<Station>::None;
let mut buffer = BTreeMap::<Station, HashMap<Observable, ObservationData>>::new();

Expand All @@ -69,11 +69,11 @@ pub(crate) fn parse_epoch(
let stations = &doris.stations;

assert!(
stations.len() > 0,
!stations.is_empty(),
"badly formed DORIS RINEX: no stations defined"
);
assert!(
observables.len() > 0,
!observables.is_empty(),
"badly formed DORIS RINEX: no observables defined"
);

Expand All @@ -83,11 +83,11 @@ pub(crate) fn parse_epoch(
/* 1st line gives TAI timestamp, flag, clock offset */
let line = line.split_at(2).1; // "> "
let offset = "YYYY MM DD HH MM SS.NNNNNNNNN 0".len();
let (date, rem) = line.split_at(offset);
let (date, _rem) = line.split_at(offset);
epoch = parse_in_timescale(date, TimeScale::TAI)?;
},
_ => {
let (id, remainder) = line.split_at(4);
let (id, _remainder) = line.split_at(4);
//println!("ID : \"{}\" - REMAINDER : \"{}\"", id, remainder); //DBEUG

if obs_idx == 0 {
Expand All @@ -110,7 +110,7 @@ pub(crate) fn parse_epoch(

// consume this line
let mut offset = 5;
let mut max_offset = line.len();
let max_offset = line.len();
while offset < line.len() {
let content = &line[offset..std::cmp::min(max_offset, offset + 16)];
let obs = &content[..12];
Expand All @@ -126,15 +126,15 @@ pub(crate) fn parse_epoch(
.parse::<f64>()
.unwrap_or_else(|e| panic!("failed to parse observation: {:?}", e));

let m1 = if m1.len() > 0 {
let m1 = if !m1.is_empty() {
Some(m1.parse::<u8>().unwrap_or_else(|e| {
panic!("failed to parse observation m1 flag: {:?}", e)
}))
} else {
None
};

let m2 = if m2.len() > 0 {
let m2 = if !m2.is_empty() {
Some(m2.parse::<u8>().unwrap_or_else(|e| {
panic!("failed to parse observation m2 flag: {:?}", e)
}))
Expand All @@ -154,7 +154,7 @@ pub(crate) fn parse_epoch(
if let Some(station) = buffer.get_mut(identified_station) {
station.insert(observable.clone(), obsdata);
} else {
let mut inner =
let inner =
HashMap::from_iter([(Observable::default(), obsdata)].into_iter());
buffer.insert(identified_station.clone(), inner);
}
Expand Down Expand Up @@ -236,25 +236,25 @@ impl Preprocessing for Record {
* Decimates only a given record subset
*/
#[cfg(feature = "processing")]
fn decimate_data_subset(record: &mut Record, subset: &Record, target: &TargetItem) {
fn decimate_data_subset(record: &mut Record, _subset: &Record, target: &TargetItem) {
match target {
TargetItem::ClockItem => {
/*
* Remove clock fields from self
* where it should now be missing
*/
for (epoch, _) in record.iter_mut() {
for (_epoch, _) in record.iter_mut() {
//if subset.get(epoch).is_none() {
// // should be missing
// // *clk = None; // now missing
//}
}
},
TargetItem::SvItem(svs) => {
TargetItem::SvItem(_svs) => {
/*
* Remove SV observations where it should now be missing
*/
for (epoch, _) in record.iter_mut() {
for (_epoch, _) in record.iter_mut() {
//if subset.get(epoch).is_none() {
// // should be missing
// for sv in svs.iter() {
Expand All @@ -263,11 +263,11 @@ fn decimate_data_subset(record: &mut Record, subset: &Record, target: &TargetIte
//}
}
},
TargetItem::ObservableItem(obs_list) => {
TargetItem::ObservableItem(_obs_list) => {
/*
* Remove given observations where it should now be missing
*/
for (epoch, _) in record.iter_mut() {
for (_epoch, _) in record.iter_mut() {
//if subset.get(epoch).is_none() {
// // should be missing
// for (_sv, observables) in vehicles.iter_mut() {
Expand All @@ -276,11 +276,11 @@ fn decimate_data_subset(record: &mut Record, subset: &Record, target: &TargetIte
//}
}
},
TargetItem::ConstellationItem(constells_list) => {
TargetItem::ConstellationItem(_constells_list) => {
/*
* Remove observations for given constellation(s) where it should now be missing
*/
for (epoch, _) in record.iter_mut() {
for (_epoch, _) in record.iter_mut() {
//if subset.get(epoch).is_none() {
// // should be missing
// vehicles.retain(|sv, _| {
Expand Down
2 changes: 0 additions & 2 deletions rinex/src/doris/station.rs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
//! DORIS Station
use crate::{domes::Domes, doris::Error};
use std::str::FromStr;

#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
Expand Down
8 changes: 3 additions & 5 deletions rinex/src/epoch.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,9 @@ pub(crate) fn parse_in_timescale(content: &str, ts: TimeScale) -> Result<Epoch,
if is_nav {
// NAV RINEX : 100ms precision
ns *= 100_000_000;
} else {
if nanos.len() != 9 {
// OBS RINEX : 100ns precision
ns *= 100;
}
} else if nanos.len() != 9 {
// OBS RINEX : 100ns precision
ns *= 100;
}
} else {
ss = item
Expand Down
167 changes: 101 additions & 66 deletions rinex/src/header.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ use crate::{
clock::WorkClock,
cospar::{Error as CosparError, COSPAR},
domes::Domes,
doris,
doris::{Error as DorisError, HeaderFields as DorisHeader, Station as DorisStation},
fmt_comment, fmt_rinex,
ground_position::GroundPosition,
Expand Down Expand Up @@ -176,8 +175,8 @@ pub enum ParsingError {
ParsePcvError(#[from] antex::pcv::Error),
#[error("unknown ionex reference")]
UnknownReferenceIonex(#[from] ionex::system::Error),
#[error("invalid crinex header \"{0}\": \"{1}\"")]
CrinexHeader(String, String),
#[error("invalid crinex header \"{0}\"")]
CrinexHeader(String),
#[error("failed to parse datetime {0} field from \"{1}\"")]
DateTimeParsing(String, String),
#[error("failed to parse {0} integer value from \"{1}\"")]
Expand Down Expand Up @@ -233,15 +232,6 @@ macro_rules! parse_float_error {
};
}

/*
* Generates a ParsingError::InvalidIonexGridError(x, y)
*/
macro_rules! grid_format_error {
($field: expr, $content: expr) => {
ParsingError::InvalidIonexGrid(String::from($field), $content.to_string())
};
}

impl Header {
/// Builds a `Header` from stream reader
pub fn new(reader: &mut BufferedReader) -> Result<Header, ParsingError> {
Expand Down Expand Up @@ -313,52 +303,7 @@ impl Header {

observation.crinex = Some(Crinex::default().with_version(crinex_revision));
} else if marker.contains("CRINEX PROG / DATE") {
let (prog, remainder) = content.split_at(20);
let (_, remainder) = remainder.split_at(20);
let date = remainder.split_at(20).0.trim();
let items: Vec<&str> = date.split_ascii_whitespace().collect();
if items.len() != 2 {
return Err(ParsingError::CrinexHeader(
String::from("CRINEX PROG/DATE"),
content.to_string(),
));
}

let date: Vec<&str> = items[0].split('-').collect();
let time: Vec<&str> = items[1].split(':').collect();

let day = date[0].trim();
let day = day.parse::<u8>().or(Err(ParsingError::DateTimeParsing(
String::from("day"),
day.to_string(),
)))?;

let month = date[1].trim();
let month = parse_formatted_month(month)?;

let y = date[2].trim();
let mut y = y.parse::<i32>().or(Err(ParsingError::DateTimeParsing(
String::from("year"),
y.to_string(),
)))?;

let h = time[0].trim();
let h = h.parse::<u8>().or(Err(ParsingError::DateTimeParsing(
String::from("hour"),
h.to_string(),
)))?;

let m = time[1].trim();
let m = m.parse::<u8>().or(Err(ParsingError::DateTimeParsing(
String::from("minute"),
m.to_string(),
)))?;

if let Some(crinex) = &mut observation.crinex {
y += 2000;
let date = Epoch::from_gregorian_utc(y, month, day, h, m, 0, 0);
*crinex = crinex.with_prog(prog.trim()).with_date(date);
}
Self::parse_crinex_prog_date(content, &mut observation)?;

////////////////////////////////////////
// [2] ANTEX special header
Expand Down Expand Up @@ -628,10 +573,8 @@ impl Header {
.or(Err(parse_int_error!("SYS / SCALE FACTOR", factor)))?;

// parse end of line
let (_num, remainder) = rem.split_at(3);

let mut items = remainder.split_ascii_whitespace();
while let Some(observable_str) = items.next() {
let (_num, rem) = rem.split_at(3);
for observable_str in rem.split_ascii_whitespace() {
let observable = Observable::from_str(observable_str)?;

// latch scaling value
Expand Down Expand Up @@ -855,19 +798,19 @@ impl Header {
}
} else if marker.contains("TYPES OF OBS") {
// these observations can serve both Observation & Meteo RINEX
Self::parse_v2_observables(&content, constellation, &mut meteo, &mut observation);
Self::parse_v2_observables(content, constellation, &mut meteo, &mut observation);
} else if marker.contains("SYS / # / OBS TYPES") {
match rinex_type {
Type::ObservationData => {
Self::parse_v3_observables(
&content,
content,
&mut current_constell,
&mut observation,
);
},
Type::DORIS => {
/* in DORIS RINEX, observations are not tied to a particular constellation */
Self::parse_doris_observables(&content, &mut doris);
Self::parse_doris_observables(content, &mut doris);
},
_ => {},
}
Expand Down Expand Up @@ -1646,6 +1589,98 @@ impl Header {
Ok(grid)
}
}
/*
* Parse CRINEX special header
*/
fn parse_crinex_prog_date(
line: &str,
observation: &mut ObservationHeader,
) -> Result<(), ParsingError> {
assert!(
observation.crinex.is_some(),
"badly formed CRINEX: CRINEX VERS/TYPE expected as first header"
);
let mut crinex = observation.crinex.clone().unwrap();

let (prog, rem) = line.split_at(20);
crinex = crinex.with_prog(prog.trim());

let (_, rem) = rem.split_at(20);
let date = rem.split_at(20).0.trim();

let date_time = date.split_ascii_whitespace();
if date_time.count() != 2 {
return Err(ParsingError::CrinexHeader(String::from("CRINEX PROG/DATE")));
}

let mut year = 2000_i32; // CRINEX: Y %02D
let mut month = 0_u8;
let mut day = 0_u8;
let mut hr = 0_u8;
let mut mins = 0_u8;
for (index, date_time) in date.split_ascii_whitespace().enumerate() {
match index {
0 => {
for (index, component) in date_time.split('-').enumerate() {
let component = component.trim();
match index {
0 => {
day = component.parse::<u8>().or(Err(
ParsingError::DateTimeParsing(
"crinex::day".to_string(),
component.to_string(),
),
))?;
},
1 => {
month = parse_formatted_month(component.trim())?;
},
2 => {
year += component.trim().parse::<i32>().or(Err(
ParsingError::DateTimeParsing(
"crinex::year".to_string(),
component.to_string(),
),
))?;
},
_ => {},
}
}
},
1 => {
for (index, component) in date_time.split(':').enumerate() {
let component = component.trim();
match index {
0 => {
hr = component.parse::<u8>().or(Err(
ParsingError::DateTimeParsing(
"crinex::hours".to_string(),
component.to_string(),
),
))?;
},
1 => {
mins = component.parse::<u8>().or(Err(
ParsingError::DateTimeParsing(
"crinex::minutes".to_string(),
component.to_string(),
),
))?;
},
_ => {},
}
}
},
_ => {},
}
}

let epoch = Epoch::from_gregorian_utc(year, month, day, hr, mins, 0, 0);
crinex = crinex.with_date(epoch);

observation.crinex = Some(crinex);
Ok(())
}
/*
* Parse list of observables (V2)
*/
Expand Down Expand Up @@ -1710,7 +1745,7 @@ impl Header {
// system correctly identified
for item in items.split_ascii_whitespace() {
if let Ok(observable) = Observable::from_str(item) {
if let Some(codes) = observation.codes.get_mut(&constell) {
if let Some(codes) = observation.codes.get_mut(constell) {
codes.push(observable);
} else {
observation.codes.insert(*constell, vec![observable]);
Expand Down
Loading

0 comments on commit b79e25b

Please sign in to comment.