introducing filegen opmode (#205)

* introducing filegen opmode
* new filegen opts
* fixed error in decimation
* library update
* improved time binning
* add two time synchronous contexts

---------

Signed-off-by: Guillaume W. Bres <[email protected]>

README.md

@@ -11,7 +11,8 @@ RINEX
 
 Rust tool suites to parse, analyze and process [RINEX Data](https://en.wikipedia.org/wiki/RINEX).
 
-The [Wiki pages](https://github.com/georust/rinex/wiki) contain all the documentation of this project, including several examples spanning different applications of GNSS.
+The [Wiki pages](https://github.com/georust/rinex/wiki) is the main documentation portal. It contains
+several examples spanning different GNSS applications.
 
 If you have any question or experience any problems, feel free to open an issue on Github.  
 You can also contact us [on our Discord channel](https://discord.gg/Fp2aape)
@@ -57,7 +58,7 @@ Some minor features in the RINEX2 or 3 revisions may not be supported.
 on RINEX post processing rather than RINEX data production. Do not hesitate to fork and submit
 your improvements
 
-## Architecture 
+## Repository 
 
 * [`rinex`](rinex/) is the core library 
 * [`rinex-cli`](rinex-cli/) : an application dedicated to RINEX post processing.
@@ -83,6 +84,16 @@ from raw uBlox GNSS receiver frames. This application is work in progress at the
 * [CGGTTS](https://github.com/gwbres/cggtts)
 * [GNSS definitions in Rust](https://github.com/rtk-rs/gnss)
 
+Formats & revisions
+===================
+
+The core library supports parsing RINEX V4.00 and the current behavior is to fail
+on higher revisions. NAV V4 is correctly supported as described in the following table.
+
+We support the latest revisions for both IONEX and Clock RINEX.
+
+We support the latest (rev D) SP3 format.
+
 RINEX formats & applications
 ============================
 
@@ -119,22 +130,6 @@ File formats
 
 :heavy_minus_sign: No restrictions: file names do not have to follow naming conventions.  
 
-Benchmarking
-============
-
-Test           | Results 
----------------|-------------------------|
-textdiff/decompression/epoch | 979.55 ns |
-textdiff/decompression/flag  | 147.16 ns | 
-numdiff/decompression/small  | 191.86 ns |
-numdiff/decompression/big    | 1.0973 µs |
-parsing/OBSv2/zegv0010.21o   | 951.40 µs |
-parsing/OBSv3/ACOR00ESP      | 4.1139 ms |
-processing/esbc00dnkr2021/mask:gnss | 352.81 ms |
-processing/esbc00dnkr2021/mask:obs |  438.73 ms |
-processing/esbc00dnkr2021/mask:sv | 341.42 ms | 
-processing/esbc00dnkr2021/smooth:hatch:l1c,l2c | 502.90 ms | 
-
 Special Thanks
 ==============
 

crx2rnx/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "crx2rnx"
-version = "2.3.0"
+version = "2.3.1"
 license = "MIT OR Apache-2.0"
 authors = ["Guillaume W. Bres <[email protected]>"]
 description = "RINEX data decompressor"
@@ -12,4 +12,4 @@ readme = "README.md"
 
 [dependencies]
 clap = { version = "4.4.13", features = ["derive", "color"] }
-rinex = { path = "../rinex", version = "=0.15.5", features = ["serde"] }
+rinex = { path = "../rinex", version = "=0.15.6", features = ["serde"] }

rinex-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rinex-cli"
-version = "0.10.1"
+version = "0.10.2"
 license = "MIT OR Apache-2.0"
 authors = ["Guillaume W. Bres <[email protected]>"]
 description = "Command line tool parse and analyze RINEX data"
@@ -31,8 +31,8 @@ horrorshow = "0.8"
 clap = { version = "4.4.13", features = ["derive", "color"] }
 hifitime = { version = "3.9.0", features = ["serde", "std"] }
 gnss-rs = { version = "2.1.3" , features = ["serde"] }
-rinex = { path = "../rinex", version = "=0.15.5", features = ["full"] }
-rinex-qc = { path = "../rinex-qc", version = "=0.1.10", features = ["serde"] } 
+rinex = { path = "../rinex", version = "=0.15.6", features = ["full"] }
+rinex-qc = { path = "../rinex-qc", version = "=0.1.11", features = ["serde"] } 
 sp3 = { path = "../sp3", version = "=1.0.7",  features = ["serde", "flate2"] }
 serde = { version = "1.0", default-features = false, features = ["derive"] }
 

rinex-cli/src/cli/filegen.rs

@@ -0,0 +1,13 @@
+// filegen opmode
+use clap::Command;
+
+pub fn subcommand() -> Command {
+    Command::new("filegen")
+        .long_flag("filegen")
+        .arg_required_else_help(false)
+        .about(
+            "RINEX Data formatting. Use this option to preprocess, 
+modify and dump resulting context in preserved RINEX format. 
+You can use this for example, to generate a decimated RINEX file from an input Observations file.",
+        )
+}

rinex-cli/src/cli/mod.rs

@@ -29,6 +29,8 @@ mod substract;
 mod qc;
 // positioning mode
 mod positioning;
+// filegen mode
+mod filegen;
 
 pub struct Cli {
     /// Arguments passed by user
@@ -304,6 +306,7 @@ Otherwise it gets automatically picked up."))
                     .value_name("\"lat,lon,alt\" coordinates in ddeg [°]")
                     .help("Define the (RX) antenna position manualy, in decimal degrees."))
                 .next_help_heading("Exclusive Opmodes: you can only run one at a time.")
+                .subcommand(filegen::subcommand())
                 .subcommand(graph::subcommand())
                 .subcommand(identify::subcommand())
                 .subcommand(merge::subcommand())

rinex-cli/src/fops.rs

@@ -8,6 +8,77 @@ use std::path::PathBuf;
 use std::process::Command;
 use std::str::FromStr;
 
+/*
+ * Dumps current context (usually preprocessed)
+ * into RINEX format maintaining consistent format
+ */
+pub fn filegen(ctx: &Context, _matches: &ArgMatches) -> Result<(), Error> {
+    // OBS RINEX processing
+    if let Some(rinex) = ctx.data.obs_data() {
+        let filename = ctx
+            .data
+            .obs_paths()
+            .expect("failed to determine observation output")
+            .get(0)
+            .expect("failed to determine observation output")
+            .file_name()
+            .expect("failed to determine observation output")
+            .to_string_lossy()
+            .to_string();
+
+        let output_path = ctx.workspace.join(filename).to_string_lossy().to_string();
+
+        rinex.to_file(&output_path).unwrap_or_else(|_| {
+            panic!("failed to generate rinex observations \"{}\"", output_path)
+        });
+
+        info!("generated RINEX observations \"{}\"", output_path);
+    }
+    // METEO RINEX processing
+    if let Some(rinex) = ctx.data.meteo_data() {
+        let filename = ctx
+            .data
+            .meteo_paths()
+            .expect("failed to determine meteo output")
+            .get(0)
+            .expect("failed to determine meteo output")
+            .file_name()
+            .expect("failed to determine meteo output")
+            .to_string_lossy()
+            .to_string();
+
+        let output_path = ctx.workspace.join(filename).to_string_lossy().to_string();
+
+        rinex.to_file(&output_path).unwrap_or_else(|_| {
+            panic!("failed to generate meteo observations \"{}\"", output_path)
+        });
+
+        info!("generated meteo observations \"{}\"", output_path);
+    }
+    // NAV RINEX processing
+    if let Some(rinex) = ctx.data.nav_data() {
+        let filename = ctx
+            .data
+            .nav_paths()
+            .expect("failed to determine nav output")
+            .get(0)
+            .expect("failed to determine nav output")
+            .file_name()
+            .expect("failed to determine nav output")
+            .to_string_lossy()
+            .to_string();
+
+        let output_path = ctx.workspace.join(filename).to_string_lossy().to_string();
+
+        rinex
+            .to_file(&output_path)
+            .unwrap_or_else(|_| panic!("failed to generate navigation data\"{}\"", output_path));
+
+        info!("generated navigation data \"{}\"", output_path);
+    }
+    Ok(())
+}
+
 /*
  * Merges proposed (single) file and generates resulting output, into the workspace
  */
@@ -158,75 +229,89 @@ pub fn time_binning(ctx: &Context, matches: &ArgMatches) -> Result<(), Error> {
         panic!("invalid duration");
     }
 
-    if let Some(rinex) = ctx.data.obs_data() {
-        let (mut first, end) = (
-            rinex
-                .first_epoch()
-                .expect("failed to determine first epoch"),
-            rinex.last_epoch().expect("failed to determine last epoch"),
-        );
-
-        let mut last = first + *duration;
-
-        let obs_path = ctx
-            .data
-            .obs_paths()
-            .expect("failed to determine output file name")
-            .get(0)
-            .unwrap();
+    // input data determination
+    let rinex = if let Some(data) = ctx.data.obs_data() {
+        data
+    } else if let Some(data) = ctx.data.meteo_data() {
+        data
+    } else if let Some(data) = ctx.data.nav_data() {
+        data
+    } else {
+        panic!("time binning is not supported on this file format (yet)");
+    };
 
-        let filename = obs_path
-            .file_stem()
-            .expect("failed to determine output file name")
-            .to_string_lossy()
-            .to_string();
+    // time framing determination
+    let (mut first, end) = (
+        rinex
+            .first_epoch()
+            .expect("failed to determine first epoch"),
+        rinex.last_epoch().expect("failed to determine last epoch"),
+    );
+
+    let mut last = first + *duration;
+
+    // filename determination
+    let data_path = if let Some(paths) = ctx.data.obs_paths() {
+        paths.get(0).expect("failed to determine OBS filename")
+    } else if let Some(paths) = ctx.data.meteo_paths() {
+        paths.get(0).expect("failed to determine OBS filename")
+    } else if let Some(paths) = ctx.data.nav_paths() {
+        paths.get(0).expect("failed to determine OBS filename")
+    } else {
+        unreachable!("non supported file format");
+    };
 
-        let mut extension = String::new();
+    let filename = data_path
+        .file_stem()
+        .expect("failed to determine output file name")
+        .to_string_lossy()
+        .to_string();
 
-        let filename = if filename.contains('.') {
-            /* .crx.gz case */
-            let mut iter = filename.split('.');
-            let filename = iter
-                .next()
-                .expect("failed to determine output file name")
-                .to_string();
-            extension.push_str(iter.next().expect("failed to determine output file name"));
-            extension.push('.');
-            filename
-        } else {
-            filename.clone()
-        };
+    let mut extension = String::new();
 
-        let file_ext = obs_path
-            .extension()
+    let filename = if filename.contains('.') {
+        /* .crx.gz case */
+        let mut iter = filename.split('.');
+        let filename = iter
+            .next()
             .expect("failed to determine output file name")
-            .to_string_lossy()
             .to_string();
+        extension.push_str(iter.next().expect("failed to determine output file name"));
+        extension.push('.');
+        filename
+    } else {
+        filename.clone()
+    };
 
-        extension.push_str(&file_ext);
+    let file_ext = data_path
+        .extension()
+        .expect("failed to determine output file name")
+        .to_string_lossy()
+        .to_string();
 
-        while last <= end {
-            let rinex = rinex
-                .filter(Filter::from_str(&format!("< {:?}", last)).unwrap())
-                .filter(Filter::from_str(&format!(">= {:?}", first)).unwrap());
+    extension.push_str(&file_ext);
 
-            let (y, m, d, hh, mm, ss, _) = first.to_gregorian_utc();
-            let file_suffix = format!("{}{}{}_{}{}{}{}", y, m, d, hh, mm, ss, first.time_scale);
+    // run time binning algorithm
+    while last <= end {
+        let rinex = rinex
+            .filter(Filter::from_str(&format!("< {:?}", last)).unwrap())
+            .filter(Filter::from_str(&format!(">= {:?}", first)).unwrap());
 
-            let output = ctx
-                .workspace
-                .join(&format!("{}-{}.{}", filename, file_suffix, extension))
-                .to_string_lossy()
-                .to_string();
+        let (y, m, d, hh, mm, ss, _) = first.to_gregorian_utc();
+        let file_suffix = format!("{}{}{}_{}{}{}{}", y, m, d, hh, mm, ss, first.time_scale);
 
-            rinex.to_file(&output)?;
-            info!("\"{}\" has been generated", output);
+        let output = ctx
+            .workspace
+            .join(&format!("{}-{}.{}", filename, file_suffix, extension))
+            .to_string_lossy()
+            .to_string();
 
-            first += *duration;
-            last += *duration;
-        }
-    }
+        rinex.to_file(&output)?;
+        info!("\"{}\" has been generated", output);
 
+        first += *duration;
+        last += *duration;
+    }
     Ok(())
 }
 

rinex-cli/src/main.rs

@@ -64,6 +64,9 @@ pub fn main() -> Result<(), Error> {
      * Exclusive opmodes
      */
     match cli.matches.subcommand() {
+        Some(("filegen", submatches)) => {
+            fops::filegen(&ctx, submatches)?;
+        },
         Some(("graph", submatches)) => {
             graph::graph_opmode(&ctx, submatches)?;
         },

rinex-cli/src/preprocessing.rs

@@ -49,10 +49,16 @@ pub fn preprocess(ctx: &mut RnxContext, cli: &Cli) {
     }
 
     for filt_str in cli.preprocessing() {
-        /* special case : only apply to observ dataset */
-        let only_obs = filt_str.starts_with("observ:");
-        let offset: usize = match only_obs {
-            true => 7, // "observ:"
+        /*
+         * TODO
+         * special case : apply to specific file format only
+         */
+        let only_obs = filt_str.starts_with("obs:");
+        let only_met = filt_str.starts_with("met:");
+        let only_nav = filt_str.starts_with("nav:");
+        let special_prefix = only_obs || only_met || only_nav;
+        let offset: usize = match special_prefix {
+            true => 4, // "obs:","met:",..,"ion:"
             false => 0,
         };
         if let Ok(filt) = Filter::from_str(&filt_str[offset..]) {

rinex-qc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "rinex-qc"
-version = "0.1.10"
+version = "0.1.11"
 license = "MIT OR Apache-2.0"
 authors = ["Guillaume W. Bres <[email protected]>"]
 description = "RINEX data analysis"
@@ -28,7 +28,7 @@ itertools = "0.12.0"
 statrs = "0.16"
 sp3 = { path = "../sp3", version = "=1.0.7", features = ["serde"] }
 rinex-qc-traits = { path = "../qc-traits", version = "=0.1.1" }
-rinex = { path = "../rinex", version = "=0.15.5", features = ["full"] }
+rinex = { path = "../rinex", version = "=0.15.6", features = ["full"] }
 gnss-rs = { version = "2.1.3", features = ["serde"] }
 
 [dev-dependencies]