diff --git a/examples/twistrs-cli/Cargo.toml b/examples/twistrs-cli/Cargo.toml index 88971f7..7c8e91f 100644 --- a/examples/twistrs-cli/Cargo.toml +++ b/examples/twistrs-cli/Cargo.toml @@ -2,10 +2,10 @@ name = "twistrs-cli" version = "0.1.0" authors = ["Juxhin Dyrmishi Brigjaj "] -edition = "2018" +edition = "2021" [dependencies] -twistrs = "0.2.2-beta" +twistrs = { path = "../../twistrs" } clap = "3.0.0-beta.1" colored = "1.9.3" tokio = { version = "0.2", features = ["full"] } diff --git a/examples/twistrs-cli/src/main.rs b/examples/twistrs-cli/src/main.rs index 5009e8f..15b48d1 100644 --- a/examples/twistrs-cli/src/main.rs +++ b/examples/twistrs-cli/src/main.rs @@ -1,13 +1,12 @@ -use colored::*; use clap::{App, Arg}; +use colored::*; +use tokio::sync::mpsc; use twistrs::enrich::DomainMetadata; use twistrs::permutate::Domain; -use tokio::sync::mpsc; -use std::time::Instant; use std::collections::HashSet; - +use std::time::Instant; #[tokio::main] async fn main() { @@ -25,7 +24,7 @@ async fn main() { let domain = Domain::new(&matches.value_of("domain").unwrap()).unwrap(); - let mut domain_permutations = domain.all().unwrap().collect::>(); + let mut domain_permutations = domain.all().collect::>(); let domain_permutation_count = domain_permutations.len(); domain_permutations.insert(String::from(domain.fqdn.clone())); @@ -37,7 +36,10 @@ async fn main() { let mut tx = tx.clone(); tokio::spawn(async move { - if let Err(_) = tx.send((i, v.clone(), domain_metadata.dns_resolvable().await)).await { + if let Err(_) = tx + .send((i, v.clone(), domain_metadata.dns_resolvable().await)) + .await + { println!("received dropped"); return; } @@ -52,21 +54,19 @@ async fn main() { while let Some(i) = rx.recv().await { match i.2 { - Ok(v) => { - match v.ips { - Some(_) => { - enumeration_count += 1; - println!( - "\n{}\nDomain: {}\n IPs: {:?}", - "Enriched Domain".bold(), - &v.fqdn, - &v.ips - ); - }, - None => {}, + Ok(v) => match v.ips { + Some(_) => { + enumeration_count += 1; + println!( + "\n{}\nDomain: {}\n IPs: {:?}", + "Enriched Domain".bold(), + &v.fqdn, + &v.ips + ); } + None => {} }, - Err(_) => {}, + Err(_) => {} } } diff --git a/examples/twistrs-grpc/Cargo.toml b/examples/twistrs-grpc/Cargo.toml index 69a092c..b5ee5e8 100644 --- a/examples/twistrs-grpc/Cargo.toml +++ b/examples/twistrs-grpc/Cargo.toml @@ -2,10 +2,10 @@ name = "twistrs-grpc" version = "0.1.0" authors = ["Juxhin Dyrmishi Brigjaj "] -edition = "2018" +edition = "2021" [dependencies] -twistrs = "0.2.2-beta" +twistrs = { path = "../../twistrs" } prost = "0.6.1" tonic = {version="0.2.0",features = ["tls"]} tokio = {version="0.2.18",features = ["stream", "macros"]} @@ -20,4 +20,4 @@ path = "src/server.rs" [[bin]] name = "client" -path = "src/client.rs" \ No newline at end of file +path = "src/client.rs" diff --git a/examples/twistrs-grpc/src/server.rs b/examples/twistrs-grpc/src/server.rs index a19014a..bdcbb4c 100644 --- a/examples/twistrs-grpc/src/server.rs +++ b/examples/twistrs-grpc/src/server.rs @@ -2,16 +2,12 @@ use tokio::sync::mpsc; use tonic::{transport::Server, Request, Response, Status}; -use twistrs::permutate::Domain; use twistrs::enrich::DomainMetadata; +use twistrs::permutate::Domain; use domain_enumeration::domain_enumeration_server::{DomainEnumeration, DomainEnumerationServer}; -use domain_enumeration::{ - Fqdn, - MxCheckResponse, - DomainEnumerationResponse -}; +use domain_enumeration::{DomainEnumerationResponse, Fqdn, MxCheckResponse}; mod domain_enumeration; @@ -20,37 +16,38 @@ pub struct DomainEnumerationService {} #[tonic::async_trait] impl DomainEnumeration for DomainEnumerationService { - type SendDnsResolutionStream=mpsc::Receiver>; - type SendMxCheckStream=mpsc::Receiver>; - - async fn send_dns_resolution(&self, - request: Request - ) -> Result, Status> { + type SendDnsResolutionStream = mpsc::Receiver>; + type SendMxCheckStream = mpsc::Receiver>; + async fn send_dns_resolution( + &self, + request: Request, + ) -> Result, Status> { let (tx, rx) = mpsc::channel(64); - for permutation in Domain::new(&request.get_ref().fqdn).unwrap().all().unwrap() { + for permutation in Domain::new(&request.get_ref().fqdn).unwrap().all() { let domain_metadata = DomainMetadata::new(permutation.clone()); let mut tx = tx.clone(); - + // Spawn DNS Resolution check tokio::spawn(async move { match domain_metadata.dns_resolvable().await { - Ok(metadata) => { - match metadata.ips { - Some(ips) => { - if let Err(_) = tx.send(Ok(DomainEnumerationResponse { - fqdn:format!("{}", permutation.clone()), + Ok(metadata) => match metadata.ips { + Some(ips) => { + if let Err(_) = tx + .send(Ok(DomainEnumerationResponse { + fqdn: format!("{}", permutation.clone()), ips: ips.into_iter().map(|x| format!("{}", x)).collect(), - })).await { - println!("receiver dropped"); - return; - } - }, - None => {}, + })) + .await + { + println!("receiver dropped"); + return; + } } + None => {} }, - Err(_) => {}, + Err(_) => {} } drop(tx); @@ -62,34 +59,36 @@ impl DomainEnumeration for DomainEnumerationService { Ok(Response::new(rx)) } - async fn send_mx_check(&self, - request: Request + async fn send_mx_check( + &self, + request: Request, ) -> Result, Status> { let (tx, rx) = mpsc::channel(64); - for permutation in Domain::new(&request.get_ref().fqdn).unwrap().all().unwrap() { + for permutation in Domain::new(&request.get_ref().fqdn).unwrap().all() { let domain_metadata = DomainMetadata::new(permutation.clone()); let mut tx = tx.clone(); - + // Spawn DNS Resolution check tokio::spawn(async move { match domain_metadata.mx_check().await { - Ok(metadata) => { - match metadata.smtp { - Some(smtp) => { - if let Err(_) = tx.send(Ok(MxCheckResponse { - fqdn:format!("{}", permutation.clone()), + Ok(metadata) => match metadata.smtp { + Some(smtp) => { + if let Err(_) = tx + .send(Ok(MxCheckResponse { + fqdn: format!("{}", permutation.clone()), is_positive: smtp.is_positive, message: smtp.message, - })).await { - println!("receiver dropped"); - return; - } - }, - None => {}, + })) + .await + { + println!("receiver dropped"); + return; + } } + None => {} }, - Err(_) => {}, + Err(_) => {} } drop(tx); @@ -98,11 +97,10 @@ impl DomainEnumeration for DomainEnumerationService { drop(tx); - Ok(Response::new(rx)) - } + Ok(Response::new(rx)) + } } - #[tokio::main] async fn main() -> Result<(), Box> { let addr = "0.0.0.0:50051".parse().unwrap(); @@ -115,6 +113,6 @@ async fn main() -> Result<(), Box> { .add_service(DomainEnumerationServer::new(rpc_service)) .serve(addr) .await?; - + Ok(()) -} \ No newline at end of file +} diff --git a/examples/twistrs-ws/Cargo.toml b/examples/twistrs-ws/Cargo.toml index e864468..f3d08a0 100644 --- a/examples/twistrs-ws/Cargo.toml +++ b/examples/twistrs-ws/Cargo.toml @@ -2,10 +2,10 @@ name = "twistrs-ws" version = "0.1.0" authors = ["Juxhin Dyrmishi Brigjaj "] -edition = "2018" +edition = "2021" [dependencies] -twistrs = "0.2.2-beta" +twistrs = { path = "../../twistrs" } tokio = { version = "0.2", features = ["macros", "sync", "rt-threaded"] } warp = "0.2" serde = {version = "1.0", features = ["derive"] } diff --git a/examples/twistrs-ws/src/main.rs b/examples/twistrs-ws/src/main.rs index 485af9b..7e44011 100644 --- a/examples/twistrs-ws/src/main.rs +++ b/examples/twistrs-ws/src/main.rs @@ -10,8 +10,8 @@ use tokio::sync::{mpsc, RwLock}; use warp::ws::{Message, WebSocket}; use warp::Filter; -use twistrs::permutate::Domain; use twistrs::enrich::DomainMetadata; +use twistrs::permutate::Domain; /// Our global unique user id counter. static NEXT_USER_ID: AtomicUsize = AtomicUsize::new(1); @@ -28,9 +28,7 @@ async fn main() { let chat = warp::path("chat") .and(warp::ws()) .and(users) - .map(|ws: warp::ws::Ws, users| { - ws.on_upgrade(move |socket| user_connected(socket, users)) - }); + .map(|ws: warp::ws::Ws, users| ws.on_upgrade(move |socket| user_connected(socket, users))); let index = warp::path::end().map(|| warp::reply::html(INDEX_HTML)); let routes = index.or(chat); @@ -95,32 +93,30 @@ async fn user_message(my_id: usize, msg: Message, users: &Users) { eprintln!("initiating dns resolution checks for user: {}", my_id); let domain = Domain::new(&msg).unwrap(); - let mut domain_permutations = domain.all().unwrap().collect::>(); + let mut domain_permutations = domain.all().collect::>(); domain_permutations.insert(String::from(domain.fqdn.clone())); - + for v in domain_permutations.into_iter() { let domain_metadata = DomainMetadata::new(v.clone()); let tx = tx.clone(); tokio::spawn(async move { match domain_metadata.dns_resolvable().await { - Ok(metadata) => { - match metadata.ips { - Some(ips) => { - if let Err(_) = tx.send(Ok(Message::text(format!("{:?}", ips)))) { - println!("received dropped"); - return; - } - - drop(tx); - }, - None => return, + Ok(metadata) => match metadata.ips { + Some(ips) => { + if let Err(_) = tx.send(Ok(Message::text(format!("{:?}", ips)))) { + println!("received dropped"); + return; + } + + drop(tx); } + None => return, }, Err(_) => return, } }); - } + } } } } @@ -176,4 +172,4 @@ static INDEX_HTML: &str = r#" -"#; \ No newline at end of file +"#; diff --git a/twistrs/Cargo.toml b/twistrs/Cargo.toml index 1833a15..256fada 100644 --- a/twistrs/Cargo.toml +++ b/twistrs/Cargo.toml @@ -1,12 +1,12 @@ [package] name = "twistrs" -version = "0.3.1-beta" +version = "0.3.2-beta" description = "An asynchronous domain name permutation and enumeration library." license = "MIT" repository = "https://github.com/JuxhinDB/twistrs" documentation = "https://docs.rs/crate/twistrs" authors = ["Juxhin Dyrmishi Brigjaj "] -edition = "2018" +edition = "2021" [package.metadata.docs.rs] all-features = true @@ -21,12 +21,12 @@ whois_lookup = [ "whois-rust" ] [dependencies] addr = "0.13.1" -lazy_static = "1.4.0" -phf = { version = "0.8.0", features = ["macros"] } -async-smtp = "0.3.4" +lazy_static = "1.4.0" +phf = { version = "0.8.0", features = ["macros"] } +async-smtp = {git = "https://github.com/JuxhinDB/async-smtp.git"} async-native-tls = "0.3.3" futures = "0.3" -tokio = { version = "0.2.22", features = ["dns", "macros"] } +tokio = { version = "0.2.22", features = ["dns", "macros"] } fancy-regex = "0.4.0" idna = "0.2.0" hyper = "0.13.8" diff --git a/twistrs/build.rs b/twistrs/build.rs index a141c7d..dd8a5c1 100644 --- a/twistrs/build.rs +++ b/twistrs/build.rs @@ -1,35 +1,37 @@ - -use punycode; - use std::io::{self, BufRead}; use std::{env, fs, path::Path}; - fn main() { // The following build script converts a number of data assets // to be embedded directly into the libraries final binaries // without incurring any runtime costs. - // + // // For more information on the internals as well as other // possible solutions, please review the following blog post. // // https://dev.to/rustyoctopus/generating-static-arrays-during-compile-time-in-rust-10d8 let mut dicionary_output = String::from(""); - let mut tld_array_string = String::from("#[allow(dead_code)] - static TLDS: [&'static str; "); - let mut keywords_array_string = String::from("#[allow(dead_code)] - static KEYWORDS: [&'static str; "); - let mut whois_servers_string = String::from("#[allow(dead_code)] - static WHOIS_RAW_JSON: &'static str = r#"); - + let mut tld_array_string = String::from( + "#[allow(dead_code)] + static TLDS: [&str; ", + ); + let mut keywords_array_string = String::from( + "#[allow(dead_code)] + static KEYWORDS: [&str; ", + ); + let mut whois_servers_string = String::from( + "#[allow(dead_code)] + static WHOIS_RAW_JSON: &str = r#", + ); + // Calculate how many TLDs we actually have in the dictionary match read_lines("./data/tlds.txt") { Ok(lines) => { // We want to unwrap to make sure that we are able to fetch all TLDs let tlds = lines.map(|l| l.unwrap()).collect::>(); - // Finalize the variable signature and break into newline to + // Finalize the variable signature and break into newline to // start populating the TLDs tld_array_string.push_str(&tlds.len().to_string()); tld_array_string.push_str("] = [\r\n"); @@ -39,24 +41,26 @@ fn main() { // Formatting some tabs (ASCII-20) tld_array_string.push_str("\u{20}\u{20}\u{20}\u{20}\""); - let tld; - - if line.chars().all(char::is_alphanumeric) { - tld = line.to_string(); + let tld = if line.chars().all(char::is_alphanumeric) { + line.to_string() } else { - tld = punycode::encode(line.to_string().as_str()).unwrap() - } + punycode::encode(line.to_string().as_str()).unwrap() + }; tld_array_string.push_str(&tld[..]); tld_array_string.push_str("\",\r\n"); } // Close off variable signature - tld_array_string.push_str("];\r\n"); - }, - Err(e) => panic!(format!( - "unable to build library due to missing dictionary file(s): {}", e - )) + tld_array_string.push_str("];\r\n"); + } + Err(e) => panic!( + "{}", + format!( + "unable to build library due to missing dictionary file(s): {}", + e + ) + ), } match read_lines("./data/keywords.txt") { @@ -64,7 +68,7 @@ fn main() { // We want to unwrap to make sure that we are able to fetch all TLDs let tlds = lines.map(|l| l.unwrap()).collect::>(); - // Finalize the variable signature and break into newline to + // Finalize the variable signature and break into newline to // start populating the TLDs keywords_array_string.push_str(&tlds.len().to_string()); keywords_array_string.push_str("] = [\r\n"); @@ -74,13 +78,11 @@ fn main() { // Formatting some tabs (ASCII-20) keywords_array_string.push_str("\u{20}\u{20}\u{20}\u{20}\""); - let tld; - - if line.chars().all(char::is_alphanumeric) { - tld = line.to_string(); + let tld = if line.chars().all(char::is_alphanumeric) { + line.to_string() } else { - tld = punycode::encode(line.to_string().as_str()).unwrap() - } + punycode::encode(line.to_string().as_str()).unwrap() + }; keywords_array_string.push_str(&tld[..]); keywords_array_string.push_str("\",\r\n"); @@ -88,36 +90,44 @@ fn main() { // Close off variable signature keywords_array_string.push_str("];\r\n"); - }, - Err(e) => panic!(format!( - "unable to build library due to missing dictionary file(s): {}", e - )) + } + Err(e) => panic!( + "{}", + format!( + "unable to build library due to missing dictionary file(s): {}", + e + ) + ), } // Compile the WhoIs server config to later perform WhoIs lookups against match read_lines("./data/whois-servers.json") { Ok(lines) => { // Construct the in-memory JSON - whois_servers_string.push_str("\""); + whois_servers_string.push('"'); lines.for_each(|l| whois_servers_string.push_str(&l.unwrap())); whois_servers_string.push_str("\"#;"); - }, - Err(e) => panic!(format!( - "unable to build library due to missing dictionary file(s): {}", e - )) + } + Err(e) => panic!( + "{}", + format!( + "unable to build library due to missing dictionary file(s): {}", + e + ) + ), } - + // Build the final output dicionary_output.push_str(&tld_array_string); - dicionary_output.push_str("\n"); + dicionary_output.push('\n'); dicionary_output.push_str(&keywords_array_string); - dicionary_output.push_str("\n"); + dicionary_output.push('\n'); dicionary_output.push_str(&whois_servers_string); // Write out contents to the final Rust file artifact let out_dir = env::var("OUT_DIR").unwrap(); let dest_path = Path::new(&out_dir).join("data.rs"); - fs::write(&dest_path, dicionary_output).unwrap(); + fs::write(&dest_path, dicionary_output).unwrap(); } // The output is wrapped in a Result to allow matching on errors @@ -126,7 +136,9 @@ fn main() { // This was taken from the official rust-lang docs: // https://doc.rust-lang.org/stable/rust-by-example/std_misc/file/read_lines.html fn read_lines

(filename: P) -> io::Result>> -where P: AsRef, { +where + P: AsRef, +{ let file = fs::File::open(filename)?; Ok(io::BufReader::new(file).lines()) -} \ No newline at end of file +} diff --git a/twistrs/src/enrich.rs b/twistrs/src/enrich.rs index 9a67b18..b9503ac 100644 --- a/twistrs/src/enrich.rs +++ b/twistrs/src/enrich.rs @@ -23,7 +23,6 @@ use std::fmt; use std::net::IpAddr; - #[cfg(feature = "geoip_lookup")] use maxminddb; #[cfg(feature = "geoip_lookup")] @@ -32,16 +31,15 @@ use maxminddb::geoip2; #[cfg(feature = "whois_lookup")] use whois_rust::WhoIsLookupOptions; -use tokio::net; +use async_smtp::{ClientSecurity, Envelope, SendableEmail, ServerAddress, SmtpClient}; use hyper::{Body, Request}; -use async_smtp::{ClientSecurity, Envelope, SendableEmail, SmtpClient}; +use tokio::net; use crate::constants::HTTP_CLIENT; #[cfg(feature = "whois_lookup")] use crate::constants::WHOIS; - /// Temporary type-alias over `EnrichmentError`. pub type Result = std::result::Result; @@ -67,7 +65,7 @@ impl fmt::Display for EnrichmentError { /// information that was derived. /// /// **N.B**—there will be cases where a single -/// domain can have multiple DomainMetadata +/// domain can have multiple `DomainMetadata` /// instancees associated with it. #[derive(Debug, Clone, Default)] pub struct DomainMetadata { @@ -103,17 +101,16 @@ pub struct SmtpMetadata { pub message: String, } - impl DomainMetadata { /// Create a new empty state for a particular FQDN. pub fn new(fqdn: String) -> DomainMetadata { - let mut d = DomainMetadata::default(); - d.fqdn = fqdn; - - d + DomainMetadata { + fqdn, + ..Default::default() + } } - /// Asynchronous DNS resolution on a DomainMetadata instance. + /// Asynchronous DNS resolution on a `DomainMetadata` instance. /// /// Returns `Ok(DomainMetadata)` is the domain was resolved, /// otherwise returns `Err(EnrichmentError)`. @@ -126,8 +123,8 @@ impl DomainMetadata { ips: Some(addrs.map(|addr| addr.ip()).collect()), smtp: None, http_banner: None, - geo_ip_lookups: None, - who_is_lookup: None, + geo_ip_lookups: None, + who_is_lookup: None, }), Err(_) => Err(EnrichmentError), } @@ -152,74 +149,69 @@ impl DomainMetadata { "And that's how the cookie crumbles\n", ); - let smtp_domain = format!("{}:25", &self.fqdn); + let smtp = SmtpClient::with_security( + ServerAddress { + host: self.fqdn.to_string(), + port: 25, + }, + ClientSecurity::None, + ); - match SmtpClient::with_security(smtp_domain.clone(), ClientSecurity::None).await { - // TODO(jdb): Figure out how to clean this up - Ok(smtp) => { - match smtp.into_transport().connect_and_send(email).await { - Ok(response) => Ok(DomainMetadata { - fqdn: self.fqdn.clone(), - ips: None, - smtp: Some(SmtpMetadata { - is_positive: response.is_positive(), - message: response - .message - .into_iter() - .map(|s| s.to_string()) - .collect::(), - }), - http_banner: None, - geo_ip_lookups: None, - who_is_lookup: None, - }), + match smtp.into_transport().connect_and_send(email).await { + Ok(response) => Ok(DomainMetadata { + fqdn: self.fqdn.clone(), + ips: None, + smtp: Some(SmtpMetadata { + is_positive: response.is_positive(), + message: response.message.into_iter().collect::(), + }), + http_banner: None, + geo_ip_lookups: None, + who_is_lookup: None, + }), - // @CLEANUP(JDB): Currently for most scenarios, the following call with return - // an `std::io::ErrorKind::ConnectionRefused` which is normal. - // - // In such a scenario, we still do not want to panic but instead - // move on. Currently lettre::smtp::error::Error does not support - // the `fn kind` function to be able to handle error variants. - // - // Try to figure out if there is another way to handle them. - Err(_) => Ok(DomainMetadata::new(self.fqdn.clone())), - } - } + // @CLEANUP(JDB): Currently for most scenarios, the following call with return + // an `std::io::ErrorKind::ConnectionRefused` which is normal. + // + // In such a scenario, we still do not want to panic but instead + // move on. Currently lettre::smtp::error::Error does not support + // the `fn kind` function to be able to handle error variants. + // + // Try to figure out if there is another way to handle them. Err(_) => Ok(DomainMetadata::new(self.fqdn.clone())), } } /// Asynchronous HTTP Banner fetch. Searches and parses `server` header - /// from an HTTP request to gather the HTTP banner. - /// + /// from an HTTP request to gather the HTTP banner. + /// /// Note that a `HEAD` request is issued to minimise bandwidth. Also note - /// that the internal [`HttpConnector`](https://docs.rs/hyper/0.13.8/hyper/client/struct.HttpConnector.html) - /// sets the response buffer window to 1024 bytes, the CONNECT timeout to + /// that the internal [`HttpConnector`](https://docs.rs/hyper/0.13.8/hyper/client/struct.HttpConnector.html) + /// sets the response buffer window to 1024 bytes, the CONNECT timeout to /// 5s and enforces HTTP scheme. - /// + /// /// ``` /// use twistrs::enrich::DomainMetadata; - /// + /// /// #[tokio::main] /// async fn main() { /// let domain_metadata = DomainMetadata::new(String::from("www.phishdeck.com")); /// println!("{:?}", domain_metadata.http_banner().await); /// } /// ``` - /// + /// /// # Panics - /// + /// /// Currently panics if the HTTP server header value is not parseable. - /// For more information please refer to the + /// For more information please refer to the /// [Hyper implementation](https://docs.rs/hyper/hyper/header/struct.HeaderValue.html#method.to_str). pub async fn http_banner(&self) -> Result { - // Construst the basic request to be sent out let request = Request::builder() .method("HEAD") .uri(format!("http://{}", &self.fqdn)) .header("User-Agent", "github-twistrs-http-banner/1.0") - .body(Body::from("")) // This is annoying + .body(Body::from("")) // This is annoying .unwrap(); match HTTP_CLIENT.request(request).await { @@ -227,20 +219,18 @@ impl DomainMetadata { dbg!(&response); match response.headers().get("server") { - Some(server) => { - Ok(DomainMetadata { - fqdn: self.fqdn.clone(), - ips: None, - smtp: None, - http_banner: Some(String::from(server.to_str().unwrap())), - geo_ip_lookups: None, - who_is_lookup: None, - }) - }, + Some(server) => Ok(DomainMetadata { + fqdn: self.fqdn.clone(), + ips: None, + smtp: None, + http_banner: Some(String::from(server.to_str().unwrap())), + geo_ip_lookups: None, + who_is_lookup: None, + }), None => Ok(DomainMetadata::new(self.fqdn.clone())), } - }, - Err(_) => Ok(DomainMetadata::new(self.fqdn.clone())), + } + Err(_) => Ok(DomainMetadata::new(self.fqdn.clone())), } } @@ -248,16 +238,16 @@ impl DomainMetadata { /// interfaces and requires the callee to pass a [`maxminddb::Reader`](https://docs.rs/maxminddb/0.15.0/maxminddb/struct.Reader.html) /// to perform the lookup through. Internally, the maxminddb call is blocking and /// may result in performance drops, however the lookups are in-memory. - /// + /// /// The only reason you would want to do this, is to be able to get back a `DomainMetadata` - /// to then process as you would with other enrichment methods. Internally the lookup will - /// try to stitch together the City, Country & Continent that the [`IpAddr`](https://doc.rust-lang.org/std/net/enum.IpAddr.html) + /// to then process as you would with other enrichment methods. Internally the lookup will + /// try to stitch together the City, Country & Continent that the [`IpAddr`](https://doc.rust-lang.org/std/net/enum.IpAddr.html) /// resolves to. - /// + /// /// ``` /// use maxminddb::Reader; /// use twistrs::enrich::DomainMetadata; - /// + /// /// #[tokio::main] /// async fn main() { /// let reader = maxminddb::Reader::open_readfile("./data/MaxMind-DB/test-data/GeoIP2-City-Test.mmdb").unwrap(); @@ -265,14 +255,14 @@ impl DomainMetadata { /// println!("{:?}", domain_metadata.geoip_lookup(&reader).await); /// } /// ``` - /// + /// /// ### Panics - /// - /// Currently assumes that if a City/Country/Continent is found, that the English ("en") + /// + /// Currently assumes that if a City/Country/Continent is found, that the English ("en") /// result is available. - /// + /// /// ### Features - /// + /// /// This function requires the `geoip_lookup` feature toggled. #[cfg(feature = "geoip_lookup")] pub async fn geoip_lookup(&self, geoip: &maxminddb::Reader>) -> Result { @@ -286,64 +276,77 @@ impl DomainMetadata { let mut geoip_string = String::new(); if lookup_result.city.is_some() { - geoip_string.push_str(lookup_result.city.unwrap().names - .unwrap() - .get("en") - .unwrap()); + geoip_string.push_str( + lookup_result + .city + .unwrap() + .names + .unwrap() + .get("en") + .unwrap(), + ); } if lookup_result.country.is_some() { if geoip_string.len() > 0 { geoip_string.push_str(", "); } - - geoip_string.push_str(lookup_result.country.unwrap().names - .unwrap() - .get("en") - .unwrap()); - } + + geoip_string.push_str( + lookup_result + .country + .unwrap() + .names + .unwrap() + .get("en") + .unwrap(), + ); + } if lookup_result.continent.is_some() { if geoip_string.len() > 0 { geoip_string.push_str(", "); } - geoip_string.push_str(lookup_result.continent.unwrap().names - .unwrap() - .get("en") - .unwrap()); + geoip_string.push_str( + lookup_result + .continent + .unwrap() + .names + .unwrap() + .get("en") + .unwrap(), + ); } result.push((*ip, geoip_string)); - }, + } Err(_) => {} } } - Ok(DomainMetadata::new(self.fqdn.clone())) - }, - None => { Ok(DomainMetadata::new(self.fqdn.clone())) - }, - } + } + None => Ok(DomainMetadata::new(self.fqdn.clone())), + } } - /// Asyncrhonous WhoIs lookup using cached WhoIs server config. Note that + /// Asyncrhonous WhoIs lookup using cached WhoIs server config. Note that /// the internal lookups are not async and so this should be considered /// a heavy/slow call. - /// + /// /// ``` /// use twistrs::enrich::DomainMetadata; - /// + /// /// #[tokio::main] /// async fn main() { /// let domain_metadata = DomainMetadata::new(String::from("www.phishdeck.com")); /// println!("{:?}", domain_metadata.whois_lookup().await); /// } /// ``` - /// + /// /// ### Features - /// + /// /// This function requires the `whois_lookup` feature toggled. #[cfg(feature = "whois_lookup")] pub async fn whois_lookup(&self) -> Result { @@ -351,32 +354,38 @@ impl DomainMetadata { match WhoIsLookupOptions::from_string(&self.fqdn) { Ok(mut lookup_options) => { - lookup_options.timeout = Some(std::time::Duration::from_secs(5)); // Change default timeout from 60s to 5s - lookup_options.follow = 1; // Only allow at most one redirect + lookup_options.timeout = Some(std::time::Duration::from_secs(5)); // Change default timeout from 60s to 5s + lookup_options.follow = 1; // Only allow at most one redirect match WHOIS.lookup(lookup_options) { Ok(lookup_result) => { - result.who_is_lookup = Some(String::from(&lookup_result.split("\r\n") - // The only entries we care about are the ones that start with 3 spaces. - // Ideally the whois_rust library would have parsed this nicely for us. - .filter(|s| s.starts_with(" ")) - .collect::>() - .join("\n"))); - }, - Err(e) => { eprintln!("{}", e) } - } - }, - Err(e) => { eprintln!("{}", e) } + result.who_is_lookup = Some(String::from( + &lookup_result + .split("\r\n") + // The only entries we care about are the ones that start with 3 spaces. + // Ideally the whois_rust library would have parsed this nicely for us. + .filter(|s| s.starts_with(" ")) + .collect::>() + .join("\n"), + )); + } + Err(e) => { + eprintln!("{}", e) + } + } + } + Err(e) => { + eprintln!("{}", e) + } } - Ok(result) + Ok(result) } - /// Performs all FQDN enrichment methods on a given FQDN. /// This is the only function that returns a `Vec`. - /// - /// **N.B** -- this is currently very slow, and serializes the + /// + /// **N.B** -- this is currently very slow, and serializes the /// operations rather than running them concurrently. It should /// only be used for testing or debugging purposes. /// @@ -386,13 +395,13 @@ impl DomainMetadata { /// an Err. pub async fn all(&self) -> Result> { // @CLEANUP(JDB): This should use try_join! in the future instead - let result = futures::join!(self.dns_resolvable(), - self.mx_check(), - self.http_banner()); + let result = futures::join!(self.dns_resolvable(), self.mx_check(), self.http_banner()); - Ok(vec![result.0.unwrap(), - result.1.unwrap(), - result.2.unwrap()]) + Ok(vec![ + result.0.unwrap(), + result.1.unwrap(), + result.2.unwrap(), + ]) } } @@ -427,23 +436,28 @@ mod tests { async fn test_http_banner() { let domain_metadata = DomainMetadata::new(String::from("example.com")); assert!(domain_metadata.http_banner().await.is_ok()); - } - + } + #[tokio::test] #[cfg(feature = "geoip_lookup")] async fn test_geoip_lookup() { - let domain_metadata = DomainMetadata::new(String::from("example.com")).dns_resolvable().await.unwrap(); + let domain_metadata = DomainMetadata::new(String::from("example.com")) + .dns_resolvable() + .await + .unwrap(); // MaxmindDB CSV entry for example.com subnet, prone to failure but saves space - let reader = maxminddb::Reader::open_readfile("./data/MaxMind-DB/test-data/GeoIP2-City-Test.mmdb").unwrap(); - + let reader = + maxminddb::Reader::open_readfile("./data/MaxMind-DB/test-data/GeoIP2-City-Test.mmdb") + .unwrap(); + assert!(domain_metadata.geoip_lookup(&reader).await.is_ok()); - } - + } + #[tokio::test] #[cfg(feature = "whois_lookup")] async fn test_whois_lookup() { let domain_metadata = DomainMetadata::new(String::from("example.com")); assert!(domain_metadata.whois_lookup().await.is_ok()); - } + } } diff --git a/twistrs/src/lib.rs b/twistrs/src/lib.rs index 8c3a927..9259737 100644 --- a/twistrs/src/lib.rs +++ b/twistrs/src/lib.rs @@ -5,7 +5,7 @@ //! for clients. //! //! The two primary structs to look into are [Domain](./permutate/struct.Domain.html) -//! and [DomainMetadata](./enrich/struct.DomainMetadata.html). +//! and [`DomainMetadata`](./enrich/struct.DomainMetadata.html). //! //! Additionally the module documentation for [permutation](./permutate/index.html) //! and [enumeration](./enrich/index.html) provides more @@ -80,9 +80,9 @@ clippy::explicit_iter_loop, clippy::expl_impl_clone_on_copy, clippy::fallible_impl_from, - clippy::filter_map, + clippy::manual_filter_map, clippy::filter_map_next, - clippy::find_map, + clippy::manual_find_map, clippy::float_arithmetic, clippy::get_unwrap, clippy::if_not_else, @@ -94,20 +94,15 @@ clippy::maybe_infinite_iter, clippy::mem_forget, clippy::module_name_repetitions, - clippy::multiple_crate_versions, clippy::multiple_inherent_impl, clippy::mut_mut, clippy::needless_borrow, clippy::needless_continue, clippy::needless_pass_by_value, - clippy::non_ascii_literal, - clippy::option_map_unwrap_or, - clippy::option_map_unwrap_or_else, + clippy::map_unwrap_or, clippy::path_buf_push_overwrite, clippy::print_stdout, - clippy::pub_enum_variant_names, clippy::redundant_closure_for_method_calls, - clippy::result_map_unwrap_or_else, clippy::shadow_reuse, clippy::shadow_same, clippy::shadow_unrelated, @@ -121,7 +116,6 @@ clippy::used_underscore_binding, clippy::wildcard_dependencies, // clippy::wildcard_enum_match_arm, - clippy::wrong_pub_self_convention, )] #![recursion_limit = "128"] diff --git a/twistrs/src/permutate.rs b/twistrs/src/permutate.rs index 3c337c5..05a2bde 100644 --- a/twistrs/src/permutate.rs +++ b/twistrs/src/permutate.rs @@ -17,9 +17,7 @@ //! //! Additionally the permutation module can be used independently //! from the enrichment module. -use crate::constants::{ - ASCII_LOWER, HOMOGLYPHS, IDNA_FILTER_REGEX, KEYBOARD_LAYOUTS, VOWELS, -}; +use crate::constants::{ASCII_LOWER, HOMOGLYPHS, IDNA_FILTER_REGEX, KEYBOARD_LAYOUTS, VOWELS}; use std::collections::HashSet; use std::fmt; @@ -109,14 +107,14 @@ impl<'a> Domain<'a> { Domain::filter_domains( ASCII_LOWER .iter() - .map(move |c| format!("{}{}.{}", self.domain, c.to_string(), self.tld)), + .map(move |c| format!("{}{}.{}", self.domain, c, self.tld)), ) } /// Following implementation takes inspiration from the following content: /// - /// - https://github.com/artemdinaburg/bitsquat-script/blob/master/bitsquat.py - /// - http://dinaburg.org/bitsquatting.html + /// - <`https://github.com/artemdinaburg/bitsquat-script/blob/master/bitsquat.py`> + /// - <`http://dinaburg.org/bitsquatting.html`> /// /// Go through each char in the domain and XOR it against 8 separate masks: /// @@ -142,8 +140,8 @@ impl<'a> Domain<'a> { let squatted_char: u8 = mask ^ (c as u8); // Make sure we remain with ASCII range that we are happy with - if (squatted_char >= 48 && squatted_char <= 57) - || (squatted_char >= 97 && squatted_char <= 122) + if ((48..=57).contains(&squatted_char)) + || ((97..=122).contains(&squatted_char)) || squatted_char == 45 { Some((1..self.fqdn.len()).map(move |idx| { @@ -178,19 +176,15 @@ impl<'a> Domain<'a> { while j < ws { let c: char = win.chars().nth(j).unwrap(); - if HOMOGLYPHS.contains_key(&c) { - for glyph in HOMOGLYPHS.get(&c) { - let _glyph = glyph.chars().collect::>(); - - for g in _glyph { - let new_win = win.replace(c, &g.to_string()); - result_first_pass.insert(format!( - "{}{}{}", - &self.fqdn[..i], - &new_win, - &self.fqdn[i + ws..] - )); - } + if let Some(glyph) = HOMOGLYPHS.get(&c) { + for g in glyph.chars().collect::>() { + let new_win = win.replace(c, &g.to_string()); + result_first_pass.insert(format!( + "{}{}{}", + &self.fqdn[..i], + &new_win, + &self.fqdn[i + ws..] + )); } } @@ -199,33 +193,29 @@ impl<'a> Domain<'a> { } } - for domain in result_first_pass.iter() { - // We need to do this as we are dealing with UTF8 characters - // meaning that we cannot simple iterate over single byte - // values (as certain characters are composed of two or more) - let _domain = domain.chars().collect::>(); - + for domain in &result_first_pass { for ws in 1..fqdn.len() { for i in 0..(fqdn.len() - ws) + 1 { - let win: String = _domain[i..i + ws].iter().collect(); + // We need to do this as we are dealing with UTF8 characters + // meaning that we cannot simple iterate over single byte + // values (as certain characters are composed of two or more) + let win: String = domain.chars().collect::>()[i..i + ws] + .iter() + .collect(); let mut j = 0; while j < ws { let c: char = win.chars().nth(j).unwrap(); - if HOMOGLYPHS.contains_key(&c) { - for glyph in HOMOGLYPHS.get(&c) { - let _glyph = glyph.chars().collect::>(); - - for g in _glyph { - let new_win = win.replace(c, &g.to_string()); - result_second_pass.insert(format!( - "{}{}{}", - &self.fqdn[..i], - &new_win, - &self.fqdn[i + ws..] - )); - } + if let Some(glyph) = HOMOGLYPHS.get(&c) { + for g in glyph.chars().collect::>() { + let new_win = win.replace(c, &g.to_string()); + result_second_pass.insert(format!( + "{}{}{}", + &self.fqdn[..i], + &new_win, + &self.fqdn[i + ws..] + )); } }