From 55071522368b771e55f4de0c9af597e2eb711295 Mon Sep 17 00:00:00 2001 From: eatradish Date: Thu, 26 Dec 2024 21:54:34 +0800 Subject: [PATCH 1/5] refactor(apt-auth-config)!: use `winnow` to parse auth.conf.d --- Cargo.lock | 2 +- apt-auth-config/Cargo.toml | 2 +- apt-auth-config/src/lib.rs | 156 +++++++++----------------- apt-auth-config/src/parser.rs | 200 ++++++++++++++++++++++++++++++++++ oma-pm/src/download.rs | 2 +- oma-refresh/src/db.rs | 4 +- src/error.rs | 4 + 7 files changed, 262 insertions(+), 108 deletions(-) create mode 100644 apt-auth-config/src/parser.rs diff --git a/Cargo.lock b/Cargo.lock index 4f4a54744..be7d57a10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,9 +157,9 @@ checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" name = "apt-auth-config" version = "0.2.0" dependencies = [ - "rustix", "thiserror 2.0.9", "tracing", + "winnow", ] [[package]] diff --git a/apt-auth-config/Cargo.toml b/apt-auth-config/Cargo.toml index 056e0efb8..72f53b65a 100644 --- a/apt-auth-config/Cargo.toml +++ b/apt-auth-config/Cargo.toml @@ -8,4 +8,4 @@ license = "MIT" [dependencies] thiserror = "2" tracing = "0.1" -rustix = { version = "0.38", features = ["process"] } +winnow = { version = "0.6.20" } diff --git a/apt-auth-config/src/lib.rs b/apt-auth-config/src/lib.rs index 565d9f56a..0525dd820 100644 --- a/apt-auth-config/src/lib.rs +++ b/apt-auth-config/src/lib.rs @@ -1,3 +1,5 @@ +mod parser; + use std::{ fs::{self, read_dir}, io::{self}, @@ -5,7 +7,7 @@ use std::{ str::FromStr, }; -use rustix::process; +use parser::{line, multiline}; use thiserror::Error; use tracing::debug; @@ -19,6 +21,8 @@ pub enum AuthConfigError { OpenFile { path: PathBuf, err: io::Error }, #[error("Auth config file missing entry: {0}")] MissingEntry(&'static str), + #[error("Parse failed, unknown line: {0}")] + ParseError(String), } #[derive(Debug, PartialEq, Eq, Clone)] @@ -37,77 +41,53 @@ impl FromStr for AuthConfigEntry { type Err = AuthConfigError; fn from_str(s: &str) -> Result { - let entry = s - .split_ascii_whitespace() - .filter(|x| !x.starts_with("#")) - .collect::>(); - - let mut host = None; - let mut login = None; - let mut password = None; - - for (i, c) in entry.iter().enumerate() { - if *c == "machine" { - let Some(h) = entry.get(i + 1) else { - return Err(AuthConfigError::MissingEntry("machine")); - }; - - host = Some(h); - continue; - } + let mut s = s; + let parse = line(&mut s).map_err(|e| AuthConfigError::ParseError(e.to_string()))?; - if *c == "login" { - let Some(l) = entry.get(i + 1) else { - return Err(AuthConfigError::MissingEntry("login")); - }; + Ok(parse_entry_inner(parse)) + } +} - login = Some(l); - continue; - } +impl FromStr for AuthConfig { + type Err = AuthConfigError; - if *c == "password" { - let Some(p) = entry.get(i + 1) else { - return Err(AuthConfigError::MissingEntry("password")); - }; + fn from_str(s: &str) -> Result { + let mut s = s; + let parse = multiline(&mut s).map_err(|e| AuthConfigError::ParseError(e.to_string()))?; + let mut res = vec![]; - password = Some(p); - continue; - } + for r in parse { + res.push(parse_entry_inner(r)); } - let Some(host) = host else { - return Err(AuthConfigError::MissingEntry("machine")); - }; - - let Some(login) = login else { - return Err(AuthConfigError::MissingEntry("login")); - }; + Ok(AuthConfig { inner: res }) + } +} - let Some(password) = password else { - return Err(AuthConfigError::MissingEntry("password")); - }; +fn parse_entry_inner(input: Vec<(&str, &str)>) -> AuthConfigEntry { + let mut machine = None; + let mut login = None; + let mut password = None; + + for i in input { + match i.0 { + "machine" => machine = Some(i.1), + "login" => login = Some(i.1), + "password" => password = Some(i.1), + x => panic!("unexcept {x}"), + } + } - Ok(Self { - host: (*host).into(), - user: (*login).into(), - password: (*password).into(), - }) + AuthConfigEntry { + host: machine.unwrap().into(), + user: login.unwrap().into(), + password: password.unwrap().into(), } } impl AuthConfig { /// Read system auth.conf.d config (/etc/apt/auth.conf.d) - /// - /// Note that this function returns empty vector if run as a non-root user. pub fn system(sysroot: impl AsRef) -> Result { - // 在 auth.conf.d 的使用惯例中 - // 配置文件的权限一般为 600，并且所有者为 root - // 以普通用户身份下载文件时，会没有权限读取 auth 配置 - // 因此，在以普通用户访问时，不读取 auth 配置 - if !process::geteuid().is_root() { - return Ok(Self { inner: vec![] }); - } - let p = sysroot.as_ref().join("etc/apt/auth.conf.d"); Self::from_path(p) } @@ -130,7 +110,7 @@ impl AuthConfig { err: e, })?; - let config = AuthConfig::from_str(&s)?; + let config: AuthConfig = s.parse()?; v.extend(config.inner); } @@ -145,7 +125,7 @@ impl AuthConfig { debug!("auth find url is: {}", url); - self.inner.iter().find(|x| { + self.inner.iter().find_map(|x| { let mut host = x.host.to_string(); while host.ends_with('/') { host.pop(); @@ -156,7 +136,11 @@ impl AuthConfig { url.pop(); } - host == url + if host == url { + Some(x) + } else { + None + } }) } @@ -168,48 +152,12 @@ impl AuthConfig { debug!("auth find package url is: {}", url); - self.inner.iter().find(|x| url.starts_with(x.host.as_ref())) - } -} - -impl FromStr for AuthConfig { - type Err = AuthConfigError; - - fn from_str(s: &str) -> Result { - let mut v = vec![]; - - for i in s.lines().filter(|x| !x.starts_with('#')) { - let entry = AuthConfigEntry::from_str(i)?; - v.push(entry); - } - - Ok(Self { inner: v }) + self.inner.iter().find_map(|x| { + if url.starts_with(x.host.as_ref()) { + Some(x) + } else { + None + } + }) } } - -#[test] -fn test_config_parser() { - let config = r#"machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # ubuntu-pro-client -machine esm.ubuntu.com/infra/ubuntu/ login bearer password qaq # ubuntu-pro-client -"#; - - let config = AuthConfig::from_str(config).unwrap(); - - assert_eq!( - config, - AuthConfig { - inner: vec![ - AuthConfigEntry { - host: "esm.ubuntu.com/apps/ubuntu/".into(), - user: "bearer".into(), - password: "qaq".into(), - }, - AuthConfigEntry { - host: "esm.ubuntu.com/infra/ubuntu/".into(), - user: "bearer".into(), - password: "qaq".into(), - }, - ] - } - ); -} diff --git a/apt-auth-config/src/parser.rs b/apt-auth-config/src/parser.rs new file mode 100644 index 000000000..ec3426ca9 --- /dev/null +++ b/apt-auth-config/src/parser.rs @@ -0,0 +1,200 @@ +use winnow::{ + ascii::{line_ending, multispace0, multispace1}, + combinator::{alt, eof, preceded, repeat, separated_pair, terminated}, + stream::AsChar, + token::take_till, + PResult, Parser, +}; + +#[inline] +fn kv<'a>(input: &mut &'a str) -> PResult<(&'a str, &'a str)> { + separated_pair(key, separator, right).parse_next(input) +} + +#[inline] +fn key<'a>(input: &mut &'a str) -> PResult<&'a str> { + alt(("machine", "login", "password")).parse_next(input) +} + +#[inline] +fn separator<'a>(input: &mut &'a str) -> PResult<()> { + multispace1.void().parse_next(input) +} + +#[inline] +fn right<'a>(input: &mut &'a str) -> PResult<&'a str> { + terminated( + take_till(1.., |c: char| c.is_whitespace()).verify(|s: &str| !s.starts_with('#')), + multispace0, + ) + .parse_next(input) +} + +#[inline] +fn comment<'a>(input: &mut &'a str) -> PResult<()> { + ('#', line_reset).void().parse_next(input) +} + +#[inline] +fn line_reset<'a>(input: &mut &'a str) -> PResult<()> { + (take_till(0.., |c: char| c.is_newline()), line_ending) + .void() + .parse_next(input) +} + +#[inline] +fn whitespace<'a>(input: &mut &'a str) -> PResult<()> { + multispace1.void().parse_next(input) +} + +#[inline] +fn garbage<'a>(input: &mut &'a str) -> PResult<()> { + alt((whitespace, comment)).parse_next(input) +} + +#[inline] +fn multi_ignore<'a>(input: &mut &'a str) -> PResult<()> { + repeat(0.., garbage).parse_next(input) +} + +#[inline] +pub(crate) fn multiline<'a>(input: &mut &'a str) -> PResult>> { + repeat(0.., preceded(multi_ignore, line)).parse_next(input) +} + +#[inline] +pub(crate) fn line<'a>(input: &mut &'a str) -> PResult> { + fn verify_line((res, ()): &(Vec<(&str, &str)>, ())) -> bool { + for i in ["machine", "login", "password"] { + if !res.iter().any(|x| x.0 == i) { + return false; + } + } + + true + } + + let (res, _) = ( + repeat(3, kv), + alt((line_ending.void(), eof.void(), comment)), + ) + .verify(verify_line) + .parse_next(input)?; + + Ok(res) +} + + +#[test] +fn test_garbage() { + let a = "# 123\n"; + let b = "#123\n"; + let c = "\n"; + let d = "#\n"; + let e = " \n"; + let f = " "; + + for mut i in [a, b, c, d, e, f] { + let out = garbage(&mut i); + assert!(out.is_ok()); + assert!(i.is_empty()); + } +} + +#[test] +fn test_multi_garbage() { + let a = "\n# 123\n"; + let b = "# 123\n\n"; + let c = "\n\n# 123"; + + for mut i in [a, b, c] { + let out = multi_ignore(&mut i); + assert!(out.is_ok()); + dbg!(i); + } +} + +#[test] +fn test_single_line() { + let s = "machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq #sdadasdas\n"; + let s2 = "machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq\n"; + let s3 = "machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq"; + let s4 = "machine esm.ubuntu.com/apps/ubuntu/ password qaq login bearer"; + let s5 = "machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # sdadasdas\n"; + for mut i in [s, s2, s3, s4, s5] { + let mut l = line(&mut i).unwrap(); + l.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + l, + vec![ + ("login", "bearer"), + ("machine", "esm.ubuntu.com/apps/ubuntu/"), + ("password", "qaq") + ] + ); + assert_eq!(i, ""); + } + + let mut i = "machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq sdadasdas\n"; + let l = line(&mut i); + assert!(l.is_err()); + + let mut i = "machine esm.ubuntu.com/apps/ubuntu/ # password qaq login bearer"; + let l = line(&mut i); + assert!(l.is_err()); + + let mut i = "machine #esm.ubuntu.com/apps/ubuntu/ password qaq login bearer"; + let l = line(&mut i); + assert!(l.is_err()); +} + +#[test] +fn test_multi_line() { + let config1 = r#"# 123 +machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # ubuntu-pro-client +machine esm.ubuntu.com/infra/ubuntu/ login bearer password qaq # ubuntu-pro-client +"#; + let config2 = r#" +#123 +machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # ubuntu-pro-client +machine esm.ubuntu.com/infra/ubuntu/ login bearer password qaq # ubuntu-pro-client +"#; + let config3 = r#" +machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # ubuntu-pro-client +machine esm.ubuntu.com/infra/ubuntu/ login bearer password qaq # ubuntu-pro-client +"#; + let config4 = r#" +machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # ubuntu-pro-client +#123 +machine esm.ubuntu.com/infra/ubuntu/ login bearer password qaq # ubuntu-pro-client +"#; + let config5 = r#" +machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # ubuntu-pro-client +machine esm.ubuntu.com/infra/ubuntu/ login bearer password qaq # ubuntu-pro-client +#123 +"#; + let config6 = r#" +machine esm.ubuntu.com/apps/ubuntu/ login bearer password qaq # ubuntu-pro-client +machine esm.ubuntu.com/infra/ubuntu/ login bearer password qaq # ubuntu-pro-client + +#123"#; + + for mut i in [config1, config2, config3, config4, config5, config6] { + let out = multiline(&mut i); + assert_eq!( + out, + Ok(vec![ + vec![ + ("machine", "esm.ubuntu.com/apps/ubuntu/",), + ("login", "bearer",), + ("password", "qaq",), + ], + vec![ + ("machine", "esm.ubuntu.com/infra/ubuntu/",), + ("login", "bearer",), + ("password", "qaq",), + ], + ],) + ); + } +} diff --git a/oma-pm/src/download.rs b/oma-pm/src/download.rs index 16c697021..f4fe6960d 100644 --- a/oma-pm/src/download.rs +++ b/oma-pm/src/download.rs @@ -50,7 +50,7 @@ where let auth = auth.find_package_url(x); DownloadSourceType::Http { - auth: auth.map(|x| (x.user.to_owned(), x.password.to_owned())), + auth: auth.map(|x| (x.host.to_owned(), x.password.to_owned())), } }; diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 9530e5556..3e657832f 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -686,7 +686,7 @@ impl<'a> OmaRefresh<'a> { source_index: &OmaSourceEntry<'_>, ) -> reqwest::RequestBuilder { let mut request = self.client.get(url); - if let Some(auth) = &source_index.auth { + if let Some(auth) = &source_index.auth.as_ref().and_then(|x| x.entry.as_ref()) { request = request.basic_auth(&auth.user, Some(&auth.password)) } @@ -986,6 +986,7 @@ fn download_flat_repo_no_release( auth: source_index .auth .as_ref() + .and_then(|auth| auth.entry.as_ref()) .map(|auth| (auth.user.clone(), auth.password.clone())), }, OmaSourceEntryFrom::Local => DownloadSourceType::Local(source_index.is_flat()), @@ -1033,6 +1034,7 @@ fn collect_download_task( auth: source_index .auth .as_ref() + .and_then(|auth| auth.entry.as_ref()) .map(|auth| (auth.user.clone(), auth.password.clone())), }, OmaSourceEntryFrom::Local => DownloadSourceType::Local(source_index.is_flat()), diff --git a/src/error.rs b/src/error.rs index 886235557..9431e6a6d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -449,6 +449,10 @@ impl From for OutputError { description: format!("Missing field: {field}"), source: None, }, + AuthConfigError::ParseError(field) => Self { + description: format!("Unknown input: {field}"), + source: None, + }, } } } From 6ceb5169d80266ed1064046d597ffae105e3f69e Mon Sep 17 00:00:00 2001 From: eatradish Date: Mon, 30 Dec 2024 15:03:09 +0800 Subject: [PATCH 2/5] refactor(oma-refresh)!: move parse InRelease logic to `FromStr` trait .. `InRelease` struct rename to `Release` --- oma-refresh/src/db.rs | 13 +++++++------ oma-refresh/src/inrelease.rs | 10 +++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 3e657832f..c5a9d3a0d 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -57,8 +57,8 @@ use tracing::{debug, warn}; use crate::{ config::{ChecksumDownloadEntry, IndexTargetConfig}, inrelease::{ - file_is_compress, split_ext_and_filename, verify_inrelease, ChecksumItem, InRelease, - InReleaseChecksum, InReleaseError, + file_is_compress, split_ext_and_filename, verify_inrelease, ChecksumItem, + InReleaseChecksum, InReleaseError, Release, }, sourceslist::{sources_lists, OmaSourceEntry, OmaSourceEntryFrom}, util::DatabaseFilenameReplacer, @@ -791,7 +791,8 @@ impl<'a> OmaRefresh<'a> { ) .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; - let inrelease = InRelease::new(&inrelease) + let inrelease: Release = inrelease + .parse() .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; if ose_list[0].is_flat() { @@ -835,7 +836,7 @@ impl<'a> OmaRefresh<'a> { get_all_need_db_from_config(download_list, &mut total, checksums, &mut handle); for i in &self.flat_repo_no_release { - download_flat_repo_no_release( + collect_flat_repo_no_release( sourcelist.get(*i).unwrap(), &self.download_dir, &mut tasks, @@ -971,7 +972,7 @@ async fn remove_unused_db(download_dir: &Path, download_list: Vec<&str>) -> Resu Ok(()) } -fn download_flat_repo_no_release( +fn collect_flat_repo_no_release( source_index: &OmaSourceEntry, download_dir: &Path, tasks: &mut Vec, @@ -1020,7 +1021,7 @@ fn collect_download_task( source_index: &OmaSourceEntry, download_dir: &Path, tasks: &mut Vec, - inrelease: &InRelease, + inrelease: &Release, replacer: &DatabaseFilenameReplacer, ) -> Result<()> { let file_type = &c.msg; diff --git a/oma-refresh/src/inrelease.rs b/oma-refresh/src/inrelease.rs index 2769703be..626e7e286 100644 --- a/oma-refresh/src/inrelease.rs +++ b/oma-refresh/src/inrelease.rs @@ -58,7 +58,7 @@ pub enum InReleaseChecksum { const COMPRESS: &[&str] = &[".gz", ".xz", ".zst", ".bz2"]; -pub struct InRelease { +pub struct Release { source: InReleaseEntry, acquire_by_hash: OnceCell, checksum_type_and_list: OnceCell<(InReleaseChecksum, Vec)>, @@ -80,8 +80,10 @@ struct InReleaseEntry { sha512: Option, } -impl InRelease { - pub fn new(input: &str) -> Result { +impl FromStr for Release { + type Err = InReleaseError; + + fn from_str(input: &str) -> Result { let source: Paragraph = input.parse().map_err(|_| InReleaseError::BrokenInRelease)?; let source: InReleaseEntry = FromDeb822Paragraph::from_paragraph(&source) .map_err(|_| InReleaseError::BrokenInRelease)?; @@ -92,7 +94,9 @@ impl InRelease { checksum_type_and_list: OnceCell::new(), }) } +} +impl Release { pub fn get_or_try_init_checksum_type_and_list( &self, ) -> Result<&(InReleaseChecksum, Vec), InReleaseError> { From cd33fc0e177b9b5474649e2c4a6514644660e843 Mon Sep 17 00:00:00 2001 From: eatradish Date: Mon, 30 Dec 2024 18:27:35 +0800 Subject: [PATCH 3/5] refactor(oma-refresh): add `MirrorSource` and `MirrorSources` wrapper --- oma-refresh/src/db.rs | 508 ++++++++++++++++----------------- oma-refresh/src/sourceslist.rs | 102 ++++++- src/error.rs | 4 + 3 files changed, 348 insertions(+), 266 deletions(-) diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index c5a9d3a0d..670bb1830 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -54,6 +54,7 @@ use tokio::{ }; use tracing::{debug, warn}; +use crate::sourceslist::{MirrorSource, MirrorSources}; use crate::{ config::{ChecksumDownloadEntry, IndexTargetConfig}, inrelease::{ @@ -103,6 +104,8 @@ pub enum RefreshError { SetLockWithProcess(String, i32), #[error("duplicate components")] DuplicateComponents(Box, String), + #[error("sources.list is empty")] + SourceListsEmpty, } type Result = std::result::Result; @@ -115,8 +118,6 @@ pub struct OmaRefresh<'a> { arch: String, download_dir: PathBuf, client: &'a Client, - #[builder(skip)] - flat_repo_no_release: Vec, #[cfg(feature = "aosc")] refresh_topics: bool, apt_config: &'a Config, @@ -124,8 +125,7 @@ pub struct OmaRefresh<'a> { auth_config: &'a AuthConfig, } -type SourceMap<'a> = AHashMap>>; - +/// Create `apt update` file lock fn get_apt_update_lock(download_dir: &Path) -> Result<()> { let lock_path = download_dir.join("lock"); @@ -195,12 +195,6 @@ pub enum Event { Done, } -#[derive(Debug)] -enum KeyOrIndex<'a> { - Key(&'a str), - Index(usize), -} - impl<'a> OmaRefresh<'a> { pub async fn start(mut self, callback: F) -> Result<()> where @@ -249,14 +243,14 @@ impl<'a> OmaRefresh<'a> { let mut download_list = vec![]; let replacer = DatabaseFilenameReplacer::new()?; - let source_map = self + let mirror_sources = self .download_releases(&sourcelist, &replacer, &callback) .await?; - download_list.extend(source_map.keys().map(|x| x.as_str())); + download_list.extend(mirror_sources.0.iter().flat_map(|x| x.file_name())); let (tasks, total) = self - .collect_all_release_entry(&sourcelist, &replacer, &source_map) + .collect_all_release_entry(&replacer, &mirror_sources) .await?; for i in &tasks { @@ -353,28 +347,21 @@ impl<'a> OmaRefresh<'a> { sourcelist: &'b [OmaSourceEntry<'b>], replacer: &DatabaseFilenameReplacer, callback: &F, - ) -> Result> + ) -> Result> where F: Fn(Event) -> Fut, Fut: Future, { - let mut source_map = SourceMap::new(); - #[cfg(feature = "aosc")] let mut not_found = vec![]; #[cfg(not(feature = "aosc"))] let not_found = vec![]; - let mut mirror_ose_map: AHashMap> = AHashMap::new(); - - for (i, c) in sourcelist.iter().enumerate() { - let name = replacer.replace(c.dist_path())?; - mirror_ose_map.entry(name).or_default().push((c, i)); - } + let mirror_sources = MirrorSources::from_sourcelist(sourcelist, replacer)?; - let tasks = mirror_ose_map.iter().enumerate().map(|(index, (k, v))| { - self.get_release_file(v[0], replacer, index, mirror_ose_map.len(), k, callback) + let tasks = mirror_sources.0.iter().enumerate().map(|(index, m)| { + self.get_release_file(m, replacer, index, mirror_sources.0.len(), callback) }); let results = futures::stream::iter(tasks) @@ -384,48 +371,22 @@ impl<'a> OmaRefresh<'a> { debug!("download_releases: results: {:?}", results); - for result in results { - match result { - Ok((Some(file_name), key_or_index)) => { - let KeyOrIndex::Key(key) = key_or_index else { - unreachable!() - }; - - source_map.insert( - file_name, - mirror_ose_map - .get(key) - .unwrap() - .iter() - .map(|x| x.0) - .collect::>(), - ); - } - Ok((None, key_or_index)) => { - let KeyOrIndex::Index(index) = key_or_index else { - unreachable!() - }; - - self.flat_repo_no_release.push(index) - } - Err(e) => { - #[cfg(feature = "aosc")] - match e { - RefreshError::ReqwestError(e) - if e.status() - .map(|x| x == StatusCode::NOT_FOUND) - .unwrap_or(false) - && self.refresh_topics => - { - let url = e.url().map(|x| x.to_owned()); - not_found.push(url.unwrap()); - } - _ => return Err(e), - } - #[cfg(not(feature = "aosc"))] - return Err(e.into()); + if let Err(e) = results.into_iter().collect::>() { + #[cfg(feature = "aosc")] + match e { + RefreshError::ReqwestError(e) + if e.status() + .map(|x| x == StatusCode::NOT_FOUND) + .unwrap_or(false) + && self.refresh_topics => + { + let url = e.url().map(|x| x.to_owned()); + not_found.push(url.unwrap()); } + _ => return Err(e), } + #[cfg(not(feature = "aosc"))] + return Err(e.into()); } #[cfg(not(feature = "aosc"))] @@ -433,7 +394,7 @@ impl<'a> OmaRefresh<'a> { self.refresh_topics(callback, not_found).await?; - Ok(source_map) + Ok(mirror_sources) } #[cfg(feature = "aosc")] @@ -491,202 +452,230 @@ impl<'a> OmaRefresh<'a> { async fn get_release_file<'b, F, Fut>( &self, - entry: (&OmaSourceEntry<'_>, usize), + entry: &MirrorSource<'_>, replacer: &DatabaseFilenameReplacer, progress_index: usize, total: usize, - key: &'b str, callback: &F, - ) -> Result<(Option, KeyOrIndex<'b>)> + ) -> Result<()> where F: Fn(Event) -> Fut, Fut: Future, { - let (entry, index) = entry; match entry.from()? { OmaSourceEntryFrom::Http => { - let dist_path = entry.dist_path(); + self.download_http_release(entry, replacer, progress_index, total, callback) + .await + } + OmaSourceEntryFrom::Local => { + self.download_local_release(entry, replacer, progress_index, total, callback) + .await + } + } + } - let mut r = None; - let mut u = None; - let mut is_release = false; + async fn download_local_release( + &self, + entry: &MirrorSource<'_>, + replacer: &DatabaseFilenameReplacer, + progress_index: usize, + total: usize, + callback: &F, + ) -> Result<()> + where + F: Fn(Event) -> Fut, + Fut: Future, + { + let dist_path_with_protocol = entry.dist_path(); + let dist_path = dist_path_with_protocol + .strip_prefix("file:") + .unwrap_or(dist_path_with_protocol); + let dist_path = Path::new(dist_path); - let msg = entry.get_human_download_url(None)?; + let mut name = None; - callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { - index: progress_index, - msg: format!("({}/{}) {}", progress_index, total, msg), - })) - .await; + let msg = entry.get_human_download_url(None)?; - for (index, file_name) in ["InRelease", "Release"].iter().enumerate() { - let url = format!("{}/{}", dist_path, file_name); - let request = self.request_get_builder(&url, entry); + callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { + index: progress_index, + msg: format!("({}/{}) {}", progress_index, total, msg), + })) + .await; - let resp = request - .send() - .await - .and_then(|resp| resp.error_for_status()); + let mut is_release = false; - r = Some(resp); + for (index, entry) in ["InRelease", "Release"].iter().enumerate() { + let p = dist_path.join(entry); - if r.as_ref().unwrap().is_ok() { - u = Some(url); - if index == 1 { - is_release = true; - } - break; - } - } + let dst = if dist_path_with_protocol.ends_with('/') { + format!("{}{}", dist_path_with_protocol, entry) + } else { + format!("{}/{}", dist_path_with_protocol, entry) + }; - let r = r.unwrap(); + let file_name = replacer.replace(&dst)?; - callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( - progress_index, - ))) - .await; + let dst = self.download_dir.join(&file_name); - if r.is_err() && entry.is_flat() { - return Ok((None, KeyOrIndex::Index(index))); + if p.exists() { + if dst.exists() { + debug!("get_release_file: Removing {}", dst.display()); + fs::remove_file(&dst).await.map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; } - let resp = r?; + debug!("get_release_file: Symlink {}", dst.display()); + fs::symlink(p, dst).await.map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; - let url = u.unwrap(); - let file_name = replacer.replace(&url)?; + if index == 1 { + is_release = true; + } - self.download_file(&file_name, resp, entry, progress_index, total, &callback) - .await?; + name = Some(file_name); + break; + } + } - if is_release && !entry.trusted() { - let url = format!("{}/{}", dist_path, "Release.gpg"); + if name.is_none() && entry.is_flat() { + // Flat repo no release + return Ok(()); + } - let request = self.request_get_builder(&url, entry); - let resp = request - .send() - .await - .and_then(|resp| resp.error_for_status())?; + if is_release { + let p = dist_path.join("Release.gpg"); + let entry = "Release.gpg"; - let file_name = replacer.replace(&url)?; + let dst = if dist_path_with_protocol.ends_with('/') { + format!("{}{}", dist_path_with_protocol, entry) + } else { + format!("{}/{}", dist_path_with_protocol, entry) + }; + + let file_name = replacer.replace(&dst)?; - self.download_file(&file_name, resp, entry, progress_index, total, &callback) - .await?; + let dst = self.download_dir.join(&file_name); + + if p.exists() { + if dst.exists() { + fs::remove_file(&dst).await.map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; } - Ok((Some(file_name), KeyOrIndex::Key(key))) + fs::symlink(p, self.download_dir.join(file_name)) + .await + .map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; } - OmaSourceEntryFrom::Local => { - let dist_path_with_protocol = entry.dist_path(); - let dist_path = dist_path_with_protocol - .strip_prefix("file:") - .unwrap_or(dist_path_with_protocol); - let dist_path = Path::new(dist_path); - - let mut name = None; - - let msg = entry.get_human_download_url(None)?; - - callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { - index: progress_index, - msg: format!("({}/{}) {}", progress_index, total, msg), - })) - .await; - - let mut is_release = false; - - for (index, entry) in ["InRelease", "Release"].iter().enumerate() { - let p = dist_path.join(entry); - - let dst = if dist_path_with_protocol.ends_with('/') { - format!("{}{}", dist_path_with_protocol, entry) - } else { - format!("{}/{}", dist_path_with_protocol, entry) - }; - - let file_name = replacer.replace(&dst)?; - - let dst = self.download_dir.join(&file_name); - - if p.exists() { - if dst.exists() { - debug!("get_release_file: Removing {}", dst.display()); - fs::remove_file(&dst).await.map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError( - entry.to_string(), - e, - )) - })?; - } + } - debug!("get_release_file: Symlink {}", dst.display()); - fs::symlink(p, dst).await.map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) - })?; + callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( + progress_index, + ))) + .await; - if index == 1 { - is_release = true; - } + let name = name.ok_or_else(|| RefreshError::NoInReleaseFile(entry.url().to_string()))?; + entry.set_release_file_name(name); - name = Some(file_name); - break; - } - } + Ok(()) + } + + async fn download_http_release( + &self, + entry: &MirrorSource<'_>, + replacer: &DatabaseFilenameReplacer, + progress_index: usize, + total: usize, + callback: &F, + ) -> Result<()> + where + F: Fn(Event) -> Fut, + Fut: Future, + { + let dist_path = entry.dist_path(); - if is_release { - let p = dist_path.join("Release.gpg"); - let entry = "Release.gpg"; + let mut r = None; + let mut u = None; + let mut is_release = false; - let dst = if dist_path_with_protocol.ends_with('/') { - format!("{}{}", dist_path_with_protocol, entry) - } else { - format!("{}/{}", dist_path_with_protocol, entry) - }; + let msg = entry.get_human_download_url(None)?; - let file_name = replacer.replace(&dst)?; + callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { + index: progress_index, + msg: format!("({}/{}) {}", progress_index, total, msg), + })) + .await; - let dst = self.download_dir.join(&file_name); + for (index, file_name) in ["InRelease", "Release"].iter().enumerate() { + let url = format!("{}/{}", dist_path, file_name); + let request = self.request_get_builder(&url, entry); - if p.exists() { - if dst.exists() { - fs::remove_file(&dst).await.map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError( - entry.to_string(), - e, - )) - })?; - } + let resp = request + .send() + .await + .and_then(|resp| resp.error_for_status()); - fs::symlink(p, self.download_dir.join(file_name)) - .await - .map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError( - entry.to_string(), - e, - )) - })?; - } + r = Some(resp); + + if r.as_ref().unwrap().is_ok() { + u = Some(url); + if index == 1 { + is_release = true; } + break; + } + } - callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( - progress_index, - ))) - .await; + let r = r.unwrap(); - let name = - name.ok_or_else(|| RefreshError::NoInReleaseFile(entry.url().to_string()))?; + callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( + progress_index, + ))) + .await; - Ok((Some(name), KeyOrIndex::Key(key))) - } + if r.is_err() && entry.is_flat() { + // Flat repo no release + return Ok(()); + } + + let resp = r?; + + let url = u.unwrap(); + let file_name = replacer.replace(&url)?; + + self.download_file(&file_name, resp, entry, progress_index, total, &callback) + .await?; + entry.set_release_file_name(file_name); + + if is_release && !entry.trusted() { + let url = format!("{}/{}", dist_path, "Release.gpg"); + + let request = self.request_get_builder(&url, entry); + let resp = request + .send() + .await + .and_then(|resp| resp.error_for_status())?; + + let file_name = replacer.replace(&url)?; + + self.download_file(&file_name, resp, entry, progress_index, total, &callback) + .await?; } + + Ok(()) } fn request_get_builder( &self, url: &str, - source_index: &OmaSourceEntry<'_>, + source_index: &MirrorSource<'_>, ) -> reqwest::RequestBuilder { let mut request = self.client.get(url); - if let Some(auth) = &source_index.auth.as_ref().and_then(|x| x.entry.as_ref()) { + if let Some(auth) = source_index.auth() { request = request.basic_auth(&auth.user, Some(&auth.password)) } @@ -697,7 +686,7 @@ impl<'a> OmaRefresh<'a> { &self, file_name: &str, mut resp: Response, - source_index: &OmaSourceEntry<'_>, + source_index: &MirrorSource<'_>, index: usize, total: usize, callback: &F, @@ -754,9 +743,8 @@ impl<'a> OmaRefresh<'a> { async fn collect_all_release_entry( &self, - sourcelist: &[OmaSourceEntry<'a>], replacer: &DatabaseFilenameReplacer, - sources_map: &AHashMap>>, + mirror_sources: &MirrorSources<'a>, ) -> Result<(Vec, u64)> { let mut total = 0; let mut tasks = vec![]; @@ -767,7 +755,17 @@ impl<'a> OmaRefresh<'a> { .await .map(|f| f.lines().map(|x| x.to_string()).collect::>()); - for (file_name, ose_list) in sources_map { + let mut flat_repo_no_release = vec![]; + + for m in &mirror_sources.0 { + let file_name = match m.file_name() { + Some(name) => name, + None => { + flat_repo_no_release.push(m); + continue; + } + }; + let inrelease_path = self.download_dir.join(file_name); let mut handle = HashSet::with_hasher(ahash::RandomState::new()); @@ -778,41 +776,35 @@ impl<'a> OmaRefresh<'a> { let inrelease = verify_inrelease( &inrelease, - ose_list.iter().find_map(|x| { - if let Some(x) = x.signed_by() { - Some(x) - } else { - None - } - }), + m.signed_by(), &self.source, &inrelease_path, - ose_list.iter().any(|x| x.trusted()), + m.trusted(), ) .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; - let inrelease: Release = inrelease + let release: Release = inrelease .parse() .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; - if ose_list[0].is_flat() { + if m.is_flat() { let now = Utc::now(); - inrelease.check_date(&now).map_err(|e| { + release.check_date(&now).map_err(|e| { RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e) })?; - inrelease.check_valid_until(&now).map_err(|e| { + release.check_valid_until(&now).map_err(|e| { RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e) })?; } - let checksums = &inrelease + let checksums = &release .get_or_try_init_checksum_type_and_list() .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))? .1; - for ose in ose_list { + for ose in &m.sources { debug!("Getted oma source entry: {:#?}", ose); let mut archs = if let Some(archs) = ose.archs() { @@ -834,26 +826,14 @@ impl<'a> OmaRefresh<'a> { )?; get_all_need_db_from_config(download_list, &mut total, checksums, &mut handle); + } - for i in &self.flat_repo_no_release { - collect_flat_repo_no_release( - sourcelist.get(*i).unwrap(), - &self.download_dir, - &mut tasks, - replacer, - )?; - } + for i in &flat_repo_no_release { + collect_flat_repo_no_release(i, &self.download_dir, &mut tasks, replacer)?; } for c in &handle { - collect_download_task( - c, - ose_list[0], - &self.download_dir, - &mut tasks, - &inrelease, - replacer, - )?; + collect_download_task(c, m, &self.download_dir, &mut tasks, &release, replacer)?; } } @@ -973,24 +953,23 @@ async fn remove_unused_db(download_dir: &Path, download_list: Vec<&str>) -> Resu } fn collect_flat_repo_no_release( - source_index: &OmaSourceEntry, + mirror_source: &MirrorSource, download_dir: &Path, tasks: &mut Vec, replacer: &DatabaseFilenameReplacer, ) -> Result<()> { - let msg = source_index.get_human_download_url(Some("Packages"))?; + let msg = mirror_source.get_human_download_url(Some("Packages"))?; - let dist_url = source_index.dist_path(); + let dist_url = mirror_source.dist_path(); - let from = match source_index.from()? { + let from = match mirror_source.from()? { OmaSourceEntryFrom::Http => DownloadSourceType::Http { - auth: source_index - .auth + auth: mirror_source + .auth() .as_ref() - .and_then(|auth| auth.entry.as_ref()) .map(|auth| (auth.user.clone(), auth.password.clone())), }, - OmaSourceEntryFrom::Local => DownloadSourceType::Local(source_index.is_flat()), + OmaSourceEntryFrom::Local => DownloadSourceType::Local(mirror_source.is_flat()), }; let download_url = format!("{}/Packages", dist_url); @@ -1018,27 +997,26 @@ fn collect_flat_repo_no_release( fn collect_download_task( c: &ChecksumDownloadEntry, - source_index: &OmaSourceEntry, + mirror_source: &MirrorSource<'_>, download_dir: &Path, tasks: &mut Vec, - inrelease: &Release, + release: &Release, replacer: &DatabaseFilenameReplacer, ) -> Result<()> { let file_type = &c.msg; - let msg = source_index.get_human_download_url(Some(file_type))?; + let msg = mirror_source.get_human_download_url(Some(file_type))?; - let dist_url = &source_index.dist_path(); + let dist_url = &mirror_source.dist_path(); - let from = match source_index.from()? { + let from = match mirror_source.from()? { OmaSourceEntryFrom::Http => DownloadSourceType::Http { - auth: source_index - .auth + auth: mirror_source + .auth() .as_ref() - .and_then(|auth| auth.entry.as_ref()) .map(|auth| (auth.user.clone(), auth.password.clone())), }, - OmaSourceEntryFrom::Local => DownloadSourceType::Local(source_index.is_flat()), + OmaSourceEntryFrom::Local => DownloadSourceType::Local(mirror_source.is_flat()), }; let not_compress_filename_before = if file_is_compress(&c.item.name) { @@ -1050,7 +1028,7 @@ fn collect_download_task( let checksum = if c.keep_compress { Some(&c.item.checksum) } else { - inrelease + release .checksum_type_and_list() .1 .iter() @@ -1059,10 +1037,10 @@ fn collect_download_task( .map(|c| &c.checksum) }; - let download_url = if inrelease.acquire_by_hash() { + let download_url = if release.acquire_by_hash() { let path = Path::new(&c.item.name); let parent = path.parent().unwrap_or(path); - let dir = match inrelease.checksum_type_and_list().0 { + let dir = match release.checksum_type_and_list().0 { InReleaseChecksum::Sha256 => "SHA256", InReleaseChecksum::Sha512 => "SHA512", InReleaseChecksum::Md5 => "MD5Sum", @@ -1083,7 +1061,7 @@ fn collect_download_task( }]; let file_path = if c.keep_compress { - if inrelease.acquire_by_hash() { + if release.acquire_by_hash() { Cow::Owned(format!("{}/{}", dist_url, c.item.name)) } else { Cow::Borrowed(&download_url) @@ -1116,7 +1094,7 @@ fn collect_download_task( } }) .maybe_hash(if let Some(checksum) = checksum { - match inrelease.checksum_type_and_list().0 { + match release.checksum_type_and_list().0 { InReleaseChecksum::Sha256 => Some(Checksum::from_sha256_str(checksum)?), InReleaseChecksum::Sha512 => Some(Checksum::from_sha512_str(checksum)?), InReleaseChecksum::Md5 => Some(Checksum::from_md5_str(checksum)?), diff --git a/oma-refresh/src/sourceslist.rs b/oma-refresh/src/sourceslist.rs index b958a3fe4..a069515df 100644 --- a/oma-refresh/src/sourceslist.rs +++ b/oma-refresh/src/sourceslist.rs @@ -1,11 +1,12 @@ use std::path::Path; +use ahash::HashMap; use apt_auth_config::AuthConfigEntry; use oma_apt_sources_lists::{Signature, SourceEntry, SourceLine, SourceListType, SourcesLists}; use once_cell::sync::OnceCell; use url::Url; -use crate::db::RefreshError; +use crate::{db::RefreshError, util::DatabaseFilenameReplacer}; #[derive(Debug, Clone)] pub struct OmaSourceEntry<'a> { @@ -161,6 +162,105 @@ impl<'a> OmaSourceEntry<'a> { } } +#[derive(Debug)] +pub(crate) struct MirrorSources<'a>(pub Vec>); + +#[derive(Debug)] +pub(crate) struct MirrorSource<'a> { + pub(crate) sources: Vec<&'a OmaSourceEntry<'a>>, + release_file_name: OnceCell, +} + +impl MirrorSource<'_> { + pub(crate) fn set_release_file_name(&self, file_name: String) { + self.release_file_name + .set(file_name) + .expect("Release file name was init"); + } + + pub(crate) fn dist_path(&self) -> &str { + self.sources.first().unwrap().dist_path() + } + + pub(crate) fn from(&self) -> Result<&OmaSourceEntryFrom, RefreshError> { + self.sources.first().unwrap().from() + } + + pub(crate) fn get_human_download_url( + &self, + file_name: Option<&str>, + ) -> Result { + self.sources + .first() + .unwrap() + .get_human_download_url(file_name) + } + + pub(crate) fn auth(&self) -> Option<&AuthConfigEntry> { + self.sources + .iter() + .find_map(|x| if let Some(x) = &x.auth { Some(x) } else { None }) + } + + pub(crate) fn signed_by(&self) -> Option<&Signature> { + self.sources.iter().find_map(|x| { + if let Some(x) = &x.signed_by() { + Some(x) + } else { + None + } + }) + } + + pub(crate) fn url(&self) -> &str { + self.sources.first().unwrap().url() + } + + pub(crate) fn is_flat(&self) -> bool { + self.sources.first().unwrap().is_flat() + } + + pub(crate) fn trusted(&self) -> bool { + self.sources.iter().any(|x| x.trusted()) + } + + pub(crate) fn file_name(&self) -> Option<&str> { + self.release_file_name.get().map(|x| x.as_str()) + } +} + +impl<'a> MirrorSources<'a> { + pub(crate) fn from_sourcelist( + sourcelist: &'a [OmaSourceEntry<'a>], + replacer: &DatabaseFilenameReplacer, + ) -> Result { + let mut map: HashMap> = + HashMap::with_hasher(ahash::RandomState::new()); + + if sourcelist.is_empty() { + return Err(RefreshError::SourceListsEmpty); + } + + for source in sourcelist { + let dist_path = source.dist_path(); + let name = replacer.replace(dist_path)?; + + map.entry(name).or_default().push(source); + } + + let mut res = vec![]; + + for (_, v) in map { + res.push(MirrorSource { + sources: v, + release_file_name: OnceCell::new(), + }); + } + + Ok(Self(res)) + } +} + #[test] fn test_ose() { use oma_utils::dpkg::dpkg_arch; diff --git a/src/error.rs b/src/error.rs index 9431e6a6d..1ea18c90e 100644 --- a/src/error.rs +++ b/src/error.rs @@ -426,6 +426,10 @@ impl From for OutputError { description: fl!("doplicate-component", url = url.to_string(), c = component), source: None, }, + RefreshError::SourceListsEmpty => Self { + description: "Source list is empty".to_string(), + source: None, + }, } } } From 79fb102b227ae5d1c7e7e6e73a78bd79bd4c4f0a Mon Sep 17 00:00:00 2001 From: eatradish Date: Mon, 30 Dec 2024 23:31:42 +0800 Subject: [PATCH 4/5] refactor(oma-repo-verify): move replace '.' logic to inner --- oma-repo-verify/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oma-repo-verify/src/lib.rs b/oma-repo-verify/src/lib.rs index 98bf35e04..7fc074236 100644 --- a/oma-repo-verify/src/lib.rs +++ b/oma-repo-verify/src/lib.rs @@ -61,6 +61,8 @@ impl InReleaseVerifier { } pub fn from_key_block(block: &str, trusted: bool) -> VerifyResult { + // 这个点存在只是表示换行，因此把它替换掉 + let block = block.replace('.', ""); let mut certs: Vec = Vec::new(); let ppr = PacketParserBuilder::from_bytes(block.as_bytes())?.build()?; let cert = CertParser::from(ppr); @@ -141,9 +143,7 @@ pub fn verify_inrelease( &p, None, if let Some(deb822_inner_signed_by_str) = deb822_inner_signed_by_str { - // 这个点存在只是表示换行，因此把它替换掉 - let signed_by_str = deb822_inner_signed_by_str.replace('.', ""); - InReleaseVerifier::from_key_block(&signed_by_str, trusted)? + InReleaseVerifier::from_key_block(deb822_inner_signed_by_str, trusted)? } else { InReleaseVerifier::from_paths(&certs, trusted)? }, From 37e0f2319932c50c0d1b903f2d43e11925afc742 Mon Sep 17 00:00:00 2001 From: eatradish Date: Wed, 1 Jan 2025 17:12:06 +0800 Subject: [PATCH 5/5] WIP --- Cargo.lock | 2 + apt-auth-config/Cargo.toml | 2 + apt-auth-config/src/lib.rs | 87 ++++++++++++---------------------- oma-refresh/src/db.rs | 12 +++-- oma-refresh/src/sourceslist.rs | 3 +- 5 files changed, 44 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be7d57a10..a10bf8198 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -157,8 +157,10 @@ checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" name = "apt-auth-config" version = "0.2.0" dependencies = [ + "ahash", "thiserror 2.0.9", "tracing", + "url", "winnow", ] diff --git a/apt-auth-config/Cargo.toml b/apt-auth-config/Cargo.toml index 72f53b65a..69143f1fb 100644 --- a/apt-auth-config/Cargo.toml +++ b/apt-auth-config/Cargo.toml @@ -9,3 +9,5 @@ license = "MIT" thiserror = "2" tracing = "0.1" winnow = { version = "0.6.20" } +url = "2.5" +ahash = "0.8.11" diff --git a/apt-auth-config/src/lib.rs b/apt-auth-config/src/lib.rs index 0525dd820..5436963c2 100644 --- a/apt-auth-config/src/lib.rs +++ b/apt-auth-config/src/lib.rs @@ -7,9 +7,10 @@ use std::{ str::FromStr, }; +use ahash::HashMap; use parser::{line, multiline}; use thiserror::Error; -use tracing::debug; +use url::Url; #[derive(Debug, Error)] pub enum AuthConfigError { @@ -26,9 +27,7 @@ pub enum AuthConfigError { } #[derive(Debug, PartialEq, Eq, Clone)] -pub struct AuthConfig { - pub inner: Vec, -} +pub struct AuthConfig(pub HashMap, AuthConfigEntry>); #[derive(Debug, PartialEq, Eq, Clone)] pub struct AuthConfigEntry { @@ -44,7 +43,7 @@ impl FromStr for AuthConfigEntry { let mut s = s; let parse = line(&mut s).map_err(|e| AuthConfigError::ParseError(e.to_string()))?; - Ok(parse_entry_inner(parse)) + Ok(parse_entry_inner(parse).1) } } @@ -54,17 +53,18 @@ impl FromStr for AuthConfig { fn from_str(s: &str) -> Result { let mut s = s; let parse = multiline(&mut s).map_err(|e| AuthConfigError::ParseError(e.to_string()))?; - let mut res = vec![]; + let mut res = HashMap::with_hasher(ahash::RandomState::new()); for r in parse { - res.push(parse_entry_inner(r)); + let (k, v) = parse_entry_inner(r); + res.insert(k, v); } - Ok(AuthConfig { inner: res }) + Ok(AuthConfig(res)) } } -fn parse_entry_inner(input: Vec<(&str, &str)>) -> AuthConfigEntry { +fn parse_entry_inner(input: Vec<(&str, &str)>) -> (Box, AuthConfigEntry) { let mut machine = None; let mut login = None; let mut password = None; @@ -78,11 +78,16 @@ fn parse_entry_inner(input: Vec<(&str, &str)>) -> AuthConfigEntry { } } - AuthConfigEntry { - host: machine.unwrap().into(), - user: login.unwrap().into(), - password: password.unwrap().into(), - } + let machine: Box = machine.unwrap().into(); + + ( + machine.clone(), + AuthConfigEntry { + host: machine, + user: login.unwrap().into(), + password: password.unwrap().into(), + }, + ) } impl AuthConfig { @@ -93,7 +98,7 @@ impl AuthConfig { } pub fn from_path(p: impl AsRef) -> Result { - let mut v = vec![]; + let mut v = HashMap::with_hasher(ahash::RandomState::new()); for i in read_dir(p.as_ref()).map_err(|e| AuthConfigError::ReadDir { path: p.as_ref().to_path_buf(), @@ -111,53 +116,19 @@ impl AuthConfig { })?; let config: AuthConfig = s.parse()?; - v.extend(config.inner); + v.extend(config.0); } - Ok(Self { inner: v }) - } - - pub fn find(&self, url: &str) -> Option<&AuthConfigEntry> { - let url = url - .strip_prefix("http://") - .or_else(|| url.strip_prefix("https://")) - .unwrap_or(url); - - debug!("auth find url is: {}", url); - - self.inner.iter().find_map(|x| { - let mut host = x.host.to_string(); - while host.ends_with('/') { - host.pop(); - } - - let mut url = url.to_string(); - while url.ends_with('/') { - url.pop(); - } - - if host == url { - Some(x) - } else { - None - } - }) + Ok(Self(v)) } - pub fn find_package_url(&self, url: &str) -> Option<&AuthConfigEntry> { - let url = url - .strip_prefix("http://") - .or_else(|| url.strip_prefix("https://")) - .unwrap_or(url); - - debug!("auth find package url is: {}", url); + pub fn get_match_auth(&self, url: Url) -> Option<&AuthConfigEntry> { + let host = url.host_str()?; + let path = url.path(); + let url_without_schema = [host, path].concat(); - self.inner.iter().find_map(|x| { - if url.starts_with(x.host.as_ref()) { - Some(x) - } else { - None - } - }) + self.0 + .values() + .find(|x| url_without_schema.starts_with(&*x.host)) } } diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 670bb1830..aa2f98609 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -53,6 +53,7 @@ use tokio::{ task::spawn_blocking, }; use tracing::{debug, warn}; +use url::Url; use crate::sourceslist::{MirrorSource, MirrorSources}; use crate::{ @@ -306,13 +307,18 @@ impl<'a> OmaRefresh<'a> { Ok(res) } - fn set_auth(&self, sourcelist: &mut [OmaSourceEntry<'_>]) { + fn set_auth(&self, sourcelist: &mut [OmaSourceEntry<'_>]) -> Result<()> { for i in sourcelist { - let auth = self.auth_config.find(i.url()); + let auth = self.auth_config.get_match_auth( + Url::parse(i.url()).map_err(|_| RefreshError::InvalidUrl(i.url().to_string()))?, + ); + if let Some(auth) = auth { i.set_auth(auth.to_owned()); } } + + Ok(()) } async fn run_success_post_invoke(&self) { @@ -358,7 +364,7 @@ impl<'a> OmaRefresh<'a> { #[cfg(not(feature = "aosc"))] let not_found = vec![]; - let mirror_sources = MirrorSources::from_sourcelist(sourcelist, replacer)?; + let mirror_sources = MirrorSources::from_sourcelist(sourcelist, replacer, self.auth_config)?; let tasks = mirror_sources.0.iter().enumerate().map(|(index, m)| { self.get_release_file(m, replacer, index, mirror_sources.0.len(), callback) diff --git a/oma-refresh/src/sourceslist.rs b/oma-refresh/src/sourceslist.rs index a069515df..d9d4abf5a 100644 --- a/oma-refresh/src/sourceslist.rs +++ b/oma-refresh/src/sourceslist.rs @@ -1,7 +1,7 @@ use std::path::Path; use ahash::HashMap; -use apt_auth_config::AuthConfigEntry; +use apt_auth_config::{AuthConfig, AuthConfigEntry}; use oma_apt_sources_lists::{Signature, SourceEntry, SourceLine, SourceListType, SourcesLists}; use once_cell::sync::OnceCell; use url::Url; @@ -233,6 +233,7 @@ impl<'a> MirrorSources<'a> { pub(crate) fn from_sourcelist( sourcelist: &'a [OmaSourceEntry<'a>], replacer: &DatabaseFilenameReplacer, + auth_config: &AuthConfig, ) -> Result { let mut map: HashMap> = HashMap::with_hasher(ahash::RandomState::new());