From 8167ec476db3139994bd0d01326016455158d246 Mon Sep 17 00:00:00 2001 From: eatradish Date: Mon, 30 Dec 2024 15:03:09 +0800 Subject: [PATCH 1/4] refactor(oma-refresh)!: move parse InRelease logic to `FromStr` trait .. `InRelease` struct rename to `Release` --- oma-refresh/src/db.rs | 13 +++++++------ oma-refresh/src/inrelease.rs | 10 +++++++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 9530e555..220ee491 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -57,8 +57,8 @@ use tracing::{debug, warn}; use crate::{ config::{ChecksumDownloadEntry, IndexTargetConfig}, inrelease::{ - file_is_compress, split_ext_and_filename, verify_inrelease, ChecksumItem, InRelease, - InReleaseChecksum, InReleaseError, + file_is_compress, split_ext_and_filename, verify_inrelease, ChecksumItem, + InReleaseChecksum, InReleaseError, Release, }, sourceslist::{sources_lists, OmaSourceEntry, OmaSourceEntryFrom}, util::DatabaseFilenameReplacer, @@ -791,7 +791,8 @@ impl<'a> OmaRefresh<'a> { ) .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; - let inrelease = InRelease::new(&inrelease) + let inrelease: Release = inrelease + .parse() .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; if ose_list[0].is_flat() { @@ -835,7 +836,7 @@ impl<'a> OmaRefresh<'a> { get_all_need_db_from_config(download_list, &mut total, checksums, &mut handle); for i in &self.flat_repo_no_release { - download_flat_repo_no_release( + collect_flat_repo_no_release( sourcelist.get(*i).unwrap(), &self.download_dir, &mut tasks, @@ -971,7 +972,7 @@ async fn remove_unused_db(download_dir: &Path, download_list: Vec<&str>) -> Resu Ok(()) } -fn download_flat_repo_no_release( +fn collect_flat_repo_no_release( source_index: &OmaSourceEntry, download_dir: &Path, tasks: &mut Vec, @@ -1019,7 +1020,7 @@ fn collect_download_task( source_index: &OmaSourceEntry, download_dir: &Path, tasks: &mut Vec, - inrelease: &InRelease, + inrelease: &Release, replacer: &DatabaseFilenameReplacer, ) -> Result<()> { let file_type = &c.msg; diff --git a/oma-refresh/src/inrelease.rs b/oma-refresh/src/inrelease.rs index 2769703b..626e7e28 100644 --- a/oma-refresh/src/inrelease.rs +++ b/oma-refresh/src/inrelease.rs @@ -58,7 +58,7 @@ pub enum InReleaseChecksum { const COMPRESS: &[&str] = &[".gz", ".xz", ".zst", ".bz2"]; -pub struct InRelease { +pub struct Release { source: InReleaseEntry, acquire_by_hash: OnceCell, checksum_type_and_list: OnceCell<(InReleaseChecksum, Vec)>, @@ -80,8 +80,10 @@ struct InReleaseEntry { sha512: Option, } -impl InRelease { - pub fn new(input: &str) -> Result { +impl FromStr for Release { + type Err = InReleaseError; + + fn from_str(input: &str) -> Result { let source: Paragraph = input.parse().map_err(|_| InReleaseError::BrokenInRelease)?; let source: InReleaseEntry = FromDeb822Paragraph::from_paragraph(&source) .map_err(|_| InReleaseError::BrokenInRelease)?; @@ -92,7 +94,9 @@ impl InRelease { checksum_type_and_list: OnceCell::new(), }) } +} +impl Release { pub fn get_or_try_init_checksum_type_and_list( &self, ) -> Result<&(InReleaseChecksum, Vec), InReleaseError> { From af56d75c4d954d4e9036df0dc91812fdd08a0729 Mon Sep 17 00:00:00 2001 From: eatradish Date: Mon, 30 Dec 2024 18:27:35 +0800 Subject: [PATCH 2/4] refactor(oma-refresh): add `MirrorSource` and `MirrorSources` wrapper --- oma-refresh/src/db.rs | 506 ++++++++++++++++----------------- oma-refresh/src/sourceslist.rs | 102 ++++++- src/error.rs | 4 + 3 files changed, 348 insertions(+), 264 deletions(-) diff --git a/oma-refresh/src/db.rs b/oma-refresh/src/db.rs index 220ee491..670bb183 100644 --- a/oma-refresh/src/db.rs +++ b/oma-refresh/src/db.rs @@ -54,6 +54,7 @@ use tokio::{ }; use tracing::{debug, warn}; +use crate::sourceslist::{MirrorSource, MirrorSources}; use crate::{ config::{ChecksumDownloadEntry, IndexTargetConfig}, inrelease::{ @@ -103,6 +104,8 @@ pub enum RefreshError { SetLockWithProcess(String, i32), #[error("duplicate components")] DuplicateComponents(Box, String), + #[error("sources.list is empty")] + SourceListsEmpty, } type Result = std::result::Result; @@ -115,8 +118,6 @@ pub struct OmaRefresh<'a> { arch: String, download_dir: PathBuf, client: &'a Client, - #[builder(skip)] - flat_repo_no_release: Vec, #[cfg(feature = "aosc")] refresh_topics: bool, apt_config: &'a Config, @@ -124,8 +125,7 @@ pub struct OmaRefresh<'a> { auth_config: &'a AuthConfig, } -type SourceMap<'a> = AHashMap>>; - +/// Create `apt update` file lock fn get_apt_update_lock(download_dir: &Path) -> Result<()> { let lock_path = download_dir.join("lock"); @@ -195,12 +195,6 @@ pub enum Event { Done, } -#[derive(Debug)] -enum KeyOrIndex<'a> { - Key(&'a str), - Index(usize), -} - impl<'a> OmaRefresh<'a> { pub async fn start(mut self, callback: F) -> Result<()> where @@ -249,14 +243,14 @@ impl<'a> OmaRefresh<'a> { let mut download_list = vec![]; let replacer = DatabaseFilenameReplacer::new()?; - let source_map = self + let mirror_sources = self .download_releases(&sourcelist, &replacer, &callback) .await?; - download_list.extend(source_map.keys().map(|x| x.as_str())); + download_list.extend(mirror_sources.0.iter().flat_map(|x| x.file_name())); let (tasks, total) = self - .collect_all_release_entry(&sourcelist, &replacer, &source_map) + .collect_all_release_entry(&replacer, &mirror_sources) .await?; for i in &tasks { @@ -353,28 +347,21 @@ impl<'a> OmaRefresh<'a> { sourcelist: &'b [OmaSourceEntry<'b>], replacer: &DatabaseFilenameReplacer, callback: &F, - ) -> Result> + ) -> Result> where F: Fn(Event) -> Fut, Fut: Future, { - let mut source_map = SourceMap::new(); - #[cfg(feature = "aosc")] let mut not_found = vec![]; #[cfg(not(feature = "aosc"))] let not_found = vec![]; - let mut mirror_ose_map: AHashMap> = AHashMap::new(); - - for (i, c) in sourcelist.iter().enumerate() { - let name = replacer.replace(c.dist_path())?; - mirror_ose_map.entry(name).or_default().push((c, i)); - } + let mirror_sources = MirrorSources::from_sourcelist(sourcelist, replacer)?; - let tasks = mirror_ose_map.iter().enumerate().map(|(index, (k, v))| { - self.get_release_file(v[0], replacer, index, mirror_ose_map.len(), k, callback) + let tasks = mirror_sources.0.iter().enumerate().map(|(index, m)| { + self.get_release_file(m, replacer, index, mirror_sources.0.len(), callback) }); let results = futures::stream::iter(tasks) @@ -384,48 +371,22 @@ impl<'a> OmaRefresh<'a> { debug!("download_releases: results: {:?}", results); - for result in results { - match result { - Ok((Some(file_name), key_or_index)) => { - let KeyOrIndex::Key(key) = key_or_index else { - unreachable!() - }; - - source_map.insert( - file_name, - mirror_ose_map - .get(key) - .unwrap() - .iter() - .map(|x| x.0) - .collect::>(), - ); - } - Ok((None, key_or_index)) => { - let KeyOrIndex::Index(index) = key_or_index else { - unreachable!() - }; - - self.flat_repo_no_release.push(index) - } - Err(e) => { - #[cfg(feature = "aosc")] - match e { - RefreshError::ReqwestError(e) - if e.status() - .map(|x| x == StatusCode::NOT_FOUND) - .unwrap_or(false) - && self.refresh_topics => - { - let url = e.url().map(|x| x.to_owned()); - not_found.push(url.unwrap()); - } - _ => return Err(e), - } - #[cfg(not(feature = "aosc"))] - return Err(e.into()); + if let Err(e) = results.into_iter().collect::>() { + #[cfg(feature = "aosc")] + match e { + RefreshError::ReqwestError(e) + if e.status() + .map(|x| x == StatusCode::NOT_FOUND) + .unwrap_or(false) + && self.refresh_topics => + { + let url = e.url().map(|x| x.to_owned()); + not_found.push(url.unwrap()); } + _ => return Err(e), } + #[cfg(not(feature = "aosc"))] + return Err(e.into()); } #[cfg(not(feature = "aosc"))] @@ -433,7 +394,7 @@ impl<'a> OmaRefresh<'a> { self.refresh_topics(callback, not_found).await?; - Ok(source_map) + Ok(mirror_sources) } #[cfg(feature = "aosc")] @@ -491,202 +452,230 @@ impl<'a> OmaRefresh<'a> { async fn get_release_file<'b, F, Fut>( &self, - entry: (&OmaSourceEntry<'_>, usize), + entry: &MirrorSource<'_>, replacer: &DatabaseFilenameReplacer, progress_index: usize, total: usize, - key: &'b str, callback: &F, - ) -> Result<(Option, KeyOrIndex<'b>)> + ) -> Result<()> where F: Fn(Event) -> Fut, Fut: Future, { - let (entry, index) = entry; match entry.from()? { OmaSourceEntryFrom::Http => { - let dist_path = entry.dist_path(); + self.download_http_release(entry, replacer, progress_index, total, callback) + .await + } + OmaSourceEntryFrom::Local => { + self.download_local_release(entry, replacer, progress_index, total, callback) + .await + } + } + } - let mut r = None; - let mut u = None; - let mut is_release = false; + async fn download_local_release( + &self, + entry: &MirrorSource<'_>, + replacer: &DatabaseFilenameReplacer, + progress_index: usize, + total: usize, + callback: &F, + ) -> Result<()> + where + F: Fn(Event) -> Fut, + Fut: Future, + { + let dist_path_with_protocol = entry.dist_path(); + let dist_path = dist_path_with_protocol + .strip_prefix("file:") + .unwrap_or(dist_path_with_protocol); + let dist_path = Path::new(dist_path); - let msg = entry.get_human_download_url(None)?; + let mut name = None; - callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { - index: progress_index, - msg: format!("({}/{}) {}", progress_index, total, msg), - })) - .await; + let msg = entry.get_human_download_url(None)?; - for (index, file_name) in ["InRelease", "Release"].iter().enumerate() { - let url = format!("{}/{}", dist_path, file_name); - let request = self.request_get_builder(&url, entry); + callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { + index: progress_index, + msg: format!("({}/{}) {}", progress_index, total, msg), + })) + .await; - let resp = request - .send() - .await - .and_then(|resp| resp.error_for_status()); + let mut is_release = false; - r = Some(resp); + for (index, entry) in ["InRelease", "Release"].iter().enumerate() { + let p = dist_path.join(entry); - if r.as_ref().unwrap().is_ok() { - u = Some(url); - if index == 1 { - is_release = true; - } - break; - } - } + let dst = if dist_path_with_protocol.ends_with('/') { + format!("{}{}", dist_path_with_protocol, entry) + } else { + format!("{}/{}", dist_path_with_protocol, entry) + }; - let r = r.unwrap(); + let file_name = replacer.replace(&dst)?; - callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( - progress_index, - ))) - .await; + let dst = self.download_dir.join(&file_name); - if r.is_err() && entry.is_flat() { - return Ok((None, KeyOrIndex::Index(index))); + if p.exists() { + if dst.exists() { + debug!("get_release_file: Removing {}", dst.display()); + fs::remove_file(&dst).await.map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; } - let resp = r?; + debug!("get_release_file: Symlink {}", dst.display()); + fs::symlink(p, dst).await.map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; - let url = u.unwrap(); - let file_name = replacer.replace(&url)?; + if index == 1 { + is_release = true; + } - self.download_file(&file_name, resp, entry, progress_index, total, &callback) - .await?; + name = Some(file_name); + break; + } + } - if is_release && !entry.trusted() { - let url = format!("{}/{}", dist_path, "Release.gpg"); + if name.is_none() && entry.is_flat() { + // Flat repo no release + return Ok(()); + } - let request = self.request_get_builder(&url, entry); - let resp = request - .send() - .await - .and_then(|resp| resp.error_for_status())?; + if is_release { + let p = dist_path.join("Release.gpg"); + let entry = "Release.gpg"; - let file_name = replacer.replace(&url)?; + let dst = if dist_path_with_protocol.ends_with('/') { + format!("{}{}", dist_path_with_protocol, entry) + } else { + format!("{}/{}", dist_path_with_protocol, entry) + }; + + let file_name = replacer.replace(&dst)?; - self.download_file(&file_name, resp, entry, progress_index, total, &callback) - .await?; + let dst = self.download_dir.join(&file_name); + + if p.exists() { + if dst.exists() { + fs::remove_file(&dst).await.map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; } - Ok((Some(file_name), KeyOrIndex::Key(key))) + fs::symlink(p, self.download_dir.join(file_name)) + .await + .map_err(|e| { + RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) + })?; } - OmaSourceEntryFrom::Local => { - let dist_path_with_protocol = entry.dist_path(); - let dist_path = dist_path_with_protocol - .strip_prefix("file:") - .unwrap_or(dist_path_with_protocol); - let dist_path = Path::new(dist_path); - - let mut name = None; - - let msg = entry.get_human_download_url(None)?; - - callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { - index: progress_index, - msg: format!("({}/{}) {}", progress_index, total, msg), - })) - .await; - - let mut is_release = false; - - for (index, entry) in ["InRelease", "Release"].iter().enumerate() { - let p = dist_path.join(entry); - - let dst = if dist_path_with_protocol.ends_with('/') { - format!("{}{}", dist_path_with_protocol, entry) - } else { - format!("{}/{}", dist_path_with_protocol, entry) - }; - - let file_name = replacer.replace(&dst)?; - - let dst = self.download_dir.join(&file_name); - - if p.exists() { - if dst.exists() { - debug!("get_release_file: Removing {}", dst.display()); - fs::remove_file(&dst).await.map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError( - entry.to_string(), - e, - )) - })?; - } + } - debug!("get_release_file: Symlink {}", dst.display()); - fs::symlink(p, dst).await.map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError(entry.to_string(), e)) - })?; + callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( + progress_index, + ))) + .await; - if index == 1 { - is_release = true; - } + let name = name.ok_or_else(|| RefreshError::NoInReleaseFile(entry.url().to_string()))?; + entry.set_release_file_name(name); - name = Some(file_name); - break; - } - } + Ok(()) + } + + async fn download_http_release( + &self, + entry: &MirrorSource<'_>, + replacer: &DatabaseFilenameReplacer, + progress_index: usize, + total: usize, + callback: &F, + ) -> Result<()> + where + F: Fn(Event) -> Fut, + Fut: Future, + { + let dist_path = entry.dist_path(); - if is_release { - let p = dist_path.join("Release.gpg"); - let entry = "Release.gpg"; + let mut r = None; + let mut u = None; + let mut is_release = false; - let dst = if dist_path_with_protocol.ends_with('/') { - format!("{}{}", dist_path_with_protocol, entry) - } else { - format!("{}/{}", dist_path_with_protocol, entry) - }; + let msg = entry.get_human_download_url(None)?; - let file_name = replacer.replace(&dst)?; + callback(Event::DownloadEvent(oma_fetch::Event::NewProgressSpinner { + index: progress_index, + msg: format!("({}/{}) {}", progress_index, total, msg), + })) + .await; - let dst = self.download_dir.join(&file_name); + for (index, file_name) in ["InRelease", "Release"].iter().enumerate() { + let url = format!("{}/{}", dist_path, file_name); + let request = self.request_get_builder(&url, entry); - if p.exists() { - if dst.exists() { - fs::remove_file(&dst).await.map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError( - entry.to_string(), - e, - )) - })?; - } + let resp = request + .send() + .await + .and_then(|resp| resp.error_for_status()); - fs::symlink(p, self.download_dir.join(file_name)) - .await - .map_err(|e| { - RefreshError::FetcherError(DownloadError::IOError( - entry.to_string(), - e, - )) - })?; - } + r = Some(resp); + + if r.as_ref().unwrap().is_ok() { + u = Some(url); + if index == 1 { + is_release = true; } + break; + } + } - callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( - progress_index, - ))) - .await; + let r = r.unwrap(); - let name = - name.ok_or_else(|| RefreshError::NoInReleaseFile(entry.url().to_string()))?; + callback(Event::DownloadEvent(oma_fetch::Event::ProgressDone( + progress_index, + ))) + .await; - Ok((Some(name), KeyOrIndex::Key(key))) - } + if r.is_err() && entry.is_flat() { + // Flat repo no release + return Ok(()); + } + + let resp = r?; + + let url = u.unwrap(); + let file_name = replacer.replace(&url)?; + + self.download_file(&file_name, resp, entry, progress_index, total, &callback) + .await?; + entry.set_release_file_name(file_name); + + if is_release && !entry.trusted() { + let url = format!("{}/{}", dist_path, "Release.gpg"); + + let request = self.request_get_builder(&url, entry); + let resp = request + .send() + .await + .and_then(|resp| resp.error_for_status())?; + + let file_name = replacer.replace(&url)?; + + self.download_file(&file_name, resp, entry, progress_index, total, &callback) + .await?; } + + Ok(()) } fn request_get_builder( &self, url: &str, - source_index: &OmaSourceEntry<'_>, + source_index: &MirrorSource<'_>, ) -> reqwest::RequestBuilder { let mut request = self.client.get(url); - if let Some(auth) = &source_index.auth { + if let Some(auth) = source_index.auth() { request = request.basic_auth(&auth.user, Some(&auth.password)) } @@ -697,7 +686,7 @@ impl<'a> OmaRefresh<'a> { &self, file_name: &str, mut resp: Response, - source_index: &OmaSourceEntry<'_>, + source_index: &MirrorSource<'_>, index: usize, total: usize, callback: &F, @@ -754,9 +743,8 @@ impl<'a> OmaRefresh<'a> { async fn collect_all_release_entry( &self, - sourcelist: &[OmaSourceEntry<'a>], replacer: &DatabaseFilenameReplacer, - sources_map: &AHashMap>>, + mirror_sources: &MirrorSources<'a>, ) -> Result<(Vec, u64)> { let mut total = 0; let mut tasks = vec![]; @@ -767,7 +755,17 @@ impl<'a> OmaRefresh<'a> { .await .map(|f| f.lines().map(|x| x.to_string()).collect::>()); - for (file_name, ose_list) in sources_map { + let mut flat_repo_no_release = vec![]; + + for m in &mirror_sources.0 { + let file_name = match m.file_name() { + Some(name) => name, + None => { + flat_repo_no_release.push(m); + continue; + } + }; + let inrelease_path = self.download_dir.join(file_name); let mut handle = HashSet::with_hasher(ahash::RandomState::new()); @@ -778,41 +776,35 @@ impl<'a> OmaRefresh<'a> { let inrelease = verify_inrelease( &inrelease, - ose_list.iter().find_map(|x| { - if let Some(x) = x.signed_by() { - Some(x) - } else { - None - } - }), + m.signed_by(), &self.source, &inrelease_path, - ose_list.iter().any(|x| x.trusted()), + m.trusted(), ) .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; - let inrelease: Release = inrelease + let release: Release = inrelease .parse() .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))?; - if ose_list[0].is_flat() { + if m.is_flat() { let now = Utc::now(); - inrelease.check_date(&now).map_err(|e| { + release.check_date(&now).map_err(|e| { RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e) })?; - inrelease.check_valid_until(&now).map_err(|e| { + release.check_valid_until(&now).map_err(|e| { RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e) })?; } - let checksums = &inrelease + let checksums = &release .get_or_try_init_checksum_type_and_list() .map_err(|e| RefreshError::InReleaseParseError(inrelease_path.to_path_buf(), e))? .1; - for ose in ose_list { + for ose in &m.sources { debug!("Getted oma source entry: {:#?}", ose); let mut archs = if let Some(archs) = ose.archs() { @@ -834,26 +826,14 @@ impl<'a> OmaRefresh<'a> { )?; get_all_need_db_from_config(download_list, &mut total, checksums, &mut handle); + } - for i in &self.flat_repo_no_release { - collect_flat_repo_no_release( - sourcelist.get(*i).unwrap(), - &self.download_dir, - &mut tasks, - replacer, - )?; - } + for i in &flat_repo_no_release { + collect_flat_repo_no_release(i, &self.download_dir, &mut tasks, replacer)?; } for c in &handle { - collect_download_task( - c, - ose_list[0], - &self.download_dir, - &mut tasks, - &inrelease, - replacer, - )?; + collect_download_task(c, m, &self.download_dir, &mut tasks, &release, replacer)?; } } @@ -973,23 +953,23 @@ async fn remove_unused_db(download_dir: &Path, download_list: Vec<&str>) -> Resu } fn collect_flat_repo_no_release( - source_index: &OmaSourceEntry, + mirror_source: &MirrorSource, download_dir: &Path, tasks: &mut Vec, replacer: &DatabaseFilenameReplacer, ) -> Result<()> { - let msg = source_index.get_human_download_url(Some("Packages"))?; + let msg = mirror_source.get_human_download_url(Some("Packages"))?; - let dist_url = source_index.dist_path(); + let dist_url = mirror_source.dist_path(); - let from = match source_index.from()? { + let from = match mirror_source.from()? { OmaSourceEntryFrom::Http => DownloadSourceType::Http { - auth: source_index - .auth + auth: mirror_source + .auth() .as_ref() .map(|auth| (auth.user.clone(), auth.password.clone())), }, - OmaSourceEntryFrom::Local => DownloadSourceType::Local(source_index.is_flat()), + OmaSourceEntryFrom::Local => DownloadSourceType::Local(mirror_source.is_flat()), }; let download_url = format!("{}/Packages", dist_url); @@ -1017,26 +997,26 @@ fn collect_flat_repo_no_release( fn collect_download_task( c: &ChecksumDownloadEntry, - source_index: &OmaSourceEntry, + mirror_source: &MirrorSource<'_>, download_dir: &Path, tasks: &mut Vec, - inrelease: &Release, + release: &Release, replacer: &DatabaseFilenameReplacer, ) -> Result<()> { let file_type = &c.msg; - let msg = source_index.get_human_download_url(Some(file_type))?; + let msg = mirror_source.get_human_download_url(Some(file_type))?; - let dist_url = &source_index.dist_path(); + let dist_url = &mirror_source.dist_path(); - let from = match source_index.from()? { + let from = match mirror_source.from()? { OmaSourceEntryFrom::Http => DownloadSourceType::Http { - auth: source_index - .auth + auth: mirror_source + .auth() .as_ref() .map(|auth| (auth.user.clone(), auth.password.clone())), }, - OmaSourceEntryFrom::Local => DownloadSourceType::Local(source_index.is_flat()), + OmaSourceEntryFrom::Local => DownloadSourceType::Local(mirror_source.is_flat()), }; let not_compress_filename_before = if file_is_compress(&c.item.name) { @@ -1048,7 +1028,7 @@ fn collect_download_task( let checksum = if c.keep_compress { Some(&c.item.checksum) } else { - inrelease + release .checksum_type_and_list() .1 .iter() @@ -1057,10 +1037,10 @@ fn collect_download_task( .map(|c| &c.checksum) }; - let download_url = if inrelease.acquire_by_hash() { + let download_url = if release.acquire_by_hash() { let path = Path::new(&c.item.name); let parent = path.parent().unwrap_or(path); - let dir = match inrelease.checksum_type_and_list().0 { + let dir = match release.checksum_type_and_list().0 { InReleaseChecksum::Sha256 => "SHA256", InReleaseChecksum::Sha512 => "SHA512", InReleaseChecksum::Md5 => "MD5Sum", @@ -1081,7 +1061,7 @@ fn collect_download_task( }]; let file_path = if c.keep_compress { - if inrelease.acquire_by_hash() { + if release.acquire_by_hash() { Cow::Owned(format!("{}/{}", dist_url, c.item.name)) } else { Cow::Borrowed(&download_url) @@ -1114,7 +1094,7 @@ fn collect_download_task( } }) .maybe_hash(if let Some(checksum) = checksum { - match inrelease.checksum_type_and_list().0 { + match release.checksum_type_and_list().0 { InReleaseChecksum::Sha256 => Some(Checksum::from_sha256_str(checksum)?), InReleaseChecksum::Sha512 => Some(Checksum::from_sha512_str(checksum)?), InReleaseChecksum::Md5 => Some(Checksum::from_md5_str(checksum)?), diff --git a/oma-refresh/src/sourceslist.rs b/oma-refresh/src/sourceslist.rs index b958a3fe..a069515d 100644 --- a/oma-refresh/src/sourceslist.rs +++ b/oma-refresh/src/sourceslist.rs @@ -1,11 +1,12 @@ use std::path::Path; +use ahash::HashMap; use apt_auth_config::AuthConfigEntry; use oma_apt_sources_lists::{Signature, SourceEntry, SourceLine, SourceListType, SourcesLists}; use once_cell::sync::OnceCell; use url::Url; -use crate::db::RefreshError; +use crate::{db::RefreshError, util::DatabaseFilenameReplacer}; #[derive(Debug, Clone)] pub struct OmaSourceEntry<'a> { @@ -161,6 +162,105 @@ impl<'a> OmaSourceEntry<'a> { } } +#[derive(Debug)] +pub(crate) struct MirrorSources<'a>(pub Vec>); + +#[derive(Debug)] +pub(crate) struct MirrorSource<'a> { + pub(crate) sources: Vec<&'a OmaSourceEntry<'a>>, + release_file_name: OnceCell, +} + +impl MirrorSource<'_> { + pub(crate) fn set_release_file_name(&self, file_name: String) { + self.release_file_name + .set(file_name) + .expect("Release file name was init"); + } + + pub(crate) fn dist_path(&self) -> &str { + self.sources.first().unwrap().dist_path() + } + + pub(crate) fn from(&self) -> Result<&OmaSourceEntryFrom, RefreshError> { + self.sources.first().unwrap().from() + } + + pub(crate) fn get_human_download_url( + &self, + file_name: Option<&str>, + ) -> Result { + self.sources + .first() + .unwrap() + .get_human_download_url(file_name) + } + + pub(crate) fn auth(&self) -> Option<&AuthConfigEntry> { + self.sources + .iter() + .find_map(|x| if let Some(x) = &x.auth { Some(x) } else { None }) + } + + pub(crate) fn signed_by(&self) -> Option<&Signature> { + self.sources.iter().find_map(|x| { + if let Some(x) = &x.signed_by() { + Some(x) + } else { + None + } + }) + } + + pub(crate) fn url(&self) -> &str { + self.sources.first().unwrap().url() + } + + pub(crate) fn is_flat(&self) -> bool { + self.sources.first().unwrap().is_flat() + } + + pub(crate) fn trusted(&self) -> bool { + self.sources.iter().any(|x| x.trusted()) + } + + pub(crate) fn file_name(&self) -> Option<&str> { + self.release_file_name.get().map(|x| x.as_str()) + } +} + +impl<'a> MirrorSources<'a> { + pub(crate) fn from_sourcelist( + sourcelist: &'a [OmaSourceEntry<'a>], + replacer: &DatabaseFilenameReplacer, + ) -> Result { + let mut map: HashMap> = + HashMap::with_hasher(ahash::RandomState::new()); + + if sourcelist.is_empty() { + return Err(RefreshError::SourceListsEmpty); + } + + for source in sourcelist { + let dist_path = source.dist_path(); + let name = replacer.replace(dist_path)?; + + map.entry(name).or_default().push(source); + } + + let mut res = vec![]; + + for (_, v) in map { + res.push(MirrorSource { + sources: v, + release_file_name: OnceCell::new(), + }); + } + + Ok(Self(res)) + } +} + #[test] fn test_ose() { use oma_utils::dpkg::dpkg_arch; diff --git a/src/error.rs b/src/error.rs index 88623555..14443d53 100644 --- a/src/error.rs +++ b/src/error.rs @@ -426,6 +426,10 @@ impl From for OutputError { description: fl!("doplicate-component", url = url.to_string(), c = component), source: None, }, + RefreshError::SourceListsEmpty => Self { + description: "Source list is empty".to_string(), + source: None, + }, } } } From 31bee5ad586ecc49c9bd1deb0211e316bf05da3a Mon Sep 17 00:00:00 2001 From: eatradish Date: Mon, 30 Dec 2024 23:31:42 +0800 Subject: [PATCH 3/4] refactor(oma-repo-verify): move replace '.' logic to inner --- oma-repo-verify/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/oma-repo-verify/src/lib.rs b/oma-repo-verify/src/lib.rs index 98bf35e0..7fc07423 100644 --- a/oma-repo-verify/src/lib.rs +++ b/oma-repo-verify/src/lib.rs @@ -61,6 +61,8 @@ impl InReleaseVerifier { } pub fn from_key_block(block: &str, trusted: bool) -> VerifyResult { + // 这个点存在只是表示换行，因此把它替换掉 + let block = block.replace('.', ""); let mut certs: Vec = Vec::new(); let ppr = PacketParserBuilder::from_bytes(block.as_bytes())?.build()?; let cert = CertParser::from(ppr); @@ -141,9 +143,7 @@ pub fn verify_inrelease( &p, None, if let Some(deb822_inner_signed_by_str) = deb822_inner_signed_by_str { - // 这个点存在只是表示换行，因此把它替换掉 - let signed_by_str = deb822_inner_signed_by_str.replace('.', ""); - InReleaseVerifier::from_key_block(&signed_by_str, trusted)? + InReleaseVerifier::from_key_block(deb822_inner_signed_by_str, trusted)? } else { InReleaseVerifier::from_paths(&certs, trusted)? }, From 42f1cc08667ef3c3ce8895d3ab2579f0a0eab72c Mon Sep 17 00:00:00 2001 From: Kaiyang Wu Date: Mon, 30 Dec 2024 03:47:53 +0000 Subject: [PATCH 4/4] Translated using Weblate (Chinese (Traditional Han script)) Currently translated at 100.0% (221 of 221 strings) Translation: Oh My Ailurus/oma Translate-URL: https://weblate.aosc.io/projects/oma/oma/zh_Hant/ --- i18n/zh-TW/oma.ftl | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/i18n/zh-TW/oma.ftl b/i18n/zh-TW/oma.ftl index 0d4a16ce..93a82416 100644 --- a/i18n/zh-TW/oma.ftl +++ b/i18n/zh-TW/oma.ftl @@ -226,3 +226,13 @@ tui-continue-tips = 請按 [c] 繼續。 dpkg-triggers-only-a-non-zero = `dpkg --triggers-only -a` 返回了錯誤。 success = 操作已成功完成。 topics-unchanged = oma 在退出測試庫時遇到錯誤，因此未儲存測試庫設定狀態。 +status-pending = 等待操作中 +status-downloading = 正在下載軟體套件 +status-package = 正在處理軟體套件 { $pkg } +update-available = 有可用的系統更新 +update-available-2 = 您的系統有如下可用的更新： +tum-1 = 您的系統有 { $updates } 個可用的更新。 +tum-1-with-security = 您的系統有 { $updates } 個重要更新（含 { $security } 個標記為 { $security_str } 的安全更新）。 +tum-2 = 根據您指定的操作，oma 需要對系統組件進行若干變更，詳情如下。 +security = 安全更新 +verify-error = 在驗證 { $p } 的簽章時遇到錯誤。