From 9a72f6c75b41c8aa55eb5048e621f45ee188a983 Mon Sep 17 00:00:00 2001 From: pynickle <2330458484@qq.com> Date: Fri, 17 Oct 2025 00:13:47 +0800 Subject: [PATCH 1/6] perf(file-validator): improve the performance of file_validator --- .../src/launch/helpers/file_validator.rs | 384 ++++++++++-------- 1 file changed, 213 insertions(+), 171 deletions(-) diff --git a/src-tauri/src/launch/helpers/file_validator.rs b/src-tauri/src/launch/helpers/file_validator.rs index b9317ae13..4c8b4552a 100644 --- a/src-tauri/src/launch/helpers/file_validator.rs +++ b/src-tauri/src/launch/helpers/file_validator.rs @@ -11,23 +11,83 @@ use crate::resource::models::{ResourceType, SourceType}; use crate::tasks::download::DownloadParam; use crate::tasks::PTaskParam; use crate::utils::fs::validate_sha1; -use futures::future::join_all; +use futures::stream::{self, StreamExt}; use semver::Version; use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::path::{Path, PathBuf}; use tauri::AppHandle; use tokio::fs; +use url::Url; use zip::ZipArchive; +const CONCURRENT_HASH_CHECKS: usize = 16; + +async fn validate_file_with_hash( + file_path: PathBuf, + expected_hash: String, + download_url: Url, + check_hash: bool, +) -> SJMCLResult> { + let exists = fs::try_exists(&file_path).await?; + + if !exists { + return Ok(Some(PTaskParam::Download(DownloadParam { + src: download_url, + dest: file_path, + filename: None, + sha1: Some(expected_hash), + }))); + } + + if check_hash { + let hash = expected_hash.clone(); + let path = file_path.clone(); + let is_valid = tokio::task::spawn_blocking(move || validate_sha1(path, hash).is_ok()).await?; + + if !is_valid { + return Ok(Some(PTaskParam::Download(DownloadParam { + src: download_url, + dest: file_path, + filename: None, + sha1: Some(expected_hash), + }))); + } + } + + Ok(None) +} + +async fn validate_files_concurrently( + items: impl Iterator, + check_hash: bool, + processor: F, +) -> SJMCLResult> +where + F: Fn(T, bool) -> Fut, + Fut: std::future::Future>>, +{ + let results: Vec>> = stream::iter(items) + .map(|item| processor(item, check_hash)) + .buffer_unordered(CONCURRENT_HASH_CHECKS) + .collect() + .await; + + let mut params = Vec::new(); + for r in results { + if let Some(p) = r? { + params.push(p); + } + } + Ok(params) +} + pub fn get_nonnative_library_artifacts(client_info: &McClientInfo) -> Vec { let mut artifacts = HashSet::new(); let feature = FeaturesInfo::default(); + for library in &client_info.libraries { - if !library.is_allowed(&feature).unwrap_or(false) { - continue; - } - if library.natives.is_some() { + if !library.is_allowed(&feature).unwrap_or(false) || library.natives.is_some() { continue; } if let Some(ref downloads) = &library.downloads { @@ -74,44 +134,75 @@ pub async fn get_invalid_library_files( artifacts.extend(get_native_library_artifacts(client_info)); artifacts.extend(get_nonnative_library_artifacts(client_info)); - let futs = artifacts.into_iter().map(move |artifact| async move { - let file_path = library_path.join(&artifact.path); - let exists = fs::try_exists(&file_path).await?; - if exists && (!check_hash || validate_sha1(file_path.clone(), artifact.sha1.clone()).is_ok()) { - Ok(None) - } else if artifact.url.is_empty() { - return Err(LaunchError::GameFilesIncomplete.into()); - } else { - let src = convert_url_to_target_source( - &url::Url::parse(&artifact.url)?, - &[ - ResourceType::Libraries, - ResourceType::FabricMaven, - ResourceType::ForgeMaven, - ResourceType::ForgeMavenNew, - ResourceType::NeoforgeMaven, - ], - &source, - )?; - Ok(Some(PTaskParam::Download(DownloadParam { - src, - dest: file_path, - filename: None, - sha1: Some(artifact.sha1.clone()), - }))) - } - }); + let library_path = library_path.to_path_buf(); + let source = source.clone(); - let results: Vec>> = join_all(futs).await; + validate_files_concurrently( + artifacts.into_iter(), + check_hash, + move |artifact, check_hash| { + let source = source.clone(); + let library_path = library_path.clone(); - let mut params = Vec::new(); - for r in results { - if let Some(p) = r? { - params.push(p); - } - } + async move { + if artifact.url.is_empty() { + return Err(LaunchError::GameFilesIncomplete.into()); + } - Ok(params) + let file_path = library_path.join(&artifact.path); + let url = Url::parse(&artifact.url)?; + + let download_url = convert_url_to_target_source( + &url, + &[ + ResourceType::Libraries, + ResourceType::FabricMaven, + ResourceType::ForgeMaven, + ResourceType::ForgeMavenNew, + ResourceType::NeoforgeMaven, + ], + &source, + )?; + + validate_file_with_hash(file_path, artifact.sha1, download_url, check_hash).await + } + }, + ) + .await +} + +pub async fn get_invalid_assets( + app: &AppHandle, + client_info: &McClientInfo, + source: SourceType, + asset_path: &Path, + check_hash: bool, +) -> SJMCLResult> { + let assets_download_api = get_download_api(source, ResourceType::Assets)?; + let asset_index_path = asset_path.join(format!("indexes/{}.json", client_info.asset_index.id)); + let asset_index = load_asset_index(app, &asset_index_path, &client_info.asset_index.url).await?; + + let base_path = asset_path.to_path_buf(); + + validate_files_concurrently( + asset_index.objects.into_values(), + check_hash, + move |item, check_hash| { + let assets_download_api = assets_download_api.clone(); + let base_path = base_path.clone(); + + async move { + let path_in_repo = format!("{}/{}", &item.hash[..2], item.hash); + let dest = base_path.join(format!("objects/{}", path_in_repo)); + let download_url = assets_download_api + .join(&path_in_repo) + .map_err(crate::error::SJMCLError::from)?; + + validate_file_with_hash(dest, item.hash, download_url, check_hash).await + } + }, + ) + .await } pub struct LibraryParts { @@ -124,34 +215,58 @@ pub struct LibraryParts { pub fn parse_library_name(name: &str, native: Option) -> SJMCLResult { let parts: Vec<&str> = name.split('@').collect(); - let file_ext = if parts.len() > 1 { - parts[1].to_string() - } else { - "jar".to_string() - }; + let file_ext = parts + .get(1) + .map(|s| s.to_string()) + .unwrap_or_else(|| "jar".to_string()); + let mut name_split: Vec = parts[0].split(':').map(|s| s.to_string()).collect(); + if name_split.len() < 3 { - Err(InstanceError::InvalidSourcePath.into()) - } else { - if let Some(native) = native { - name_split.push(native); - } - let pack_name = name_split[1].clone(); - let pack_version = name_split[2].clone(); - let classifier = if name_split.len() > 3 { - Some(name_split[3].clone()) - } else { - None - }; - let path = name_split[0].replace('.', "/"); - Ok(LibraryParts { - path, - pack_name, - pack_version, - classifier, - extension: file_ext, - }) + return Err(InstanceError::InvalidSourcePath.into()); + } + + if let Some(native) = native { + name_split.push(native); } + + let path = name_split[0].replace('.', "/"); + let pack_name = name_split[1].clone(); + let pack_version = name_split[2].clone(); + let classifier = name_split.get(3).cloned(); + + Ok(LibraryParts { + path, + pack_name, + pack_version, + classifier, + extension: file_ext, + }) +} + +pub fn convert_library_name_to_path(name: &str, native: Option) -> SJMCLResult { + let LibraryParts { + path, + pack_name, + pack_version, + classifier, + extension: file_ext, + } = parse_library_name(name, native)?; + + let file_name = [ + pack_name.clone(), + pack_version.clone(), + classifier.unwrap_or_default(), + ] + .iter() + .filter(|s| !s.is_empty()) + .map(|s| s.as_str()) + .collect::>() + .join("-") + + "." + + &file_ext; + + Ok(format!("{path}/{pack_name}/{pack_version}/{file_name}")) } #[derive(Debug, Hash, Eq, PartialEq)] @@ -178,14 +293,14 @@ pub fn merge_library_lists( extension: library_parts.extension, }; - let new_version = library_parts.pack_version; + let new_version = &library_parts.pack_version; if let Some(existing_library) = library_map.get(&key) { let existing_version = parse_library_name(&existing_library.name, None) .map(|parts| parts.pack_version) .unwrap_or("0.1.0".to_string()); - if parse_sem_version(&new_version) > parse_sem_version(&existing_version) { + if parse_sem_version(new_version) > parse_sem_version(&existing_version) { library_map.insert(key, library.clone()); } } else { @@ -198,60 +313,34 @@ pub fn merge_library_lists( } fn parse_sem_version(version: &str) -> Version { - Version::parse(version).unwrap_or({ + Version::parse(version).unwrap_or_else(|_| { let mut parts = version.split('.').collect::>(); while parts.len() < 3 { parts.push("0"); } - Version::parse(&parts[..3].join(".")).unwrap_or(Version::new(0, 1, 0)) + Version::parse(&parts[..3].join(".")).unwrap_or_else(|_| Version::new(0, 1, 0)) }) } -pub fn convert_library_name_to_path(name: &str, native: Option) -> SJMCLResult { - let LibraryParts { - path, - pack_name, - pack_version, - classifier, - extension: file_ext, - } = parse_library_name(name, native)?; - - let file_name = [ - pack_name.clone(), - pack_version.clone(), - classifier.unwrap_or_default(), - ] - .iter() - .filter(|s| !s.is_empty()) - .map(|s| s.as_str()) - .collect::>() - .join("-") - + "." - + &file_ext; - Ok(format!("{path}/{pack_name}/{pack_version}/{file_name}")) -} - pub fn get_nonnative_library_paths( client_info: &McClientInfo, library_path: &Path, ) -> SJMCLResult> { let mut libraries = Vec::new(); let feature = FeaturesInfo::default(); + for library in &client_info.libraries { - if !library.is_allowed(&feature).unwrap_or(false) { - continue; - } - if library.natives.is_some() { - continue; + if library.is_allowed(&feature).unwrap_or(false) && library.natives.is_none() { + libraries.push(library.clone()); } - libraries.push(library.clone()); } - libraries = merge_library_lists(&libraries, &[]); // remove duplicates to prevent launch errors - let mut result = Vec::new(); - for library in libraries { - result.push(library_path.join(convert_library_name_to_path(&library.name, None)?)); - } - Ok(result) + + libraries = merge_library_lists(&libraries, &[]); + + libraries + .iter() + .map(|lib| Ok(library_path.join(convert_library_name_to_path(&lib.name, None)?))) + .collect() } pub fn get_native_library_paths( @@ -261,17 +350,17 @@ pub fn get_native_library_paths( let mut result = Vec::new(); let feature = FeaturesInfo::default(); for library in &client_info.libraries { - if !library.is_allowed(&feature).unwrap_or(false) { + if !library.is_allowed(&feature).unwrap_or(false) || library.natives.is_none() { continue; } - if let Some(natives) = &library.natives { - if let Some(native) = get_natives_string(natives) { - let path = convert_library_name_to_path(&library.name, Some(native))?; - result.push(library_path.join(path)); - } else { - println!("natives is None"); - } - } + let native_str = if let Some(native_fn) = Some(&get_natives_string) { + library.natives.as_ref().and_then(native_fn) + } else { + None + }; + + let path = convert_library_name_to_path(&library.name, native_str)?; + result.push(library_path.join(path)); } Ok(result) } @@ -284,22 +373,23 @@ pub async fn extract_native_libraries( if !natives_dir.exists() { fs::create_dir(natives_dir).await?; } + let native_libraries = get_native_library_paths(client_info, library_path)?; - let tasks: Vec>> = native_libraries - .into_iter() + + let results: Vec<_> = stream::iter(native_libraries) .map(|library_path| { let patches_dir_clone = natives_dir.clone(); - tokio::spawn(async move { - let file = Cursor::new(fs::read(library_path).await?); + async move { + let file = Cursor::new(fs::read(&library_path).await?); let mut jar = ZipArchive::new(file)?; jar.extract(&patches_dir_clone)?; - Ok(()) - }) + Ok::<_, crate::error::SJMCLError>(()) + } }) - .collect(); - - let results = futures::future::join_all(tasks).await; + .buffer_unordered(4) + .collect::>() + .await; for result in results { if let Err(e) = result { @@ -310,51 +400,3 @@ pub async fn extract_native_libraries( Ok(()) } - -pub async fn get_invalid_assets( - app: &AppHandle, - client_info: &McClientInfo, - source: SourceType, - asset_path: &Path, - check_hash: bool, -) -> SJMCLResult> { - let assets_download_api = get_download_api(source, ResourceType::Assets)?; - - let asset_index_path = asset_path.join(format!("indexes/{}.json", client_info.asset_index.id)); - let asset_index = load_asset_index(app, &asset_index_path, &client_info.asset_index.url).await?; - - let futs = asset_index.objects.into_values().map(|item| { - let assets_download_api = assets_download_api.clone(); - let base_path = asset_path.to_path_buf(); - - async move { - let path_in_repo = format!("{}/{}", &item.hash[..2], item.hash); - let dest = base_path.join(format!("objects/{}", path_in_repo)); - let exists = fs::try_exists(&dest).await?; - - if exists && (!check_hash || validate_sha1(dest.clone(), item.hash.clone()).is_ok()) { - Ok::, crate::error::SJMCLError>(None) - } else { - let src = assets_download_api - .join(&path_in_repo) - .map_err(crate::error::SJMCLError::from)?; - Ok(Some(PTaskParam::Download(DownloadParam { - src, - dest, - filename: None, - sha1: Some(item.hash.clone()), - }))) - } - } - }); - - let results: Vec>> = join_all(futs).await; - - let mut params = Vec::new(); - for r in results { - if let Some(p) = r? { - params.push(p); - } - } - Ok(params) -} From d2f7fc650c55e1ac04a58ac5d99e648ae2cbcf0c Mon Sep 17 00:00:00 2001 From: pynickle <2330458484@qq.com> Date: Mon, 3 Nov 2025 22:03:12 +0800 Subject: [PATCH 2/6] imp: adjust function order --- .../src/launch/helpers/file_validator.rs | 374 +++++++++--------- 1 file changed, 187 insertions(+), 187 deletions(-) diff --git a/src-tauri/src/launch/helpers/file_validator.rs b/src-tauri/src/launch/helpers/file_validator.rs index 4c8b4552a..90dbb278d 100644 --- a/src-tauri/src/launch/helpers/file_validator.rs +++ b/src-tauri/src/launch/helpers/file_validator.rs @@ -23,6 +23,88 @@ use zip::ZipArchive; const CONCURRENT_HASH_CHECKS: usize = 16; +#[derive(Debug, Hash, Eq, PartialEq)] +struct LibraryKey { + path: String, + pack_name: String, + classifier: Option, + extension: String, +} + +pub struct LibraryParts { + pub path: String, + pub pack_name: String, + pub pack_version: String, + pub classifier: Option, + pub extension: String, +} + +fn parse_sem_version(version: &str) -> Version { + Version::parse(version).unwrap_or_else(|_| { + let mut parts = version.split('.').collect::>(); + while parts.len() < 3 { + parts.push("0"); + } + Version::parse(&parts[..3].join(".")).unwrap_or_else(|_| Version::new(0, 1, 0)) + }) +} + +pub fn parse_library_name(name: &str, native: Option) -> SJMCLResult { + let parts: Vec<&str> = name.split('@').collect(); + let file_ext = parts + .get(1) + .map(|s| s.to_string()) + .unwrap_or_else(|| "jar".to_string()); + + let mut name_split: Vec = parts[0].split(':').map(|s| s.to_string()).collect(); + + if name_split.len() < 3 { + return Err(InstanceError::InvalidSourcePath.into()); + } + + if let Some(native) = native { + name_split.push(native); + } + + let path = name_split[0].replace('.', "/"); + let pack_name = name_split[1].clone(); + let pack_version = name_split[2].clone(); + let classifier = name_split.get(3).cloned(); + + Ok(LibraryParts { + path, + pack_name, + pack_version, + classifier, + extension: file_ext, + }) +} + +pub fn convert_library_name_to_path(name: &str, native: Option) -> SJMCLResult { + let LibraryParts { + path, + pack_name, + pack_version, + classifier, + extension: file_ext, + } = parse_library_name(name, native)?; + + let file_name = [ + pack_name.clone(), + pack_version.clone(), + classifier.unwrap_or_default(), + ] + .iter() + .filter(|s| !s.is_empty()) + .map(|s| s.as_str()) + .collect::>() + .join("-") + + "." + + &file_ext; + + Ok(format!("{path}/{pack_name}/{pack_version}/{file_name}")) +} + async fn validate_file_with_hash( file_path: PathBuf, expected_hash: String, @@ -124,157 +206,47 @@ pub fn get_native_library_artifacts(client_info: &McClientInfo) -> Vec SJMCLResult> { - let mut artifacts = Vec::new(); - artifacts.extend(get_native_library_artifacts(client_info)); - artifacts.extend(get_nonnative_library_artifacts(client_info)); - - let library_path = library_path.to_path_buf(); - let source = source.clone(); - - validate_files_concurrently( - artifacts.into_iter(), - check_hash, - move |artifact, check_hash| { - let source = source.clone(); - let library_path = library_path.clone(); - - async move { - if artifact.url.is_empty() { - return Err(LaunchError::GameFilesIncomplete.into()); - } + library_path: &Path, +) -> SJMCLResult> { + let mut libraries = Vec::new(); + let feature = FeaturesInfo::default(); - let file_path = library_path.join(&artifact.path); - let url = Url::parse(&artifact.url)?; + for library in &client_info.libraries { + if library.is_allowed(&feature).unwrap_or(false) && library.natives.is_none() { + libraries.push(library.clone()); + } + } - let download_url = convert_url_to_target_source( - &url, - &[ - ResourceType::Libraries, - ResourceType::FabricMaven, - ResourceType::ForgeMaven, - ResourceType::ForgeMavenNew, - ResourceType::NeoforgeMaven, - ], - &source, - )?; + libraries = merge_library_lists(&libraries, &[]); - validate_file_with_hash(file_path, artifact.sha1, download_url, check_hash).await - } - }, - ) - .await + libraries + .iter() + .map(|lib| Ok(library_path.join(convert_library_name_to_path(&lib.name, None)?))) + .collect() } -pub async fn get_invalid_assets( - app: &AppHandle, +pub fn get_native_library_paths( client_info: &McClientInfo, - source: SourceType, - asset_path: &Path, - check_hash: bool, -) -> SJMCLResult> { - let assets_download_api = get_download_api(source, ResourceType::Assets)?; - let asset_index_path = asset_path.join(format!("indexes/{}.json", client_info.asset_index.id)); - let asset_index = load_asset_index(app, &asset_index_path, &client_info.asset_index.url).await?; - - let base_path = asset_path.to_path_buf(); - - validate_files_concurrently( - asset_index.objects.into_values(), - check_hash, - move |item, check_hash| { - let assets_download_api = assets_download_api.clone(); - let base_path = base_path.clone(); - - async move { - let path_in_repo = format!("{}/{}", &item.hash[..2], item.hash); - let dest = base_path.join(format!("objects/{}", path_in_repo)); - let download_url = assets_download_api - .join(&path_in_repo) - .map_err(crate::error::SJMCLError::from)?; - - validate_file_with_hash(dest, item.hash, download_url, check_hash).await - } - }, - ) - .await -} - -pub struct LibraryParts { - pub path: String, - pub pack_name: String, - pub pack_version: String, - pub classifier: Option, - pub extension: String, -} - -pub fn parse_library_name(name: &str, native: Option) -> SJMCLResult { - let parts: Vec<&str> = name.split('@').collect(); - let file_ext = parts - .get(1) - .map(|s| s.to_string()) - .unwrap_or_else(|| "jar".to_string()); - - let mut name_split: Vec = parts[0].split(':').map(|s| s.to_string()).collect(); - - if name_split.len() < 3 { - return Err(InstanceError::InvalidSourcePath.into()); - } + library_path: &Path, +) -> SJMCLResult> { + let mut result = Vec::new(); + let feature = FeaturesInfo::default(); + for library in &client_info.libraries { + if !library.is_allowed(&feature).unwrap_or(false) || library.natives.is_none() { + continue; + } + let native_str = if let Some(native_fn) = Some(&get_natives_string) { + library.natives.as_ref().and_then(native_fn) + } else { + None + }; - if let Some(native) = native { - name_split.push(native); + let path = convert_library_name_to_path(&library.name, native_str)?; + result.push(library_path.join(path)); } - - let path = name_split[0].replace('.', "/"); - let pack_name = name_split[1].clone(); - let pack_version = name_split[2].clone(); - let classifier = name_split.get(3).cloned(); - - Ok(LibraryParts { - path, - pack_name, - pack_version, - classifier, - extension: file_ext, - }) -} - -pub fn convert_library_name_to_path(name: &str, native: Option) -> SJMCLResult { - let LibraryParts { - path, - pack_name, - pack_version, - classifier, - extension: file_ext, - } = parse_library_name(name, native)?; - - let file_name = [ - pack_name.clone(), - pack_version.clone(), - classifier.unwrap_or_default(), - ] - .iter() - .filter(|s| !s.is_empty()) - .map(|s| s.as_str()) - .collect::>() - .join("-") - + "." - + &file_ext; - - Ok(format!("{path}/{pack_name}/{pack_version}/{file_name}")) -} - -#[derive(Debug, Hash, Eq, PartialEq)] -struct LibraryKey { - path: String, - pack_name: String, - classifier: Option, - extension: String, + Ok(result) } // merge two vectors of libraries, remove duplicates by name, keep the one with the highest version. also remove libraries with invalid names @@ -312,57 +284,51 @@ pub fn merge_library_lists( library_map.into_values().collect() } -fn parse_sem_version(version: &str) -> Version { - Version::parse(version).unwrap_or_else(|_| { - let mut parts = version.split('.').collect::>(); - while parts.len() < 3 { - parts.push("0"); - } - Version::parse(&parts[..3].join(".")).unwrap_or_else(|_| Version::new(0, 1, 0)) - }) -} - -pub fn get_nonnative_library_paths( - client_info: &McClientInfo, +pub async fn get_invalid_library_files( + source: SourceType, library_path: &Path, -) -> SJMCLResult> { - let mut libraries = Vec::new(); - let feature = FeaturesInfo::default(); + client_info: &McClientInfo, + check_hash: bool, +) -> SJMCLResult> { + let mut artifacts = Vec::new(); + artifacts.extend(get_native_library_artifacts(client_info)); + artifacts.extend(get_nonnative_library_artifacts(client_info)); - for library in &client_info.libraries { - if library.is_allowed(&feature).unwrap_or(false) && library.natives.is_none() { - libraries.push(library.clone()); - } - } + let library_path = library_path.to_path_buf(); + let source = source.clone(); - libraries = merge_library_lists(&libraries, &[]); + validate_files_concurrently( + artifacts.into_iter(), + check_hash, + move |artifact, check_hash| { + let source = source.clone(); + let library_path = library_path.clone(); - libraries - .iter() - .map(|lib| Ok(library_path.join(convert_library_name_to_path(&lib.name, None)?))) - .collect() -} + async move { + if artifact.url.is_empty() { + return Err(LaunchError::GameFilesIncomplete.into()); + } -pub fn get_native_library_paths( - client_info: &McClientInfo, - library_path: &Path, -) -> SJMCLResult> { - let mut result = Vec::new(); - let feature = FeaturesInfo::default(); - for library in &client_info.libraries { - if !library.is_allowed(&feature).unwrap_or(false) || library.natives.is_none() { - continue; - } - let native_str = if let Some(native_fn) = Some(&get_natives_string) { - library.natives.as_ref().and_then(native_fn) - } else { - None - }; + let file_path = library_path.join(&artifact.path); + let url = Url::parse(&artifact.url)?; - let path = convert_library_name_to_path(&library.name, native_str)?; - result.push(library_path.join(path)); - } - Ok(result) + let download_url = convert_url_to_target_source( + &url, + &[ + ResourceType::Libraries, + ResourceType::FabricMaven, + ResourceType::ForgeMaven, + ResourceType::ForgeMavenNew, + ResourceType::NeoforgeMaven, + ], + &source, + )?; + + validate_file_with_hash(file_path, artifact.sha1, download_url, check_hash).await + } + }, + ) + .await } pub async fn extract_native_libraries( @@ -400,3 +366,37 @@ pub async fn extract_native_libraries( Ok(()) } + +pub async fn get_invalid_assets( + app: &AppHandle, + client_info: &McClientInfo, + source: SourceType, + asset_path: &Path, + check_hash: bool, +) -> SJMCLResult> { + let assets_download_api = get_download_api(source, ResourceType::Assets)?; + let asset_index_path = asset_path.join(format!("indexes/{}.json", client_info.asset_index.id)); + let asset_index = load_asset_index(app, &asset_index_path, &client_info.asset_index.url).await?; + + let base_path = asset_path.to_path_buf(); + + validate_files_concurrently( + asset_index.objects.into_values(), + check_hash, + move |item, check_hash| { + let assets_download_api = assets_download_api.clone(); + let base_path = base_path.clone(); + + async move { + let path_in_repo = format!("{}/{}", &item.hash[..2], item.hash); + let dest = base_path.join(format!("objects/{}", path_in_repo)); + let download_url = assets_download_api + .join(&path_in_repo) + .map_err(crate::error::SJMCLError::from)?; + + validate_file_with_hash(dest, item.hash, download_url, check_hash).await + } + }, + ) + .await +} From 8830e34c0e0894add40513c405518fac5025a2f5 Mon Sep 17 00:00:00 2001 From: pynickle <2330458484@qq.com> Date: Mon, 3 Nov 2025 22:14:43 +0800 Subject: [PATCH 3/6] feat: use dynamic concurrent limit --- .../src/launch/helpers/file_validator.rs | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src-tauri/src/launch/helpers/file_validator.rs b/src-tauri/src/launch/helpers/file_validator.rs index 90dbb278d..c1c2f5e53 100644 --- a/src-tauri/src/launch/helpers/file_validator.rs +++ b/src-tauri/src/launch/helpers/file_validator.rs @@ -16,13 +16,13 @@ use semver::Version; use std::collections::{HashMap, HashSet}; use std::io::Cursor; use std::path::{Path, PathBuf}; +use std::sync::OnceLock; +use sysinfo::{CpuRefreshKind, RefreshKind, System}; use tauri::AppHandle; use tokio::fs; use url::Url; use zip::ZipArchive; -const CONCURRENT_HASH_CHECKS: usize = 16; - #[derive(Debug, Hash, Eq, PartialEq)] struct LibraryKey { path: String, @@ -39,6 +39,19 @@ pub struct LibraryParts { pub extension: String, } +fn get_concurrent_hash_checks() -> usize { + static CONCURRENT_LIMIT: OnceLock = OnceLock::new(); + + *CONCURRENT_LIMIT.get_or_init(|| { + let mut sys = + System::new_with_specifics(RefreshKind::nothing().with_cpu(CpuRefreshKind::everything())); + std::thread::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL); + sys.refresh_cpu_usage(); + let cpu_count = sys.cpus().len(); + (cpu_count * 3).max(8).min(32) + }) +} + fn parse_sem_version(version: &str) -> Version { Version::parse(version).unwrap_or_else(|_| { let mut parts = version.split('.').collect::>(); @@ -149,9 +162,11 @@ where F: Fn(T, bool) -> Fut, Fut: std::future::Future>>, { + let concurrent_limit = get_concurrent_hash_checks(); + let results: Vec>> = stream::iter(items) .map(|item| processor(item, check_hash)) - .buffer_unordered(CONCURRENT_HASH_CHECKS) + .buffer_unordered(concurrent_limit) .collect() .await; From bccccc5765fdfb8a3fec1bc560ecc62babcae8b6 Mon Sep 17 00:00:00 2001 From: pynickle <2330458484@qq.com> Date: Sun, 16 Nov 2025 17:00:46 +0800 Subject: [PATCH 4/6] imp --- .../src/launch/helpers/file_validator.rs | 28 ++++++++----------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src-tauri/src/launch/helpers/file_validator.rs b/src-tauri/src/launch/helpers/file_validator.rs index c1c2f5e53..86915a0e1 100644 --- a/src-tauri/src/launch/helpers/file_validator.rs +++ b/src-tauri/src/launch/helpers/file_validator.rs @@ -126,7 +126,18 @@ async fn validate_file_with_hash( ) -> SJMCLResult> { let exists = fs::try_exists(&file_path).await?; - if !exists { + let needs_download = !exists || { + if check_hash { + let hash = expected_hash.clone(); + let path = file_path.clone(); + let is_valid = tokio::task::spawn_blocking(move || validate_sha1(path, hash).is_ok()).await?; + !is_valid + } else { + false + } + }; + + if needs_download { return Ok(Some(PTaskParam::Download(DownloadParam { src: download_url, dest: file_path, @@ -135,21 +146,6 @@ async fn validate_file_with_hash( }))); } - if check_hash { - let hash = expected_hash.clone(); - let path = file_path.clone(); - let is_valid = tokio::task::spawn_blocking(move || validate_sha1(path, hash).is_ok()).await?; - - if !is_valid { - return Ok(Some(PTaskParam::Download(DownloadParam { - src: download_url, - dest: file_path, - filename: None, - sha1: Some(expected_hash), - }))); - } - } - Ok(None) } From 48fa7f2e76add441a10234ffef21e9e83e2b405b Mon Sep 17 00:00:00 2001 From: pynickle <2330458484@qq.com> Date: Thu, 20 Nov 2025 00:37:44 +0800 Subject: [PATCH 5/6] imp: improve validate files concurrently use semaphore --- .../src/launch/helpers/file_validator.rs | 49 ++++++++++--------- src-tauri/src/utils/sys_info.rs | 21 +++++++- 2 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src-tauri/src/launch/helpers/file_validator.rs b/src-tauri/src/launch/helpers/file_validator.rs index 86915a0e1..8278f4db2 100644 --- a/src-tauri/src/launch/helpers/file_validator.rs +++ b/src-tauri/src/launch/helpers/file_validator.rs @@ -11,6 +11,7 @@ use crate::resource::models::{ResourceType, SourceType}; use crate::tasks::download::DownloadParam; use crate::tasks::PTaskParam; use crate::utils::fs::validate_sha1; +use crate::utils::sys_info::get_concurrent_limit; use futures::stream::{self, StreamExt}; use semver::Version; use std::collections::{HashMap, HashSet}; @@ -20,6 +21,7 @@ use std::sync::OnceLock; use sysinfo::{CpuRefreshKind, RefreshKind, System}; use tauri::AppHandle; use tokio::fs; +use tokio::sync::Semaphore; use url::Url; use zip::ZipArchive; @@ -39,19 +41,6 @@ pub struct LibraryParts { pub extension: String, } -fn get_concurrent_hash_checks() -> usize { - static CONCURRENT_LIMIT: OnceLock = OnceLock::new(); - - *CONCURRENT_LIMIT.get_or_init(|| { - let mut sys = - System::new_with_specifics(RefreshKind::nothing().with_cpu(CpuRefreshKind::everything())); - std::thread::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL); - sys.refresh_cpu_usage(); - let cpu_count = sys.cpus().len(); - (cpu_count * 3).max(8).min(32) - }) -} - fn parse_sem_version(version: &str) -> Version { Version::parse(version).unwrap_or_else(|_| { let mut parts = version.split('.').collect::>(); @@ -150,20 +139,34 @@ async fn validate_file_with_hash( } async fn validate_files_concurrently( - items: impl Iterator, + items: impl IntoIterator, check_hash: bool, processor: F, ) -> SJMCLResult> where - F: Fn(T, bool) -> Fut, - Fut: std::future::Future>>, + T: Send + Sync + 'static, + F: Fn(T, bool) -> Fut + Send + Sync + Clone + 'static, + Fut: std::future::Future>> + Send, { - let concurrent_limit = get_concurrent_hash_checks(); + let max_concurrent = get_concurrent_limit(3.0); + let semaphore = std::sync::Arc::new(Semaphore::new(max_concurrent)); - let results: Vec>> = stream::iter(items) - .map(|item| processor(item, check_hash)) - .buffer_unordered(concurrent_limit) - .collect() + let items_vec: Vec = items.into_iter().collect(); + let processor = std::sync::Arc::new(processor); + + let results = stream::iter(items_vec) + .map(|item| { + let permit = semaphore.clone().acquire_owned(); + let processor = processor.clone(); + + async move { + let _permit = permit.await; + + processor(item, check_hash).await + } + }) + .buffer_unordered(max_concurrent) + .collect::>() .await; let mut params = Vec::new(); @@ -351,6 +354,8 @@ pub async fn extract_native_libraries( fs::create_dir(natives_dir).await?; } + let max_concurrent = get_concurrent_limit(1.5); + let native_libraries = get_native_library_paths(client_info, library_path)?; let results: Vec<_> = stream::iter(native_libraries) @@ -364,7 +369,7 @@ pub async fn extract_native_libraries( Ok::<_, crate::error::SJMCLError>(()) } }) - .buffer_unordered(4) + .buffer_unordered(max_concurrent) .collect::>() .await; diff --git a/src-tauri/src/utils/sys_info.rs b/src-tauri/src/utils/sys_info.rs index 4a4a25bfb..7553bfbaf 100644 --- a/src-tauri/src/utils/sys_info.rs +++ b/src-tauri/src/utils/sys_info.rs @@ -3,7 +3,8 @@ use crate::launcher_config::models::MemoryInfo; use serde_json::json; use std::net::{SocketAddr, TcpListener}; use std::path::PathBuf; -use sysinfo::{Disk, Disks}; +use std::sync::OnceLock; +use sysinfo::{CpuRefreshKind, Disk, Disks, RefreshKind, System}; use systemstat::{saturating_sub_bytes, Platform}; use tauri_plugin_http::reqwest; use tauri_plugin_os::locale; @@ -151,3 +152,21 @@ pub fn find_free_port(start_port: Option) -> SJMCLResult { log::error!("No free port found."); Err(SJMCLError("No free port found".to_string())) } + +pub fn get_concurrent_limit(multiplier: f64) -> usize { + static CONCURRENT_LIMIT: OnceLock = OnceLock::new(); + + *CONCURRENT_LIMIT.get_or_init(|| { + let mut sys = + System::new_with_specifics(RefreshKind::nothing().with_cpu(CpuRefreshKind::everything())); + std::thread::sleep(sysinfo::MINIMUM_CPU_UPDATE_INTERVAL); + sys.refresh_cpu_usage(); + let cpu_count = sys.cpus().len() as f64; + + let raw = cpu_count * multiplier; + + let threads = raw.round() as usize; + + threads.max(8).min(32) + }) +} From 132e508e43c8b539dcb111fb8650623514c54bf5 Mon Sep 17 00:00:00 2001 From: pynickle <2330458484@qq.com> Date: Sun, 23 Nov 2025 21:12:30 +0800 Subject: [PATCH 6/6] style: add comment --- src-tauri/src/launch/helpers/file_validator.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src-tauri/src/launch/helpers/file_validator.rs b/src-tauri/src/launch/helpers/file_validator.rs index 8278f4db2..74a182d6c 100644 --- a/src-tauri/src/launch/helpers/file_validator.rs +++ b/src-tauri/src/launch/helpers/file_validator.rs @@ -148,6 +148,7 @@ where F: Fn(T, bool) -> Fut + Send + Sync + Clone + 'static, Fut: std::future::Future>> + Send, { + // 3.0 → Used for downloads & hash validation (lightweight I/O-bound — higher concurrency is safe and efficient) let max_concurrent = get_concurrent_limit(3.0); let semaphore = std::sync::Arc::new(Semaphore::new(max_concurrent)); @@ -354,6 +355,7 @@ pub async fn extract_native_libraries( fs::create_dir(natives_dir).await?; } + // 1.5 → Used for native library extraction (mixed I/O + CPU, heavier work — keep conservative to avoid blocking) let max_concurrent = get_concurrent_limit(1.5); let native_libraries = get_native_library_paths(client_info, library_path)?;