From d724b2d7a9bd79965d7260c812b6a39bb4bf1a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E5=A4=A9=E5=B0=8F=E7=A6=BD=E5=85=BD?= Date: Sun, 31 Mar 2024 02:49:44 +0800 Subject: [PATCH] Support 4-thread download for archive download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 归档下载切换为4线程下载 --- lib/src/network/eh_request.dart | 8 +- lib/src/service/archive_download_service.dart | 82 ++++++++++++------- lib/src/utils/archive_util.dart | 5 +- pubspec.lock | 2 +- 4 files changed, 62 insertions(+), 35 deletions(-) diff --git a/lib/src/network/eh_request.dart b/lib/src/network/eh_request.dart index d8d5a4b1f..ded971bd0 100644 --- a/lib/src/network/eh_request.dart +++ b/lib/src/network/eh_request.dart @@ -859,8 +859,12 @@ emyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc= return _parseResponse(response, parser); } - static Future head({required String url, Options? options}) { - return _dio.head(url, options: options); + static Future head({required String url, CancelToken? cancelToken, Options? options}) { + return _dio.head( + url, + cancelToken: cancelToken, + options: options, + ); } static Future _parseResponse(Response response, EHHtmlParser? parser) async { diff --git a/lib/src/service/archive_download_service.dart b/lib/src/service/archive_download_service.dart index 01df58c23..6f4901cad 100644 --- a/lib/src/service/archive_download_service.dart +++ b/lib/src/service/archive_download_service.dart @@ -248,6 +248,8 @@ class ArchiveDownloadService extends GetxController with GridBasePageServiceMixi archiveDownloadInfo.downloadPageUrl = null; archiveDownloadInfo.downloadUrl = null; archiveDownloadInfo.cancelToken = CancelToken(); + archiveDownloadInfo.downloadTask?.pause(); + archiveDownloadInfo.downloadTask = null; await _updateArchiveInDatabase(archive); update(['$archiveStatusId::${archive.gid}']); @@ -440,7 +442,7 @@ class ArchiveDownloadService extends GetxController with GridBasePageServiceMixi String computePackingFileDownloadPath(ArchiveDownloadedData archive) { String title = _computeArchiveTitle(archive.title); - return join(DownloadSetting.downloadPath.value, 'Archive - ${archive.gid} - $title.zip'); + return join(DownloadSetting.downloadPath.value, 'Archive - v2 - ${archive.gid} - $title.zip'); } String computeArchiveUnpackingPath(ArchiveDownloadedData archive) { @@ -486,6 +488,7 @@ class ArchiveDownloadService extends GetxController with GridBasePageServiceMixi url: url, savePath: computePackingFileDownloadPath(archive), isolateCount: isolateCount, + deleteWhenUrlMismatch: false, onProgress: (current, total) { archiveDownloadInfos[archive.gid]!.speedComputer.downloadedBytes = current; }, @@ -511,9 +514,44 @@ class ArchiveDownloadService extends GetxController with GridBasePageServiceMixi ); } - bool _invalidDownload(Response response) { - Headers headers = response.headers; + Future _check410Reason(String url, ArchiveDownloadedData archive) async { + try { + await EHRequest.get( + url: url, + cancelToken: archiveDownloadInfos[archive.gid]?.cancelToken, + ); + } on DioException catch (e) { + if (e.type == DioExceptionType.cancel) { + return; + } + + if (e.response!.data is String && e.response!.data.contains('You have clocked too many downloaded bytes on this gallery')) { + Log.download('${'410Hints'.tr} Archive: ${archive.title}'); + snack('archiveError'.tr, '${'410Hints'.tr} : ${archive.title}', longDuration: true); + + return await pauseDownloadArchive(archive, needReUnlock: true); + } else if (e.response!.data is String && e.response!.data.contains('IP quota exhausted')) { + Log.download('IP quota exhausted! Archive: ${archive.title}'); + snack('archiveError'.tr, 'IP quota exhausted!', longDuration: true); + + return await pauseDownloadArchive(archive, needReUnlock: true); + } else { + Log.download('Download archive 410, try re-parse. Archive: ${archive.title}'); + return await _reParseDownloadUrlAndDownload(archive); + } + } + } + + Future _reParseDownloadUrlAndDownload(ArchiveDownloadedData archive) async { + ArchiveDownloadInfo archiveDownloadInfo = archiveDownloadInfos[archive.gid]!; + archiveDownloadInfo.downloadUrl = null; + _updateArchiveInDatabase(archive); + + await _getDownloadUrl(archive); + await _doDownloadArchiveViaMultiIsolate(archive); + } + bool _invalidDownload(Headers headers) { if (headers['content-transfer-encoding']?.contains('binary') ?? false) { return false; } @@ -611,6 +649,7 @@ class ArchiveDownloadService extends GetxController with GridBasePageServiceMixi if (e.type == DioExceptionType.cancel) { return; } + return await _getDownloadUrl(archive); } on EHSiteException catch (e) { Log.download('Download error, reason: ${e.message}'); @@ -660,33 +699,25 @@ class ArchiveDownloadService extends GetxController with GridBasePageServiceMixi /// check if archive link is invalid Response response; try { - response = await EHRequest.head(url: archiveDownloadInfo.downloadUrl!); + response = await EHRequest.head( + url: archiveDownloadInfo.downloadUrl!, + cancelToken: archiveDownloadInfo.cancelToken, + ); } on DioException catch (e) { + if (e.type == DioExceptionType.cancel) { + return; + } + /// download too many bytes will cause 410 if (e.response?.statusCode == 410) { - if (e.response!.data is String && e.response!.data.contains('You have clocked too many downloaded bytes on this gallery')) { - Log.download('${'410Hints'.tr} Archive: ${archive.title}'); - snack('archiveError'.tr, '${'410Hints'.tr} : ${archive.title}', longDuration: true); - - return await pauseDownloadArchive(archive, needReUnlock: true); - } else if (e.response!.data is String && e.response!.data.contains('IP quota exhausted')) { - Log.download('IP quota exhausted! Archive: ${archive.title}'); - snack('archiveError'.tr, 'IP quota exhausted!', longDuration: true); - - return await pauseDownloadArchive(archive, needReUnlock: true); - } else { - Log.download('Download archive 410, try re-parse. Archive: ${archive.title}'); - return await _reParseDownloadUrlAndDownload(archive); - } + return await _check410Reason(archiveDownloadInfo.downloadUrl!, archive); } - Log.download('Check archive ${archive.title} available failed, retry. Reason: ${e.message}, url:${archiveDownloadInfo.downloadUrl!}'); - await Future.delayed(const Duration(milliseconds: 1000)); return await _doDownloadArchiveViaMultiIsolate(archive); } - if (_invalidDownload(response)) { + if (_invalidDownload(response.headers)) { Log.error('Invalid archive!'); Log.uploadError(Exception('Invalid archive!'), extraInfos: { 'code': response.statusCode, @@ -711,15 +742,6 @@ class ArchiveDownloadService extends GetxController with GridBasePageServiceMixi } } - Future _reParseDownloadUrlAndDownload(ArchiveDownloadedData archive) async { - ArchiveDownloadInfo archiveDownloadInfo = archiveDownloadInfos[archive.gid]!; - archiveDownloadInfo.downloadUrl = null; - _updateArchiveInDatabase(archive); - - await _getDownloadUrl(archive); - await _doDownloadArchiveViaMultiIsolate(archive); - } - Future _unpackingArchive(ArchiveDownloadedData archive) async { ArchiveDownloadInfo archiveDownloadInfo = archiveDownloadInfos[archive.gid]!; Log.info('Unpacking archive: ${archive.title}, original: ${archive.isOriginal}'); diff --git a/lib/src/utils/archive_util.dart b/lib/src/utils/archive_util.dart index 8a41eeeb3..00a3e72ed 100644 --- a/lib/src/utils/archive_util.dart +++ b/lib/src/utils/archive_util.dart @@ -4,13 +4,14 @@ import 'package:flutter/foundation.dart'; Future extractZipArchive(String archivePath, String extractPath) { return compute( (List path) async { - InputFileStream inputStream = InputFileStream(path[0]); + InputFileStream? inputStream; try { + inputStream = InputFileStream(path[0]); extractArchiveToDisk(ZipDecoder().decodeBuffer(inputStream), path[1]); } on Exception catch (_) { return false; } finally { - inputStream.close(); + inputStream?.close(); } return true; }, diff --git a/pubspec.lock b/pubspec.lock index 8ab5306f9..aabc6d30a 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -820,7 +820,7 @@ packages: description: path: "." ref: HEAD - resolved-ref: "12eade4727588e5617749aea2393568cb97c6824" + resolved-ref: "56dd1dfe666fad690f0265e41fc5cd7bb13ecb4e" url: "https://github.com/jiangtian616/JDownloader" source: git version: "0.0.1"