From 1ffe209958907452d907ce33bd272586ca081835 Mon Sep 17 00:00:00 2001 From: yma Date: Mon, 2 Dec 2024 11:45:48 +0800 Subject: [PATCH] Fix checksum validation for archive generation download --- .../archive/controller/ArchiveController.java | 90 ++++++++++++++++--- .../util/HistoricalContentListReader.java | 63 ++++++------- .../util/HistoricalContentListReaderTest.java | 18 ++-- 3 files changed, 122 insertions(+), 49 deletions(-) diff --git a/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java b/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java index b62e345..09a6838 100644 --- a/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java +++ b/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java @@ -33,6 +33,7 @@ import org.commonjava.indy.service.archive.config.PreSeedConfig; import org.commonjava.indy.service.archive.model.ArchiveStatus; import org.commonjava.indy.service.archive.model.dto.HistoricalContentDTO; +import org.commonjava.indy.service.archive.model.dto.HistoricalEntryDTO; import org.commonjava.indy.service.archive.util.HistoricalContentListReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,10 +48,15 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorCompletionService; @@ -79,6 +85,15 @@ public class ArchiveController private final String PART_ARCHIVE_SUFFIX = PART_SUFFIX + ARCHIVE_SUFFIX; + private static final Set CHECKSUMS = Collections.unmodifiableSet( new HashSet() + { + { + add( ".md5" ); + add( ".sha1" ); + add( ".sha256" ); + } + } ); + @Inject HistoricalContentListReader reader; @@ -146,11 +161,14 @@ protected Boolean doGenerate( HistoricalContentDTO content ) content.getBuildConfigId() ); recordInProgress( content.getBuildConfigId() ); - Map downloadPaths = reader.readPaths( content ); + Map entryDTOs = reader.readEntries( content ); + Map downloadPaths = new HashMap<>(); + entryDTOs.forEach( ( key, value ) -> downloadPaths.put( key, value.getPath() ) ); + Optional archive; try { - downloadArtifacts( downloadPaths, content ); + downloadArtifacts( entryDTOs, downloadPaths, content ); archive = generateArchive( new ArrayList<>( downloadPaths.values() ), content ); } catch ( final InterruptedException e ) @@ -226,7 +244,8 @@ public String getStatus( String buildConfigId ) return treated.get( buildConfigId ); } - private void downloadArtifacts( final Map downloadPaths, final HistoricalContentDTO content ) + private void downloadArtifacts( final Map entryDTOs, + final Map downloadPaths, final HistoricalContentDTO content ) throws InterruptedException, ExecutionException, IOException { BasicCookieStore cookieStore = new BasicCookieStore(); @@ -236,13 +255,23 @@ private void downloadArtifacts( final Map downloadPaths, final H File dir = new File( contentBuildDir ); dir.delete(); - fileTrackedContent( contentBuildDir, content ); - unpackHistoricalArchive( contentBuildDir, content.getBuildConfigId() ); + HistoricalContentDTO originalTracked = unpackHistoricalArchive( contentBuildDir, content.getBuildConfigId() ); + Map> originalChecksumsMap = new HashMap<>(); + if ( originalTracked != null ) + { + Map originalEntries = reader.readEntries( originalTracked ); + originalEntries.forEach( ( key, entry ) -> originalChecksumsMap.put( key, new ArrayList<>( + Arrays.asList( entry.getSha1(), entry.getSha256(), entry.getMd5() ) ) ) ); + } for ( String path : downloadPaths.keySet() ) { String filePath = downloadPaths.get( path ); - executor.submit( download( contentBuildDir, path, filePath, cookieStore ) ); + HistoricalEntryDTO entry = entryDTOs.get( path ); + List checksums = + new ArrayList<>( Arrays.asList( entry.getSha1(), entry.getSha256(), entry.getMd5() ) ); + List originalChecksums = originalChecksumsMap.get( path ); + executor.submit( download( contentBuildDir, path, filePath, checksums, originalChecksums, cookieStore ) ); } int success = 0; int failed = 0; @@ -257,6 +286,8 @@ private void downloadArtifacts( final Map downloadPaths, final H failed++; } } + // file the latest tracked json at the end + fileTrackedContent( contentBuildDir, content ); logger.info( "Artifacts download completed, success:{}, failed:{}", success, failed ); } @@ -369,14 +400,14 @@ private void fileTrackedContent( String contentBuildDir, final HistoricalContent } } - private void unpackHistoricalArchive( String contentBuildDir, String buildConfigId ) + private HistoricalContentDTO unpackHistoricalArchive( String contentBuildDir, String buildConfigId ) throws IOException { final File archive = new File( archiveDir, buildConfigId + ARCHIVE_SUFFIX ); if ( !archive.exists() ) { logger.debug( "Don't find historical archive for buildConfigId: {}.", buildConfigId ); - return; + return null; } logger.info( "Start unpacking historical archive for buildConfigId: {}.", buildConfigId ); @@ -393,16 +424,53 @@ private void unpackHistoricalArchive( String contentBuildDir, String buildConfig } inputStream.close(); + + File originalTracked = new File( contentBuildDir, buildConfigId ); + if ( originalTracked.exists() ) + { + return objectMapper.readValue( originalTracked, HistoricalContentDTO.class ); + } + else + { + logger.debug( "No tracked json file found after zip unpack for buildConfigId {}", buildConfigId ); + return null; + } } - private Callable download( String contentBuildDir, final String path, final String filePath, + private boolean validateChecksum( final String filePath, final List current, final List original ) + { + if ( CHECKSUMS.stream().anyMatch( suffix -> filePath.toLowerCase().endsWith( "." + suffix ) ) ) + { + // skip to validate checksum files + return false; + } + if ( original == null || original.isEmpty() || original.stream().allMatch( Objects::isNull ) ) + { + return false; + } + if ( original.get( 0 ) != null && original.get( 0 ).equals( current.get( 0 ) ) ) + { + return true; + } + if ( original.get( 1 ) != null && original.get( 1 ).equals( current.get( 1 ) ) ) + { + return true; + } + return original.get( 2 ) != null && original.get( 2 ).equals( current.get( 2 ) ); + } + + private Callable download( final String contentBuildDir, final String path, final String filePath, + final List checksums, final List originalChecksums, final CookieStore cookieStore ) { return () -> { final File target = new File( contentBuildDir, filePath ); - if ( target.exists() ) + + if ( target.exists() && validateChecksum( filePath, checksums, originalChecksums ) ) { - logger.trace( "<< readPaths( HistoricalContentDTO content ) + public Map readEntries( HistoricalContentDTO content ) { - Map pathMap = new HashMap<>(); + Map pathMap = new HashMap<>(); HistoricalEntryDTO[] downloads = content.getDownloads(); - if ( downloads != null ) + if ( downloads == null ) { - for ( HistoricalEntryDTO download : downloads ) - { - String path = download.getPath(); - String packageType = download.getStoreKey().getPackageType(); + return pathMap; + } + for ( HistoricalEntryDTO download : downloads ) + { + String path = download.getPath(); + String packageType = download.getStoreKey().getPackageType(); - if ( packageType.equals( NPM_PKG_KEY ) && !path.endsWith( ".tgz" ) ) - { - // Ignore the npm package metadata in archive - continue; - } - if ( path.contains( "maven-metadata.xml" ) ) - { - // Ignore maven-metadata.xml in archive - continue; - } - // ensure every entry has an available localUrl - buildDownloadUrl( download ); + if ( packageType.equals( NPM_PKG_KEY ) && !path.endsWith( ".tgz" ) ) + { + // Ignore the npm package metadata in archive + continue; + } + if ( path.contains( "maven-metadata.xml" ) ) + { + // Ignore maven-metadata.xml in archive + continue; + } + // ensure every entry has an available localUrl + buildDownloadUrl( download ); - // local url would be preferred to download artifact - String url = download.getLocalUrl(); - if ( url == null ) - { - url = download.getOriginUrl(); - } - if ( url != null ) - { - pathMap.put( url, download.getPath() ); - } + // local url would be preferred to download artifact + String url = download.getLocalUrl(); + if ( url == null ) + { + url = download.getOriginUrl(); + } + if ( url != null ) + { + pathMap.put( url, download ); } } return pathMap; diff --git a/src/test/java/org/commonjava/indy/service/archive/util/HistoricalContentListReaderTest.java b/src/test/java/org/commonjava/indy/service/archive/util/HistoricalContentListReaderTest.java index b7743da..2e76fe3 100644 --- a/src/test/java/org/commonjava/indy/service/archive/util/HistoricalContentListReaderTest.java +++ b/src/test/java/org/commonjava/indy/service/archive/util/HistoricalContentListReaderTest.java @@ -16,7 +16,6 @@ package org.commonjava.indy.service.archive.util; import io.quarkus.test.junit.QuarkusTest; -import org.commonjava.indy.service.archive.config.PreSeedConfig; import org.commonjava.indy.service.archive.model.StoreKey; import org.commonjava.indy.service.archive.model.StoreType; import org.commonjava.indy.service.archive.model.dto.HistoricalContentDTO; @@ -24,6 +23,7 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -58,13 +58,15 @@ public void testMavenReadPaths() entryDTOs.add( entry ); entryDTOs.add( metaEntry ); - HistoricalContentDTO contentDTO = new HistoricalContentDTO( "8888", entryDTOs.toArray( - new HistoricalEntryDTO[entryDTOs.size()] ) ); + HistoricalContentDTO contentDTO = + new HistoricalContentDTO( "8888", entryDTOs.toArray( new HistoricalEntryDTO[entryDTOs.size()] ) ); TestPreSeedConfig preSeedConfig = new TestPreSeedConfig( Optional.of( MAIN_INDY ) ); HistoricalContentListReader reader = new HistoricalContentListReader( preSeedConfig ); - Map paths = reader.readPaths( contentDTO ); + Map entryMaps = reader.readEntries( contentDTO ); + Map paths = new HashMap<>(); + entryMaps.forEach( ( key, value ) -> paths.put( key, value.getPath() ) ); assertThat( paths.size(), equalTo( 1 ) ); String storePath = MAIN_INDY + "/api/content" + entry.getStorePath(); @@ -86,13 +88,15 @@ public void testNPMReadPaths() entryDTOs.add( npmEntry ); entryDTOs.add( npmMetaEntry ); - HistoricalContentDTO contentDTO = new HistoricalContentDTO( "8888", entryDTOs.toArray( - new HistoricalEntryDTO[entryDTOs.size()] ) ); + HistoricalContentDTO contentDTO = + new HistoricalContentDTO( "8888", entryDTOs.toArray( new HistoricalEntryDTO[entryDTOs.size()] ) ); TestPreSeedConfig preSeedConfig = new TestPreSeedConfig( Optional.of( MAIN_INDY ) ); HistoricalContentListReader reader = new HistoricalContentListReader( preSeedConfig ); - Map paths = reader.readPaths( contentDTO ); + Map entryMaps = reader.readEntries( contentDTO ); + Map paths = new HashMap<>(); + entryMaps.forEach( ( key, value ) -> paths.put( key, value.getPath() ) ); assertThat( paths.size(), equalTo( 1 ) ); String storePath = MAIN_INDY + "/api/content" + npmEntry.getStorePath();