From 6c61b709b6b9cc635d74cbd0aa5545fc31653823 Mon Sep 17 00:00:00 2001
From: yma <yma@redhat.com>
Date: Tue, 3 Dec 2024 00:48:38 +0800
Subject: [PATCH] Add API to delete archive with checksum validation
---
.../archive/controller/ArchiveController.java | 95 ++++++++++++++++---
.../archive/jaxrs/ArchiveManageResources.java | 21 ++++
2 files changed, 104 insertions(+), 12 deletions(-)
diff --git a/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java b/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java
index 2560f2e..85ffc69 100644
--- a/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java
+++ b/src/main/java/org/commonjava/indy/service/archive/controller/ArchiveController.java
@@ -29,6 +29,7 @@
import org.apache.http.impl.client.BasicCookieStore;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
+import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.commonjava.indy.service.archive.config.PreSeedConfig;
import org.commonjava.indy.service.archive.model.ArchiveStatus;
import org.commonjava.indy.service.archive.model.dto.HistoricalContentDTO;
@@ -43,11 +44,12 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.security.KeyManagementException;
-import java.security.KeyStoreException;
+import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -88,6 +90,16 @@ public class ArchiveController
private final String PART_ARCHIVE_SUFFIX = PART_SUFFIX + ARCHIVE_SUFFIX;
+ private static final int threads = 4 * Runtime.getRuntime().availableProcessors();
+
+ private final ExecutorService generateExecutor =
+ Executors.newFixedThreadPool( threads, ( final Runnable r ) -> {
+ final Thread t = new Thread( r );
+ t.setName( "Generate-" + t.getName() );
+ t.setDaemon( true );
+ return t;
+ } );
+
private static final Set<String> CHECKSUMS = Collections.unmodifiableSet( new HashSet<String>()
{
{
@@ -99,15 +111,13 @@ public class ArchiveController
private static final Map<String, Object> buildConfigLocks = new ConcurrentHashMap<>();
- private static final int threads = 4 * Runtime.getRuntime().availableProcessors();
+ private static final String SHA_256 = "SHA-256";
- private static final ExecutorService generateExecutor =
- Executors.newFixedThreadPool( threads, ( final Runnable r ) -> {
- final Thread t = new Thread( r );
- t.setName( "Generate-" + t.getName() );
- t.setDaemon( true );
- return t;
- } );
+ private static final Long BOLCK_SIZE = 100 * 1024 * 1024L;
+
+ private static final String HEX_DIGITS = "0123456789abcdef";
+
+ private static final char[] HEX_ARRAY = HEX_DIGITS.toCharArray();
@Inject
HistoricalContentListReader reader;
@@ -130,7 +140,7 @@ public class ArchiveController
@PostConstruct
public void init()
- throws IOException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException
+ throws IOException
{
int threads = 4 * Runtime.getRuntime().availableProcessors();
executorService = Executors.newFixedThreadPool( threads, ( final Runnable r ) -> {
@@ -140,8 +150,11 @@ public void init()
return t;
} );
+ final PoolingHttpClientConnectionManager ccm = new PoolingHttpClientConnectionManager();
+ ccm.setMaxTotal( 500 );
+
RequestConfig rc = RequestConfig.custom().build();
- client = HttpClients.custom().setDefaultRequestConfig( rc ).build();
+ client = HttpClients.custom().setConnectionManager( ccm ).setDefaultRequestConfig( rc ).build();
String storeDir = preSeedConfig.storageDir().orElse( "data" );
contentDir = String.format( "%s%s", storeDir, CONTENT_DIR );
@@ -259,6 +272,64 @@ public void deleteArchive( final String buildConfigId )
logger.info( "Historical archive for build config id: {} is deleted.", buildConfigId );
}
+ public void deleteArchiveWithChecksum( final String buildConfigId, final String checksum )
+ throws IOException
+ {
+ logger.info( "Start to delete archive with checksum validation, buildConfigId {}, checksum {}", buildConfigId,
+ checksum );
+ File zip = new File( archiveDir, buildConfigId + ARCHIVE_SUFFIX );
+ if ( !zip.exists() )
+ {
+ return;
+ }
+
+ try (FileChannel channel = new FileInputStream( zip ).getChannel())
+ {
+ MessageDigest digest = MessageDigest.getInstance( SHA_256 );
+ long position = 0;
+ long size = channel.size();
+
+ while ( position < size )
+ {
+ long remaining = size - position;
+ long currentBlock = Math.min( remaining, BOLCK_SIZE );
+ MappedByteBuffer buffer = channel.map( FileChannel.MapMode.READ_ONLY, position, currentBlock );
+ digest.update( buffer );
+ position += currentBlock;
+ }
+
+ String stored = bytesToHex( digest.digest() );
+ // only delete the zip once checksum is equaled
+ if ( stored.equals( checksum ) )
+ {
+ zip.delete();
+ logger.info( "Historical archive for build config id: {} is deleted, checksum {}.", buildConfigId,
+ stored );
+ }
+ else
+ {
+ logger.info( "Don't delete the {} zip, transferred checksum {}, but stored checksum {}.", buildConfigId,
+ checksum, stored );
+ }
+ }
+ catch ( NoSuchAlgorithmException e )
+ {
+ logger.error( "No such algorithm SHA-256 Exception", e );
+ }
+ }
+
+ private String bytesToHex( byte[] hash )
+ {
+ char[] hexChars = new char[hash.length * 2];
+ for ( int i = 0; i < hash.length; i++ )
+ {
+ int v = hash[i] & 0xFF;
+ hexChars[i * 2] = HEX_ARRAY[v >>> 4];
+ hexChars[i * 2 + 1] = HEX_ARRAY[v & 0x0F];
+ }
+ return new String( hexChars );
+ }
+
public void cleanup()
throws IOException
{
diff --git a/src/main/java/org/commonjava/indy/service/archive/jaxrs/ArchiveManageResources.java b/src/main/java/org/commonjava/indy/service/archive/jaxrs/ArchiveManageResources.java
index b01d36d..96a8c60 100644
--- a/src/main/java/org/commonjava/indy/service/archive/jaxrs/ArchiveManageResources.java
+++ b/src/main/java/org/commonjava/indy/service/archive/jaxrs/ArchiveManageResources.java
@@ -188,6 +188,27 @@ public Uni<Response> delete( final @PathParam( "buildConfigId" ) String buildCon
return Uni.createFrom().item( noContent().build() );
}
+ @Operation( description = "Delete the archive by buildConfigId and checksum" )
+ @APIResponse( responseCode = "204", description = "The history archive is deleted or doesn't exist" )
+ @Path( "{buildConfigId}/{checksum}" )
+ @DELETE
+ public Uni<Response> deleteWithChecksum( final @PathParam( "buildConfigId" ) String buildConfigId,
+ final @PathParam( "checksum" ) String checksum,
+ final @Context UriInfo uriInfo )
+ {
+ try
+ {
+ controller.deleteArchiveWithChecksum( buildConfigId, checksum );
+ }
+ catch ( final IOException e )
+ {
+ final String message = "Failed to delete historical archive for build config id: " + buildConfigId;
+ logger.error( message, e );
+ return fromResponse( message );
+ }
+ return Uni.createFrom().item( noContent().build() );
+ }
+
@Operation( description = "Clean up all the temp workplace" )
@APIResponse( responseCode = "204", description = "The workplace cleanup is finished" )
@Path( "cleanup" )