Skip to content

Release 6.9.8 #1492

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MancunianSam merged 4 commits into from
Oct 16, 2025
Merged

Release 6.9.8 #1492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
83f66b3
Revert change to not increment loop
MancunianSam Oct 9, 2025
0414fc3
Dependency updates
dependabot[bot] Oct 6, 2025
aec9d17
Various S3 identification fixes.
MancunianSam Oct 10, 2025
b80f834
Remove calls to HeadObject.
MancunianSam Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion droid-api/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>droid-parent</artifactId>
<groupId>uk.gov.nationalarchives</groupId>
<version>6.9.7-SNAPSHOT</version>
<version>6.9.8-SNAPSHOT</version>
<relativePath>../droid-parent</relativePath>
</parent>

Expand Down
5 changes: 3 additions & 2 deletions droid-api/src/main/java/uk/gov/nationalarchives/droid/internal/api/DroidAPI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ public final class DroidAPI implements AutoCloseable {
private static final String S3_SCHEME = "s3";
private static final String GZIP_PUID = "x-fmt/266";
private static final long DEFAULT_MAX_BYTES_TO_SCAN = -1;
private static final int DEFAULT_WINDOW_SIZE = 4 * 1024 * 1024;

private static final AtomicLong ID_GENERATOR = new AtomicLong();

Expand Down Expand Up @@ -360,8 +361,8 @@ private List<APIResult> submitS3Identification(final URI uri, String extension)
Map<HashAlgorithm, String> hashResults = generateHashResults(s3Uri, this::getS3Hash);

final RequestIdentifier id = getRequestIdentifier(s3Uri.uri());
RequestMetaData metaData = new RequestMetaData(s3Object.size(), s3Object.lastModified().getEpochSecond(), s3Uri.uri().toString());
try (final S3IdentificationRequest request = new S3IdentificationRequest(metaData, id, s3Client)) {
RequestMetaData metaData = new RequestMetaData(s3Object.size(), s3Object.lastModified().toEpochMilli(), s3Uri.uri().toString());
try (final S3IdentificationRequest request = new S3IdentificationRequest(metaData, id, s3Client, DEFAULT_WINDOW_SIZE)) {
request.setExtension(extension);
request.open(s3Uri);
apiResults.add(new APIResult(getIdentificationResults(request), hashResults));
Expand Down
10 changes: 0 additions & 10 deletions droid-api/src/test/java/uk/gov/nationalarchives/droid/internal/api/DroidAPITestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -169,16 +169,6 @@ static HttpServer createS3Server() throws IOException {
OutputStream responseBody = exchange.getResponseBody();
responseBody.write(response.getBytes());
responseBody.close();
} else if (exchange.getRequestMethod().equals("HEAD")) {
String fullPath = exchange.getRequestURI().getPath().substring(1);
Path filePath = getFilePathFromUriPath(fullPath.substring(fullPath.indexOf("/")));
long size = Files.size(filePath);
exchange.getResponseHeaders().add("Content-Length", Long.toString(size));
exchange.getResponseHeaders().add("Last-Modified", "Mon, 03 Mar 2025 17:29:48 GMT");
exchange.sendResponseHeaders(200, -1);
OutputStream responseBody = exchange.getResponseBody();
responseBody.write("".getBytes());
responseBody.close();
} else if (exchange.getRequestMethod().equals("GET")) {
String fullPath = exchange.getRequestURI().getPath().substring(1);
Path filePath = getFilePathFromUriPath(fullPath.substring(fullPath.indexOf("/")));
Expand Down
2 changes: 1 addition & 1 deletion droid-binary/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>droid-parent</artifactId>
<groupId>uk.gov.nationalarchives</groupId>
<version>6.9.7-SNAPSHOT</version>
<version>6.9.8-SNAPSHOT</version>
<relativePath>../droid-parent</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion droid-build-tools/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>droid-parent</artifactId>
<groupId>uk.gov.nationalarchives</groupId>
<version>6.9.7-SNAPSHOT</version>
<version>6.9.8-SNAPSHOT</version>
<relativePath>../droid-parent</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion droid-command-line/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>droid-parent</artifactId>
<groupId>uk.gov.nationalarchives</groupId>
<version>6.9.7-SNAPSHOT</version>
<version>6.9.8-SNAPSHOT</version>
<relativePath>../droid-parent</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion droid-container/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>droid-parent</artifactId>
<groupId>uk.gov.nationalarchives</groupId>
<version>6.9.7-SNAPSHOT</version>
<version>6.9.8-SNAPSHOT</version>
<relativePath>../droid-parent</relativePath>
</parent>

Expand Down
2 changes: 1 addition & 1 deletion droid-core-interfaces/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>droid-parent</artifactId>
<groupId>uk.gov.nationalarchives</groupId>
<version>6.9.7-SNAPSHOT</version>
<version>6.9.8-SNAPSHOT</version>
<relativePath>../droid-parent</relativePath>
</parent>

Expand Down
9 changes: 6 additions & 3 deletions ...ces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/http/S3ClientFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,15 @@ public class S3ClientFactory implements ProxySubscriber {

private S3Client s3Client;

private final Region region;
private Region region;

public S3ClientFactory(ProxySettings proxySettings) {
public S3ClientFactory() {
}

public void init(ProxySettings proxySettings) {
this.region = DefaultAwsRegionProviderChain.builder().build().getRegion();
proxySettings.addProxySubscriber(this);
setS3Client(proxySettings);
this.region = DefaultAwsRegionProviderChain.builder().build().getRegion();
}

@Override
Expand Down
16 changes: 13 additions & 3 deletions .../java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,25 +51,27 @@ public class S3IdentificationRequest implements IdentificationRequest<S3Uri> {
private WindowReader s3Reader;
private final RequestIdentifier identifier;
private final RequestMetaData requestMetaData;
private final int windowSize;

private final S3Client s3client;
private final S3Utils.S3ObjectMetadata s3ObjectMetadata;
private String extension;

public S3IdentificationRequest(final RequestMetaData requestMetaData, final RequestIdentifier identifier, final S3Client s3Client) {
public S3IdentificationRequest(final RequestMetaData requestMetaData, final RequestIdentifier identifier, final S3Client s3Client, final int windowSize) {
this.identifier = identifier;
this.s3client = s3Client;
this.requestMetaData = requestMetaData;
this.windowSize = windowSize;
S3Utils s3Utils = new S3Utils(s3Client);

this.s3ObjectMetadata = s3Utils.getS3ObjectMetadata(identifier.getUri());
this.s3ObjectMetadata = s3Utils.getS3ObjectMetadata(identifier.getUri(), requestMetaData);
this.s3Reader = buildWindowReader();

}

private WindowReader buildWindowReader() {
final WindowCache cache = new TopAndTailFixedLengthCache(this.s3ObjectMetadata.contentLength(), TOP_TAIL_BUFFER_CAPACITY);
return new S3WindowReader(cache, s3ObjectMetadata, s3client);
return new S3WindowReader(cache, s3ObjectMetadata, s3client, windowSize);
}

/**
Expand Down Expand Up @@ -158,4 +160,12 @@ public byte getByte(long position) throws IOException {
public WindowReader getWindowReader() {
return this.s3Reader;
}

/**
* Gets the window size.
* @return the window size
*/
public int getWindowSize() {
return windowSize;
}
}
18 changes: 4 additions & 14 deletions ...erfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3Utils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,26 +63,16 @@ public record S3ObjectMetadata(String bucket, Optional<String> key, S3Uri uri, L

public record S3ObjectList(String bucket, Iterable<S3Object> contents) {}

public S3ObjectMetadata getS3ObjectMetadata(final URI uri) {
public S3ObjectMetadata getS3ObjectMetadata(final URI uri, RequestMetaData requestMetaData) {

S3Uri s3Uri = S3Utilities.builder().region(region).build().parseUri(uri);
return getS3ObjectMetadata(s3Uri);
return getS3ObjectMetadata(s3Uri, requestMetaData);
}

public S3ObjectMetadata getS3ObjectMetadata(final S3Uri s3Uri) {
public S3ObjectMetadata getS3ObjectMetadata(final S3Uri s3Uri, RequestMetaData requestMetaData) {
String bucket = s3Uri.bucket().orElseThrow(() -> new RuntimeException(BUCKET_NOT_FOUND + s3Uri));
Optional<String> key = s3Uri.key();
long contentLength = 0L;
long lastModified = 0L;
if (key.isPresent()) {
try {
HeadObjectRequest headObjectRequest = HeadObjectRequest.builder().bucket(bucket).key(key.get()).build();
HeadObjectResponse headObjectResponse = this.s3Client.headObject(headObjectRequest);
contentLength = headObjectResponse.contentLength();
lastModified = headObjectResponse.lastModified().getEpochSecond();
} catch (NoSuchKeyException ignored) {}
}
return new S3ObjectMetadata(bucket, key, s3Uri, contentLength, lastModified);
return new S3ObjectMetadata(bucket, key, s3Uri, requestMetaData.getSize(), requestMetaData.getTime());
}

public S3ObjectList listObjects(final URI uri) {
Expand Down
6 changes: 2 additions & 4 deletions .../src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,16 +49,14 @@ public class S3WindowReader extends AbstractReader implements SoftWindowRecovery

private static final int BUFFER_LENGTH = 8192;

private static final int WINDOW_SIZE = 4 * 1024 * 1024;

private final S3Utils.S3ObjectMetadata s3ObjectMetadata;

private final S3Client s3Client;

private final Long length;

public S3WindowReader(WindowCache cache, S3Utils.S3ObjectMetadata s3ObjectMetadata, S3Client s3Client) {
super(WINDOW_SIZE, cache);
public S3WindowReader(WindowCache cache, S3Utils.S3ObjectMetadata s3ObjectMetadata, S3Client s3Client, int windowSize) {
super(windowSize, cache);
this.s3Client = s3Client;
this.length = s3ObjectMetadata.contentLength();
this.s3ObjectMetadata = s3ObjectMetadata;
Expand Down
19 changes: 3 additions & 16 deletions ...a/uk/gov/nationalarchives/droid/core/interfaces/resource/S3IdentificationRequestTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,17 +33,12 @@

import org.junit.jupiter.api.Test;
import org.mockito.ArgumentCaptor;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.exception.SdkException;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Uri;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;
import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.net.URI;

Expand All @@ -63,11 +58,10 @@ public void testS3IdentificationRequestGetsFirstWindowOnCreation() throws IOExce
request.open(s3Uri);

ArgumentCaptor<GetObjectRequest> getObjectRequestCaptor = ArgumentCaptor.forClass(GetObjectRequest.class);
verify(mockS3Client, times(1)).headObject(any(HeadObjectRequest.class));
verify(mockS3Client, times(1)).getObject(getObjectRequestCaptor.capture());

GetObjectRequest getRequestValue = getObjectRequestCaptor.getValue();
assertEquals(getRequestValue.range(), "bytes=0-4194303");
assertEquals(getRequestValue.range(), "bytes=0-1023");
}

@Test
Expand All @@ -78,7 +72,7 @@ public void testS3IdentificationRequestHasCorrectAttributes() throws IOException

request.open(s3Uri);

assertEquals(request.size(), 1);
assertEquals(request.size(), 2);
assertEquals(request.getFileName(), "entry.txt");
assertEquals(request.getExtension(), "txt");
}
Expand Down Expand Up @@ -126,18 +120,11 @@ public void testErrorIfS3GetObjectCallsFail() {
assertThrows(SdkException.class, () -> request.open(s3Uri));
}

@Test
public void testErrorIfS3HeadObjectCallsFail() {
S3Client mockS3Client = mock(S3Client.class);
when(mockS3Client.headObject(any(HeadObjectRequest.class))).thenThrow(SdkException.class);
assertThrows(SdkException.class, () -> createRequest(mockS3Client));
}

private S3IdentificationRequest createRequest(S3Client mockS3Client) {
RequestMetaData requestMetaData = new RequestMetaData(2L, 1L, "entry.txt");
URI uri = URI.create("s3://bucket/test");

RequestIdentifier requestIdentifier = new RequestIdentifier(uri);
return new S3IdentificationRequest(requestMetaData, requestIdentifier, mockS3Client);
return new S3IdentificationRequest(requestMetaData, requestIdentifier, mockS3Client, 1024);
}
}
6 changes: 0 additions & 6 deletions ...ces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3TestUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,8 @@
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.HeadObjectRequest;
import software.amazon.awssdk.services.s3.model.HeadObjectResponse;

import java.io.ByteArrayInputStream;
import java.time.Instant;

import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.mock;
Expand All @@ -49,10 +46,7 @@ public class S3TestUtils {

static S3Client mockS3Client() {
S3Client mockS3Client = mock(S3Client.class);
HeadObjectResponse response = HeadObjectResponse.builder().contentLength(1L).lastModified(Instant.ofEpochSecond(1)).build();
GetObjectResponse getObjectResponse = GetObjectResponse.builder().build();
ResponseInputStream<GetObjectResponse> responseInputStream = new ResponseInputStream<>(getObjectResponse, new ByteArrayInputStream("test".getBytes()));
when(mockS3Client.headObject(any(HeadObjectRequest.class))).thenReturn(response);
when(mockS3Client.getObject(any(GetObjectRequest.class))).thenAnswer(invocation -> {
GetObjectRequest argument = invocation.getArgument(0, GetObjectRequest.class);
String range = argument.range();
Expand Down
5 changes: 3 additions & 2 deletions ...ces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3UtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,10 @@ public void getObjectMetadataReturnsCorrectMetadata() {
S3Client s3Client = mockS3Client();
S3Utils s3Utils = new S3Utils(s3Client, Region.EU_WEST_2);
URI uri = URI.create("s3://bucket/key");
S3Utils.S3ObjectMetadata s3ObjectMetadataFromUri = s3Utils.getS3ObjectMetadata(uri);
RequestMetaData requestMetaData = new RequestMetaData(1L, 1L, "key");
S3Utils.S3ObjectMetadata s3ObjectMetadataFromUri = s3Utils.getS3ObjectMetadata(uri, requestMetaData);
S3Uri s3Uri = S3Uri.builder().uri(uri).bucket("bucket").key("key").build();
S3Utils.S3ObjectMetadata s3ObjectMetadataFromS3Uri = s3Utils.getS3ObjectMetadata(s3Uri);
S3Utils.S3ObjectMetadata s3ObjectMetadataFromS3Uri = s3Utils.getS3ObjectMetadata(s3Uri, requestMetaData);

for (S3Utils.S3ObjectMetadata s3ObjectMetadata: List.of(s3ObjectMetadataFromUri, s3ObjectMetadataFromS3Uri)) {
assertTrue(s3ObjectMetadata.key().isPresent());
Expand Down
12 changes: 6 additions & 6 deletions .../test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/S3WindowReaderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ public void testWindowReaderReturnsExpectedWindow() throws Exception {
S3Client s3Client = mockS3Client();
S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build();
S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 4L, 1L);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client, 1);

byte[] testResponse = "test".getBytes();
for (int i = 0; i < testResponse.length; i++) {
Expand All @@ -79,12 +79,12 @@ public void testWindowReaderReturnsExpectedWindowForLargeFile() throws Exception
S3Client s3Client = mockS3Client();
S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build();
S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 4L, 1L);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client, 10);

s3WindowReader.createWindow(0);
Collection<Invocation> invocations = Mockito.mockingDetails(s3Client).getInvocations();
GetObjectRequest getObjectRequest = (GetObjectRequest)invocations.stream().toList().getFirst().getArguments()[0];
assertEquals(getObjectRequest.range(), "bytes=0-" + ((4 * 1024 * 1024) - 1));
assertEquals(getObjectRequest.range(), "bytes=0-" + 9);
}

@Test
Expand All @@ -93,7 +93,7 @@ public void testWindowReaderReturnsNullIfPositionLessThanZero() throws Exception
S3Client s3Client = mock(S3Client.class);
S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build();
S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 1L, 1L);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client, 1);

assertNull(s3WindowReader.createWindow(-1));
}
Expand All @@ -104,7 +104,7 @@ public void testWindowReaderReturnsNullIfPositionGreaterOrEqualToLength() throws
S3Client s3Client = mockS3Client();
S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build();
S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 4L, 1L);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client, 1);

assertNotNull(s3WindowReader.createWindow(3));
assertNull(s3WindowReader.createWindow(4));
Expand All @@ -119,7 +119,7 @@ public void testWindowReaderReturnsErrorOnS3Failure() throws Exception {
when(s3Client.getObject(any(GetObjectRequest.class))).thenThrow(S3Exception.builder().message("Error contacting s3").build());
S3Uri s3Uri = S3Uri.builder().uri(URI.create("s3://bucket/key")).build();
S3Utils.S3ObjectMetadata s3ObjectMetadata = new S3Utils.S3ObjectMetadata("bucket", Optional.of("key"), s3Uri, 1L, 1L);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client);
S3WindowReader s3WindowReader = new S3WindowReader(windowCache, s3ObjectMetadata, s3Client, 1);

assertThrows(S3Exception.class, () -> s3WindowReader.createWindow(0));
}
Expand Down
8 changes: 1 addition & 7 deletions droid-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<parent>
<artifactId>droid-parent</artifactId>
<groupId>uk.gov.nationalarchives</groupId>
<version>6.9.7-SNAPSHOT</version>
<version>6.9.8-SNAPSHOT</version>
<relativePath>../droid-parent</relativePath>
</parent>

Expand Down Expand Up @@ -114,12 +114,6 @@
<version>${aws.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>aws-core</artifactId>
<version>${aws.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>regions</artifactId>
Expand Down
Loading
Loading