Release DROID

.github/workflows/ci.yml

@@ -59,4 +59,4 @@ jobs:
       - run: |
           mvn clean package -DskipTests=true
           docker build -t cli-tests -f cli-tests/Dockerfile .
-          docker run -v "$PWD:/code" cli-tests /code/cli-tests/test
+          docker run -v "$PWD:/code" cli-tests /code/cli-tests/test

cli-tests/test/cli-test.bats

@@ -54,24 +54,26 @@ setup() {
 
 @test "check for signature updates" {
   run $DROID_BIN -c
-  assert_output 'No signature updates are available'
+  assert_output -p 'Container signature update Version 20250925 is available'
+  assert_output -p 'Binary signature update Version 121 is available'
 }
 
 @test "download signature updates" {
   run $DROID_BIN -d
-  assert_output 'No signature updates are available'
+  assert_output -p 'Signature update version 20,250,925 has been downloaded'
+  assert_output -p 'Signature update version 121 has been downloaded'
 }
 
 @test "list signature files" {
   run $DROID_BIN -X
-  assert_output -p "Type: Binary Version:  120  File name: DROID_SignatureFile_V120.xml"
-  assert_output -p "Type: Container Version:  20240715  File name: container-signature-20240715.xml"
+  assert_output -p "Type: Binary Version:  121  File name: DROID_SignatureFile_V121.xml"
+  assert_output -p "Type: Container Version:  20250925  File name: container-signature-20250925.xml"
 }
 
 @test "display signature file" {
   run $DROID_BIN -x
-  assert_output -p "Type: Container Version:  20240715  File name: container-signature-20240715.xml"
-  assert_output -p "Type: Binary Version:  120  File name: DROID_SignatureFile_V120.xml"
+  assert_output -p "Type: Container Version:  20250925  File name: container-signature-20250925.xml"
+  assert_output -p "Type: Binary Version:  121  File name: DROID_SignatureFile_V121.xml"
 }
 
 @test "set signature file" {

droid-api/pom.xml

@@ -4,7 +4,7 @@
 	<parent>
 		<artifactId>droid-parent</artifactId>
 		<groupId>uk.gov.nationalarchives</groupId>
-		<version>6.9.5-SNAPSHOT</version>
+		<version>6.9.6-SNAPSHOT</version>
 		<relativePath>../droid-parent</relativePath>
 	</parent>
 

droid-binary/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.9.5-SNAPSHOT</version>
+        <version>6.9.6-SNAPSHOT</version>
   	    <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-build-tools/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.9.5-SNAPSHOT</version>
+        <version>6.9.6-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-command-line/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.9.5-SNAPSHOT</version>
+        <version>6.9.6-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-container/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.9.5-SNAPSHOT</version>
+        <version>6.9.6-SNAPSHOT</version>
   	    <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-core-interfaces/pom.xml

@@ -4,7 +4,7 @@
     <parent>
         <artifactId>droid-parent</artifactId>
         <groupId>uk.gov.nationalarchives</groupId>
-        <version>6.9.5-SNAPSHOT</version>
+        <version>6.9.6-SNAPSHOT</version>
         <relativePath>../droid-parent</relativePath>
     </parent>
 

droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequest.java

@@ -109,4 +109,12 @@ public interface IdentificationRequest<T> extends Closeable {
      * @return the identifier
      */
     RequestIdentifier getIdentifier();
+
+    /**
+     * Is this a debug request.
+     * @return if it is a debug request.
+     */
+    default boolean isDebug() {
+        return false;
+    }
 }

droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java

@@ -168,7 +168,10 @@ public enum DroidGlobalProperty {
     FILES_FROM_S3("profile.s3", PropertyType.BOOLEAN, true),
 
     /** The last updated time of the binary signature file .*/
-    BINARY_SIGNATURE_LAST_UPDATED("profile.binarySignatureLastUpdated", PropertyType.LONG, true);
+    BINARY_SIGNATURE_LAST_UPDATED("profile.binarySignatureLastUpdated", PropertyType.LONG, true),
+
+    /** Output CSV which shows the number of bytes read per signature. */
+    DEBUG("profile.debug", PropertyType.BOOLEAN, true);
 
     private static Map<String, DroidGlobalProperty> allValues = new HashMap<String, DroidGlobalProperty>(); 
 

droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileReader.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2016, The National Archives <[email protected]>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following
+ * conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ *  * Neither the name of the The National Archives nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package uk.gov.nationalarchives.droid.core.interfaces.resource;
+
+import net.byteseek.io.reader.FileReader;
+import net.byteseek.io.reader.cache.WindowCache;
+import net.byteseek.io.reader.windows.Window;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+public class DebugFileReader extends FileReader {
+
+    private long bytesReadFromFile;
+
+    private long bytesReadFromCache;
+
+    public DebugFileReader(File file, WindowCache cache) throws FileNotFoundException {
+        super(file, cache);
+    }
+
+    @Override
+    protected Window createWindow(long windowStart) throws IOException {
+        bytesReadFromFile += windowSize;
+        return super.createWindow(windowStart);
+    }
+
+    @Override
+    public Window getWindow(final long position) throws IOException {
+        if (this.cache.getWindow(position) != null) {
+            bytesReadFromCache += windowSize;
+        }
+        return super.getWindow(position);
+    }
+
+    public long getBytesReadFromFile() {
+        return bytesReadFromFile;
+    }
+
+    public long getBytesReadFromCache() {
+        return bytesReadFromCache;
+    }
+
+    public void resetBytesRead() {
+        this.bytesReadFromFile = 0;
+        this.bytesReadFromCache = 0;
+    }
+}

droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileSystemIdentificationRequest.java

@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2016, The National Archives <[email protected]>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following
+ * conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ *  * Neither the name of the The National Archives nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package uk.gov.nationalarchives.droid.core.interfaces.resource;
+
+import net.byteseek.io.reader.cache.TopAndTailFixedLengthCache;
+import net.byteseek.io.reader.cache.WindowCache;
+import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public class DebugFileSystemIdentificationRequest extends FileSystemIdentificationRequest {
+
+    private static final int TOP_TAIL_BUFFER_CAPACITY = 8 * 1024 * 1024; // buffer 8Mb on the top and tail of files.
+
+    /**
+     * Constructs a new identification request.
+     *
+     * @param metaData   the metaData about the binary.
+     * @param identifier the request's identifier
+     */
+    public DebugFileSystemIdentificationRequest(RequestMetaData metaData, RequestIdentifier identifier) {
+        super(metaData, identifier);
+    }
+
+    @Override
+    public final void open(final Path theFile) throws IOException {
+        super.open(theFile);
+        final WindowCache cache = new TopAndTailFixedLengthCache(theFile.toFile().length(), TOP_TAIL_BUFFER_CAPACITY);
+        setFileReader(new DebugFileReader(theFile.toFile(), cache));
+    }
+
+    @Override
+    public boolean isDebug() {
+        return true;
+    }
+}

droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequest.java

@@ -56,10 +56,10 @@ public class FileSystemIdentificationRequest implements IdentificationRequest<Pa
     private String extension;
     private final String fileName;
     private final long size;
-    private WindowReader fileReader;
     private final RequestIdentifier identifier;
     private RequestMetaData requestMetaData;
     private Path file;
+    private WindowReader fileReader;
 
     /**
      * Constructs a new identification request.
@@ -77,7 +77,7 @@ public FileSystemIdentificationRequest(final RequestMetaData metaData, final Req
      * {@inheritDoc}
      */
     @Override
-    public final void open(final Path theFile) throws IOException {
+    public void open(final Path theFile) throws IOException {
         // Use a caching strategy that uses soft references, to allow the GC to reclaim
         // cached file bytes in low memory conditions.
         final WindowCache cache = new TopAndTailFixedLengthCache(theFile.toFile().length(), TOP_TAIL_BUFFER_CAPACITY);
@@ -124,7 +124,7 @@ public final long size() {
      * {@inheritDoc}
      */
     @Override
-    public final void close() throws IOException {
+    public void close() throws IOException {
         file = null;
         fileReader.close();
     }
@@ -176,4 +176,8 @@ public WindowReader getWindowReader() {
     public Path getFile() {
         return file;
     }
+
+    protected void setFileReader(WindowReader fileReader) {
+        this.fileReader = fileReader;
+    }
 }

droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileReaderTest.java

@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2016, The National Archives <[email protected]>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following
+ * conditions are met:
+ *
+ *  * Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ *  * Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ *  * Neither the name of the The National Archives nor the
+ *    names of its contributors may be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package uk.gov.nationalarchives.droid.core.interfaces.resource;
+
+import net.byteseek.io.reader.cache.WindowCache;
+import net.byteseek.io.reader.windows.Window;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Objects;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DebugFileReaderTest {
+
+    private DebugFileReader createDebugFileReader() throws IOException {
+        return createDebugFileReader(null);
+    }
+
+    private DebugFileReader createDebugFileReader(Window cacheResponse) throws IOException {
+        WindowCache windowCache = mock(WindowCache.class);
+        when(windowCache.getWindow(anyLong())).thenReturn(cacheResponse);
+        File file = new File(Objects.requireNonNull(DebugFileReaderTest.class.getResource("/testXmlFile.xml")).getPath());
+        return new DebugFileReader(file, windowCache);
+    }
+
+    @Test
+    public void testDebugFileReaderIncrementsReadBytesOnCreateWindow() throws IOException {
+        try (DebugFileReader debugFileReader = createDebugFileReader()) {
+            assertEquals(0, debugFileReader.getBytesReadFromFile());
+            debugFileReader.createWindow(0);
+            assertEquals(4096, debugFileReader.getBytesReadFromFile());
+        }
+    }
+
+    @Test
+    public void testDebugFileReaderDoesNotIncrementCacheBytesIfWindowNotInCache() throws IOException {
+        try (DebugFileReader debugFileReader = createDebugFileReader()) {
+            assertEquals(0, debugFileReader.getBytesReadFromCache());
+            debugFileReader.getWindow(0);
+            assertEquals(0, debugFileReader.getBytesReadFromCache());
+        }
+    }
+
+    @Test
+    public void testDebugFileReaderIncrementsCacheBytesIfWindowInCache() throws IOException {
+        try(DebugFileReader debugFileReader = createDebugFileReader(mock(Window.class))) {
+            assertEquals(0, debugFileReader.getBytesReadFromCache());
+            debugFileReader.getWindow(0);
+            assertEquals(4096, debugFileReader.getBytesReadFromCache());
+        }
+    }
+}