diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b9726ebe..e31ddd9d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,4 +59,4 @@ jobs: - run: | mvn clean package -DskipTests=true docker build -t cli-tests -f cli-tests/Dockerfile . - docker run -v "$PWD:/code" cli-tests /code/cli-tests/test \ No newline at end of file + docker run -v "$PWD:/code" cli-tests /code/cli-tests/test diff --git a/cli-tests/test/cli-test.bats b/cli-tests/test/cli-test.bats index e1caff8f0..02a42d6f5 100644 --- a/cli-tests/test/cli-test.bats +++ b/cli-tests/test/cli-test.bats @@ -54,24 +54,26 @@ setup() { @test "check for signature updates" { run $DROID_BIN -c - assert_output 'No signature updates are available' + assert_output -p 'Container signature update Version 20250925 is available' + assert_output -p 'Binary signature update Version 121 is available' } @test "download signature updates" { run $DROID_BIN -d - assert_output 'No signature updates are available' + assert_output -p 'Signature update version 20,250,925 has been downloaded' + assert_output -p 'Signature update version 121 has been downloaded' } @test "list signature files" { run $DROID_BIN -X - assert_output -p "Type: Binary Version: 120 File name: DROID_SignatureFile_V120.xml" - assert_output -p "Type: Container Version: 20240715 File name: container-signature-20240715.xml" + assert_output -p "Type: Binary Version: 121 File name: DROID_SignatureFile_V121.xml" + assert_output -p "Type: Container Version: 20250925 File name: container-signature-20250925.xml" } @test "display signature file" { run $DROID_BIN -x - assert_output -p "Type: Container Version: 20240715 File name: container-signature-20240715.xml" - assert_output -p "Type: Binary Version: 120 File name: DROID_SignatureFile_V120.xml" + assert_output -p "Type: Container Version: 20250925 File name: container-signature-20250925.xml" + assert_output -p "Type: Binary Version: 121 File name: DROID_SignatureFile_V121.xml" } @test "set signature file" { diff --git a/droid-api/pom.xml b/droid-api/pom.xml index 6abb3d33f..e9d2c08d4 100644 --- a/droid-api/pom.xml +++ b/droid-api/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-binary/pom.xml b/droid-binary/pom.xml index 6cee74689..0601d764d 100644 --- a/droid-binary/pom.xml +++ b/droid-binary/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-build-tools/pom.xml b/droid-build-tools/pom.xml index 53dc00d88..db149abf2 100644 --- a/droid-build-tools/pom.xml +++ b/droid-build-tools/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-command-line/pom.xml b/droid-command-line/pom.xml index 75ab6a477..ea99ee09e 100644 --- a/droid-command-line/pom.xml +++ b/droid-command-line/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-container/pom.xml b/droid-container/pom.xml index 14ff512fc..7264f0cbd 100644 --- a/droid-container/pom.xml +++ b/droid-container/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-core-interfaces/pom.xml b/droid-core-interfaces/pom.xml index 9a2ab0c54..d6e537f22 100644 --- a/droid-core-interfaces/pom.xml +++ b/droid-core-interfaces/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequest.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequest.java index 203b75029..887b80ac7 100644 --- a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequest.java +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/IdentificationRequest.java @@ -109,4 +109,12 @@ public interface IdentificationRequest extends Closeable { * @return the identifier */ RequestIdentifier getIdentifier(); + + /** + * Is this a debug request. + * @return if it is a debug request. + */ + default boolean isDebug() { + return false; + } } diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java index fc54ff249..6db583f12 100644 --- a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/config/DroidGlobalProperty.java @@ -168,7 +168,10 @@ public enum DroidGlobalProperty { FILES_FROM_S3("profile.s3", PropertyType.BOOLEAN, true), /** The last updated time of the binary signature file .*/ - BINARY_SIGNATURE_LAST_UPDATED("profile.binarySignatureLastUpdated", PropertyType.LONG, true); + BINARY_SIGNATURE_LAST_UPDATED("profile.binarySignatureLastUpdated", PropertyType.LONG, true), + + /** Output CSV which shows the number of bytes read per signature. */ + DEBUG("profile.debug", PropertyType.BOOLEAN, true); private static Map allValues = new HashMap(); diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileReader.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileReader.java new file mode 100644 index 000000000..2da64f9bc --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileReader.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.FileReader; +import net.byteseek.io.reader.cache.WindowCache; +import net.byteseek.io.reader.windows.Window; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; + +public class DebugFileReader extends FileReader { + + private long bytesReadFromFile; + + private long bytesReadFromCache; + + public DebugFileReader(File file, WindowCache cache) throws FileNotFoundException { + super(file, cache); + } + + @Override + protected Window createWindow(long windowStart) throws IOException { + bytesReadFromFile += windowSize; + return super.createWindow(windowStart); + } + + @Override + public Window getWindow(final long position) throws IOException { + if (this.cache.getWindow(position) != null) { + bytesReadFromCache += windowSize; + } + return super.getWindow(position); + } + + public long getBytesReadFromFile() { + return bytesReadFromFile; + } + + public long getBytesReadFromCache() { + return bytesReadFromCache; + } + + public void resetBytesRead() { + this.bytesReadFromFile = 0; + this.bytesReadFromCache = 0; + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileSystemIdentificationRequest.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileSystemIdentificationRequest.java new file mode 100644 index 000000000..7cd43b783 --- /dev/null +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileSystemIdentificationRequest.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.cache.TopAndTailFixedLengthCache; +import net.byteseek.io.reader.cache.WindowCache; +import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; + +import java.io.IOException; +import java.nio.file.Path; + +public class DebugFileSystemIdentificationRequest extends FileSystemIdentificationRequest { + + private static final int TOP_TAIL_BUFFER_CAPACITY = 8 * 1024 * 1024; // buffer 8Mb on the top and tail of files. + + /** + * Constructs a new identification request. + * + * @param metaData the metaData about the binary. + * @param identifier the request's identifier + */ + public DebugFileSystemIdentificationRequest(RequestMetaData metaData, RequestIdentifier identifier) { + super(metaData, identifier); + } + + @Override + public final void open(final Path theFile) throws IOException { + super.open(theFile); + final WindowCache cache = new TopAndTailFixedLengthCache(theFile.toFile().length(), TOP_TAIL_BUFFER_CAPACITY); + setFileReader(new DebugFileReader(theFile.toFile(), cache)); + } + + @Override + public boolean isDebug() { + return true; + } +} diff --git a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequest.java b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequest.java index d4d140f96..eb2207bc3 100644 --- a/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequest.java +++ b/droid-core-interfaces/src/main/java/uk/gov/nationalarchives/droid/core/interfaces/resource/FileSystemIdentificationRequest.java @@ -56,10 +56,10 @@ public class FileSystemIdentificationRequest implements IdentificationRequest + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import net.byteseek.io.reader.cache.WindowCache; +import net.byteseek.io.reader.windows.Window; +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class DebugFileReaderTest { + + private DebugFileReader createDebugFileReader() throws IOException { + return createDebugFileReader(null); + } + + private DebugFileReader createDebugFileReader(Window cacheResponse) throws IOException { + WindowCache windowCache = mock(WindowCache.class); + when(windowCache.getWindow(anyLong())).thenReturn(cacheResponse); + File file = new File(Objects.requireNonNull(DebugFileReaderTest.class.getResource("/testXmlFile.xml")).getPath()); + return new DebugFileReader(file, windowCache); + } + + @Test + public void testDebugFileReaderIncrementsReadBytesOnCreateWindow() throws IOException { + try (DebugFileReader debugFileReader = createDebugFileReader()) { + assertEquals(0, debugFileReader.getBytesReadFromFile()); + debugFileReader.createWindow(0); + assertEquals(4096, debugFileReader.getBytesReadFromFile()); + } + } + + @Test + public void testDebugFileReaderDoesNotIncrementCacheBytesIfWindowNotInCache() throws IOException { + try (DebugFileReader debugFileReader = createDebugFileReader()) { + assertEquals(0, debugFileReader.getBytesReadFromCache()); + debugFileReader.getWindow(0); + assertEquals(0, debugFileReader.getBytesReadFromCache()); + } + } + + @Test + public void testDebugFileReaderIncrementsCacheBytesIfWindowInCache() throws IOException { + try(DebugFileReader debugFileReader = createDebugFileReader(mock(Window.class))) { + assertEquals(0, debugFileReader.getBytesReadFromCache()); + debugFileReader.getWindow(0); + assertEquals(4096, debugFileReader.getBytesReadFromCache()); + } + } +} diff --git a/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileSystemIdentificationRequestTest.java b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileSystemIdentificationRequestTest.java new file mode 100644 index 000000000..1dce7d374 --- /dev/null +++ b/droid-core-interfaces/src/test/java/uk/gov/nationalarchives/droid/core/interfaces/resource/DebugFileSystemIdentificationRequestTest.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2016, The National Archives + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.core.interfaces.resource; + +import org.junit.jupiter.api.Test; +import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Objects; + +import static org.junit.jupiter.api.Assertions.assertInstanceOf; + +public class DebugFileSystemIdentificationRequestTest { + + @Test + public void testFileReaderIsOfCorrectType() throws URISyntaxException, IOException { + Path file = Paths.get(Objects.requireNonNull(FileSystemIdentificationRequestTest.class.getResource("/testXmlFile.xml")).toURI()); + RequestMetaData metaData = new RequestMetaData(Files.size(file), Files.getLastModifiedTime(file).toMillis(), "testXmlFile.xml"); + RequestIdentifier identifier = new RequestIdentifier(file.toUri()); + try (DebugFileSystemIdentificationRequest request = new DebugFileSystemIdentificationRequest(metaData, identifier)) { + request.open(file); + assertInstanceOf(DebugFileReader.class, request.getWindowReader()); + } + } +} diff --git a/droid-core/pom.xml b/droid-core/pom.xml index 37e15f7c0..af79798fb 100644 --- a/droid-core/pom.xml +++ b/droid-core/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/IdentificationRequestByteReaderAdapter.java b/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/IdentificationRequestByteReaderAdapter.java index c4b7278b0..4140f757b 100644 --- a/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/IdentificationRequestByteReaderAdapter.java +++ b/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/IdentificationRequestByteReaderAdapter.java @@ -47,11 +47,12 @@ * Adapts an IdentificationRequest to the ByteReader interface. * @author rflitcroft * + * @param The type of the identification request. */ -public class IdentificationRequestByteReaderAdapter implements ByteReader { +public class IdentificationRequestByteReaderAdapter implements ByteReader { private Logger log = LoggerFactory.getLogger(this.getClass()); - private IdentificationRequest request; + private final IdentificationRequest request; private long fileMarker; private List hits = new ArrayList(); @@ -60,7 +61,7 @@ public class IdentificationRequestByteReaderAdapter implements ByteReader { * * @param request the request to wrap. */ - public IdentificationRequestByteReaderAdapter(IdentificationRequest request) { + public IdentificationRequestByteReaderAdapter(IdentificationRequest request) { this.request = request; } @@ -84,6 +85,11 @@ public final void close() { } } + @Override + public IdentificationRequest getRequest() { + return request; + } + /** * Not supported. * @return Not Supported diff --git a/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/ByteReader.java b/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/ByteReader.java index 2fa79d474..6e4992fd1 100644 --- a/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/ByteReader.java +++ b/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/ByteReader.java @@ -70,15 +70,18 @@ package uk.gov.nationalarchives.droid.core.signature; import net.byteseek.io.reader.WindowReader; +import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; /** * Interface for accessing the bytes from a file, URL or stream. * *

Create an instance with AbstractByteReader.newByteReader().

* + * @param The type of the identification request + * * @author linb, boreilly */ -public interface ByteReader extends AutoCloseable { +public interface ByteReader extends AutoCloseable { /* Setters for identification status */ /** @@ -215,4 +218,10 @@ public interface ByteReader extends AutoCloseable { * Closes any files associated with the ByteReader. */ void close(); + + /** + * Gets the request. + * @return The identification request. + */ + IdentificationRequest getRequest(); } diff --git a/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/droid6/InternalSignatureCollection.java b/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/droid6/InternalSignatureCollection.java index b68791e71..f35570caa 100644 --- a/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/droid6/InternalSignatureCollection.java +++ b/droid-core/src/main/java/uk/gov/nationalarchives/droid/core/signature/droid6/InternalSignatureCollection.java @@ -53,14 +53,12 @@ */ package uk.gov.nationalarchives.droid.core.signature.droid6; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import uk.gov.nationalarchives.droid.core.interfaces.resource.DebugFileReader; +import uk.gov.nationalarchives.droid.core.interfaces.resource.DebugFileSystemIdentificationRequest; import uk.gov.nationalarchives.droid.core.signature.ByteReader; import uk.gov.nationalarchives.droid.core.signature.xml.SimpleElement; @@ -81,9 +79,9 @@ public class InternalSignatureCollection extends SimpleElement { //BNO there is one instance of this for the entire profile - not each request private static final int DEFAULT_COLLECTION_SIZE = 10; - private List intSigs = new ArrayList(DEFAULT_COLLECTION_SIZE); - private Map sigsByID = new HashMap(); - + private List intSigs = new ArrayList<>(DEFAULT_COLLECTION_SIZE); + private Map sigsByID = new HashMap<>(); + /** * Runs all the signatures against the target file, * adding a hit for each of them, if any of them match. @@ -99,15 +97,33 @@ public List getMatchingSignatures(ByteReader targetFile, long final int stop = intSigs.size(); for (int sigIndex = 0; sigIndex < stop; sigIndex++) { final InternalSignature internalSig = intSigs.get(sigIndex); - if (internalSig.matches(targetFile, maxBytesToScan)) { + boolean matches = internalSig.matches(targetFile, maxBytesToScan); + if (matches) { matchingSigs.add(internalSig); } + + if (targetFile.getRequest().isDebug()) { + outputDebugInformation((DebugFileSystemIdentificationRequest) targetFile.getRequest(), sigIndex, internalSig, matches); + } } } return matchingSigs; } - - + + private void outputDebugInformation(DebugFileSystemIdentificationRequest request, int sigIndex, InternalSignature internalSig, boolean matches) { + if (sigIndex == 0) { + System.out.println("SignatureID,BytesReadFromFile,BytesReadFromCache,Matched"); + } + DebugFileReader debugFileReader = (DebugFileReader) request.getWindowReader(); + + String row = Stream.of(internalSig.getID(), debugFileReader.getBytesReadFromFile(), debugFileReader.getBytesReadFromCache(), matches) + .map(Object::toString) + .collect(Collectors.joining(",")); + System.out.println(row); + debugFileReader.resetBytesRead(); + } + + /** * Prepares the internal signatures in the collection for use. */ diff --git a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java index 00c86efeb..a8a774439 100644 --- a/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java +++ b/droid-core/src/test/java/uk/gov/nationalarchives/droid/core/SkeletonSuiteTest.java @@ -57,10 +57,7 @@ import uk.gov.nationalarchives.droid.core.interfaces.IdentificationResult; import uk.gov.nationalarchives.droid.core.interfaces.IdentificationResultCollection; import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; -import uk.gov.nationalarchives.droid.core.interfaces.resource.FileSystemIdentificationRequest; -import uk.gov.nationalarchives.droid.core.interfaces.resource.HttpIdentificationRequest; -import uk.gov.nationalarchives.droid.core.interfaces.resource.RequestMetaData; -import uk.gov.nationalarchives.droid.core.interfaces.resource.S3IdentificationRequest; +import uk.gov.nationalarchives.droid.core.interfaces.resource.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -189,9 +186,14 @@ public static Stream> identificationRequests() { s3Identifier.setParentId(1L); RequestIdentifier httpIdentifier = new RequestIdentifier(httpUri); httpIdentifier.setParentId(1L); + DebugFileSystemIdentificationRequest debugFileSystemIdentificationRequest = new DebugFileSystemIdentificationRequest(metaData, fileIdentifier); + debugFileSystemIdentificationRequest.open(skeletonPath); + builder.add(debugFileSystemIdentificationRequest); + FileSystemIdentificationRequest fileSystemIdentificationRequest = new FileSystemIdentificationRequest(metaData, fileIdentifier); fileSystemIdentificationRequest.open(skeletonPath); builder.add(fileSystemIdentificationRequest); + S3IdentificationRequest s3IdentificationRequest = new S3IdentificationRequest(metaData, s3Identifier, new TestS3Client()); s3IdentificationRequest.open(s3Uri); builder.add(s3IdentificationRequest); diff --git a/droid-export-interfaces/pom.xml b/droid-export-interfaces/pom.xml index 0e96c3845..60310076c 100644 --- a/droid-export-interfaces/pom.xml +++ b/droid-export-interfaces/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-export/pom.xml b/droid-export/pom.xml index be546228f..eda8b210f 100644 --- a/droid-export/pom.xml +++ b/droid-export/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-help/pom.xml b/droid-help/pom.xml index 5cd0bcfe6..270d87646 100644 --- a/droid-help/pom.xml +++ b/droid-help/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/droid-parent/pom.xml b/droid-parent/pom.xml index 5b330bb59..f571d6a57 100644 --- a/droid-parent/pom.xml +++ b/droid-parent/pom.xml @@ -4,7 +4,7 @@ uk.gov.nationalarchives droid-parent - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT pom droid-parent @@ -98,17 +98,17 @@ 10.17.1.0 4.1.3 2.1.0 - 4.0.5 - 4.0.2 + 4.0.6 + 4.0.4 2.19.2 9.7.2 0.14.0 1.1.3 3.0 2.0.17 - 2.25.1 + 2.25.2 10.21.2 - 2.33.7 + 2.34.1 5.13.4 2.20.0 @@ -161,7 +161,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.14.0 + 3.14.1 org.jacoco @@ -171,12 +171,12 @@ org.apache.maven.plugins maven-surefire-plugin - 3.5.3 + 3.5.4 org.apache.maven.plugins maven-surefire-report-plugin - 3.5.3 + 3.5.4 org.apache.maven.plugins @@ -192,7 +192,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.11.2 + 3.12.0 @@ -280,21 +280,6 @@ - - org.owasp - dependency-check-maven - 12.1.3 - - 8 - - - - - check - - - - org.apache.maven.plugins maven-resources-plugin @@ -792,7 +777,7 @@ Copyright © ${project.inceptionYear}-{currentYear} + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following + * conditions are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * * Neither the name of the The National Archives nor the + * names of its contributors may be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package uk.gov.nationalarchives.droid.submitter; + +import uk.gov.nationalarchives.droid.core.interfaces.IdentificationRequest; +import uk.gov.nationalarchives.droid.core.interfaces.RequestIdentifier; +import uk.gov.nationalarchives.droid.core.interfaces.archive.IdentificationRequestFactory; +import uk.gov.nationalarchives.droid.core.interfaces.resource.DebugFileSystemIdentificationRequest; +import uk.gov.nationalarchives.droid.core.interfaces.resource.RequestMetaData; + +import java.nio.file.Path; + +/** + * @author rflitcroft + * + */ +public class DebugFileIdentificationRequestFactory implements IdentificationRequestFactory { + + /** + * {@inheritDoc} + */ + @Override + public final IdentificationRequest newRequest(RequestMetaData metaData, + RequestIdentifier identifier) { + return new DebugFileSystemIdentificationRequest(metaData, identifier); + } + +} diff --git a/droid-results/src/main/resources/META-INF/spring-results.xml b/droid-results/src/main/resources/META-INF/spring-results.xml index a35b0da1c..ef13ab80a 100644 --- a/droid-results/src/main/resources/META-INF/spring-results.xml +++ b/droid-results/src/main/resources/META-INF/spring-results.xml @@ -386,4 +386,16 @@ http://www.springframework.org/schema/context http://www.springframework.org/sch + + + + + + + + + + + + diff --git a/droid-swing-ui/pom.xml b/droid-swing-ui/pom.xml index d41367990..c27199629 100644 --- a/droid-swing-ui/pom.xml +++ b/droid-swing-ui/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent @@ -191,7 +191,7 @@ org.netbeans.api org-netbeans-swing-outline - RELEASE260 + RELEASE270 org.jdesktop diff --git a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java index a92e55581..90204a82f 100644 --- a/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java +++ b/droid-swing-ui/src/main/java/uk/gov/nationalarchives/droid/gui/DroidMainFrame.java @@ -298,7 +298,7 @@ public void windowClosing(WindowEvent e) { helpMenuItem.addActionListener(evt -> { try { - URI uri = globalContext.getGlobalConfig().getHelpPagesDir().resolve("indexs.html").toUri(); + URI uri = globalContext.getGlobalConfig().getHelpPagesDir().resolve("index.html").toUri(); Desktop.getDesktop().browse(uri); } catch (IOException e) { throw new RuntimeException(e); diff --git a/droid-tools/pom.xml b/droid-tools/pom.xml index 294787436..0e9fdce3c 100644 --- a/droid-tools/pom.xml +++ b/droid-tools/pom.xml @@ -5,7 +5,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT ../droid-parent diff --git a/pom.xml b/pom.xml index 8beee9ce5..a5ac98d82 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ droid-parent uk.gov.nationalarchives - 6.9.5-SNAPSHOT + 6.9.6-SNAPSHOT droid-parent