Feature/add blurry analysis

.gitignore

@@ -5,6 +5,7 @@ application-*.properties
 !application-test.properties
 document_analysis.properties
 .env
+dossierfacile-process-file/testResult/
 target/
 mockstorage/
 mock-storage/

Aptfile

@@ -1,3 +1,5 @@
 ttf-mscorefonts-installer
 tesseract-ocr
 imagemagick
+libblas-dev
+libopencv-dev

Procfile

@@ -1 +1 @@
-web: java $JVM_OPTIONS -Djna.library.path=$JNA_LIBRARY_PATH -jar $APP_DIR/target/$APP_DIR.jar
+web: bash bin/start.sh

bin/start.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+ln -s /app/.apt/usr/lib/x86_64-linux-gnu/lapack/liblapack.so.3 /app/.apt/usr/lib/x86_64-linux-gnu/
+ln -s /app/.apt/usr/lib/x86_64-linux-gnu/blas/libblas.so.3 /app/.apt/usr/lib/x86_64-linux-gnu/
+
+java $JVM_OPTIONS -Djna.library.path=$JNA_LIBRARY_PATH -jar $APP_DIR/target/$APP_DIR.jar

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/BlurryFileAnalysis.java

@@ -0,0 +1,43 @@
+package fr.dossierfacile.common.entity;
+
+import fr.dossierfacile.common.entity.ocr.BlurryResult;
+import fr.dossierfacile.common.enums.BlurryFileAnalysisStatus;
+import jakarta.persistence.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.hibernate.annotations.JdbcTypeCode;
+import org.hibernate.type.SqlTypes;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.List;
+
+@Data
+@Builder
+@Entity
+@Table(name = "blurry_file_analysis")
+@AllArgsConstructor
+@NoArgsConstructor
+public class BlurryFileAnalysis implements Serializable {
+
+    @Serial
+    private static final long serialVersionUID = 2405172041950251807L;
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @OneToOne(targetEntity = File.class, fetch = FetchType.LAZY)
+    @JoinColumn(name = "file_id")
+    private File file;
+
+    @Enumerated(EnumType.STRING)
+    private BlurryFileAnalysisStatus analysisStatus;
+
+    @JdbcTypeCode(SqlTypes.JSON)
+    @Column(columnDefinition = "jsonb")
+    private List<BlurryResult> blurryResults;
+
+}

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/DocumentRule.java

@@ -34,7 +34,9 @@ public enum DocumentRule {
     R_RENT_RECEIPT_NB_DOCUMENTS(Level.WARN, "Un seul document a été détecté"),
 
     R_FRANCE_IDENTITE_NAMES(Level.CRITICAL, "Les noms et prénoms ne correspondent pas"),
-    R_FRANCE_IDENTITE_STATUS(Level.CRITICAL, "Ce document n'a pas pu être validé par France Identité");
+    R_FRANCE_IDENTITE_STATUS(Level.CRITICAL, "Ce document n'a pas pu être validé par France Identité"),
+
+    R_BLURRY_FILE(Level.CRITICAL, "Votre document semble flou");
 
     public enum Level {
         CRITICAL, WARN

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/File.java

@@ -3,12 +3,7 @@
 import fr.dossierfacile.common.enums.FileStorageStatus;
 import jakarta.annotation.Nullable;
 import jakarta.persistence.*;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.ToString;
+import lombok.*;
 import org.hibernate.Hibernate;
 
 import java.io.Serial;
@@ -56,9 +51,13 @@ public class File implements Serializable {
     private BarCodeFileAnalysis fileAnalysis;
 
     @Nullable
-    @OneToOne(mappedBy= "file", fetch = FetchType.LAZY)
+    @OneToOne(mappedBy = "file", fetch = FetchType.LAZY)
     private ParsedFileAnalysis parsedFileAnalysis;
 
+    @Nullable
+    @OneToOne(mappedBy = "file", fetch = FetchType.LAZY)
+    private BlurryFileAnalysis blurryFileAnalysis;
+
     @PreRemove
     void deleteCascade() {
         if (storageFile != null)

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ocr/BlurryAlgorithmType.java

@@ -0,0 +1,8 @@
+package fr.dossierfacile.common.entity.ocr;
+
+public enum BlurryAlgorithmType {
+    LAPLACIEN,
+    FFT,
+    SOBEL,
+    DOG
+}

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/ocr/BlurryResult.java

@@ -0,0 +1,13 @@
+package fr.dossierfacile.common.entity.ocr;
+
+import java.io.Serial;
+import java.io.Serializable;
+
+public record BlurryResult(
+        BlurryAlgorithmType algorithm,
+        double score
+) implements Serializable {
+
+    @Serial
+    private static final long serialVersionUID = 8347582394758234758L;
+}

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/enums/BlurryFileAnalysisStatus.java

@@ -0,0 +1,6 @@
+package fr.dossierfacile.common.enums;
+
+public enum BlurryFileAnalysisStatus {
+    COMPLETED,
+    FAILED
+}

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/repository/BlurryFileAnalysisRepository.java

@@ -0,0 +1,7 @@
+package fr.dossierfacile.common.repository;
+
+import fr.dossierfacile.common.entity.BlurryFileAnalysis;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface BlurryFileAnalysisRepository extends JpaRepository<BlurryFileAnalysis, Long> {
+}

dossierfacile-common-library/src/main/resources/db/changelog/databaseChangeLog.xml

@@ -166,5 +166,6 @@
     <include file="db/migration/202503170000-set-default-category_step-for-financial-doc.xml"/>
     <include file="db/migration/202504230000-update-cdi-trial.xml"/>
     <include file="db/migration/202504290000-update-document-denied-reason.xml"/>
+    <include file="db/migration/202504220000-create-blurry-file-analysis-table.xml" />
 
 </databaseChangeLog>

dossierfacile-common-library/src/main/resources/db/migration/202504220000-create-blurry-file-analysis-table.xml

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+                                       http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd">
+
+    <changeSet id="202504220000-01" author="Nicolas Sagon">
+        <createTable tableName="blurry_file_analysis">
+            <column name="id" type="BIGINT" autoIncrement="true">
+                <constraints primaryKey="true" nullable="false" />
+            </column>
+            <column name="file_id" type="BIGINT">
+                <constraints nullable="false" />
+            </column>
+            <column name="analysis_status" type="VARCHAR(32)">
+                <constraints nullable="true" />
+            </column>
+            <column name="blurry_results" type="jsonb">
+                <constraints nullable="true" />
+            </column>
+        </createTable>
+    </changeSet>
+    <changeSet id="202504220000-02" author="Nicolas Sagon">
+        <addForeignKeyConstraint baseTableName="blurry_file_analysis"
+                                 baseColumnNames="file_id"
+                                 constraintName="fk_file_blurry_file_analysis"
+                                 referencedTableName="file"
+                                 referencedColumnNames="id"
+                                 onDelete="CASCADE" />
+    </changeSet>
+    <changeSet id="202504220000-03" author="Nicolas Sagon">
+        <addUniqueConstraint
+                tableName="blurry_file_analysis"
+                columnNames="file_id"
+                constraintName="uc_blurry_file_analysis_file_id"
+        />
+    </changeSet>
+</databaseChangeLog>

dossierfacile-common-test-library/src/main/java/fr/dossierfacile/fileAnalysis/FileAnalysisTestData.kt

@@ -35,16 +35,21 @@ data class FileAnalysisTestData<V, I, F>(
     val invalidDocuments: List<InvalidDocumentData<I, F>>
 )
 
-abstract class DocumentData<F>(open val bucketPath: String, open val fileDescription: F)
+abstract class DocumentData<F>(open val bucketPath: String, open val fileDescription: F?)
+
 
 data class ValidDocumentData<T, F>(
     override val bucketPath: String,
-    override val fileDescription: F,
-    val expectedResult: T
-) : DocumentData<F>(bucketPath, fileDescription)
+    override val fileDescription: F?,
+    val expectedResult: T?
+) : DocumentData<F>(bucketPath, fileDescription) {
+    constructor(bucketPath: String) : this(bucketPath, null, null)
+}
 
 data class InvalidDocumentData<T, F>(
     override val bucketPath: String,
-    override val fileDescription: F,
-    val expectedError: T
-) : DocumentData<F>(bucketPath, fileDescription)
+    override val fileDescription: F?,
+    val expectedError: T?
+) : DocumentData<F>(bucketPath, fileDescription) {
+    constructor(bucketPath: String) : this(bucketPath, null, null)
+}

dossierfacile-process-file/pom.xml

@@ -112,6 +112,13 @@
             <artifactId>commons-imaging</artifactId>
             <version>1.0-alpha3</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.bytedeco</groupId>
+            <artifactId>opencv</artifactId>
+            <version>4.10.0-1.5.11</version>
+        </dependency>
+
     </dependencies>
     <build>
         <finalName>dossierfacile-process-file</finalName>

dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/AnalyzeFileService.java

@@ -4,6 +4,7 @@
 import fr.dossierfacile.process.file.repository.FileRepository;
 import fr.dossierfacile.process.file.service.processors.BarCodeFileProcessor;
 import fr.dossierfacile.process.file.service.processors.FileParserProcessor;
+import fr.dossierfacile.process.file.service.processors.blurry.BlurryProcessor;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
@@ -16,13 +17,15 @@
 public class AnalyzeFileService {
     private final BarCodeFileProcessor barCodeFileProcessor;
     private final FileParserProcessor fileParserProcessor;
+    private final BlurryProcessor blurryProcessor;
     private final FileRepository fileRepository;
 
     public void processFile(Long fileId) {
         Optional<File> optFile = fileRepository.findById(fileId);
         if (optFile.isPresent()) {
             barCodeFileProcessor.process(optFile.get());
             fileParserProcessor.process(optFile.get());
+            blurryProcessor.process(optFile.get());
         }
     }
 }

dossierfacile-process-file/src/main/java/fr/dossierfacile/process/file/service/documentrules/BlurryRulesValidationService.java

@@ -0,0 +1,102 @@
+package fr.dossierfacile.process.file.service.documentrules;
+
+import fr.dossierfacile.common.entity.*;
+import fr.dossierfacile.common.entity.ocr.BlurryAlgorithmType;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+import org.springframework.util.CollectionUtils;
+
+@Service
+@RequiredArgsConstructor
+@Slf4j
+public class BlurryRulesValidationService implements RulesValidationService {
+
+    @Value("${blurry.laplacian.threshold:400}")
+    private double laplacianThreshold;
+    @Value("${blurry.sobel.threshold:30}")
+    private double sobelThreshold;
+    @Value("${blurry.fft.threshold:170}")
+    private double fftThreshold;
+    @Value("${blurry.dog.threshold:20}")
+    private double dogThreshold;
+
+    @Override
+    public boolean shouldBeApplied(Document document) {
+        return document.getFiles().stream().allMatch(file -> file.getBlurryFileAnalysis() != null)
+                && !CollectionUtils.isEmpty(document.getFiles());
+    }
+
+    @Override
+    public DocumentAnalysisReport process(Document document, DocumentAnalysisReport report) {
+        var isBlurryDocument = false;
+        for (var file : document.getFiles()) {
+            if (isFileBlurry(file.getBlurryFileAnalysis())) {
+                isBlurryDocument = true;
+            }
+        }
+        if (isBlurryDocument) {
+            report.getBrokenRules().add(DocumentBrokenRule.builder()
+                    .rule(DocumentRule.R_BLURRY_FILE)
+                    .message(DocumentRule.R_BLURRY_FILE.getDefaultMessage())
+                    .build());
+            report.setAnalysisStatus(DocumentAnalysisStatus.DENIED);
+        }
+        if (report.getBrokenRules().isEmpty()) {
+            report.setAnalysisStatus(DocumentAnalysisStatus.CHECKED);
+        } else if (report.getBrokenRules().stream().anyMatch(r -> r.getRule().getLevel() == DocumentRule.Level.CRITICAL)) {
+            report.setAnalysisStatus(DocumentAnalysisStatus.DENIED);
+        } else {
+            report.setAnalysisStatus(DocumentAnalysisStatus.UNDEFINED);
+        }
+
+        return report;
+    }
+
+    private boolean isFileBlurry(BlurryFileAnalysis blurryFileAnalysis) {
+        var score = 0;
+        if (checkLaplacianBlurryFile(blurryFileAnalysis)) {
+            score++;
+        }
+        if (checkSobelBlurryFile(blurryFileAnalysis)) {
+            score++;
+        }
+        if (checkFFTBlurryFile(blurryFileAnalysis)) {
+            score++;
+        }
+        if (checkDogBlurryFile(blurryFileAnalysis)) {
+            score++;
+        }
+        return score >= 3;
+    }
+
+    private boolean checkLaplacianBlurryFile(BlurryFileAnalysis blurryFileAnalysis) {
+        var laplacianResult = blurryFileAnalysis.getBlurryResults().stream().filter(item -> item.algorithm() == BlurryAlgorithmType.LAPLACIEN).findFirst();
+        return laplacianResult
+                .filter(blurryResult -> blurryResult.score() < laplacianThreshold)
+                .isPresent();
+    }
+
+    private boolean checkSobelBlurryFile(BlurryFileAnalysis blurryFileAnalysis) {
+        var sobelResult = blurryFileAnalysis.getBlurryResults().stream().filter(item -> item.algorithm() == BlurryAlgorithmType.SOBEL).findFirst();
+        return sobelResult
+                .filter(blurryResult -> blurryResult.score() < sobelThreshold)
+                .isPresent();
+    }
+
+    private boolean checkFFTBlurryFile(BlurryFileAnalysis blurryFileAnalysis) {
+        var fftResult = blurryFileAnalysis.getBlurryResults().stream().filter(item -> item.algorithm() == BlurryAlgorithmType.FFT).findFirst();
+        return fftResult
+                .filter(blurryResult -> blurryResult.score() > fftThreshold)
+                .isPresent();
+    }
+
+    private boolean checkDogBlurryFile(BlurryFileAnalysis blurryFileAnalysis) {
+        var fftResult = blurryFileAnalysis.getBlurryResults().stream().filter(item -> item.algorithm() == BlurryAlgorithmType.DOG).findFirst();
+        return fftResult
+                .filter(blurryResult -> blurryResult.score() < dogThreshold)
+                .isPresent();
+    }
+
+}