Feature: Add tests for tenant residency files

.gitignore

@@ -5,6 +5,7 @@ application-*.properties
 !application-test.properties
 document_analysis.properties
 .env
+dossierfacile-process-file/testResult/
 target/
 mockstorage/
 mock-storage/
@@ -14,6 +15,7 @@ logs/
 hs_err_pid*.log
 tmp
 **/certs/*
+elastalert/dist
 
 ### STS ###
 .apt_generated

Aptfile

@@ -1,3 +1,5 @@
 ttf-mscorefonts-installer
 tesseract-ocr
 imagemagick
+libblas-dev
+libopencv-dev

Procfile

@@ -1 +1 @@
-web: java $JVM_OPTIONS -Djna.library.path=$JNA_LIBRARY_PATH -jar $APP_DIR/target/$APP_DIR.jar
+web: bash bin/start.sh

bin/start.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+ln -s /app/.apt/usr/lib/x86_64-linux-gnu/lapack/liblapack.so.3 /app/.apt/usr/lib/x86_64-linux-gnu/
+ln -s /app/.apt/usr/lib/x86_64-linux-gnu/blas/libblas.so.3 /app/.apt/usr/lib/x86_64-linux-gnu/
+
+java $JVM_OPTIONS -Djna.library.path=$JNA_LIBRARY_PATH -jar $APP_DIR/target/$APP_DIR.jar

dossierfacile-bo/src/main/java/fr/gouv/bo/controller/BODocumentDeniedOptionsController.java

@@ -23,7 +23,7 @@
 public class BODocumentDeniedOptionsController {
 
     private static final String EMAIL = "email";
-    private static final List<String> DOCUMENT_USER_TYPES = List.of("tenant", "guarantor");
+    private static final List<String> DOCUMENT_USER_TYPES = List.of("tenant", "guarantor", "all");
 
     private final DocumentDeniedOptionsService service;
 

dossierfacile-bo/src/main/java/fr/gouv/bo/controller/BOTenantController.java

@@ -218,10 +218,10 @@ private static String redirectToTenantPage(Tenant tenant) {
         return "redirect:/bo/colocation/" + tenant.getApartmentSharing().getId() + "#tenant" + tenant.getId();
     }
 
-    private List<ItemDetail> getItemDetailForSubcategoryOfDocument(DocumentSubCategory documentSubCategory, String tenantOrGuarantor) {
+    private List<ItemDetail> getItemDetailForSubcategoryOfDocument(DocumentSubCategory documentSubCategory, String documentUserType) {
 
         List<ItemDetail> itemDetails = new ArrayList<>();
-        for (DocumentDeniedOptions documentDeniedOptions : documentService.findDocumentDeniedOptionsByDocumentSubCategoryAndDocumentUserType(documentSubCategory, tenantOrGuarantor)) {
+        for (DocumentDeniedOptions documentDeniedOptions : documentService.findDocumentDeniedOptionsByDocumentSubCategoryAndDocumentUserTypeIncludeGeneric(documentSubCategory, documentUserType)) {
             ItemDetail itemDetail1 = ItemDetail.builder()
                     .check(false)
                     .message(documentDeniedOptions.getMessageValue())

dossierfacile-bo/src/main/java/fr/gouv/bo/repository/DocumentDeniedOptionsRepository.java

@@ -3,6 +3,7 @@
 import fr.dossierfacile.common.entity.DocumentDeniedOptions;
 import fr.dossierfacile.common.enums.DocumentSubCategory;
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Query;
 
 import java.util.List;
 
@@ -12,4 +13,17 @@ public interface DocumentDeniedOptionsRepository extends JpaRepository<DocumentD
 
     List<DocumentDeniedOptions> findAllByDocumentSubCategory(DocumentSubCategory documentSubCategory);
 
+    @Query(value = """
+            select *
+            from document_denied_options
+            where (document_user_type = 'all' or document_user_type = :documentUserType)
+              and (document_sub_category = 'UNDEFINED' or document_sub_category = :documentSubCategory)
+            ORDER BY
+                CASE
+                WHEN document_sub_category = 'UNDEFINED' THEN 1
+                ELSE 0
+            END, code
+            """, nativeQuery = true)
+    List<DocumentDeniedOptions> findAllByDocumentSubCategoryAndDocumentUserTypeIncludeGeneric(String documentSubCategory, String documentUserType);
+
 }

dossierfacile-bo/src/main/java/fr/gouv/bo/service/DocumentService.java

@@ -108,7 +108,7 @@ public void updateDocumentWithDocumentDeniedReasons(DocumentDeniedReasons docume
         documentRepository.updateDocumentWithDocumentDeniedReasons(documentDeniedReasons, documentId);
     }
 
-    public List<DocumentDeniedOptions> findDocumentDeniedOptionsByDocumentSubCategoryAndDocumentUserType(DocumentSubCategory documentSubCategory, String tenantOrGuarantor) {
-        return documentDeniedOptionsRepository.findAllByDocumentSubCategoryAndDocumentUserTypeOrderByCode(documentSubCategory, tenantOrGuarantor);
+    public List<DocumentDeniedOptions> findDocumentDeniedOptionsByDocumentSubCategoryAndDocumentUserTypeIncludeGeneric(DocumentSubCategory documentSubCategory, String documentUserType) {
+        return documentDeniedOptionsRepository.findAllByDocumentSubCategoryAndDocumentUserTypeIncludeGeneric(documentSubCategory.name(), documentUserType);
     }
 }

dossierfacile-bo/src/main/java/fr/gouv/bo/service/TenantLogService.java

@@ -1,10 +1,14 @@
 package fr.gouv.bo.service;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import fr.dossierfacile.common.entity.Document;
 import fr.dossierfacile.common.entity.Tenant;
 import fr.dossierfacile.common.entity.TenantLog;
 import fr.dossierfacile.common.enums.LogType;
+import fr.dossierfacile.common.model.log.EditedDocument;
+import fr.dossierfacile.common.model.log.EditionType;
 import fr.gouv.bo.repository.BoTenantLogRepository;
 import lombok.AllArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
@@ -48,11 +52,22 @@ public void addOperatorCommentLog(Tenant tenant, String operatorComment) {
                 .logType(LogType.OPERATOR_COMMENT)
                 .tenantId(tenant.getId())
                 .creationDateTime(LocalDateTime.now())
-                .logDetails(writeAsString(Map.of("comment", operatorComment)))
+                .logDetails(writeAsObjectNode(Map.of("comment", operatorComment)))
                 .build();
         saveByLog(log);
     }
 
+    public void addDeleteDocumentLog(Long tenantId, Long operatorId, Document document) {
+        TenantLog log = TenantLog.builder()
+            .logType(LogType.ACCOUNT_EDITED)
+            .tenantId(tenantId)
+            .operatorId(operatorId)
+            .creationDateTime(LocalDateTime.now())
+            .logDetails(writeAsObjectNode(EditedDocument.from(document, EditionType.DELETE)))
+            .build();
+        saveByLog(log);
+    }
+
     public List<Object[]> listLastTreatedFilesByOperator(Long operatorId, int minusDays) {
         return logRepository.countTreatedFromXDaysGroupByDate(operatorId, minusDays);
     }
@@ -61,12 +76,14 @@ public List<Object[]> listDailyTreatedFilesByOperator() {
         return logRepository.countTreatedFromTodayGroupByOperator();
     }
 
-    private String writeAsString(Object object) {
+    private ObjectNode writeAsObjectNode(Object object) {
         try {
-            return objectMapper.writeValueAsString(object);
-        } catch (JsonProcessingException e) {
-            log.error("FATAL: Cannot write log details as string", e);
+            return (ObjectNode) objectMapper.valueToTree(object);
+        } catch (IllegalArgumentException e) {
+            log.error("FATAL: Cannot write log details as object node", e);
         }
         return null;
     }
+
+
 }

dossierfacile-bo/src/main/java/fr/gouv/bo/service/TenantService.java

@@ -261,6 +261,14 @@ private void processDocumentDeniedReasons(MessageItem messageItem, List<Long> do
         if (!documentDeniedReasons.getCheckedOptionsId().isEmpty() || (documentDeniedReasons.getComment() != null && !documentDeniedReasons.getComment().isBlank())) {
             Document document = documentRepository.findById(messageItem.getDocumentId()).orElseThrow(() -> new DocumentNotFoundException(messageItem.getDocumentId()));
             documentDeniedReasons.setDocument(document);
+            documentDeniedReasons.setDocumentCategory(document.getDocumentCategory());
+            documentDeniedReasons.setDocumentSubCategory(document.getDocumentSubCategory());
+            documentDeniedReasons.setDocumentCategoryStep(document.getDocumentCategoryStep());
+            if (document.getGuarantor() != null) {
+                documentDeniedReasons.setDocumentTenantType("guarantor");
+            } else {
+                documentDeniedReasons.setDocumentTenantType("tenant");
+            }
             documentDeniedReasonsRepository.save(documentDeniedReasons);
             DocumentDeniedReasons documentDeniedReasonsToDelete = document.getDocumentDeniedReasons();
             documentService.updateDocumentWithDocumentDeniedReasons(documentDeniedReasons, messageItem.getDocumentId());
@@ -773,7 +781,9 @@ public Optional<Tenant> getOldestToProcessApplication() {
 
     @Transactional
     public Tenant deleteDocument(Long id, User operator) {
+        Document document = documentService.findDocumentById(id);
         Tenant tenant = documentService.deleteDocument(id);
+        tenantLogService.addDeleteDocumentLog(tenant.getId(), operator.getId(), document);
         apartmentSharingService.resetDossierPdfGenerated(tenant.getApartmentSharing());
         updateTenantStatus(tenant, operator);
         return tenant;

dossierfacile-bo/src/main/java/fr/gouv/bo/service/UserService.java

@@ -1,7 +1,7 @@
 package fr.gouv.bo.service;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.google.gson.Gson;
 import fr.dossierfacile.common.entity.*;
 import fr.dossierfacile.common.enums.*;
@@ -89,7 +89,7 @@ private void saveAndDeleteInfoByTenant(Tenant tenant, BOUser operator) {
                         .logType(LogType.ACCOUNT_DELETE)
                         .tenantId(tenant.getId())
                         .operatorId(operator.getId())
-                        .jsonProfile(writeValueAsString(tenantMapper.toTenantModel(tenant)))
+                        .jsonProfile(writeAsObjectNode(tenantMapper.toTenantModel(tenant)))
                         .creationDateTime(LocalDateTime.now())
                         .build());
     }
@@ -184,11 +184,11 @@ public void createUserByEmail(String email, Role role) {
         addRoles(userRepository.save(user), Collections.singletonList(role));
     }
 
-    private String writeValueAsString(Object object) {
+    private ObjectNode writeAsObjectNode(Object object) {
         try {
-            return objectMapper.writeValueAsString(object);
-        } catch (JsonProcessingException e) {
-            log.error("FATAL: Something very bad happened " + object.getClass(), e);
+            return objectMapper.valueToTree(object);
+        } catch (IllegalArgumentException e) {
+            log.error("FATAL: cannot write jsonProfile as object node", e);
         }
         return null;
     }

dossierfacile-bo/src/main/resources/templates/bo/apartment-sharing-view.html

@@ -346,7 +346,7 @@ <h2>Liste des liens envoyés:</h2>
                                 <div aria-hidden="true" aria-labelledby="exampleModalScrollableTitle" class="modal fade"
                                      role="dialog"
                                      tabindex="-1" th:id="'target'+${tenant.getId()}">
-                                    <div class="modal-dialog modal-dialog-scrollable" role="document">
+                                    <div class="modal-dialog modal-dialog-scrollable" role="document" style="--bs-modal-width: 600px">
                                         <div class="modal-content">
                                             <div class="modal-header" style="background: cadetblue;">
                                                 <h3 class="modal-title" id="exampleModalScrollableTitle"
@@ -360,9 +360,14 @@ <h2>Liste des liens envoyés:</h2>
                                                 <div class="bg-warning-subtle" style="display: grid; padding: 7px;margin-bottom: 10px;"
                                                      th:each="log: ${logser.getLogByTenantId(tenant.getId())}">
                                                     <div style="font-weight: bold;"
-                                                       th:text="${log.getLogType()} + ${ (log.getOperatorId() != null)? ' by opId: ' + log.getOperatorId() : ''}">
+                                                       th:text="${log.getLogType()} + ${ (log.getOperatorId() != null)? ' by opId: ' + log.getOperatorId() : ''} + ${log.getLogDetails() != null && log.getLogDetails().get('editionType') != null ? ' : DOCUMENT ' + (log.getLogDetails().get('editionType').asText().equals('ADD') ? 'ADDED' : 'DELETED'): ''}">
                                                     </div>
-                                                    <span style="padding-left: 5px;"
+                                                    <span th:if="${log != null && log.getLogDetails() != null && log.getLogDetails().get('documentCategory') != null && log.getLogDetails().get('documentSubCategory') != null}" class="bold" style="font-size: 14px">
+                                                        <span th:if="${log.getLogDetails().get('guarantorId') != null}">GUARANTOR - </span>
+                                                        <span th:text="${log.getLogDetails().get('documentCategory').asText()}"></span> -
+                                                        <span th:text="${log.getLogDetails().get('documentSubCategory').asText()}"></span>
+                                                    </span>
+                                                    <span
                                                           th:text="'date: ' + ${log.getCreationDateTime().getYear()} + '-' + ${log.getCreationDateTime().getMonthValue()} + '-' + ${log.getCreationDateTime().getDayOfMonth()} + ' ' + ${log.getCreationDateTime().getHour()} + ':' + ${log.getCreationDateTime().getMinute()} + ':' + ${log.getCreationDateTime().getSecond()}">
                                                     </span>
                                                 </div>

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/BlurryFileAnalysis.java

@@ -0,0 +1,43 @@
+package fr.dossierfacile.common.entity;
+
+import fr.dossierfacile.common.entity.ocr.BlurryResult;
+import fr.dossierfacile.common.enums.BlurryFileAnalysisStatus;
+import jakarta.persistence.*;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.hibernate.annotations.JdbcTypeCode;
+import org.hibernate.type.SqlTypes;
+
+import java.io.Serial;
+import java.io.Serializable;
+import java.util.List;
+
+@Data
+@Builder
+@Entity
+@Table(name = "blurry_file_analysis")
+@AllArgsConstructor
+@NoArgsConstructor
+public class BlurryFileAnalysis implements Serializable {
+
+    @Serial
+    private static final long serialVersionUID = 2405172041950251807L;
+
+    @Id
+    @GeneratedValue(strategy = GenerationType.IDENTITY)
+    private Long id;
+
+    @OneToOne(targetEntity = File.class, fetch = FetchType.LAZY)
+    @JoinColumn(name = "file_id")
+    private File file;
+
+    @Enumerated(EnumType.STRING)
+    private BlurryFileAnalysisStatus analysisStatus;
+
+    @JdbcTypeCode(SqlTypes.JSON)
+    @Column(columnDefinition = "jsonb")
+    private List<BlurryResult> blurryResults;
+
+}

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/DocumentDeniedReasons.java

@@ -1,13 +1,9 @@
 package fr.dossierfacile.common.entity;
 
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.FetchType;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.JoinColumn;
-import jakarta.persistence.ManyToOne;
+import fr.dossierfacile.common.enums.DocumentCategory;
+import fr.dossierfacile.common.enums.DocumentCategoryStep;
+import fr.dossierfacile.common.enums.DocumentSubCategory;
+import jakarta.persistence.*;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Getter;
@@ -69,6 +65,21 @@ public class DocumentDeniedReasons implements Serializable {
     @Column(name = "creation_date")
     private LocalDateTime creationDate = LocalDateTime.now();
 
+    @Enumerated(EnumType.STRING)
+    @Column(name = "document_category")
+    private DocumentCategory documentCategory;
+
+    @Enumerated(EnumType.STRING)
+    @Column(name = "document_sub_category")
+    private DocumentSubCategory documentSubCategory;
+
+    @Enumerated(EnumType.STRING)
+    @Column(name = "document_category_step")
+    private DocumentCategoryStep documentCategoryStep;
+
+    @Column(name = "document_tenant_type")
+    private String documentTenantType;
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/DocumentRule.java

@@ -34,7 +34,9 @@ public enum DocumentRule {
     R_RENT_RECEIPT_NB_DOCUMENTS(Level.WARN, "Un seul document a été détecté"),
 
     R_FRANCE_IDENTITE_NAMES(Level.CRITICAL, "Les noms et prénoms ne correspondent pas"),
-    R_FRANCE_IDENTITE_STATUS(Level.CRITICAL, "Ce document n'a pas pu être validé par France Identité");
+    R_FRANCE_IDENTITE_STATUS(Level.CRITICAL, "Ce document n'a pas pu être validé par France Identité"),
+
+    R_BLURRY_FILE(Level.CRITICAL, "Votre document semble flou");
 
     public enum Level {
         CRITICAL, WARN

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/File.java

@@ -3,12 +3,7 @@
 import fr.dossierfacile.common.enums.FileStorageStatus;
 import jakarta.annotation.Nullable;
 import jakarta.persistence.*;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
-import lombok.ToString;
+import lombok.*;
 import org.hibernate.Hibernate;
 
 import java.io.Serial;
@@ -56,9 +51,13 @@ public class File implements Serializable {
     private BarCodeFileAnalysis fileAnalysis;
 
     @Nullable
-    @OneToOne(mappedBy= "file", fetch = FetchType.LAZY)
+    @OneToOne(mappedBy = "file", fetch = FetchType.LAZY)
     private ParsedFileAnalysis parsedFileAnalysis;
 
+    @Nullable
+    @OneToOne(mappedBy = "file", fetch = FetchType.LAZY)
+    private BlurryFileAnalysis blurryFileAnalysis;
+
     @PreRemove
     void deleteCascade() {
         if (storageFile != null)

dossierfacile-common-library/src/main/java/fr/dossierfacile/common/entity/TenantLog.java

@@ -1,19 +1,9 @@
 package fr.dossierfacile.common.entity;
 
+import com.fasterxml.jackson.databind.node.ObjectNode;
 import fr.dossierfacile.common.enums.LogType;
-import jakarta.persistence.Column;
-import jakarta.persistence.Entity;
-import jakarta.persistence.EnumType;
-import jakarta.persistence.Enumerated;
-import jakarta.persistence.GeneratedValue;
-import jakarta.persistence.GenerationType;
-import jakarta.persistence.Id;
-import jakarta.persistence.Table;
-import lombok.AllArgsConstructor;
-import lombok.Builder;
-import lombok.Getter;
-import lombok.NoArgsConstructor;
-import lombok.Setter;
+import jakarta.persistence.*;
+import lombok.*;
 import org.hibernate.annotations.JdbcTypeCode;
 import org.hibernate.type.SqlTypes;
 
@@ -53,11 +43,11 @@ public class TenantLog implements Serializable {
 
     @JdbcTypeCode(SqlTypes.JSON)
     @Column(columnDefinition = "jsonb")
-    private Object jsonProfile;
+    private ObjectNode jsonProfile;
 
     @JdbcTypeCode(SqlTypes.JSON)
     @Column(columnDefinition = "jsonb")
-    private Object logDetails;
+    private ObjectNode logDetails;
 
     @Column
     private Long messageId;