diff --git a/modules/odf_data_quality_dashboard/__init__.py b/modules/odf_data_quality_dashboard/__init__.py new file mode 100644 index 00000000..0650744f --- /dev/null +++ b/modules/odf_data_quality_dashboard/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/modules/odf_data_quality_dashboard/__manifest__.py b/modules/odf_data_quality_dashboard/__manifest__.py new file mode 100644 index 00000000..64a1c910 --- /dev/null +++ b/modules/odf_data_quality_dashboard/__manifest__.py @@ -0,0 +1,19 @@ +{ + "name": "ODF Data Quality Dashboard", + "version": "18.0.1.0.0", + "summary": "Data Quality Dashboard for Odoo", + "author": "OdooDataFlow", + "website": "https://github.com/OdooDataFlow/odoo-data-flow", + "license": "AGPL-3", + "category": "Tools", + "depends": ["base", "base_vat"], + "data": [ + "security/security.xml", + "security/ir.model.access.csv", + "data/res_users.xml", + "data/scheduled_actions.xml", + "views/data_quality_issue_views.xml", + "views/menus.xml", + ], + "installable": True, +} diff --git a/modules/odf_data_quality_dashboard/data/res_users.xml b/modules/odf_data_quality_dashboard/data/res_users.xml new file mode 100644 index 00000000..1d3f32fd --- /dev/null +++ b/modules/odf_data_quality_dashboard/data/res_users.xml @@ -0,0 +1,14 @@ + + + + + + Data Quality Cron User + data_quality_cron_user + + + + + + + diff --git a/modules/odf_data_quality_dashboard/data/scheduled_actions.xml b/modules/odf_data_quality_dashboard/data/scheduled_actions.xml new file mode 100644 index 00000000..4ea4d513 --- /dev/null +++ b/modules/odf_data_quality_dashboard/data/scheduled_actions.xml @@ -0,0 +1,16 @@ + + + + + + Data Quality: Run Checks + + code + model._run_data_quality_checks() + + 1 + days + + + + diff --git a/modules/odf_data_quality_dashboard/models/__init__.py b/modules/odf_data_quality_dashboard/models/__init__.py new file mode 100644 index 00000000..ec9328e5 --- /dev/null +++ b/modules/odf_data_quality_dashboard/models/__init__.py @@ -0,0 +1 @@ +from . import data_quality_issue diff --git a/modules/odf_data_quality_dashboard/models/data_quality_issue.py b/modules/odf_data_quality_dashboard/models/data_quality_issue.py new file mode 100644 index 00000000..c11dcaf4 --- /dev/null +++ b/modules/odf_data_quality_dashboard/models/data_quality_issue.py @@ -0,0 +1,137 @@ +import time +from datetime import timedelta + +from odoo import api, fields, models + + +class DataQualityIssue(models.Model): + """Represents a data quality issue found in the system. + This model stores records of data inconsistencies or errors, + allowing users to track and resolve them in a structured manner. + """ + + _name = "odf.data.quality.issue" + _description = "Data Quality Issue" + + name = fields.Char( + string="Name", + required=True, + ) + issue_type = fields.Char( + string="Issue Type", + ) + related_record = fields.Reference( + selection="_selection_related_record", + string="Related Record", + ) + status = fields.Selection( + selection=[ + ("new", "New"), + ("in_progress", "In Progress"), + ("resolved", "Resolved"), + ], + string="Status", + default="new", + ) + notes = fields.Text( + string="Notes", + ) + + @api.model + def _selection_related_record(self): + """Return the list of models that can be checked.""" + return [ + ("res.partner", "Partner"), + ("product.product", "Product"), + ] + + @api.model + def _run_data_quality_checks(self): + """Dispatcher for all data quality checks.""" + self._check_partner_vat() + + @api.model + def _check_partner_vat(self): + """Check for partners with invalid VAT numbers using a performant, + batch-oriented approach. + """ + yesterday = fields.Datetime.now() - timedelta(days=1) + + # 1. Fetch all partners modified recently that have a VAT number. + partners_to_check = self.env["res.partner"].search( + [ + ("write_date", ">=", fields.Datetime.to_string(yesterday)), + ("vat", "!=", False), + ("vat", "!=", ""), + ] + ) + + if not partners_to_check: + return + + # 2. Fetch all existing, unresolved "Invalid VAT" issues for the partners. + existing_issues = self.env["odf.data.quality.issue"].search( + [ + ( + "related_record", + "in", + [f"res.partner,{pid}" for pid in partners_to_check.ids], + ), + ("issue_type", "=", "Invalid VAT"), + ("status", "!=", "resolved"), + ] + ) + partners_with_existing_issue = set( + issue.related_record.id + for issue in existing_issues + if issue.related_record + ) + partners_to_validate = partners_to_check.filtered( + lambda p: p.id not in partners_with_existing_issue + ) + + if not partners_to_validate: + return + + # 3. Perform validation checks. + invalid_partners = self.env["res.partner"] + + # 3a. First, run the VIES check in batches if enabled. This is the most + # reliable check, but it is remote and requires careful handling. + if self.env.company.vat_check_vies: + batch_size = 10 # Process 10 partners per batch + for i in range(0, len(partners_to_validate), batch_size): + batch = partners_to_validate[i : i + batch_size] + try: + # The button_vies_check method from base_vat is designed to + # be called on a recordset and handles iteration internally. + batch.button_vies_check() + except Exception as e: + # If the VIES service fails, we can't validate this batch. + # We could log this, but for now we'll just continue. + _logger.warning( + "VIES check failed for batch with error: %s", e + ) + time.sleep(1) # Wait 1 second between batches to be safe. + + # 3b. Now, check the validation status. The `check_vat` method in + # `base_vat` provides the definitive status, incorporating the + # result of the VIES check if it was performed. + partners_to_validate.refresh() # Refresh to get latest status + for partner in partners_to_validate: + if not partner.check_vat(): + invalid_partners |= partner + + # 4. Create issues for all invalid partners found. + issues_to_create = [] + for partner in invalid_partners: + issues_to_create.append( + { + "name": f"Invalid VAT number for '{partner.display_name}'", + "issue_type": "Invalid VAT", + "related_record": f"res.partner,{partner.id}", + } + ) + + if issues_to_create: + self.env["odf.data.quality.issue"].create(issues_to_create) diff --git a/modules/odf_data_quality_dashboard/readme/CONFIGURE.md b/modules/odf_data_quality_dashboard/readme/CONFIGURE.md new file mode 100644 index 00000000..d9193a75 --- /dev/null +++ b/modules/odf_data_quality_dashboard/readme/CONFIGURE.md @@ -0,0 +1,8 @@ + +To configure this module, you need to assign users to the appropriate security groups. This module introduces two new groups to manage access: + +* **Data Quality User:** Users in this group can view, create, and edit data quality issues. They have read, write, and create permissions. This group is intended for functional users who are responsible for correcting data. + +* **Data Quality Manager:** Users in this group have full access to all data quality issues, including the ability to delete them. This group inherits all rights from the 'User' group and is intended for team leads or administrators. + +To assign users to these groups, go to **Settings > Users & Companies > Users**, select a user, and under the 'Access Rights' tab, find the 'Data Quality' section to grant the desired permissions. diff --git a/modules/odf_data_quality_dashboard/readme/CONTEXT.md b/modules/odf_data_quality_dashboard/readme/CONTEXT.md new file mode 100644 index 00000000..3e15708d --- /dev/null +++ b/modules/odf_data_quality_dashboard/readme/CONTEXT.md @@ -0,0 +1,3 @@ +The primary business need for this module is to streamline and accelerate data import processes. In many data migration or integration scenarios, a single invalid record (e.g., a contact with an incorrect VAT number) can cause an entire import batch to fail. This creates a bottleneck and requires technical intervention to fix the data and re-run the import. + +This module solves that problem by adopting a non-blocking approach. Data is imported first, ensuring speed and efficiency. A separate, automated process then runs nightly to check for common data quality issues. These issues are flagged and presented in a user-friendly dashboard, transforming data validation from a blocking, technical task into a manageable, asynchronous workflow for functional users. This ensures that the data import pipeline remains fast and resilient, while still providing a robust mechanism for maintaining high data quality. diff --git a/modules/odf_data_quality_dashboard/readme/CONTRIBUTORS.md b/modules/odf_data_quality_dashboard/readme/CONTRIBUTORS.md new file mode 100644 index 00000000..1b22f2a4 --- /dev/null +++ b/modules/odf_data_quality_dashboard/readme/CONTRIBUTORS.md @@ -0,0 +1 @@ +- bosd diff --git a/modules/odf_data_quality_dashboard/readme/CREDITS.md b/modules/odf_data_quality_dashboard/readme/CREDITS.md new file mode 100644 index 00000000..e69de29b diff --git a/modules/odf_data_quality_dashboard/readme/DESCRIPTION.md b/modules/odf_data_quality_dashboard/readme/DESCRIPTION.md new file mode 100644 index 00000000..f10f246b --- /dev/null +++ b/modules/odf_data_quality_dashboard/readme/DESCRIPTION.md @@ -0,0 +1 @@ +This module provides a Data Quality Dashboard for Odoo. It helps improve data import workflows by decoupling data validation from the import process. Instead of blocking imports, it allows data to be imported quickly and then runs validation checks asynchronously. Issues found are logged as records in a 'Data Quality Issue' model, which are then displayed on a dashboard for users to review and resolve. diff --git a/modules/odf_data_quality_dashboard/readme/HISTORY.md b/modules/odf_data_quality_dashboard/readme/HISTORY.md new file mode 100644 index 00000000..139597f9 --- /dev/null +++ b/modules/odf_data_quality_dashboard/readme/HISTORY.md @@ -0,0 +1,2 @@ + + diff --git a/modules/odf_data_quality_dashboard/readme/INSTALL.md b/modules/odf_data_quality_dashboard/readme/INSTALL.md new file mode 100644 index 00000000..4d949ce8 --- /dev/null +++ b/modules/odf_data_quality_dashboard/readme/INSTALL.md @@ -0,0 +1,5 @@ +[ This file must only be present if there are very specific + installation instructions, such as installing non-python + dependencies. The audience is systems administrators. ] + + diff --git a/modules/odf_data_quality_dashboard/readme/ROADMAP.md b/modules/odf_data_quality_dashboard/readme/ROADMAP.md new file mode 100644 index 00000000..e69de29b diff --git a/modules/odf_data_quality_dashboard/readme/USAGE.md b/modules/odf_data_quality_dashboard/readme/USAGE.md new file mode 100644 index 00000000..4fc769f0 --- /dev/null +++ b/modules/odf_data_quality_dashboard/readme/USAGE.md @@ -0,0 +1,4 @@ + +Once installed, a 'Data Quality Dashboard' menu will appear under the 'Data Flow' top-level menu. This dashboard displays data quality issues in a Kanban view, categorized by their status (New, In Progress, Resolved). + +Users can click on an issue to view its details, including a direct link to the problematic record (e.g., the Partner with an invalid VAT). From there, users can correct the data. Once the issue is fixed, the status of the quality issue record can be moved to 'Resolved'. diff --git a/modules/odf_data_quality_dashboard/security/ir.model.access.csv b/modules/odf_data_quality_dashboard/security/ir.model.access.csv new file mode 100644 index 00000000..f756cb68 --- /dev/null +++ b/modules/odf_data_quality_dashboard/security/ir.model.access.csv @@ -0,0 +1,3 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_odf_data_quality_issue_user,odf.data.quality.issue.user,model_odf_data_quality_issue,odf_data_quality_dashboard.group_odf_data_quality_user,1,1,1,0 +access_odf_data_quality_issue_manager,odf.data.quality.issue.manager,model_odf_data_quality_issue,odf_data_quality_dashboard.group_odf_data_quality_manager,1,1,1,1 diff --git a/modules/odf_data_quality_dashboard/security/security.xml b/modules/odf_data_quality_dashboard/security/security.xml new file mode 100644 index 00000000..71e0c19b --- /dev/null +++ b/modules/odf_data_quality_dashboard/security/security.xml @@ -0,0 +1,23 @@ + + + + + Data Quality + User access levels for the Data Quality Dashboard module. + 20 + + + + User + + + + + + Manager + + + + + + diff --git a/modules/odf_data_quality_dashboard/static/description/icon.png b/modules/odf_data_quality_dashboard/static/description/icon.png new file mode 100644 index 00000000..415f7f89 Binary files /dev/null and b/modules/odf_data_quality_dashboard/static/description/icon.png differ diff --git a/modules/odf_data_quality_dashboard/views/data_quality_issue_views.xml b/modules/odf_data_quality_dashboard/views/data_quality_issue_views.xml new file mode 100644 index 00000000..26c7162e --- /dev/null +++ b/modules/odf_data_quality_dashboard/views/data_quality_issue_views.xml @@ -0,0 +1,72 @@ + + + + + + + odf.data.quality.issue.form + odf.data.quality.issue + +
+ + + + + + + + + + + + + +
+ + + + + + odf.data.quality.issue.kanban + odf.data.quality.issue + + + + + +
+
+ +
Type:
+
Record:
+
+
+ + + + + + + + + odf.data.quality.issue.list + odf.data.quality.issue + + + + + + + + + + + + + Data Quality Issues + odf.data.quality.issue + kanban,list,form + + + + diff --git a/modules/odf_data_quality_dashboard/views/menus.xml b/modules/odf_data_quality_dashboard/views/menus.xml new file mode 100644 index 00000000..31f26015 --- /dev/null +++ b/modules/odf_data_quality_dashboard/views/menus.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + +