diff --git a/attribute_set_jsonb/README.rst b/attribute_set_jsonb/README.rst new file mode 100644 index 000000000..e14bfaf7b --- /dev/null +++ b/attribute_set_jsonb/README.rst @@ -0,0 +1,124 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + +=================== +Attribute Set JSONB +=================== + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:d286206bd64f4c1b478ec8b2ae6c5c943bd6a1b0d786362ef06055dcb606b2b7 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fodoo--pim-lightgray.png?logo=github + :target: https://github.com/OCA/odoo-pim/tree/19.0/attribute_set_jsonb + :alt: OCA/odoo-pim +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/odoo-pim-19-0/odoo-pim-19-0-attribute_set_jsonb + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/odoo-pim&target_branch=19.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This module provides JSONB optimization for the ``attribute_set`` module +by integrating with ``base_sparse_field_jsonb`` from OCA/server-tools. + +Features +-------- + +Automatic JSONB Migration +~~~~~~~~~~~~~~~~~~~~~~~~~ + +On installation, the module automatically migrates ``attribute_set``'s +``x_custom_json_attrs`` columns from TEXT to PostgreSQL JSONB format, +providing: + +- **Faster filtering**: GIN indexes enable efficient key/value lookups +- **Native JSON operators**: Database-level filtering instead of Python +- **Better storage**: Binary format with automatic compression + +Expression-Based Indexing +~~~~~~~~~~~~~~~~~~~~~~~~~ + +For frequently filtered attributes, you can enable per-attribute +expression indexes that provide even faster filtering performance: + +1. Go to PIM → Attributes → Product Attributes +2. Edit an attribute and check "Create Expression Index" +3. The module creates an optimized partial index for that specific + attribute + +This is recommended for attributes used heavily in e-commerce filtering +(e.g., brand, color, material). + +When to Use +~~~~~~~~~~~ + +Install this module if you: + +- Have many serialized attributes (100+) +- Use attribute filtering on e-commerce pages +- Need faster attribute-based searches + +Technical Details +~~~~~~~~~~~~~~~~~ + +The module creates two types of indexes: + +1. **GIN index on JSONB column**: Enables fast key existence checks and + value lookups across all attributes in the column +2. **Expression indexes (optional)**: Per-attribute indexes for specific + value extraction, optimal for equality queries on high-traffic + attributes + +**Table of contents** + +.. contents:: + :local: + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Contributors +------------ + +- Stefcy hello@stefcy.com + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/odoo-pim `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/attribute_set_jsonb/__init__.py b/attribute_set_jsonb/__init__.py new file mode 100644 index 000000000..1d353d71e --- /dev/null +++ b/attribute_set_jsonb/__init__.py @@ -0,0 +1,2 @@ +from .hooks import post_init_hook +from . import models diff --git a/attribute_set_jsonb/__manifest__.py b/attribute_set_jsonb/__manifest__.py new file mode 100644 index 000000000..3761f64cd --- /dev/null +++ b/attribute_set_jsonb/__manifest__.py @@ -0,0 +1,19 @@ +{ + "name": "Attribute Set JSONB", + "version": "19.0.1.0.0", + "category": "Technical", + "summary": "JSONB optimization and expression indexing for attribute_set", + "author": "Odoo Community Association (OCA)", + "website": "https://github.com/OCA/odoo-pim", + "license": "AGPL-3", + "depends": [ + "attribute_set", + "base_sparse_field_jsonb", + ], + "data": [ + "views/attribute_attribute_views.xml", + ], + "post_init_hook": "post_init_hook", + "installable": True, + "auto_install": True, +} diff --git a/attribute_set_jsonb/hooks.py b/attribute_set_jsonb/hooks.py new file mode 100644 index 000000000..f2da5826b --- /dev/null +++ b/attribute_set_jsonb/hooks.py @@ -0,0 +1,111 @@ +"""Installation hooks for attribute_set_jsonb. + +The post_init_hook handles migration of attribute_set's serialized field +columns from TEXT to JSONB and creates GIN indexes for filtering. +""" + +import logging + +from psycopg2 import Error as Psycopg2Error +from psycopg2 import sql + +_logger = logging.getLogger(__name__) + + +def post_init_hook(env): + """Post-installation hook to migrate attribute_set columns to JSONB. + + This hook specifically targets the x_custom_json_attrs columns used by + attribute_set for serialized attribute storage. + """ + cr = env.cr + + _logger.info("attribute_set_jsonb: Starting post-install migration...") + + # Find all x_custom_json_attrs columns (attribute_set's serialization field) + cr.execute( + """ + SELECT table_name, column_name, data_type + FROM information_schema.columns + WHERE column_name = 'x_custom_json_attrs' + ORDER BY table_name + """ + ) + columns_to_migrate = cr.fetchall() + + migrated_count = 0 + index_count = 0 + + for table_name, column_name, data_type in columns_to_migrate: + # Migrate TEXT to JSONB if needed + if data_type == "text": + _logger.info( + "Migrating %s.%s from TEXT to JSONB...", table_name, column_name + ) + try: + alter_query = sql.SQL( + """ + ALTER TABLE {table} + ALTER COLUMN {column} + TYPE jsonb + USING CASE + WHEN {column} IS NULL THEN NULL + WHEN {column} = '' THEN '{{}}'::jsonb + ELSE {column}::jsonb + END + """ + ).format( + table=sql.Identifier(table_name), + column=sql.Identifier(column_name), + ) + cr.execute(alter_query) + migrated_count += 1 + _logger.info( + "Successfully migrated %s.%s to JSONB", table_name, column_name + ) + except Psycopg2Error as e: + _logger.warning( + "Could not migrate %s.%s to JSONB: %s", table_name, column_name, e + ) + cr.rollback() + continue + + # Create GIN index if not exists + index_name = f"idx_{table_name}_{column_name}_gin" + cr.execute( + """ + SELECT 1 FROM pg_indexes + WHERE tablename = %s AND indexname = %s + """, + (table_name, index_name), + ) + if not cr.fetchone(): + _logger.info("Creating GIN index on %s.%s...", table_name, column_name) + try: + create_index_query = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index} + ON {table} USING GIN ({column}) + """ + ).format( + index=sql.Identifier(index_name), + table=sql.Identifier(table_name), + column=sql.Identifier(column_name), + ) + cr.execute(create_index_query) + index_count += 1 + _logger.info("Created GIN index %s", index_name) + except Psycopg2Error as e: + _logger.warning( + "Could not create GIN index on %s.%s: %s", + table_name, + column_name, + e, + ) + + _logger.info( + "attribute_set_jsonb: Migration complete. " + "Migrated %d columns, created %d GIN indexes.", + migrated_count, + index_count, + ) diff --git a/attribute_set_jsonb/models/__init__.py b/attribute_set_jsonb/models/__init__.py new file mode 100644 index 000000000..74467765a --- /dev/null +++ b/attribute_set_jsonb/models/__init__.py @@ -0,0 +1 @@ +from . import attribute_attribute diff --git a/attribute_set_jsonb/models/attribute_attribute.py b/attribute_set_jsonb/models/attribute_attribute.py new file mode 100644 index 000000000..5516a85a4 --- /dev/null +++ b/attribute_set_jsonb/models/attribute_attribute.py @@ -0,0 +1,287 @@ +"""Extension to attribute.attribute for expression-based GIN indexes. + +This module adds the ability to create expression-based GIN indexes on +individual serialized attributes for optimized filtering performance. +""" + +import logging +import re + +from psycopg2 import Error as Psycopg2Error +from psycopg2 import sql + +from odoo import api, fields, models + +_logger = logging.getLogger(__name__) + + +class AttributeAttribute(models.Model): + """Extend attribute.attribute with expression-based GIN index support.""" + + _inherit = "attribute.attribute" + + create_gin_index = fields.Boolean( + string="Create Expression Index", + default=False, + help="Create an expression-based index for this attribute. " + "Recommended for frequently filtered serialized attributes. " + "This creates an index on the extracted JSON value for faster queries.", + ) + + def _get_index_name(self): + """Generate a safe index name for this attribute. + + Returns: + str: Index name in format idx_{table}_{field_name}_expr + """ + self.ensure_one() + if not self.serialization_field_id or not self.model: + return None + + # Get table name from model + table_name = self.model.replace(".", "_") + + # Sanitize field name (remove x_ prefix for readability) + field_name = self.name + if field_name.startswith("x_"): + field_name = field_name[2:] + + # Ensure name is safe for PostgreSQL identifier + safe_name = re.sub(r"[^a-z0-9_]", "", field_name.lower()) + + # PostgreSQL has a 63-character limit for identifiers + # idx_ (4) + table + _ (1) + field + _expr (5) = 10 + table + field + max_total = 63 + max_table_field = max_total - 10 + combined = f"{table_name}_{safe_name}" + if len(combined) > max_table_field: + combined = combined[:max_table_field] + + return f"idx_{combined}_expr" + + def _get_table_name(self): + """Get the PostgreSQL table name for this attribute's model.""" + self.ensure_one() + if not self.model: + return None + return self.model.replace(".", "_") + + def _get_jsonb_column_name(self): + """Get the JSONB column name for this attribute.""" + self.ensure_one() + if not self.serialization_field_id: + return None + return self.serialization_field_id.name + + def _create_expression_index(self): + """Create an expression-based index for this attribute. + + Creates an index like: + CREATE INDEX idx_product_template_color_expr + ON product_template ((x_custom_json_attrs->>'x_color')) + WHERE x_custom_json_attrs ? 'x_color'; + """ + self.ensure_one() + + if not self.serialized: + _logger.debug( + "Skipping index creation for non-serialized attribute %s", + self.name, + ) + return False + + index_name = self._get_index_name() + table_name = self._get_table_name() + jsonb_column = self._get_jsonb_column_name() + + if not all([index_name, table_name, jsonb_column]): + _logger.warning( + "Cannot create index for attribute %s: missing required info", + self.name, + ) + return False + + cr = self.env.cr + + # Check if index already exists + cr.execute( + """ + SELECT 1 FROM pg_indexes + WHERE tablename = %s AND indexname = %s + """, + (table_name, index_name), + ) + if cr.fetchone(): + _logger.info("Index %s already exists", index_name) + return True + + # Check if table exists + cr.execute( + """ + SELECT 1 FROM information_schema.tables + WHERE table_name = %s + """, + (table_name,), + ) + if not cr.fetchone(): + _logger.warning( + "Table %s does not exist, skipping index creation", + table_name, + ) + return False + + # Check if column exists + cr.execute( + """ + SELECT 1 FROM information_schema.columns + WHERE table_name = %s AND column_name = %s + """, + (table_name, jsonb_column), + ) + if not cr.fetchone(): + _logger.warning( + "Column %s.%s does not exist, skipping index creation", + table_name, + jsonb_column, + ) + return False + + try: + # Create expression-based index with partial index condition + # This index is optimal for equality queries on extracted values + create_index_query = sql.SQL( + """ + CREATE INDEX IF NOT EXISTS {index} + ON {table} (({column}->>{attr_name})) + WHERE {column} ? {attr_name} + """ + ).format( + index=sql.Identifier(index_name), + table=sql.Identifier(table_name), + column=sql.Identifier(jsonb_column), + attr_name=sql.Literal(self.name), + ) + cr.execute(create_index_query) + _logger.info( + "Created expression index %s on %s.%s for attribute %s", + index_name, + table_name, + jsonb_column, + self.name, + ) + return True + except Psycopg2Error as e: + _logger.warning( + "Could not create expression index %s: %s", + index_name, + e, + ) + return False + + def _drop_expression_index(self): + """Drop the expression-based index for this attribute.""" + self.ensure_one() + + index_name = self._get_index_name() + if not index_name: + return False + + cr = self.env.cr + + try: + drop_index_query = sql.SQL("DROP INDEX IF EXISTS {index}").format( + index=sql.Identifier(index_name), + ) + cr.execute(drop_index_query) + _logger.info("Dropped expression index %s", index_name) + return True + except Psycopg2Error as e: + _logger.warning( + "Could not drop expression index %s: %s", + index_name, + e, + ) + return False + + @api.model_create_multi + def create(self, vals_list): + """Override create to handle index creation.""" + records = super().create(vals_list) + + for record in records: + if record.create_gin_index and record.serialized: + record._create_expression_index() + + return records + + def write(self, vals): + """Override write to handle index creation/deletion.""" + # Track which records need index changes + records_to_index = self.env["attribute.attribute"] + records_to_drop_index = self.env["attribute.attribute"] + + if "create_gin_index" in vals: + if vals["create_gin_index"]: + # Will need to create indexes for serialized attributes + records_to_index = self.filtered(lambda r: r.serialized) + else: + # Will need to drop indexes + records_to_drop_index = self.filtered( + lambda r: r.create_gin_index and r.serialized + ) + + result = super().write(vals) + + # Handle index changes after write + for record in records_to_index: + record._create_expression_index() + + for record in records_to_drop_index: + record._drop_expression_index() + + return result + + def unlink(self): + """Override unlink to drop indexes before deletion.""" + for record in self: + if record.create_gin_index and record.serialized: + record._drop_expression_index() + + return super().unlink() + + def action_regenerate_all_indexes(self): + """Regenerate all expression indexes for serialized attributes. + + This action can be triggered manually to recreate all indexes, + useful after database migration or recovery. + """ + attributes = self.search( + [ + ("create_gin_index", "=", True), + ("serialized", "=", True), + ] + ) + + created = 0 + failed = 0 + for attr in attributes: + if attr._create_expression_index(): + created += 1 + else: + failed += 1 + + _logger.info( + "Index regeneration complete: %d created, %d failed", + created, + failed, + ) + + return { + "type": "ir.actions.client", + "tag": "display_notification", + "params": { + "title": "Index Regeneration Complete", + "message": f"Created {created} indexes, {failed} failed.", + "type": "success" if failed == 0 else "warning", + }, + } diff --git a/attribute_set_jsonb/pyproject.toml b/attribute_set_jsonb/pyproject.toml new file mode 100644 index 000000000..4231d0ccc --- /dev/null +++ b/attribute_set_jsonb/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/attribute_set_jsonb/readme/CONTRIBUTORS.md b/attribute_set_jsonb/readme/CONTRIBUTORS.md new file mode 100644 index 000000000..5be626ed8 --- /dev/null +++ b/attribute_set_jsonb/readme/CONTRIBUTORS.md @@ -0,0 +1 @@ +* Stefcy diff --git a/attribute_set_jsonb/readme/DESCRIPTION.md b/attribute_set_jsonb/readme/DESCRIPTION.md new file mode 100644 index 000000000..0dc33937f --- /dev/null +++ b/attribute_set_jsonb/readme/DESCRIPTION.md @@ -0,0 +1,41 @@ +This module provides JSONB optimization for the `attribute_set` module by +integrating with `base_sparse_field_jsonb` from OCA/server-tools. + +## Features + +### Automatic JSONB Migration +On installation, the module automatically migrates `attribute_set`'s +`x_custom_json_attrs` columns from TEXT to PostgreSQL JSONB format, +providing: + +* **Faster filtering**: GIN indexes enable efficient key/value lookups +* **Native JSON operators**: Database-level filtering instead of Python +* **Better storage**: Binary format with automatic compression + +### Expression-Based Indexing +For frequently filtered attributes, you can enable per-attribute expression +indexes that provide even faster filtering performance: + +1. Go to PIM → Attributes → Product Attributes +2. Edit an attribute and check "Create Expression Index" +3. The module creates an optimized partial index for that specific attribute + +This is recommended for attributes used heavily in e-commerce filtering +(e.g., brand, color, material). + +### When to Use + +Install this module if you: + +* Have many serialized attributes (100+) +* Use attribute filtering on e-commerce pages +* Need faster attribute-based searches + +### Technical Details + +The module creates two types of indexes: + +1. **GIN index on JSONB column**: Enables fast key existence checks and + value lookups across all attributes in the column +2. **Expression indexes (optional)**: Per-attribute indexes for specific + value extraction, optimal for equality queries on high-traffic attributes diff --git a/attribute_set_jsonb/static/description/icon.png b/attribute_set_jsonb/static/description/icon.png new file mode 100644 index 000000000..1dcc49c24 Binary files /dev/null and b/attribute_set_jsonb/static/description/icon.png differ diff --git a/attribute_set_jsonb/static/description/index.html b/attribute_set_jsonb/static/description/index.html new file mode 100644 index 000000000..60556ef03 --- /dev/null +++ b/attribute_set_jsonb/static/description/index.html @@ -0,0 +1,466 @@ + + + + + +README.rst + + + +
+ + + +Odoo Community Association + +
+

Attribute Set JSONB

+ +

Beta License: AGPL-3 OCA/odoo-pim Translate me on Weblate Try me on Runboat

+

This module provides JSONB optimization for the attribute_set module +by integrating with base_sparse_field_jsonb from OCA/server-tools.

+
+

Features

+
+

Automatic JSONB Migration

+

On installation, the module automatically migrates attribute_set’s +x_custom_json_attrs columns from TEXT to PostgreSQL JSONB format, +providing:

+
    +
  • Faster filtering: GIN indexes enable efficient key/value lookups
    • +
  • Native JSON operators: Database-level filtering instead of Python
    • +
  • Better storage: Binary format with automatic compression
    • +
+
+
+

Expression-Based Indexing

+

For frequently filtered attributes, you can enable per-attribute +expression indexes that provide even faster filtering performance:

+
    +
  1. Go to PIM → Attributes → Product Attributes
    2. +
  2. Edit an attribute and check “Create Expression Index”
    3. +
  3. The module creates an optimized partial index for that specific +attribute
    4. +
+

This is recommended for attributes used heavily in e-commerce filtering +(e.g., brand, color, material).

+
+
+

When to Use

+

Install this module if you:

+
    +
  • Have many serialized attributes (100+)
    • +
  • Use attribute filtering on e-commerce pages
    • +
  • Need faster attribute-based searches
    • +
+
+
+

Technical Details

+

The module creates two types of indexes:

+
    +
  1. GIN index on JSONB column: Enables fast key existence checks and +value lookups across all attributes in the column
    2. +
  2. Expression indexes (optional): Per-attribute indexes for specific +value extraction, optimal for equality queries on high-traffic +attributes
    3. +
+

Table of contents

+ +
+

Bug Tracker

+

Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

+

Do not contact contributors directly about support or help with technical issues.

+
+ +
+
+
+

Contributors

+ +
+
+

Maintainers

+

This module is maintained by the OCA.

+ +Odoo Community Association + +

OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

+

This module is part of the OCA/odoo-pim project on GitHub.

+

You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

+
+
+
+ + diff --git a/attribute_set_jsonb/views/attribute_attribute_views.xml b/attribute_set_jsonb/views/attribute_attribute_views.xml new file mode 100644 index 000000000..68d48c012 --- /dev/null +++ b/attribute_set_jsonb/views/attribute_attribute_views.xml @@ -0,0 +1,65 @@ + + + + + attribute.attribute.form.jsonb + attribute.attribute + + + + + + + + + + + + attribute.attribute.tree.jsonb + attribute.attribute + + + + + + + + + + + attribute.attribute.search.jsonb + attribute.attribute + + + + + + + + + + + + Regenerate Expression Indexes + + + list + code + +if records: + records.action_regenerate_all_indexes() + + + diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 000000000..ef2920682 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,2 @@ +odoo-addon-attribute_set @ git+https://github.com/OCA/odoo-pim.git@refs/pull/226/head#subdirectory=attribute_set +odoo-addon-base_sparse_field_jsonb @ git+https://github.com/OCA/server-tools@refs/pull/3480/head#subdirectory=base_sparse_field_jsonb