[ADD] product_attribute_lazy: performance module for product attribute template rel

product_attribute_lazy/__init__.py

@@ -0,0 +1 @@
+from . import models

product_attribute_lazy/__manifest__.py

@@ -0,0 +1,29 @@
+{
+    'name': 'Product Attribute Lazy',
+    'author': 'Hibou Corp.',
+    'version': '16.0.1.0.0',
+    'category': 'Product',
+    'sequence': 95,
+    'summary': 'Performance module to change behavior of attribute-product-template relationship.',
+    'description': """
+Performance module to change behavior of attribute-product-template relationship.
+
+Adds scheduled action to reindex all attributes.
+
+Adds scheduled action that will kill queries to this table, 
+you can deactive if you're not having issues with long running table queries.
+
+By default, a new pure SQL version of the index is created.  If you would like to use the slower ORM 
+computation, set system config parameter `product_attribute_lazy.indexer_use_sql` to '0'.
+    """,
+    'website': 'https://hibou.io/',
+    'depends': [
+        'product',
+    ],
+    'data': [
+        'data/product_data.xml',
+    ],
+    'installable': True,
+    'application': False,
+    'license': 'LGPL-3',
+}

product_attribute_lazy/data/product_data.xml

@@ -0,0 +1,50 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <data noupdate="1">
+
+        <record id="action_product_attribute_index" model="ir.actions.server">
+            <field name="name">Reindex Related Products</field>
+            <field name="model_id" ref="product.model_product_attribute"/>
+            <field name="binding_model_id" ref="product.model_product_attribute"/>
+            <field name="state">code</field>
+            <field name="code">
+records.run_indexer_manual()
+            </field>
+        </record>
+
+        <!-- Product Attribute Indexer -->
+        <record forcecreate="True" id="ir_cron_product_attribute_indexer" model="ir.cron">
+            <field name="name">Product Attribute Indexer</field>
+            <field name="model_id" ref="model_product_attribute"/>
+            <field name="state">code</field>
+            <field name="code">
+model.run_indexer(True)
+            </field>
+            <field name="user_id" ref="base.user_root"/>
+            <field name="interval_number">1</field>
+            <field name="interval_type">days</field>
+            <field name="numbercall">-1</field>
+            <field eval="False" name="doall"/>
+            <field name="priority" eval="5"/>
+        </record>
+
+        <!-- Product Attribute Indexer -->
+        <record forcecreate="True" id="ir_cron_product_attribute_rel_query_watchdog" model="ir.cron">
+            <field name="name">Query Watchdog: product_attribute_product_template_rel</field>
+            <field name="model_id" ref="model_product_attribute"/>
+            <field name="state">code</field>
+            <field name="code">
+dbname = env.cr.dbname
+query = f'select pg_cancel_backend(pid) from pg_stat_activity where state=\'active\' and datname = \'{dbname}\' and pid != pg_backend_pid() and query like \'%product_attribute_product_template_rel%\';'
+env.cr.execute(query)                
+            </field>
+            <field name="user_id" ref="base.user_root"/>
+            <field name="interval_number">1</field>
+            <field name="interval_type">minutes</field>
+            <field name="numbercall">-1</field>
+            <field eval="False" name="doall"/>
+            <field name="priority" eval="10"/>
+        </record>
+
+    </data>
+</odoo>

product_attribute_lazy/models/__init__.py

@@ -0,0 +1,2 @@
+from . import product_attribute_patch
+from . import product_attribute

product_attribute_lazy/models/product_attribute.py

@@ -0,0 +1,158 @@
+import logging
+from datetime import datetime
+from odoo import api, fields, models, _, registry, tools
+from odoo.exceptions import UserError
+
+
+_logger = logging.getLogger(__name__)
+
+
+class ProductAttribute(models.Model):
+    _inherit = 'product.attribute'
+    
+    number_related_products = fields.Integer(compute=False, store=True) # compute='_compute_number_related_products')
+    product_tmpl_ids = fields.Many2many('product.template', string="Related Products", compute=False, store=True)  #compute='_compute_products', store=True)
+
+    def action_open_related_products(self):
+        self.ensure_one()
+        self.env.cr.execute('SELECT product_template_id FROM product_attribute_product_template_rel WHERE product_attribute_id = %s' % (self.id, ))
+        tmpl_res = self.env.cr.fetchall()
+        tmpl_ids = [t[0] for t in tmpl_res]
+        return {
+            'type': 'ir.actions.act_window',
+            'name': _("Related Products"),
+            'res_model': 'product.template',
+            'view_mode': 'tree,form',
+            'domain': [('id', 'in', tmpl_ids)],
+        }
+
+    def _compute_products_sql(self):
+        if not self:
+            raise UserError('Index with SQL requires attributes.')
+        # ORIGINAL for replicating logic
+        # for pa in self:
+        #     product_tmpls = pa.attribute_line_ids.product_tmpl_id
+        #     pa.with_context(active_test=False).product_tmpl_ids = product_tmpls
+        #     pa.number_related_products = len(product_tmpls)
+        self_ids = ','.join(str(i) for i in self.ids)
+        query = """
+WITH current_rel AS (
+    SELECT 
+      attribute_id AS product_attribute_id,
+      product_tmpl_id AS product_template_id
+    FROM product_template_attribute_line
+    WHERE attribute_id in (%s)
+    GROUP BY 1, 2
+),
+expanded_rel AS (
+    SELECT
+      cr.product_attribute_id as cr_product_attribute_id,
+      cr.product_template_id as cr_product_template_id,
+      paptr.product_attribute_id as paptr_product_attribute_id,
+      paptr.product_template_id as paptr_product_template_id
+    FROM current_rel cr
+    LEFT JOIN product_attribute_product_template_rel paptr ON paptr.product_attribute_id = cr.product_attribute_id AND paptr.product_template_id = cr.product_template_id
+),
+not_in_rel AS (
+    SELECT
+      cr_product_attribute_id as product_attribute_id,
+      cr_product_template_id as product_template_id
+    FROM expanded_rel
+    WHERE paptr_product_attribute_id is null and paptr_product_template_id is null
+)
+
+INSERT INTO product_attribute_product_template_rel (product_attribute_id, product_template_id)
+SELECT product_attribute_id, product_template_id
+FROM not_in_rel;
+        """ % (self_ids, )
+        query += """
+WITH needs_del AS (
+          SELECT
+            t.product_attribute_id,
+            t.product_template_id
+          FROM product_attribute_product_template_rel AS t
+          LEFT JOIN product_template_attribute_line AS real ON
+            real.attribute_id = t.product_attribute_id
+            AND real.product_tmpl_id = t.product_template_id
+          WHERE t.product_attribute_id in (%s)
+            AND real.product_tmpl_id is null
+            AND real.attribute_id is null
+)
+DELETE FROM product_attribute_product_template_rel
+WHERE product_attribute_id in (%s) AND (
+    (product_attribute_product_template_rel.product_attribute_id,
+    product_attribute_product_template_rel.product_template_id) IN (SELECT * FROM needs_del)
+);
+        """ % (self_ids, self_ids)
+        for i in self.ids:
+            query += """
+UPDATE product_attribute 
+SET number_related_products = (SELECT COUNT(*) FROM product_attribute_product_template_rel WHERE product_attribute_id = %s)
+WHERE id = %s;
+        """ % (i, i)
+
+        self.env.cr.execute(query)
+
+    def _run_indexer(self, use_new_cursor=False):
+        indexer_use_sql = self.env['ir.config_parameter'].sudo().get_param('product_attribute_lazy.indexer_use_sql', '1') != '0'
+        for pa in self:
+            if indexer_use_sql:
+                pa._compute_products_sql()
+            else:
+                pa._compute_products()
+            if use_new_cursor:
+                pa._cr.commit()
+                _logger.info("_run_indexer is finished and committed for %s" % (pa.id, ))
+
+    @api.model
+    def run_indexer(self, use_new_cursor=False):
+        start_time = datetime.now()
+        attributes = None
+        try:
+            if use_new_cursor:
+                cr = registry(self._cr.dbname).cursor()
+                self = self.with_env(self.env(cr=cr))
+
+            # We want to freeze the cron that kills long running relationship queries...
+            watchdog_cron = self.sudo().env.ref('product_attribute_lazy.ir_cron_product_attribute_rel_query_watchdog', raise_if_not_found=False)
+            if watchdog_cron:
+                try:
+                    with tools.mute_logger('odoo.sql_db'):
+                        self._cr.execute("SELECT id FROM ir_cron WHERE id = %s FOR UPDATE NOWAIT", (watchdog_cron.id, ))
+                except Exception:
+                    _logger.info('Attempt to run indexer aborted, as the query watchdog is already running')
+                    self._cr.rollback()
+                    raise UserError('Attempt to run indexer aborted, as the query watchdog is already running')
+
+            # if we could tell that it needs re-indexed....
+            attributes = self.env['product.attribute'].search([])
+            attributes._run_indexer(use_new_cursor=use_new_cursor)
+        except Exception:
+            _logger.error("Error during product attribute indexer", exc_info=True)
+            raise
+        finally:
+            if use_new_cursor:
+                try:
+                    self._cr.close()
+                except Exception:
+                    pass
+        if attributes:
+            _logger.warning('Indexer took %s seconds total for %s attributes.' % ((datetime.now()-start_time).seconds, len(attributes)))
+        return {}
+
+    def run_indexer_manual(self):
+        # intended to be called by server action.  Don't allow it to overlap with cron
+        if not self.exists():
+            raise UserError('One or more selected Product Attributes are required.')
+        
+        indexer_cron = self.sudo().env.ref('product_attribute_lazy.ir_cron_product_attribute_indexer')
+        # Avoid repeated and overlapping index processes
+        try:
+            with tools.mute_logger('odoo.sql_db'):
+                self._cr.execute("SELECT id FROM ir_cron WHERE id = %s FOR UPDATE NOWAIT", (indexer_cron.id, ))
+        except Exception:
+            _logger.info('Attempt to run indexer aborted, as already running')
+            self._cr.rollback()
+            raise UserError('Attempt to run indexer aborted, as already running')
+        
+        self._run_indexer(True)

product_attribute_lazy/models/product_attribute_patch.py

@@ -0,0 +1,79 @@
+from odoo import api, _
+from odoo.exceptions import UserError
+from odoo.addons.product.models.product_attribute import ProductAttribute
+
+
+def write(self, vals):
+    """Override to make sure attribute type can't be changed if it's used on
+    a product template.
+
+    This is important to prevent because changing the type would make
+    existing combinations invalid without recomputing them, and recomputing
+    them might take too long and we don't want to change products without
+    the user knowing about it."""
+    if 'create_variant' in vals:
+        for pa in self:
+            
+            if vals['create_variant'] != pa.create_variant:
+                if not pa.number_related_products:
+                    # index this attribute, this will be free if there are truely no related products
+                    pa._compute_products()
+                if pa.number_related_products:
+                    raise UserError(
+                        _("You cannot change the Variants Creation Mode of the attribute %s because it is used on the following products:\n%s") %
+                        (pa.display_name, ", ".join(pa.product_tmpl_ids.mapped('display_name')))
+                    )
+    invalidate = 'sequence' in vals and any(record.sequence != vals['sequence'] for record in self)
+    res = super(ProductAttribute, self).write(vals)
+    if invalidate:
+        # prefetched o2m have to be resequenced
+        # (eg. product.template: attribute_line_ids)
+        self.env.flush_all()
+        self.env.invalidate_all()
+    return res
+
+ProductAttribute.write = write
+
+# eventually may want to only display if count is <100 otherwise this takes a long long time
+@api.ondelete(at_uninstall=False)
+def _unlink_except_used_on_product(self):
+    for pa in self:
+        if not pa.number_related_products:
+             # index this attribute, this will be free if there are truely no related products
+            pa._compute_products()
+        if pa.number_related_products:
+            if pa.number_related_products > 100:
+                raise UserError(
+                    _("You cannot delete the attribute %s because it is used on many products.") %
+                    (pa.display_name, )
+                )
+            else:
+                raise UserError(
+                    _("You cannot delete the attribute %s because it is used on the following products:\n%s") %
+                    (pa.display_name, ", ".join(pa.product_tmpl_ids.mapped('display_name')))
+                )
+
+ProductAttribute._unlink_except_used_on_product = _unlink_except_used_on_product
+
+# this method should never be called as it is not a computed field anymore
+# replace it to replace decoration
+# @api.depends('product_tmpl_ids')
+def _compute_number_related_products(self):
+    pass
+    # raise Exception('in patched _compute_number_related_products')
+    # for pa in self:
+    #     pa.number_related_products = len(pa.product_tmpl_ids)
+
+ProductAttribute._compute_number_related_products = _compute_number_related_products
+
+# this method should be called on a schedule to "index" these...
+# replace it to replace decoration
+# @api.depends('attribute_line_ids.active', 'attribute_line_ids.product_tmpl_id')
+def _compute_products(self):
+    for pa in self:
+        product_tmpls = pa.attribute_line_ids.product_tmpl_id
+        pa.with_context(active_test=False).product_tmpl_ids = product_tmpls
+        pa.number_related_products = len(product_tmpls)
+        
+
+ProductAttribute._compute_products = _compute_products