diff --git a/product_attribute_lazy/__init__.py b/product_attribute_lazy/__init__.py new file mode 100644 index 00000000..0650744f --- /dev/null +++ b/product_attribute_lazy/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/product_attribute_lazy/__manifest__.py b/product_attribute_lazy/__manifest__.py new file mode 100644 index 00000000..08fef9c0 --- /dev/null +++ b/product_attribute_lazy/__manifest__.py @@ -0,0 +1,29 @@ +{ + 'name': 'Product Attribute Lazy', + 'author': 'Hibou Corp.', + 'version': '16.0.1.0.0', + 'category': 'Product', + 'sequence': 95, + 'summary': 'Performance module to change behavior of attribute-product-template relationship.', + 'description': """ +Performance module to change behavior of attribute-product-template relationship. + +Adds scheduled action to reindex all attributes. + +Adds scheduled action that will kill queries to this table, +you can deactive if you're not having issues with long running table queries. + +By default, a new pure SQL version of the index is created. If you would like to use the slower ORM +computation, set system config parameter `product_attribute_lazy.indexer_use_sql` to '0'. + """, + 'website': 'https://hibou.io/', + 'depends': [ + 'product', + ], + 'data': [ + 'data/product_data.xml', + ], + 'installable': True, + 'application': False, + 'license': 'LGPL-3', +} diff --git a/product_attribute_lazy/data/product_data.xml b/product_attribute_lazy/data/product_data.xml new file mode 100644 index 00000000..66c1905f --- /dev/null +++ b/product_attribute_lazy/data/product_data.xml @@ -0,0 +1,50 @@ + + + + + + Reindex Related Products + + + code + +records.run_indexer_manual() + + + + + + Product Attribute Indexer + + code + +model.run_indexer(True) + + + 1 + days + -1 + + + + + + + Query Watchdog: product_attribute_product_template_rel + + code + +dbname = env.cr.dbname +query = f'select pg_cancel_backend(pid) from pg_stat_activity where state=\'active\' and datname = \'{dbname}\' and pid != pg_backend_pid() and query like \'%product_attribute_product_template_rel%\';' +env.cr.execute(query) + + + 1 + minutes + -1 + + + + + + diff --git a/product_attribute_lazy/models/__init__.py b/product_attribute_lazy/models/__init__.py new file mode 100644 index 00000000..46e0ea07 --- /dev/null +++ b/product_attribute_lazy/models/__init__.py @@ -0,0 +1,2 @@ +from . import product_attribute_patch +from . import product_attribute diff --git a/product_attribute_lazy/models/product_attribute.py b/product_attribute_lazy/models/product_attribute.py new file mode 100644 index 00000000..acc159be --- /dev/null +++ b/product_attribute_lazy/models/product_attribute.py @@ -0,0 +1,158 @@ +import logging +from datetime import datetime +from odoo import api, fields, models, _, registry, tools +from odoo.exceptions import UserError + + +_logger = logging.getLogger(__name__) + + +class ProductAttribute(models.Model): + _inherit = 'product.attribute' + + number_related_products = fields.Integer(compute=False, store=True) # compute='_compute_number_related_products') + product_tmpl_ids = fields.Many2many('product.template', string="Related Products", compute=False, store=True) #compute='_compute_products', store=True) + + def action_open_related_products(self): + self.ensure_one() + self.env.cr.execute('SELECT product_template_id FROM product_attribute_product_template_rel WHERE product_attribute_id = %s' % (self.id, )) + tmpl_res = self.env.cr.fetchall() + tmpl_ids = [t[0] for t in tmpl_res] + return { + 'type': 'ir.actions.act_window', + 'name': _("Related Products"), + 'res_model': 'product.template', + 'view_mode': 'tree,form', + 'domain': [('id', 'in', tmpl_ids)], + } + + def _compute_products_sql(self): + if not self: + raise UserError('Index with SQL requires attributes.') + # ORIGINAL for replicating logic + # for pa in self: + # product_tmpls = pa.attribute_line_ids.product_tmpl_id + # pa.with_context(active_test=False).product_tmpl_ids = product_tmpls + # pa.number_related_products = len(product_tmpls) + self_ids = ','.join(str(i) for i in self.ids) + query = """ +WITH current_rel AS ( + SELECT + attribute_id AS product_attribute_id, + product_tmpl_id AS product_template_id + FROM product_template_attribute_line + WHERE attribute_id in (%s) + GROUP BY 1, 2 +), +expanded_rel AS ( + SELECT + cr.product_attribute_id as cr_product_attribute_id, + cr.product_template_id as cr_product_template_id, + paptr.product_attribute_id as paptr_product_attribute_id, + paptr.product_template_id as paptr_product_template_id + FROM current_rel cr + LEFT JOIN product_attribute_product_template_rel paptr ON paptr.product_attribute_id = cr.product_attribute_id AND paptr.product_template_id = cr.product_template_id +), +not_in_rel AS ( + SELECT + cr_product_attribute_id as product_attribute_id, + cr_product_template_id as product_template_id + FROM expanded_rel + WHERE paptr_product_attribute_id is null and paptr_product_template_id is null +) + +INSERT INTO product_attribute_product_template_rel (product_attribute_id, product_template_id) +SELECT product_attribute_id, product_template_id +FROM not_in_rel; + """ % (self_ids, ) + query += """ +WITH needs_del AS ( + SELECT + t.product_attribute_id, + t.product_template_id + FROM product_attribute_product_template_rel AS t + LEFT JOIN product_template_attribute_line AS real ON + real.attribute_id = t.product_attribute_id + AND real.product_tmpl_id = t.product_template_id + WHERE t.product_attribute_id in (%s) + AND real.product_tmpl_id is null + AND real.attribute_id is null +) +DELETE FROM product_attribute_product_template_rel +WHERE product_attribute_id in (%s) AND ( + (product_attribute_product_template_rel.product_attribute_id, + product_attribute_product_template_rel.product_template_id) IN (SELECT * FROM needs_del) +); + """ % (self_ids, self_ids) + for i in self.ids: + query += """ +UPDATE product_attribute +SET number_related_products = (SELECT COUNT(*) FROM product_attribute_product_template_rel WHERE product_attribute_id = %s) +WHERE id = %s; + """ % (i, i) + + self.env.cr.execute(query) + + def _run_indexer(self, use_new_cursor=False): + indexer_use_sql = self.env['ir.config_parameter'].sudo().get_param('product_attribute_lazy.indexer_use_sql', '1') != '0' + for pa in self: + if indexer_use_sql: + pa._compute_products_sql() + else: + pa._compute_products() + if use_new_cursor: + pa._cr.commit() + _logger.info("_run_indexer is finished and committed for %s" % (pa.id, )) + + @api.model + def run_indexer(self, use_new_cursor=False): + start_time = datetime.now() + attributes = None + try: + if use_new_cursor: + cr = registry(self._cr.dbname).cursor() + self = self.with_env(self.env(cr=cr)) + + # We want to freeze the cron that kills long running relationship queries... + watchdog_cron = self.sudo().env.ref('product_attribute_lazy.ir_cron_product_attribute_rel_query_watchdog', raise_if_not_found=False) + if watchdog_cron: + try: + with tools.mute_logger('odoo.sql_db'): + self._cr.execute("SELECT id FROM ir_cron WHERE id = %s FOR UPDATE NOWAIT", (watchdog_cron.id, )) + except Exception: + _logger.info('Attempt to run indexer aborted, as the query watchdog is already running') + self._cr.rollback() + raise UserError('Attempt to run indexer aborted, as the query watchdog is already running') + + # if we could tell that it needs re-indexed.... + attributes = self.env['product.attribute'].search([]) + attributes._run_indexer(use_new_cursor=use_new_cursor) + except Exception: + _logger.error("Error during product attribute indexer", exc_info=True) + raise + finally: + if use_new_cursor: + try: + self._cr.close() + except Exception: + pass + if attributes: + _logger.warning('Indexer took %s seconds total for %s attributes.' % ((datetime.now()-start_time).seconds, len(attributes))) + return {} + + def run_indexer_manual(self): + # intended to be called by server action. Don't allow it to overlap with cron + if not self.exists(): + raise UserError('One or more selected Product Attributes are required.') + + indexer_cron = self.sudo().env.ref('product_attribute_lazy.ir_cron_product_attribute_indexer') + # Avoid repeated and overlapping index processes + try: + with tools.mute_logger('odoo.sql_db'): + self._cr.execute("SELECT id FROM ir_cron WHERE id = %s FOR UPDATE NOWAIT", (indexer_cron.id, )) + except Exception: + _logger.info('Attempt to run indexer aborted, as already running') + self._cr.rollback() + raise UserError('Attempt to run indexer aborted, as already running') + + self._run_indexer(True) diff --git a/product_attribute_lazy/models/product_attribute_patch.py b/product_attribute_lazy/models/product_attribute_patch.py new file mode 100644 index 00000000..fdedc557 --- /dev/null +++ b/product_attribute_lazy/models/product_attribute_patch.py @@ -0,0 +1,79 @@ +from odoo import api, _ +from odoo.exceptions import UserError +from odoo.addons.product.models.product_attribute import ProductAttribute + + +def write(self, vals): + """Override to make sure attribute type can't be changed if it's used on + a product template. + + This is important to prevent because changing the type would make + existing combinations invalid without recomputing them, and recomputing + them might take too long and we don't want to change products without + the user knowing about it.""" + if 'create_variant' in vals: + for pa in self: + + if vals['create_variant'] != pa.create_variant: + if not pa.number_related_products: + # index this attribute, this will be free if there are truely no related products + pa._compute_products() + if pa.number_related_products: + raise UserError( + _("You cannot change the Variants Creation Mode of the attribute %s because it is used on the following products:\n%s") % + (pa.display_name, ", ".join(pa.product_tmpl_ids.mapped('display_name'))) + ) + invalidate = 'sequence' in vals and any(record.sequence != vals['sequence'] for record in self) + res = super(ProductAttribute, self).write(vals) + if invalidate: + # prefetched o2m have to be resequenced + # (eg. product.template: attribute_line_ids) + self.env.flush_all() + self.env.invalidate_all() + return res + +ProductAttribute.write = write + +# eventually may want to only display if count is <100 otherwise this takes a long long time +@api.ondelete(at_uninstall=False) +def _unlink_except_used_on_product(self): + for pa in self: + if not pa.number_related_products: + # index this attribute, this will be free if there are truely no related products + pa._compute_products() + if pa.number_related_products: + if pa.number_related_products > 100: + raise UserError( + _("You cannot delete the attribute %s because it is used on many products.") % + (pa.display_name, ) + ) + else: + raise UserError( + _("You cannot delete the attribute %s because it is used on the following products:\n%s") % + (pa.display_name, ", ".join(pa.product_tmpl_ids.mapped('display_name'))) + ) + +ProductAttribute._unlink_except_used_on_product = _unlink_except_used_on_product + +# this method should never be called as it is not a computed field anymore +# replace it to replace decoration +# @api.depends('product_tmpl_ids') +def _compute_number_related_products(self): + pass + # raise Exception('in patched _compute_number_related_products') + # for pa in self: + # pa.number_related_products = len(pa.product_tmpl_ids) + +ProductAttribute._compute_number_related_products = _compute_number_related_products + +# this method should be called on a schedule to "index" these... +# replace it to replace decoration +# @api.depends('attribute_line_ids.active', 'attribute_line_ids.product_tmpl_id') +def _compute_products(self): + for pa in self: + product_tmpls = pa.attribute_line_ids.product_tmpl_id + pa.with_context(active_test=False).product_tmpl_ids = product_tmpls + pa.number_related_products = len(product_tmpls) + + +ProductAttribute._compute_products = _compute_products