From 193b1faa766f6b1ac5bfe346ab8e912b4d937682 Mon Sep 17 00:00:00 2001 From: Alexis de Lattre Date: Mon, 8 Aug 2022 00:05:18 +0200 Subject: [PATCH] Add module account_statement_import_base The 2 modules account_statement_import_online and account_statement_import depend on account_statement_import_base (and not on each other) and share common code, in particular a hook to update the statement line. So we can now have reconciliation modules that use this hook and therefore work both on file import and online import. More details on https://github.com/OCA/bank-statement-import/issues/481. Improve bank statement line form view and journal form view. --- account_statement_import/__manifest__.py | 5 +- account_statement_import/models/__init__.py | 1 - .../views/account_bank_statement_line.xml | 25 ------- .../wizard/account_statement_import.py | 38 ++-------- account_statement_import_base/README.rst | 1 + account_statement_import_base/__init__.py | 1 + account_statement_import_base/__manifest__.py | 19 +++++ .../models/__init__.py | 2 + .../models/account_bank_statement_line.py | 8 +-- .../models/account_journal.py | 69 +++++++++++++++++++ .../readme/CONTRIBUTORS.rst | 1 + .../readme/DESCRIPTION.rst | 4 ++ .../views/account_bank_statement_line.xml | 61 ++++++++++++++++ .../__manifest__.py | 6 +- .../migrations/15.0.3.0.0/post-migration.py | 17 +++++ .../migrations/15.0.3.0.0/pre-migration.py | 19 +++++ .../models/__init__.py | 1 - .../models/account_bank_statement_line.py | 18 ----- .../models/online_bank_statement_provider.py | 46 ++----------- .../views/account_bank_statement_line.xml | 25 ------- .../views/account_journal.xml | 4 +- .../online_bank_statement_provider_ponto.py | 2 +- .../odoo/addons/account_statement_import_base | 1 + setup/account_statement_import_base/setup.py | 6 ++ 24 files changed, 223 insertions(+), 157 deletions(-) delete mode 100644 account_statement_import/views/account_bank_statement_line.xml create mode 100644 account_statement_import_base/README.rst create mode 100644 account_statement_import_base/__init__.py create mode 100644 account_statement_import_base/__manifest__.py create mode 100644 account_statement_import_base/models/__init__.py rename {account_statement_import => account_statement_import_base}/models/account_bank_statement_line.py (68%) create mode 100644 account_statement_import_base/models/account_journal.py create mode 100644 account_statement_import_base/readme/CONTRIBUTORS.rst create mode 100644 account_statement_import_base/readme/DESCRIPTION.rst create mode 100644 account_statement_import_base/views/account_bank_statement_line.xml create mode 100644 account_statement_import_online/migrations/15.0.3.0.0/post-migration.py create mode 100644 account_statement_import_online/migrations/15.0.3.0.0/pre-migration.py delete mode 100644 account_statement_import_online/models/account_bank_statement_line.py delete mode 100644 account_statement_import_online/views/account_bank_statement_line.xml create mode 120000 setup/account_statement_import_base/odoo/addons/account_statement_import_base create mode 100644 setup/account_statement_import_base/setup.py diff --git a/account_statement_import/__manifest__.py b/account_statement_import/__manifest__.py index d3788024..706871b3 100644 --- a/account_statement_import/__manifest__.py +++ b/account_statement_import/__manifest__.py @@ -6,9 +6,9 @@ { "name": "Import Statement Files", "category": "Accounting", - "version": "15.0.2.2.1", + "version": "15.0.3.0.0", "license": "LGPL-3", - "depends": ["account"], + "depends": ["account_statement_import_base"], "author": "Odoo SA, Akretion, Odoo Community Association (OCA)", "maintainers": ["alexis-via"], "development_status": "Mature", @@ -17,7 +17,6 @@ "security/ir.model.access.csv", "wizard/account_statement_import_view.xml", "views/account_journal.xml", - "views/account_bank_statement_line.xml", ], "demo": [ "demo/partner_bank.xml", diff --git a/account_statement_import/models/__init__.py b/account_statement_import/models/__init__.py index 16a78ca3..2388e119 100644 --- a/account_statement_import/models/__init__.py +++ b/account_statement_import/models/__init__.py @@ -1,2 +1 @@ from . import account_journal -from . import account_bank_statement_line diff --git a/account_statement_import/views/account_bank_statement_line.xml b/account_statement_import/views/account_bank_statement_line.xml deleted file mode 100644 index cac1e93f..00000000 --- a/account_statement_import/views/account_bank_statement_line.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - account.bank.statement.line - - - - - - - - - - - - - - - diff --git a/account_statement_import/wizard/account_statement_import.py b/account_statement_import/wizard/account_statement_import.py index cf15becd..03f19722 100644 --- a/account_statement_import/wizard/account_statement_import.py +++ b/account_statement_import/wizard/account_statement_import.py @@ -283,43 +283,15 @@ class AccountStatementImport(models.TransientModel): ) return journal - @api.model - def _update_partner_from_account_number(self, lvals): - partner_bank = self.env["res.partner.bank"].search( - [("acc_number", "=", lvals["account_number"])], limit=1 - ) - if partner_bank: - lvals["partner_bank_id"] = partner_bank.id - lvals["partner_id"] = partner_bank.partner_id.id - def _complete_stmts_vals(self, stmts_vals, journal, account_number): + speeddict = journal._statement_line_import_speeddict() for st_vals in stmts_vals: st_vals["journal_id"] = journal.id for lvals in st_vals["transactions"]: - unique_import_id = lvals.get("unique_import_id") - if unique_import_id: - sanitized_account_number = sanitize_account_number(account_number) - lvals["unique_import_id"] = ( - ( - sanitized_account_number - and sanitized_account_number + "-" - or "" - ) - + str(journal.id) - + "-" - + unique_import_id - ) - - if ( - not lvals.get("partner_bank_id") - and lvals.get("account_number") - and not lvals.get("partner_id") - ): - # Find the partner from his bank account number - # The partner selected during the - # reconciliation process will be linked to the bank account - # when the statement is closed (code in the account module) - self._update_partner_from_account_number(lvals) + journal._statement_line_import_update_unique_import_id( + lvals, account_number + ) + journal._statement_line_import_update_hook(lvals, speeddict) if not lvals.get("payment_ref"): raise UserError(_("Missing payment_ref on a transaction.")) return stmts_vals diff --git a/account_statement_import_base/README.rst b/account_statement_import_base/README.rst new file mode 100644 index 00000000..b04982bf --- /dev/null +++ b/account_statement_import_base/README.rst @@ -0,0 +1 @@ +Will be autogenerated from the readme subdir diff --git a/account_statement_import_base/__init__.py b/account_statement_import_base/__init__.py new file mode 100644 index 00000000..0650744f --- /dev/null +++ b/account_statement_import_base/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/account_statement_import_base/__manifest__.py b/account_statement_import_base/__manifest__.py new file mode 100644 index 00000000..0cfbf924 --- /dev/null +++ b/account_statement_import_base/__manifest__.py @@ -0,0 +1,19 @@ +# Copyright 2022 Akretion France (http://www.akretion.com/) +# @author: Alexis de Lattre +# Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0). + +{ + "name": "Base module for Bank Statement Import", + "category": "Accounting", + "version": "14.0.1.0.0", + "license": "LGPL-3", + "depends": ["account"], + "author": "Akretion, Odoo Community Association (OCA)", + "maintainers": ["alexis-via"], + "development_status": "Mature", + "website": "https://github.com/OCA/bank-statement-import", + "data": [ + "views/account_bank_statement_line.xml", + ], + "installable": True, +} diff --git a/account_statement_import_base/models/__init__.py b/account_statement_import_base/models/__init__.py new file mode 100644 index 00000000..4c252e45 --- /dev/null +++ b/account_statement_import_base/models/__init__.py @@ -0,0 +1,2 @@ +from . import account_bank_statement_line +from . import account_journal diff --git a/account_statement_import/models/account_bank_statement_line.py b/account_statement_import_base/models/account_bank_statement_line.py similarity index 68% rename from account_statement_import/models/account_bank_statement_line.py rename to account_statement_import_base/models/account_bank_statement_line.py index 71be92c0..0058331f 100644 --- a/account_statement_import/models/account_bank_statement_line.py +++ b/account_statement_import_base/models/account_bank_statement_line.py @@ -1,7 +1,6 @@ -# Copyright 2004-2020 Odoo S.A. -# Copyright 2020 Akretion France (http://www.akretion.com/) +# Copyright 2022 Akretion France (http://www.akretion.com/) # @author: Alexis de Lattre -# Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0). +# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl). from odoo import fields, models @@ -10,8 +9,9 @@ class AccountBankStatementLine(models.Model): _inherit = "account.bank.statement.line" # Ensure transactions can be imported only once - # (if the import format provides unique transaction ids) + # if the import format provides unique transaction IDs unique_import_id = fields.Char(string="Import ID", readonly=True, copy=False) + raw_data = fields.Text(readonly=True, copy=False) _sql_constraints = [ ( diff --git a/account_statement_import_base/models/account_journal.py b/account_statement_import_base/models/account_journal.py new file mode 100644 index 00000000..3080fe74 --- /dev/null +++ b/account_statement_import_base/models/account_journal.py @@ -0,0 +1,69 @@ +# Copyright 2022 Akretion France (http://www.akretion.com/) +# @author: Alexis de Lattre +# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl). + +from odoo import api, models + +from odoo.addons.base.models.res_bank import sanitize_account_number + + +class AccountJournal(models.Model): + _inherit = "account.journal" + + def _statement_line_import_speeddict(self): + """This method is designed to be inherited by reconciliation modules. + These modules can take advantage of this method to pre-fetch data + that will later be used for many statement lines (to avoid + searching data for each statement line). + The goal is to improve performances. + """ + self.ensure_one() + speeddict = {"account_number": {}} + partner_banks = self.env["res.partner.bank"].search_read( + [("company_id", "in", (False, self.company_id.id))], + ["acc_number", "partner_id"], + ) + for partner_bank in partner_banks: + speeddict["account_number"][partner_bank["acc_number"]] = { + "partner_id": partner_bank["partner_id"][0], + "partner_bank_id": partner_bank["id"], + } + return speeddict + + def _statement_line_import_update_hook(self, st_line_vals, speeddict): + """This method is designed to be inherited by reconciliation modules. + In this method you can: + - update the partner of the line by writing st_line_vals['partner_id'] + - set an automated counter-part via counterpart_account_id by writing + st_line_vals['counterpart_account_id'] + - do anythink you want with the statement line + """ + self.ensure_one() + if st_line_vals.get("account_number"): + st_line_vals["account_number"] = self._sanitize_bank_account_number( + st_line_vals["account_number"] + ) + if not st_line_vals.get("partner_id") and speeddict["account_number"].get( + st_line_vals["account_number"] + ): + st_line_vals.update( + speeddict["account_number"][st_line_vals["account_number"]] + ) + + def _statement_line_import_update_unique_import_id( + self, st_line_vals, account_number + ): + self.ensure_one() + if st_line_vals.get("unique_import_id"): + sanitized_acc_number = self._sanitize_bank_account_number(account_number) + st_line_vals["unique_import_id"] = ( + (sanitized_acc_number and sanitized_acc_number + "-" or "") + + str(self.id) + + "-" + + st_line_vals["unique_import_id"] + ) + + @api.model + def _sanitize_bank_account_number(self, account_number): + """Hook for extension""" + return sanitize_account_number(account_number) diff --git a/account_statement_import_base/readme/CONTRIBUTORS.rst b/account_statement_import_base/readme/CONTRIBUTORS.rst new file mode 100644 index 00000000..ff65d68c --- /dev/null +++ b/account_statement_import_base/readme/CONTRIBUTORS.rst @@ -0,0 +1 @@ +* Alexis de Lattre diff --git a/account_statement_import_base/readme/DESCRIPTION.rst b/account_statement_import_base/readme/DESCRIPTION.rst new file mode 100644 index 00000000..4bf07f19 --- /dev/null +++ b/account_statement_import_base/readme/DESCRIPTION.rst @@ -0,0 +1,4 @@ +This is a technical module designed to share code between 2 other modules: + +* **account_statement_import** that allows to import bank statements from files, +* **account_statement_import_online** that allows to import bank statements from webservices/APIs. diff --git a/account_statement_import_base/views/account_bank_statement_line.xml b/account_statement_import_base/views/account_bank_statement_line.xml new file mode 100644 index 00000000..f2067d07 --- /dev/null +++ b/account_statement_import_base/views/account_bank_statement_line.xml @@ -0,0 +1,61 @@ + + + + + + account.bank.statement.line + + + + not context.get('statement_line_main_view') + + + 2 + + + 1 + + + + 1 + + + 1 + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/account_statement_import_online/__manifest__.py b/account_statement_import_online/__manifest__.py index bc2ad5dd..04c85dd3 100644 --- a/account_statement_import_online/__manifest__.py +++ b/account_statement_import_online/__manifest__.py @@ -4,7 +4,7 @@ { "name": "Online Bank Statements", - "version": "15.0.2.0.0", + "version": "15.0.3.0.0", "author": "CorporateHub, Odoo Community Association (OCA)", "maintainers": ["alexey-pelykh"], "website": "https://github.com/OCA/bank-statement-import", @@ -12,8 +12,7 @@ "category": "Accounting", "summary": "Online bank statements update", "depends": [ - "account", - "account_statement_import", + "account_statement_import_base", "web_widget_dropdown_dynamic", ], "data": [ @@ -23,7 +22,6 @@ "wizards/online_bank_statement_pull_wizard.xml", "views/actions.xml", "views/account_journal.xml", - "views/account_bank_statement_line.xml", "views/online_bank_statement_provider.xml", ], "installable": True, diff --git a/account_statement_import_online/migrations/15.0.3.0.0/post-migration.py b/account_statement_import_online/migrations/15.0.3.0.0/post-migration.py new file mode 100644 index 00000000..d0b52831 --- /dev/null +++ b/account_statement_import_online/migrations/15.0.3.0.0/post-migration.py @@ -0,0 +1,17 @@ +# Copyright 2022 Akretion France (http://www.akretion.com/) +# @author: Alexis de Lattre +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from openupgradelib import openupgrade + + +@openupgrade.migrate() +def migrate(env, version): + column = openupgrade.get_legacy_name("online_raw_data") + if openupgrade.column_exists(env.cr, "account_bank_statement_line", column): + openupgrade.logged_query( + env.cr, + "UPDATE account_bank_statement_line SET raw_data={online_raw_data}".format( + online_raw_data=column, + ), + ) diff --git a/account_statement_import_online/migrations/15.0.3.0.0/pre-migration.py b/account_statement_import_online/migrations/15.0.3.0.0/pre-migration.py new file mode 100644 index 00000000..8eb0fd85 --- /dev/null +++ b/account_statement_import_online/migrations/15.0.3.0.0/pre-migration.py @@ -0,0 +1,19 @@ +# Copyright 2022 Akretion France (http://www.akretion.com/) +# @author: Alexis de Lattre +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from openupgradelib import openupgrade + +_column_renames = { + "account_bank_statement_line": [ + ("online_raw_data", None), + ] +} + + +@openupgrade.migrate() +def migrate(env, version): + if openupgrade.column_exists( + env.cr, "account_bank_statement_line", "online_raw_data" + ): + openupgrade.rename_columns(env.cr, _column_renames) diff --git a/account_statement_import_online/models/__init__.py b/account_statement_import_online/models/__init__.py index d3f9d0d6..56bd827c 100644 --- a/account_statement_import_online/models/__init__.py +++ b/account_statement_import_online/models/__init__.py @@ -1,5 +1,4 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). from . import account_journal -from . import account_bank_statement_line from . import online_bank_statement_provider diff --git a/account_statement_import_online/models/account_bank_statement_line.py b/account_statement_import_online/models/account_bank_statement_line.py deleted file mode 100644 index 7f0c7f08..00000000 --- a/account_statement_import_online/models/account_bank_statement_line.py +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2021 Therp BV . -# @author: Ronald Portier . -# Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0). -"""Add raw data to statement line, to solve import issues.""" - -from odoo import fields, models - - -class AccountBankStatementLine(models.Model): - """Add raw data to statement line, to solve import issues.""" - - _inherit = "account.bank.statement.line" - - online_raw_data = fields.Text( - help="The complete data retrieved online for this transaction", - readonly=True, - copy=False, - ) diff --git a/account_statement_import_online/models/online_bank_statement_provider.py b/account_statement_import_online/models/online_bank_statement_provider.py index 85e21af1..64af9344 100644 --- a/account_statement_import_online/models/online_bank_statement_provider.py +++ b/account_statement_import_online/models/online_bank_statement_provider.py @@ -12,7 +12,6 @@ from pytz import timezone, utc from odoo import _, api, fields, models -from odoo.addons.base.models.res_bank import sanitize_account_number from odoo.addons.base.models.res_partner import _tz_get _logger = logging.getLogger(__name__) @@ -257,6 +256,8 @@ class OnlineBankStatementProvider(models.Model): """Get lines from line data, but only for the right date.""" AccountBankStatementLine = self.env["account.bank.statement.line"] provider_tz = timezone(self.tz) if self.tz else utc + journal = self.journal_id + speeddict = journal._statement_line_import_speeddict() filtered_lines = [] for line_values in lines_data: date = line_values["date"] @@ -280,23 +281,18 @@ class OnlineBankStatementProvider(models.Model): date = date.replace(tzinfo=utc) date = date.astimezone(provider_tz).replace(tzinfo=None) line_values["date"] = date + journal._statement_line_import_update_unique_import_id( + line_values, self.account_number + ) unique_import_id = line_values.get("unique_import_id") if unique_import_id: - unique_import_id = self._generate_unique_import_id(unique_import_id) - line_values.update({"unique_import_id": unique_import_id}) if AccountBankStatementLine.sudo().search( [("unique_import_id", "=", unique_import_id)], limit=1 ): continue - bank_account_number = line_values.get("account_number") - if bank_account_number: - sanitized_account_number = self._sanitize_bank_account_number( - bank_account_number - ) - line_values["account_number"] = sanitized_account_number - self._update_partner_from_account_number(line_values) if not line_values.get("payment_ref"): line_values["payment_ref"] = line_values.get("ref") + journal._statement_line_import_update_hook(line_values, speeddict) filtered_lines.append(line_values) return filtered_lines @@ -347,36 +343,6 @@ class OnlineBankStatementProvider(models.Model): date_since = date_since.replace(tzinfo=utc).astimezone(tz) return date_since.date() - def _generate_unique_import_id(self, unique_import_id): - self.ensure_one() - return ( - (self.account_number and self.account_number + "-" or "") - + str(self.journal_id.id) - + "-" - + unique_import_id - ) - - def _sanitize_bank_account_number(self, bank_account_number): - """Hook for extension""" - self.ensure_one() - return sanitize_account_number(bank_account_number) - - def _update_partner_from_account_number(self, line_values): - """Lookup partner using account number.""" - self.ensure_one() - partner_bank = self.env["res.partner.bank"].search( - [ - ("acc_number", "=", line_values["account_number"]), - "|", - ("company_id", "=", False), - ("company_id", "=", self.company_id.id), - ], - limit=1, - ) - if partner_bank: - line_values["partner_bank_id"] = partner_bank.id - line_values["partner_id"] = partner_bank.partner_id.id - def _get_next_run_period(self): self.ensure_one() if self.interval_type == "minutes": diff --git a/account_statement_import_online/views/account_bank_statement_line.xml b/account_statement_import_online/views/account_bank_statement_line.xml deleted file mode 100644 index 1b7dd2f9..00000000 --- a/account_statement_import_online/views/account_bank_statement_line.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - account.bank.statement.line - - - - - - - - - - - - - - diff --git a/account_statement_import_online/views/account_journal.xml b/account_statement_import_online/views/account_journal.xml index f2b7bf83..98044288 100644 --- a/account_statement_import_online/views/account_journal.xml +++ b/account_statement_import_online/views/account_journal.xml @@ -11,7 +11,7 @@ account.journal - + - +