Add module account_statement_import_base

The 2 modules account_statement_import_online and
account_statement_import depend on account_statement_import_base (and
not on each other) and share common code, in particular a hook to update
the statement line. So we can now have reconciliation modules that use
this hook and therefore work both on file import and online import. More
details on https://github.com/OCA/bank-statement-import/issues/481.

Improve bank statement line form view and journal form view.

account_statement_import/__manifest__.py

@@ -8,7 +8,7 @@
     "category": "Accounting",
     "version": "14.0.2.1.0",
     "license": "LGPL-3",
-    "depends": ["account"],
+    "depends": ["account_statement_import_base"],
     "author": "Odoo SA, Akretion, Odoo Community Association (OCA)",
     "maintainers": ["alexis-via"],
     "development_status": "Mature",
@@ -17,7 +17,6 @@
         "security/ir.model.access.csv",
         "wizard/account_statement_import_view.xml",
         "views/account_journal.xml",
-        "views/account_bank_statement_line.xml",
     ],
     "demo": [
         "demo/partner_bank.xml",

account_statement_import/models/__init__.py

@@ -1,2 +1 @@
 from . import account_journal
-from . import account_bank_statement_line

account_statement_import/views/account_bank_statement_line.xml

@@ -1,25 +0,0 @@
-<?xml version="1.0" ?>
-<!--
-  Copyright 2020 Akretion France (http://www.akretion.com/)
-  @author: Alexis de Lattre <alexis.delattre@akretion.com>
-  Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0).
--->
-<odoo>
-
-<record id="view_bank_statement_line_form" model="ir.ui.view">
-    <field name="model">account.bank.statement.line</field>
-    <field name="inherit_id" ref="account.view_bank_statement_line_form" />
-    <field name="arch" type="xml">
-        <field name="transaction_type" position="before">
-            <field name="account_number" />
-        </field>
-        <field name="transaction_type" position="after">
-            <field name="partner_bank_id" />
-        </field>
-        <field name="move_id" position="after">
-            <field name="unique_import_id" groups="base.group_no_one" />
-        </field>
-    </field>
-</record>
-
-</odoo>

account_statement_import/wizard/account_statement_import.py

@@ -277,43 +277,15 @@ class AccountStatementImport(models.TransientModel):
             )
         return journal
 
-    @api.model
-    def _update_partner_from_account_number(self, lvals):
-        partner_bank = self.env["res.partner.bank"].search(
-            [("acc_number", "=", lvals["account_number"])], limit=1
-        )
-        if partner_bank:
-            lvals["partner_bank_id"] = partner_bank.id
-            lvals["partner_id"] = partner_bank.partner_id.id
-
     def _complete_stmts_vals(self, stmts_vals, journal, account_number):
+        speeddict = journal._statement_line_import_speeddict()
         for st_vals in stmts_vals:
             st_vals["journal_id"] = journal.id
             for lvals in st_vals["transactions"]:
-                unique_import_id = lvals.get("unique_import_id")
-                if unique_import_id:
-                    sanitized_account_number = sanitize_account_number(account_number)
-                    lvals["unique_import_id"] = (
-                        (
-                            sanitized_account_number
-                            and sanitized_account_number + "-"
-                            or ""
-                        )
-                        + str(journal.id)
-                        + "-"
-                        + unique_import_id
-                    )
-
-                if (
-                    not lvals.get("partner_bank_id")
-                    and lvals.get("account_number")
-                    and not lvals.get("partner_id")
-                ):
-                    # Find the partner from his bank account number
-                    # The partner selected during the
-                    # reconciliation process will be linked to the bank account
-                    # when the statement is closed (code in the account module)
-                    self._update_partner_from_account_number(lvals)
+                journal._statement_line_import_update_unique_import_id(
+                    lvals, account_number
+                )
+                journal._statement_line_import_update_hook(lvals, speeddict)
                 if not lvals.get("payment_ref"):
                     raise UserError(_("Missing payment_ref on a transaction."))
         return stmts_vals

account_statement_import_base/README.rst

@@ -0,0 +1 @@
+Will be autogenerated from the readme subdir

account_statement_import_base/__init__.py

@@ -0,0 +1 @@
+from . import models

account_statement_import_base/__manifest__.py

@@ -0,0 +1,19 @@
+# Copyright 2022 Akretion France (http://www.akretion.com/)
+# @author: Alexis de Lattre <alexis.delattre@akretion.com>
+# Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0).
+
+{
+    "name": "Base module for Bank Statement Import",
+    "category": "Accounting",
+    "version": "14.0.1.0.0",
+    "license": "LGPL-3",
+    "depends": ["account"],
+    "author": "Akretion, Odoo Community Association (OCA)",
+    "maintainers": ["alexis-via"],
+    "development_status": "Mature",
+    "website": "https://github.com/OCA/bank-statement-import",
+    "data": [
+        "views/account_bank_statement_line.xml",
+    ],
+    "installable": True,
+}

account_statement_import_base/models/__init__.py

@@ -0,0 +1,2 @@
+from . import account_bank_statement_line
+from . import account_journal

account_statement_import_base/models/account_bank_statement_line.py

@@ -1,7 +1,6 @@
-# Copyright 2004-2020 Odoo S.A.
-# Copyright 2020 Akretion France (http://www.akretion.com/)
+# Copyright 2022 Akretion France (http://www.akretion.com/)
 # @author: Alexis de Lattre <alexis.delattre@akretion.com>
-# Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0).
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl).
 
 from odoo import fields, models
 
@@ -10,8 +9,9 @@ class AccountBankStatementLine(models.Model):
     _inherit = "account.bank.statement.line"
 
     # Ensure transactions can be imported only once
-    # (if the import format provides unique transaction ids)
+    # if the import format provides unique transaction IDs
     unique_import_id = fields.Char(string="Import ID", readonly=True, copy=False)
+    raw_data = fields.Text(readonly=True, copy=False)
 
     _sql_constraints = [
         (

account_statement_import_base/models/account_journal.py

@@ -0,0 +1,69 @@
+# Copyright 2022 Akretion France (http://www.akretion.com/)
+# @author: Alexis de Lattre <alexis.delattre@akretion.com>
+# License LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl).
+
+from odoo import api, models
+
+from odoo.addons.base.models.res_bank import sanitize_account_number
+
+
+class AccountJournal(models.Model):
+    _inherit = "account.journal"
+
+    def _statement_line_import_speeddict(self):
+        """This method is designed to be inherited by reconciliation modules.
+        These modules can take advantage of this method to pre-fetch data
+        that will later be used for many statement lines (to avoid
+        searching data for each statement line).
+        The goal is to improve performances.
+        """
+        self.ensure_one()
+        speeddict = {"account_number": {}}
+        partner_banks = self.env["res.partner.bank"].search_read(
+            [("company_id", "in", (False, self.company_id.id))],
+            ["acc_number", "partner_id"],
+        )
+        for partner_bank in partner_banks:
+            speeddict["account_number"][partner_bank["acc_number"]] = {
+                "partner_id": partner_bank["partner_id"][0],
+                "partner_bank_id": partner_bank["id"],
+            }
+        return speeddict
+
+    def _statement_line_import_update_hook(self, st_line_vals, speeddict):
+        """This method is designed to be inherited by reconciliation modules.
+        In this method you can:
+        - update the partner of the line by writing st_line_vals['partner_id']
+        - set an automated counter-part via counterpart_account_id by writing
+          st_line_vals['counterpart_account_id']
+        - do anythink you want with the statement line
+        """
+        self.ensure_one()
+        if st_line_vals.get("account_number"):
+            st_line_vals["account_number"] = self._sanitize_bank_account_number(
+                st_line_vals["account_number"]
+            )
+            if not st_line_vals.get("partner_id") and speeddict["account_number"].get(
+                st_line_vals["account_number"]
+            ):
+                st_line_vals.update(
+                    speeddict["account_number"][st_line_vals["account_number"]]
+                )
+
+    def _statement_line_import_update_unique_import_id(
+        self, st_line_vals, account_number
+    ):
+        self.ensure_one()
+        if st_line_vals.get("unique_import_id"):
+            sanitized_acc_number = self._sanitize_bank_account_number(account_number)
+            st_line_vals["unique_import_id"] = (
+                (sanitized_acc_number and sanitized_acc_number + "-" or "")
+                + str(self.id)
+                + "-"
+                + st_line_vals["unique_import_id"]
+            )
+
+    @api.model
+    def _sanitize_bank_account_number(self, account_number):
+        """Hook for extension"""
+        return sanitize_account_number(account_number)

account_statement_import_base/readme/CONTRIBUTORS.rst

@@ -0,0 +1 @@
+* Alexis de Lattre <alexis.delattre@akretion.com>

account_statement_import_base/readme/DESCRIPTION.rst

@@ -0,0 +1,4 @@
+This is a technical module designed to share code between 2 other modules:
+
+* **account_statement_import** that allows to import bank statements from files,
+* **account_statement_import_online** that allows to import bank statements from webservices/APIs.

account_statement_import_base/views/account_bank_statement_line.xml

@@ -0,0 +1,61 @@
+<?xml version="1.0" ?>
+<!--
+  Copyright 2022 Akretion France (http://www.akretion.com/)
+  @author: Alexis de Lattre <alexis.delattre@akretion.com>
+  Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0).
+-->
+<odoo>
+
+<record id="view_bank_statement_line_form" model="ir.ui.view">
+    <field name="model">account.bank.statement.line</field>
+    <field name="inherit_id" ref="account.view_bank_statement_line_form" />
+    <field name="arch" type="xml">
+        <field name="statement_id" position="attributes">
+            <attribute
+                    name="invisible"
+                >not context.get('statement_line_main_view')</attribute>
+        </field>
+        <xpath expr="//field[@name='company_id']/.." position="attributes">
+            <attribute name="col">2</attribute>
+        </xpath>
+        <field name="sequence" position="attributes">
+            <attribute name="invisible">1</attribute>
+        </field>
+        <!--
+            I don't see anything interesting in the 'ref' field of the
+            move linked to the statement line
+            and the field move_id is displayed in the view.
+            But if you think it's useful to see this field,
+            don't hesitate to remove the 2 lines below
+        -->
+        <field name="ref" position="attributes">
+            <attribute name="invisible">1</attribute>
+        </field>
+        <field name="narration" position="attributes">
+            <attribute name="invisible">1</attribute>
+        </field>
+        <field name="transaction_type" position="after">
+            <field name="partner_bank_id" />
+        </field>
+        <sheet position="inside">
+            <notebook>
+                <page name="narration" string="Notes">
+                    <field name="narration" nolabel="1" />
+                </page>
+                <page name="technical" string="Technical Information">
+                    <group name="tech-fields">
+                        <field name="unique_import_id" />
+                        <field name="partner_name" />
+                        <field name="account_number" />
+                        <field name="is_reconciled" />
+                    </group>
+                    <group name="raw_data" string="Raw Data">
+                        <field name="raw_data" nolabel="1" />
+                    </group>
+                </page>
+            </notebook>
+        </sheet>
+    </field>
+</record>
+
+</odoo>

account_statement_import_online/__manifest__.py

@@ -4,7 +4,7 @@
 
 {
     "name": "Online Bank Statements",
-    "version": "14.0.2.1.1",
+    "version": "14.0.3.0.0",
     "author": "CorporateHub, Odoo Community Association (OCA)",
     "maintainers": ["alexey-pelykh"],
     "website": "https://github.com/OCA/bank-statement-import",
@@ -13,8 +13,7 @@
     "summary": "Online bank statements update",
     "external_dependencies": {"python": ["odoo_test_helper"]},
     "depends": [
-        "account",
-        "account_statement_import",
+        "account_statement_import_base",
         "web_widget_dropdown_dynamic",
     ],
     "data": [
@@ -24,7 +23,6 @@
         "wizards/online_bank_statement_pull_wizard.xml",
         "views/actions.xml",
         "views/account_journal.xml",
-        "views/account_bank_statement_line.xml",
         "views/online_bank_statement_provider.xml",
     ],
     "installable": True,

account_statement_import_online/models/__init__.py

@@ -1,5 +1,4 @@
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
 from . import account_journal
-from . import account_bank_statement_line
 from . import online_bank_statement_provider

account_statement_import_online/models/account_bank_statement_line.py

@@ -1,18 +0,0 @@
-# Copyright 2021 Therp BV <https://therp.nl>.
-# @author: Ronald Portier <ronald@therp.nl>.
-# Licence LGPL-3.0 or later (https://www.gnu.org/licenses/lgpl-3.0).
-"""Add raw data to statement line, to solve import issues."""
-
-from odoo import fields, models
-
-
-class AccountBankStatementLine(models.Model):
-    """Add raw data to statement line, to solve import issues."""
-
-    _inherit = "account.bank.statement.line"
-
-    online_raw_data = fields.Text(
-        help="The complete data retrieved online for this transaction",
-        readonly=True,
-        copy=False,
-    )

account_statement_import_online/models/online_bank_statement_provider.py

@@ -12,7 +12,6 @@ from pytz import timezone, utc
 
 from odoo import _, api, fields, models
 
-from odoo.addons.base.models.res_bank import sanitize_account_number
 from odoo.addons.base.models.res_partner import _tz_get
 
 _logger = logging.getLogger(__name__)
@@ -259,6 +258,8 @@ class OnlineBankStatementProvider(models.Model):
         """Get lines from line data, but only for the right date."""
         AccountBankStatementLine = self.env["account.bank.statement.line"]
         provider_tz = timezone(self.tz) if self.tz else utc
+        journal = self.journal_id
+        speeddict = journal._statement_line_import_speeddict()
         filtered_lines = []
         for line_values in lines_data:
             date = line_values["date"]
@@ -282,23 +283,18 @@ class OnlineBankStatementProvider(models.Model):
             date = date.replace(tzinfo=utc)
             date = date.astimezone(provider_tz).replace(tzinfo=None)
             line_values["date"] = date
+            journal._statement_line_import_update_unique_import_id(
+                line_values, self.account_number
+            )
             unique_import_id = line_values.get("unique_import_id")
             if unique_import_id:
-                unique_import_id = self._generate_unique_import_id(unique_import_id)
-                line_values.update({"unique_import_id": unique_import_id})
                 if AccountBankStatementLine.sudo().search(
                     [("unique_import_id", "=", unique_import_id)], limit=1
                 ):
                     continue
-            bank_account_number = line_values.get("account_number")
-            if bank_account_number:
-                sanitized_account_number = self._sanitize_bank_account_number(
-                    bank_account_number
-                )
-                line_values["account_number"] = sanitized_account_number
-                self._update_partner_from_account_number(line_values)
             if not line_values.get("payment_ref"):
                 line_values["payment_ref"] = line_values.get("ref")
+            journal._statement_line_import_update_hook(line_values, speeddict)
             filtered_lines.append(line_values)
         return filtered_lines
 
@@ -349,36 +345,6 @@ class OnlineBankStatementProvider(models.Model):
         date_since = date_since.replace(tzinfo=utc).astimezone(tz)
         return date_since.date()
 
-    def _generate_unique_import_id(self, unique_import_id):
-        self.ensure_one()
-        return (
-            (self.account_number and self.account_number + "-" or "")
-            + str(self.journal_id.id)
-            + "-"
-            + unique_import_id
-        )
-
-    def _sanitize_bank_account_number(self, bank_account_number):
-        """Hook for extension"""
-        self.ensure_one()
-        return sanitize_account_number(bank_account_number)
-
-    def _update_partner_from_account_number(self, line_values):
-        """Lookup partner using account number."""
-        self.ensure_one()
-        partner_bank = self.env["res.partner.bank"].search(
-            [
-                ("acc_number", "=", line_values["account_number"]),
-                "|",
-                ("company_id", "=", False),
-                ("company_id", "=", self.company_id.id),
-            ],
-            limit=1,
-        )
-        if partner_bank:
-            line_values["partner_bank_id"] = partner_bank.id
-            line_values["partner_id"] = partner_bank.partner_id.id
-
     def _get_next_run_period(self):
         self.ensure_one()
         if self.interval_type == "minutes":

account_statement_import_online/views/account_bank_statement_line.xml

@@ -1,25 +0,0 @@
-<?xml version="1.0" ?>
-<odoo>
-    <record id="view_bank_statement_line_form" model="ir.ui.view">
-        <field name="model">account.bank.statement.line</field>
-        <field
-            name="inherit_id"
-            ref="account_statement_import.view_bank_statement_line_form"
-        />
-        <field name="arch" type="xml">
-            <field name="move_id" position="after">
-                <field name="partner_name" />
-            </field>
-            <xpath expr="//sheet" position="inside">
-                <group colspan="4" col="1">
-                    <separator string="Raw Data" />
-                    <field
-                        name="online_raw_data"
-                        nolabel="1"
-                        groups="base.group_no_one"
-                    />
-                </group>
-            </xpath>
-        </field>
-    </record>
-</odoo>

account_statement_import_online/views/account_journal.xml

@@ -11,7 +11,7 @@
         <field name="model">account.journal</field>
         <field name="inherit_id" ref="account.view_account_journal_form" />
         <field name="arch" type="xml">
-            <page name="bank_account" position="inside">
+            <xpath expr="//field[@name='bank_statements_source']/.." position="after">
                 <group
                     name="online_bank_statements"
                     string="Online Bank Statements (OCA)"
@@ -47,7 +47,7 @@
                         class="oe_read_only"
                     />
                 </group>
-            </page>
+            </xpath>
             <xpath expr="/form/sheet" position="before">
                 <header>
                     <button

account_statement_import_online_paypal/models/online_bank_statement_provider_paypal.py

@@ -297,7 +297,7 @@ class OnlineBankStatementProviderPayPal(models.Model):
             "date": date,
             "payment_ref": note,
             "unique_import_id": unique_import_id,
-            "online_raw_data": transaction,
+            "raw_data": transaction,
         }
         payer_full_name = payer_name.get("full_name") or payer_name.get(
             "alternate_full_name"

account_statement_import_online_paypal/tests/test_account_statement_import_online_paypal.py

@@ -569,7 +569,7 @@ class TestAccountBankAccountStatementImportOnlinePayPal(common.TransactionCase):
             )
 
         self.assertEqual(len(data[0]), 2)
-        del data[0][0]["online_raw_data"]
+        del data[0][0]["raw_data"]
         self.assertEqual(
             data[0][0],
             {
@@ -645,7 +645,7 @@ class TestAccountBankAccountStatementImportOnlinePayPal(common.TransactionCase):
             )
         )
         self.assertEqual(len(lines), 1)
-        del lines[0]["online_raw_data"]
+        del lines[0]["raw_data"]
         self.assertEqual(
             lines[0],
             {
@@ -709,7 +709,7 @@ class TestAccountBankAccountStatementImportOnlinePayPal(common.TransactionCase):
             )
         )
         self.assertEqual(len(lines), 1)
-        del lines[0]["online_raw_data"]
+        del lines[0]["raw_data"]
         self.assertEqual(
             lines[0],
             {
@@ -773,7 +773,7 @@ class TestAccountBankAccountStatementImportOnlinePayPal(common.TransactionCase):
             )
         )
         self.assertEqual(len(lines), 2)
-        del lines[0]["online_raw_data"]
+        del lines[0]["raw_data"]
         self.assertEqual(
             lines[0],
             {
@@ -844,7 +844,7 @@ class TestAccountBankAccountStatementImportOnlinePayPal(common.TransactionCase):
             )
         )
         self.assertEqual(len(lines), 1)
-        del lines[0]["online_raw_data"]
+        del lines[0]["raw_data"]
         self.assertEqual(
             lines[0],
             {

account_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py

@@ -237,7 +237,7 @@ class OnlineBankStatementProviderPonto(models.Model):
             "payment_ref": attributes.get("remittanceInformation", ref),
             "unique_import_id": transaction["id"],
             "amount": attributes["amount"],
-            "online_raw_data": transaction,
+            "raw_data": transaction,
         }
         if attributes.get("counterpartReference"):
             vals_line["account_number"] = attributes["counterpartReference"]

setup/account_statement_import_base/odoo/addons/account_statement_import_base

@@ -0,0 +1 @@
+../../../../account_statement_import_base

setup/account_statement_import_base/setup.py

@@ -0,0 +1,6 @@
+import setuptools
+
+setuptools.setup(
+    setup_requires=['setuptools-odoo'],
+    odoo_addon=True,
+)