[MIG] account_statement_import_online: Migration to 16.0

account_statement_import_online/models/account_journal.py

@@ -1,7 +1,7 @@
 # Copyright 2019-2020 Brainbean Apps (https://brainbeanapps.com)
 # Copyright 2019-2020 Dataplug (https://dataplug.io)
+# Copyright 2023 Therp BV (https://therp.nl)
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
-
 import logging
 
 from odoo import _, api, fields, models
@@ -12,20 +12,21 @@ _logger = logging.getLogger(__name__)
 class AccountJournal(models.Model):
     _inherit = "account.journal"
 
+    @api.model
+    def _selection_service(self):
+        OnlineBankStatementProvider = self.env["online.bank.statement.provider"]
+        return OnlineBankStatementProvider._get_available_services() + [
+            ("dummy", "Dummy")
+        ]
+
+    # Keep provider fields for compatibility with other modules.
     online_bank_statement_provider = fields.Selection(
-        selection=lambda self: self.env[
-            "account.journal"
-        ]._selection_online_bank_statement_provider(),
-        help="Select the type of service provider (a model)",
+        selection=lambda self: self._selection_service(),
     )
     online_bank_statement_provider_id = fields.Many2one(
         string="Statement Provider",
         comodel_name="online.bank.statement.provider",
-        ondelete="restrict",
         copy=False,
-        help="Select the actual instance of a configured provider (a record).\n"
-        "Selecting a type of provider will automatically create a provider"
-        " record linked to this journal.",
     )
 
     def __get_bank_statements_available_sources(self):
@@ -33,52 +34,65 @@ class AccountJournal(models.Model):
         result.append(("online", _("Online (OCA)")))
         return result
 
-    @api.model
-    def _selection_online_bank_statement_provider(self):
-        return self.env["online.bank.statement.provider"]._get_available_services() + [
-            ("dummy", "Dummy")
-        ]
+    def _update_providers(self):
+        """Automatically create service.
 
-    @api.model
-    def values_online_bank_statement_provider(self):
-        """Return values for provider type selection in the form view."""
-        res = self.env["online.bank.statement.provider"]._get_available_services()
-        if self.user_has_groups("base.group_no_one"):
-            res += [("dummy", "Dummy")]
-        return res
-
-    def _update_online_bank_statement_provider_id(self):
-        """Keep provider synchronized with journal."""
+        This method exists for compatibility reasons. The preferred method
+        to create an online provider is directly through the menu,
+        """
         OnlineBankStatementProvider = self.env["online.bank.statement.provider"]
-        for journal in self.filtered("id"):
-            provider_id = journal.online_bank_statement_provider_id
-            if journal.bank_statements_source != "online":
-                journal.online_bank_statement_provider_id = False
-                if provider_id:
-                    provider_id.unlink()
+        for journal in self.filtered("online_bank_statement_provider"):
+            service = journal.online_bank_statement_provider
+            if (
+                journal.online_bank_statement_provider_id
+                and service == journal.online_bank_statement_provider_id.service
+            ):
+                _logger.info(
+                    "Journal %s already linked to service %s", journal.name, service
+                )
+                # Provider already exists.
                 continue
-            if provider_id.service == journal.online_bank_statement_provider:
-                continue
-            journal.online_bank_statement_provider_id = False
-            if provider_id:
-                provider_id.unlink()
-            # fmt: off
-            journal.online_bank_statement_provider_id = \
-                OnlineBankStatementProvider.create({
+            # Use existing or create new provider for service.
+            provider = OnlineBankStatementProvider.search(
+                [
+                    ("journal_id", "=", journal.id),
+                    ("service", "=", service),
+                ],
+                limit=1,
+            ) or OnlineBankStatementProvider.create(
+                {
                     "journal_id": journal.id,
-                    "service": journal.online_bank_statement_provider,
-                })
-            # fmt: on
+                    "service": service,
+                }
+            )
+            journal.online_bank_statement_provider_id = provider
+            _logger.info("Journal %s now linked to service %s", journal.name, service)
 
-    @api.model
-    def create(self, vals):
-        rec = super().create(vals)
-        if "bank_statements_source" in vals or "online_bank_statement_provider" in vals:
-            rec._update_online_bank_statement_provider_id()
-        return rec
+    @api.model_create_multi
+    def create(self, vals_list):
+        for vals in vals_list:
+            self._update_vals(vals)
+        journals = super().create(vals_list)
+        journals._update_providers()
+        return journals
 
     def write(self, vals):
+        self._update_vals(vals)
         res = super().write(vals)
-        if "bank_statements_source" in vals or "online_bank_statement_provider" in vals:
-            self._update_online_bank_statement_provider_id()
+        self._update_providers()
         return res
+
+    def _update_vals(self, vals):
+        """Ensure consistent values."""
+        if (
+            "bank_statements_source" in vals
+            and vals.get("bank_statements_source") != "online"
+        ):
+            vals["online_bank_statement_provider"] = False
+            vals["online_bank_statement_provider_id"] = False
+
+    def action_online_bank_statements_pull_wizard(self):
+        """This method is also kept for compatibility reasons."""
+        self.ensure_one()
+        provider = self.online_bank_statement_provider_id
+        return provider.action_online_bank_statements_pull_wizard()

account_statement_import_online/models/online_bank_statement_provider.py

@@ -1,5 +1,6 @@
 # Copyright 2019-2020 Brainbean Apps (https://brainbeanapps.com)
 # Copyright 2019-2020 Dataplug (https://dataplug.io)
+# Copyright 2022-2023 Therp BV (https://therp.nl)
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).
 
 import logging
@@ -28,7 +29,6 @@ class OnlineBankStatementProvider(models.Model):
     journal_id = fields.Many2one(
         comodel_name="account.journal",
         required=True,
-        readonly=True,
         ondelete="cascade",
         domain=[("type", "=", "bank")],
     )
@@ -48,7 +48,6 @@ class OnlineBankStatementProvider(models.Model):
     service = fields.Selection(
         selection=lambda self: self._selection_service(),
         required=True,
-        readonly=True,
     )
     interval_type = fields.Selection(
         selection=[
@@ -93,7 +92,6 @@ class OnlineBankStatementProvider(models.Model):
     certificate_public_key = fields.Text()
     certificate_private_key = fields.Text()
     certificate_chain = fields.Text()
-    allow_empty_statements = fields.Boolean()
 
     _sql_constraints = [
         (
@@ -108,6 +106,46 @@ class OnlineBankStatementProvider(models.Model):
         ),
     ]
 
+    @api.model
+    def create(self, vals):
+        """Set provider_id on journal after creation."""
+        records = super().create(vals)
+        records._update_journals()
+        return records
+
+    def write(self, vals):
+        """Set provider_id on journal after creation."""
+        result = super().write(vals)
+        self._update_journals()
+        return result
+
+    def _update_journals(self):
+        """Update journal with this provider.
+
+        This is for compatibility reasons.
+        """
+        for this in self:
+            this.journal_id.write(
+                {
+                    "online_bank_statement_provider_id": this.id,
+                    "online_bank_statement_provider": this.service,
+                    "bank_statements_source": "online",
+                }
+            )
+
+    def unlink(self):
+        """Reset journals."""
+        journals = self.mapped("journal_id")
+        if journals:
+            vals = {
+                "bank_statements_source": "undefined",
+                "online_bank_statement_provider": False,
+                "online_bank_statement_provider_id": False,
+            }
+            journals.write(vals)
+        result = super().unlink()
+        return result
+
     @api.model
     def _get_available_services(self):
         """Hook for extension"""
@@ -145,10 +183,14 @@ class OnlineBankStatementProvider(models.Model):
             }
 
     def _pull(self, date_since, date_until):
+        """Pull data for all providers within requested period."""
         is_scheduled = self.env.context.get("scheduled")
         for provider in self:
             statement_date_since = provider._get_statement_date_since(date_since)
             while statement_date_since < date_until:
+                # Note that statement_date_until is exclusive, while date_until is
+                # inclusive. So if we have daily statements date_until might
+                # be 2020-01-31, while statement_date_until is 2020-02-01.
                 statement_date_until = (
                     statement_date_since + provider._get_statement_date_step()
                 )
@@ -156,32 +198,13 @@ class OnlineBankStatementProvider(models.Model):
                     data = provider._obtain_statement_data(
                         statement_date_since, statement_date_until
                     )
-                except BaseException as e:
-                    if is_scheduled:
-                        _logger.warning(
-                            'Online Bank Statement Provider "%s" failed to'
-                            " obtain statement data since %s until %s"
-                            % (
-                                provider.name,
-                                statement_date_since,
-                                statement_date_until,
-                            ),
-                            exc_info=True,
-                        )
-                        provider.message_post(
-                            body=_(
-                                "Failed to obtain statement data for period "
-                                "since {since} until {until}: {exception}. See server logs for "
-                                "more details."
-                            ).format(
-                                since=statement_date_since,
-                                until=statement_date_until,
-                                exception=escape(str(e)) or _("N/A"),
-                            ),
-                            subject=_("Issue with Online Bank Statement Provider"),
-                        )
-                        break
-                    raise
+                except BaseException as exception:
+                    if not is_scheduled:
+                        raise
+                    provider._log_provider_exception(
+                        exception, statement_date_since, statement_date_until
+                    )
+                    break  # Continue with next provider.
                 provider._create_or_update_statement(
                     data, statement_date_since, statement_date_until
                 )
@@ -189,69 +212,112 @@ class OnlineBankStatementProvider(models.Model):
             if is_scheduled:
                 provider._schedule_next_run()
 
+    def _log_provider_exception(
+        self, exception, statement_date_since, statement_date_until
+    ):
+        """Both log error, and post a message on the provider record."""
+        self.ensure_one()
+        _logger.warning(
+            _(
+                'Online Bank Statement provider "%(name)s" failed to'
+                " obtain statement data since %(since)s until %(until)s"
+            ),
+            dict(
+                name=self.name,
+                since=statement_date_since,
+                until=statement_date_until,
+            ),
+            exc_info=True,
+        )
+        self.message_post(
+            body=_(
+                "Failed to obtain statement data for period "
+                "since {since} until {until}: {exception}. See server logs for "
+                "more details."
+            ).format(
+                since=statement_date_since,
+                until=statement_date_until,
+                exception=escape(str(exception)) or _("N/A"),
+            ),
+            subject=_("Issue with Online Bank Statement self"),
+        )
+
     def _create_or_update_statement(
         self, data, statement_date_since, statement_date_until
     ):
-        """Create or update bank statement with the data retrieved from provider."""
+        """Create or update bank statement with the data retrieved from provider.
+
+        We can not use statement.date as a unique key within the statements
+        of a journal, because this is now a computed field based on the last date in
+        the statement lines.
+
+        However we can still ensure unique and predictable names, so we wil use that
+        to find existing statements.
+        """
         self.ensure_one()
+        if not data:
+            data = ([], {})
+        unfiltered_lines, statement_values = data
+        if not unfiltered_lines:
+            unfiltered_lines = []
+        if not statement_values:
+            statement_values = {}
+        statement_values["name"] = self.make_statement_name(statement_date_since)
+        filtered_lines = self._get_statement_filtered_lines(
+            unfiltered_lines,
+            statement_values,
+            statement_date_since,
+            statement_date_until,
+        )
+        if not filtered_lines:
+            return self.env["account.bank.statement"]
+        if filtered_lines:
+            statement_values.update(
+                {"line_ids": [[0, False, line] for line in filtered_lines]}
+            )
+        self._update_statement_balances(statement_values)
+        statement = self._statement_create_or_write(statement_values)
+        return statement
+
+    def make_statement_name(self, statement_date_since):
+        """Make name for statement using date and journal name."""
+        self.ensure_one()
+        return "%s/%s" % (
+            self.journal_id.code,
+            statement_date_since.strftime("%Y-%m-%d"),
+        )
+
+    def _statement_create_or_write(self, statement_values):
+        """Final creation of statement if new, else write."""
         AccountBankStatement = self.env["account.bank.statement"]
         is_scheduled = self.env.context.get("scheduled")
         if is_scheduled:
             AccountBankStatement = AccountBankStatement.with_context(
                 tracking_disable=True,
             )
-        if not data:
-            data = ([], {})
-        if not data[0] and not data[1] and not self.allow_empty_statements:
-            return
-        lines_data, statement_values = data
-        if not lines_data:
-            lines_data = []
-        if not statement_values:
-            statement_values = {}
-        statement_date = self._get_statement_date(
-            statement_date_since,
-            statement_date_until,
-        )
+        statement_name = statement_values["name"]
         statement = AccountBankStatement.search(
             [
                 ("journal_id", "=", self.journal_id.id),
-                ("state", "=", "open"),
-                ("date", "=", statement_date),
+                ("name", "=", statement_name),
             ],
             limit=1,
         )
         if not statement:
-            statement_values.update(
-                {
-                    "name": "%s/%s"
-                    % (self.journal_id.code, statement_date.strftime("%Y-%m-%d")),
-                    "journal_id": self.journal_id.id,
-                    "date": statement_date,
-                }
-            )
+            statement_values["journal_id"] = self.journal_id.id
             statement = AccountBankStatement.with_context(
                 journal_id=self.journal_id.id,
-            ).create(
-                # NOTE: This is needed since create() alters values
-                statement_values.copy()
-            )
-        filtered_lines = self._get_statement_filtered_lines(
-            lines_data, statement_values, statement_date_since, statement_date_until
-        )
-        statement_values.update(
-            {"line_ids": [[0, False, line] for line in filtered_lines]}
-        )
-        if "balance_start" in statement_values:
-            statement_values["balance_start"] = float(statement_values["balance_start"])
-        if "balance_end_real" in statement_values:
-            statement_values["balance_end_real"] = float(
-                statement_values["balance_end_real"]
-            )
-        statement.write(statement_values)
+            ).create(statement_values)
+        else:
+            statement.write(statement_values)
+        return statement
 
     def _get_statement_filtered_lines(
-        self, lines_data, statement_values, statement_date_since, statement_date_until
+        self,
+        unfiltered_lines,
+        statement_values,
+        statement_date_since,
+        statement_date_until,
     ):
         """Get lines from line data, but only for the right date."""
         AccountBankStatementLine = self.env["account.bank.statement.line"]
@@ -259,7 +325,10 @@ class OnlineBankStatementProvider(models.Model):
         journal = self.journal_id
         speeddict = journal._statement_line_import_speeddict()
         filtered_lines = []
-        for line_values in lines_data:
+        lines_before_since = 0
+        lines_after_until = 0
+        lines_not_unique = 0
+        for line_values in unfiltered_lines:
             date = line_values["date"]
             if not isinstance(date, datetime):
                 date = fields.Datetime.from_string(date)
@@ -271,12 +340,14 @@ class OnlineBankStatementProvider(models.Model):
                     statement_values["balance_start"] = Decimal(
                         statement_values["balance_start"]
                     ) + Decimal(line_values["amount"])
+                lines_before_since += 1
                 continue
             elif date >= statement_date_until:
                 if "balance_end_real" in statement_values:
                     statement_values["balance_end_real"] = Decimal(
                         statement_values["balance_end_real"]
                     ) - Decimal(line_values["amount"])
+                lines_after_until += 1
                 continue
             date = date.replace(tzinfo=utc)
             date = date.astimezone(provider_tz).replace(tzinfo=None)
@@ -289,13 +360,56 @@ class OnlineBankStatementProvider(models.Model):
                 if AccountBankStatementLine.sudo().search(
                     [("unique_import_id", "=", unique_import_id)], limit=1
                 ):
+                    lines_not_unique += 1
                     continue
             if not line_values.get("payment_ref"):
                 line_values["payment_ref"] = line_values.get("ref")
+            line_values["journal_id"] = self.journal_id.id
             journal._statement_line_import_update_hook(line_values, speeddict)
             filtered_lines.append(line_values)
+        if unfiltered_lines:
+            if len(unfiltered_lines) == len(filtered_lines):
+                _logger.debug(_("All lines passed filtering"))
+            else:
+                _logger.debug(
+                    _(
+                        "Of %(lines_provided)s lines provided"
+                        ", %(before)s where before %(since)s"
+                        ", %(after)s where on or after %(until)s"
+                        "and %(duplicate)s where not unique."
+                    ),
+                    dict(
+                        lines_provided=len(unfiltered_lines),
+                        before=lines_before_since,
+                        since=statement_date_since,
+                        after=lines_after_until,
+                        until=statement_date_until,
+                        duplicate=lines_not_unique,
+                    ),
+                )
         return filtered_lines
 
+    def _update_statement_balances(self, statement_values):
+        """Update statement balance_ start/end/end_real."""
+        AccountBankStatement = self.env["account.bank.statement"]
+        if "balance_start" in statement_values:
+            statement_values["balance_start"] = float(statement_values["balance_start"])
+        else:
+            # Take balance_end of previous statement as start of this one.
+            previous_statement = AccountBankStatement.search(
+                [
+                    ("journal_id", "=", self.journal_id.id),
+                    ("name", "<", statement_values["name"]),
+                ],
+                limit=1,
+            )
+            if previous_statement and previous_statement.balance_end:
+                statement_values["balance_start"] = previous_statement.balance_end
+        if "balance_end_real" in statement_values:
+            statement_values["balance_end_real"] = float(
+                statement_values["balance_end_real"]
+            )
+
     def _schedule_next_run(self):
         self.ensure_one()
         self.last_successful_run = self.next_run
@@ -334,15 +448,6 @@ class OnlineBankStatementProvider(models.Model):
                 microsecond=0,
             )
 
-    def _get_statement_date(self, date_since, date_until):
-        self.ensure_one()
-        # NOTE: Statement date is treated by Odoo as start of period. Details
-        #  - addons/account/models/account_journal_dashboard.py
-        #  - def get_line_graph_datas()
-        tz = timezone(self.tz) if self.tz else utc
-        date_since = date_since.replace(tzinfo=utc).astimezone(tz)
-        return date_since.date()
-
     def _get_next_run_period(self):
         self.ensure_one()
         if self.interval_type == "minutes":
@@ -356,17 +461,19 @@ class OnlineBankStatementProvider(models.Model):
 
     @api.model
     def _scheduled_pull(self):
-        _logger.info("Scheduled pull of online bank statements...")
-
+        _logger.info(_("Scheduled pull of online bank statements..."))
         providers = self.search(
             [("active", "=", True), ("next_run", "<=", fields.Datetime.now())]
         )
         if providers:
             _logger.info(
-                "Pulling online bank statements of: %s"
-                % ", ".join(providers.mapped("journal_id.name"))
+                _("Pulling online bank statements of: %(provider_names)s"),
+                dict(provider_names=", ".join(providers.mapped("journal_id.name"))),
             )
-            for provider in providers.with_context(**{"scheduled": True}):
+            for provider in providers.with_context(
+                scheduled=True, tracking_disable=True
+            ):
+                provider._adjust_schedule()
                 date_since = (
                     (provider.last_successful_run)
                     if provider.last_successful_run
@@ -374,11 +481,38 @@ class OnlineBankStatementProvider(models.Model):
                 )
                 date_until = provider.next_run
                 provider._pull(date_since, date_until)
+        _logger.info(_("Scheduled pull of online bank statements complete."))
 
-        _logger.info("Scheduled pull of online bank statements complete.")
+    def _adjust_schedule(self):
+        """Make sure next_run is current.
+
+        Current means adding one more period would put if after the
+        current moment. This will be done at the end of the run.
+        The net effect of this method and the adjustment after the run
+        will be for the next_run to be in the future.
+        """
+        self.ensure_one()
+        delta = self._get_next_run_period()
+        now = datetime.now()
+        next_run = self.next_run + delta
+        while next_run < now:
+            self.next_run = next_run
+            next_run = self.next_run + delta
 
     def _obtain_statement_data(self, date_since, date_until):
         """Hook for extension"""
         # Check tests/online_bank_statement_provider_dummy.py for reference
         self.ensure_one()
         return []
+
+    def action_online_bank_statements_pull_wizard(self):
+        self.ensure_one()
+        WIZARD_MODEL = "online.bank.statement.pull.wizard"
+        wizard = self.env[WIZARD_MODEL].create([{"provider_ids": [(6, 0, [self.id])]}])
+        return {
+            "type": "ir.actions.act_window",
+            "res_model": WIZARD_MODEL,
+            "res_id": wizard.id,
+            "view_mode": "form",
+            "target": "new",
+        }