From 0920960d1e87e3eb47faa9b2de8c8faf8ed42e60 Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Tue, 13 Sep 2022 22:46:34 +0200 Subject: [PATCH] [IMP] *_online_ponto: Transactions are read backwards from Ponto. The streamlined logic means we no longer have to store (or clear) the last identifier retrieved from Ponto. We will always read from the latest data, backward until we hit an execution date that is before the date we are interested in. --- .../__manifest__.py | 5 +- .../data/ir_cron.xml | 20 ++++++ .../online_bank_statement_provider_ponto.py | 63 +++++++++++++++---- .../models/ponto_buffer.py | 6 +- .../models/ponto_buffer_line.py | 1 + .../models/ponto_interface.py | 8 ++- .../online_bank_statement_provider.xml | 17 +++-- 7 files changed, 95 insertions(+), 25 deletions(-) create mode 100644 account_bank_statement_import_online_ponto/data/ir_cron.xml rename account_bank_statement_import_online_ponto/{view => views}/online_bank_statement_provider.xml (52%) diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py index 0a5b371d..76d197cd 100644 --- a/account_bank_statement_import_online_ponto/__manifest__.py +++ b/account_bank_statement_import_online_ponto/__manifest__.py @@ -1,4 +1,4 @@ -# Copyright 2020 Florent de Labarre +# Copyright 2020 Florent de Labarre. # Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { @@ -14,7 +14,8 @@ "installable": True, "depends": ["account_bank_statement_import_online"], "data": [ + "data/ir_cron.xml", "security/ir.model.access.csv", - "view/online_bank_statement_provider.xml", + "views/online_bank_statement_provider.xml", ], } diff --git a/account_bank_statement_import_online_ponto/data/ir_cron.xml b/account_bank_statement_import_online_ponto/data/ir_cron.xml new file mode 100644 index 00000000..0e1b3c25 --- /dev/null +++ b/account_bank_statement_import_online_ponto/data/ir_cron.xml @@ -0,0 +1,20 @@ + + + + + + Remove old data from ponto buffers + 1 + days + -1 + code + 2019-01-01 00:20:00 + + + model._ponto_buffer_purge() + + + diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py index 7f25be7c..17413361 100644 --- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py +++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py @@ -1,7 +1,8 @@ # Copyright 2020 Florent de Labarre # Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). -from datetime import datetime +from datetime import date, datetime +from dateutil.relativedelta import relativedelta import json import pytz @@ -16,10 +17,12 @@ _logger = logging.getLogger(__name__) class OnlineBankStatementProviderPonto(models.Model): _inherit = "online.bank.statement.provider" - ponto_last_identifier = fields.Char(readonly=True) - - def ponto_reset_last_identifier(self): - self.write({"ponto_last_identifier": False}) + ponto_buffer_retain_days = fields.Integer( + string="Number of days to keep Ponto Buffers", + default=61, + help="By default buffers will be kept for 61 days.\n" + "Set this to 0 to keep buffers indefinitely.", + ) @api.model def _get_available_services(self): @@ -31,17 +34,21 @@ class OnlineBankStatementProviderPonto(models.Model): def _pull(self, date_since, date_until): """Override pull to first retrieve data from Ponto.""" if self.service == "ponto": - self._ponto_retrieve_data() + self._ponto_retrieve_data(date_since) super()._pull(date_since, date_until) - def _ponto_retrieve_data(self): - """Fill buffer with data from Ponto.""" + def _ponto_retrieve_data(self, date_since): + """Fill buffer with data from Ponto. + + We will retrieve data from the latest transactions present in Ponto + backwards, until we find data that has an execution date before date_since. + """ interface_model = self.env["ponto.interface"] buffer_model = self.env["ponto.buffer"] access_data = interface_model._login(self.username, self.password) interface_model._set_access_account(access_data, self.account_number) interface_model._ponto_synchronisation(access_data) - latest_identifier = self.ponto_last_identifier + latest_identifier = False transactions = interface_model._get_transactions( access_data, latest_identifier @@ -49,6 +56,9 @@ class OnlineBankStatementProviderPonto(models.Model): while transactions: buffer_model.sudo()._store_transactions(self, transactions) latest_identifier = transactions[-1].get("id") + earliest_datetime = self._ponto_get_execution_datetime(transactions[-1]) + if earliest_datetime < date_since: + break transactions = interface_model._get_transactions( access_data, latest_identifier @@ -107,7 +117,7 @@ class OnlineBankStatementProviderPonto(models.Model): if attributes.get(x) ] ref = " ".join(ref_list) - date = self._ponto_date_from_string(attributes.get("executionDate")) + date = self._ponto_get_execution_datetime(transaction) vals_line = { "sequence": sequence, "date": date, @@ -122,10 +132,41 @@ class OnlineBankStatementProviderPonto(models.Model): vals_line["partner_name"] = attributes["counterpartName"] return vals_line - def _ponto_date_from_string(self, date_str): + def _ponto_get_execution_datetime(self, transaction): + """Get execution datetime for a transaction. + + Odoo often names variables containing date and time just xxx_date or + date_xxx. We try to avoid this misleading naming by using datetime as + much for variables and fields of type datetime. + """ + attributes = transaction.get("attributes", {}) + return self._ponto_datetime_from_string(attributes.get("executionDate")) + + def _ponto_datetime_from_string(self, date_str): """Dates in Ponto are expressed in UTC, so we need to convert them to supplied tz for proper classification. """ dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ") dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc")) return dt.replace(tzinfo=None) + + def _ponto_buffer_purge(self): + """Remove buffers from Ponto no longer needed to import statements.""" + _logger.info("Scheduled purge of old ponto buffers...") + today = date.today() + buffer_model = self.env["ponto.buffer"] + providers = self.search([ + ("active", "=", True), + ]) + for provider in providers: + if not provider.ponto_buffer_retain_days: + continue + cutoff_date = today - relativedelta(days=provider.ponto_buffer_retain_days) + old_buffers = buffer_model.search( + [ + ("provider_id", "=", provider.id), + ("effective_date", "<", cutoff_date), + ] + ) + old_buffers.unlink() + _logger.info("Scheduled purge of old ponto buffers complete.") diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer.py b/account_bank_statement_import_online_ponto/models/ponto_buffer.py index 5acffa3d..25157300 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_buffer.py +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer.py @@ -24,7 +24,6 @@ class PontoBuffer(models.Model): comodel_name="ponto.buffer.line", inverse_name="buffer_id", readonly=True, - ondelete="cascade", ) def _store_transactions(self, provider, transactions): @@ -32,10 +31,7 @@ class PontoBuffer(models.Model): # Start by sorting all transactions per date. transactions_per_date = {} for transaction in transactions: - ponto_execution_date = transaction.get( - "attributes", {} - ).get("executionDate") - effective_date_time = provider._ponto_date_from_string(ponto_execution_date) + effective_date_time = provider._ponto_get_execution_datetime(transaction) transaction["effective_date_time"] = effective_date_time.isoformat() key = effective_date_time.isoformat()[0:10] if key not in transactions_per_date: diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py index 54527476..fcf0a953 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py +++ b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py @@ -13,6 +13,7 @@ class PontoBuffer(models.Model): comodel_name="ponto.buffer", required=True, readonly=True, + ondelete="cascade", ) ponto_id = fields.Char( required=True, diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py index 2c979d86..c84626c0 100644 --- a/account_bank_statement_import_online_ponto/models/ponto_interface.py +++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py @@ -137,7 +137,13 @@ class PontoInterface(models.AbstractModel): time.sleep(40) def _get_transactions(self, access_data, last_identifier): - """Get transactions from ponto, using last_identifier as pointer.""" + """Get transactions from ponto, using last_identifier as pointer. + + Note that Ponto has the transactions in descending order. The first + transaction, retrieved by not passing an identifier, is the latest + present in Ponto. If you read transactions 'after' a certain identifier + (Ponto id), you will get transactions with an earlier date. + """ url = ( PONTO_ENDPOINT + "/accounts/" diff --git a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml similarity index 52% rename from account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml rename to account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml index e856cf56..93e98e87 100644 --- a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml +++ b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml @@ -6,12 +6,17 @@ - - - - -