audrius/bank-statement-import
1
0
Fork 1
mirror of https://github.com/OCA/bank-statement-import.git synced 2025-01-20 12:37:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

[IMP] *_online_ponto: Transactions are read backwards from Ponto.

Browse Source 
The streamlined logic means we no longer have to store (or clear) the
last identifier retrieved from Ponto. We will always read from the latest data,
backward until we hit an execution date that is before the date we are interested in.
This commit is contained in:
Ronald Portier (Therp BV)
2022-09-13 22:46:34 +02:00
parent a33edfb75f
commit 0920960d1e
7 changed files with 95 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
account_bank_statement_import_online_ponto/__manifest__.py
View File

@@ -1,4 +1,4 @@
# Copyright 2020 Florent de Labarre
# Copyright 2020 Florent de Labarre.
# Copyright 2022 Therp BV <https://therp.nl>.
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
{
@@ -14,7 +14,8 @@
"installable": True,
"depends": ["account_bank_statement_import_online"],
"data": [
"data/ir_cron.xml",
"security/ir.model.access.csv",
"view/online_bank_statement_provider.xml",
"views/online_bank_statement_provider.xml",
],
}

20
account_bank_statement_import_online_ponto/data/ir_cron.xml Normal file
View File

@@ -0,0 +1,20 @@
<?xml version="1.0" ?>
<!--
Copyright 2022 Therp BV (https://therp.nl)
License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-->
<odoo noupdate="1">
<record model="ir.cron" id="ir_cron_purge_ponto_buffer">
<field name="name">Remove old data from ponto buffers</field>
<field name="interval_number">1</field>
<field name="interval_type">days</field>
<field name="numbercall">-1</field>
<field name="state">code</field>
<field name="nextcall">2019-01-01 00:20:00</field>
<field name="doall" eval="False"/>
<field name="model_id" ref="account_bank_statement_import_online_ponto.model_online_bank_statement_provider"/>
<field name="code">model._ponto_buffer_purge()</field>
</record>
</odoo>

63
account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
View File

@@ -1,7 +1,8 @@
# Copyright 2020 Florent de Labarre
# Copyright 2022 Therp BV <https://therp.nl>.
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
from datetime import datetime
from datetime import date, datetime
from dateutil.relativedelta import relativedelta
import json
import pytz
@@ -16,10 +17,12 @@ _logger = logging.getLogger(__name__)
class OnlineBankStatementProviderPonto(models.Model):
_inherit = "online.bank.statement.provider"
ponto_last_identifier = fields.Char(readonly=True)
def ponto_reset_last_identifier(self):
self.write({"ponto_last_identifier": False})
ponto_buffer_retain_days = fields.Integer(
string="Number of days to keep Ponto Buffers",
default=61,
help="By default buffers will be kept for 61 days.\n"
"Set this to 0 to keep buffers indefinitely.",
)
@api.model
def _get_available_services(self):
@@ -31,17 +34,21 @@ class OnlineBankStatementProviderPonto(models.Model):
def _pull(self, date_since, date_until):
"""Override pull to first retrieve data from Ponto."""
if self.service == "ponto":
self._ponto_retrieve_data()
self._ponto_retrieve_data(date_since)
super()._pull(date_since, date_until)
def _ponto_retrieve_data(self):
"""Fill buffer with data from Ponto."""
def _ponto_retrieve_data(self, date_since):
"""Fill buffer with data from Ponto.
We will retrieve data from the latest transactions present in Ponto
backwards, until we find data that has an execution date before date_since.
"""
interface_model = self.env["ponto.interface"]
buffer_model = self.env["ponto.buffer"]
access_data = interface_model._login(self.username, self.password)
interface_model._set_access_account(access_data, self.account_number)
interface_model._ponto_synchronisation(access_data)
latest_identifier = self.ponto_last_identifier
latest_identifier = False
transactions = interface_model._get_transactions(
access_data,
latest_identifier
@@ -49,6 +56,9 @@ class OnlineBankStatementProviderPonto(models.Model):
while transactions:
buffer_model.sudo()._store_transactions(self, transactions)
latest_identifier = transactions[-1].get("id")
earliest_datetime = self._ponto_get_execution_datetime(transactions[-1])
if earliest_datetime < date_since:
break
transactions = interface_model._get_transactions(
access_data,
latest_identifier
@@ -107,7 +117,7 @@ class OnlineBankStatementProviderPonto(models.Model):
if attributes.get(x)
]
ref = " ".join(ref_list)
date = self._ponto_date_from_string(attributes.get("executionDate"))
date = self._ponto_get_execution_datetime(transaction)
vals_line = {
"sequence": sequence,
"date": date,
@@ -122,10 +132,41 @@ class OnlineBankStatementProviderPonto(models.Model):
vals_line["partner_name"] = attributes["counterpartName"]
return vals_line
def _ponto_date_from_string(self, date_str):
def _ponto_get_execution_datetime(self, transaction):
"""Get execution datetime for a transaction.
Odoo often names variables containing date and time just xxx_date or
date_xxx. We try to avoid this misleading naming by using datetime as
much for variables and fields of type datetime.
"""
attributes = transaction.get("attributes", {})
return self._ponto_datetime_from_string(attributes.get("executionDate"))
def _ponto_datetime_from_string(self, date_str):
"""Dates in Ponto are expressed in UTC, so we need to convert them
to supplied tz for proper classification.
"""
dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
return dt.replace(tzinfo=None)
def _ponto_buffer_purge(self):
"""Remove buffers from Ponto no longer needed to import statements."""
_logger.info("Scheduled purge of old ponto buffers...")
today = date.today()
buffer_model = self.env["ponto.buffer"]
providers = self.search([
("active", "=", True),
])
for provider in providers:
if not provider.ponto_buffer_retain_days:
continue
cutoff_date = today - relativedelta(days=provider.ponto_buffer_retain_days)
old_buffers = buffer_model.search(
[
("provider_id", "=", provider.id),
("effective_date", "<", cutoff_date),
]
)
old_buffers.unlink()
_logger.info("Scheduled purge of old ponto buffers complete.")

6
account_bank_statement_import_online_ponto/models/ponto_buffer.py
View File

@@ -24,7 +24,6 @@ class PontoBuffer(models.Model):
comodel_name="ponto.buffer.line",
inverse_name="buffer_id",
readonly=True,
ondelete="cascade",
)
def _store_transactions(self, provider, transactions):
@@ -32,10 +31,7 @@ class PontoBuffer(models.Model):
# Start by sorting all transactions per date.
transactions_per_date = {}
for transaction in transactions:
ponto_execution_date = transaction.get(
"attributes", {}
).get("executionDate")
effective_date_time = provider._ponto_date_from_string(ponto_execution_date)
effective_date_time = provider._ponto_get_execution_datetime(transaction)
transaction["effective_date_time"] = effective_date_time.isoformat()
key = effective_date_time.isoformat()[0:10]
if key not in transactions_per_date:

1
account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
View File

@@ -13,6 +13,7 @@ class PontoBuffer(models.Model):
comodel_name="ponto.buffer",
required=True,
readonly=True,
ondelete="cascade",
)
ponto_id = fields.Char(
required=True,

8
account_bank_statement_import_online_ponto/models/ponto_interface.py
View File

@@ -137,7 +137,13 @@ class PontoInterface(models.AbstractModel):
time.sleep(40)
def _get_transactions(self, access_data, last_identifier):
"""Get transactions from ponto, using last_identifier as pointer."""
"""Get transactions from ponto, using last_identifier as pointer.
Note that Ponto has the transactions in descending order. The first
transaction, retrieved by not passing an identifier, is the latest
present in Ponto. If you read transactions 'after' a certain identifier
(Ponto id), you will get transactions with an earlier date.
"""
url = (
PONTO_ENDPOINT
+ "/accounts/"

17
account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml → account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml
View File

@@ -6,12 +6,17 @@
<field name="inherit_id" ref="account_bank_statement_import_online.online_bank_statement_provider_form"/>
<field name="arch" type="xml">
<xpath expr="//page[@name='configuration']" position="inside">
<group name="qonto" attrs="{'invisible':[('service','!=','ponto')]}">
<field name="username" string="Login"/>
<field name="password" string="Secret Key"/>
<field name="ponto_last_identifier"/>
<button name="ponto_reset_last_identifier" string="Reset Last identifier." type="object"
attrs="{'invisible':[('ponto_last_identifier','=',False)]}"/>
<group
name="ponto_configuration"
attrs="{'invisible':[('service','!=','ponto')]}"
colspan="6"
col="3"
>
<group colspan="2" col="2" name="ponto_login">
<field name="username" string="Login"/>
<field name="password" string="Secret Key"/>
<field name="ponto_buffer_retain_days"/>
</group>
</group>
</xpath>
</field>