From f9a1f8e738d84f9613ce3583ff5bfb13f063d775 Mon Sep 17 00:00:00 2001 From: "Ronald Portier (Therp BV)" Date: Fri, 27 Jan 2023 23:44:16 +0100 Subject: [PATCH] [MIG] account_statement_import_online_ponto: Migration to 16.0 --- .../__manifest__.py | 2 +- .../online_bank_statement_provider_ponto.py | 58 +++--- .../models/ponto_interface.py | 41 +++-- ...t_account_statement_import_online_ponto.py | 165 ++++++++++++++---- .../account_statement_import_online_ponto | 1 + .../setup.py | 6 + 6 files changed, 202 insertions(+), 71 deletions(-) create mode 120000 setup/account_statement_import_online_ponto/odoo/addons/account_statement_import_online_ponto create mode 100644 setup/account_statement_import_online_ponto/setup.py diff --git a/account_statement_import_online_ponto/__manifest__.py b/account_statement_import_online_ponto/__manifest__.py index ea938f98..0e400569 100644 --- a/account_statement_import_online_ponto/__manifest__.py +++ b/account_statement_import_online_ponto/__manifest__.py @@ -4,7 +4,7 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). { "name": "Online Bank Statements: MyPonto.com", - "version": "15.0.1.0.0", + "version": "16.0.1.0.0", "category": "Account", "website": "https://github.com/OCA/bank-statement-import", "author": "Florent de Labarre, Therp BV, Odoo Community Association (OCA)", diff --git a/account_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py index fe247214..9914b410 100644 --- a/account_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py +++ b/account_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py @@ -23,12 +23,9 @@ class OnlineBankStatementProvider(models.Model): ("execution_date", "Execution Date"), ("value_date", "Value Date"), ], - string="Ponto Date Field", default="execution_date", help="Select the Ponto date field that will be used for " - "the Odoo bank statement line date. If you change this parameter " - "on a provider that already has transactions, you will have to " - "purge the Ponto buffers.", + "the Odoo bank statement line date.", ) @api.model @@ -50,6 +47,7 @@ class OnlineBankStatementProvider(models.Model): stop retrieving when either we get before date_since or there is no more data available. """ + # pylint: disable=missing-return ponto_providers = self.filtered(lambda provider: provider.service == "ponto") super(OnlineBankStatementProvider, self - ponto_providers)._pull( date_since, date_until @@ -60,13 +58,18 @@ class OnlineBankStatementProvider(models.Model): def _ponto_pull(self, date_since, date_until): """Translate information from Ponto to Odoo bank statement lines.""" self.ensure_one() - is_scheduled = self.env.context.get("scheduled") + is_scheduled = self.env.context.get("scheduled", False) if is_scheduled: _logger.debug( - _("Ponto obtain statement data for journal %s from %s to %s"), - self.journal_id.name, - date_since, - date_until, + _( + "Ponto obtain statement data for journal %(journal)s" + " from %(date_since)s to %(date_until)s" + ), + dict( + journal=self.journal_id.name, + date_since=date_since, + date_until=date_until, + ), ) else: _logger.debug( @@ -74,10 +77,13 @@ class OnlineBankStatementProvider(models.Model): self.journal_id.name, ) lines = self._ponto_retrieve_data(date_since, date_until) - # For scheduled runs, store latest identifier. - if is_scheduled and lines: - self.ponto_last_identifier = lines[0].get("id") - self._ponto_store_lines(lines) + if not lines: + _logger.info(_("No lines were retrieved from Ponto")) + else: + # For scheduled runs, store latest identifier. + if is_scheduled: + self.ponto_last_identifier = lines[0].get("id") + self._ponto_store_lines(lines) def _ponto_retrieve_data(self, date_since, date_until): """Fill buffer with data from Ponto. @@ -89,12 +95,12 @@ class OnlineBankStatementProvider(models.Model): Note: when reading data they are likely to be in descending order of execution_date (not seen a guarantee for this in Ponto API). When using value date, they may well be out of order. So we cannot simply stop - when we have foundd a transaction date before the date_since. + when we have found a transaction date before the date_since. We will not read transactions more then a week before before date_since. """ date_stop = date_since - timedelta(days=7) - is_scheduled = self.env.context.get("scheduled") + is_scheduled = self.env.context.get("scheduled", False) lines = [] interface_model = self.env["ponto.interface"] access_data = interface_model._login(self.username, self.password) @@ -105,15 +111,21 @@ class OnlineBankStatementProvider(models.Model): for line in transactions: identifier = line.get("id") transaction_datetime = self._ponto_get_transaction_datetime(line) - if (is_scheduled and identifier == self.ponto_last_identifier) or ( - transaction_datetime < date_stop - and (not self.ponto_last_identifier or not is_scheduled) - ): - return lines - if not is_scheduled: - if transaction_datetime < date_since: + if is_scheduled: + # Handle all stop conditions for scheduled runs. + if identifier == self.ponto_last_identifier or ( + not self.ponto_last_identifier + and transaction_datetime < date_stop + ): return lines - if transaction_datetime > date_until: + else: + # Handle stop conditions for non scheduled runs. + if transaction_datetime < date_stop: + return lines + if ( + transaction_datetime < date_since + or transaction_datetime > date_until + ): continue line["transaction_datetime"] = transaction_datetime lines.append(line) diff --git a/account_statement_import_online_ponto/models/ponto_interface.py b/account_statement_import_online_ponto/models/ponto_interface.py index f116d6af..9b5b3f1f 100644 --- a/account_statement_import_online_ponto/models/ponto_interface.py +++ b/account_statement_import_online_ponto/models/ponto_interface.py @@ -28,14 +28,14 @@ class PontoInterface(models.AbstractModel): url = PONTO_ENDPOINT + "/oauth2/token" if not (username and password): raise UserError(_("Please fill login and key.")) - login = "%s:%s" % (username, password) + login = ":".join([username, password]) login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8") login_headers = { "Content-Type": "application/x-www-form-urlencoded", "Accept": "application/json", - "Authorization": "Basic %s" % login, + "Authorization": "Basic {login}".format(login=login), } - _logger.debug(_("POST request on %s"), url) + _logger.debug(_("POST request on %(url)s"), dict(url=url)) response = requests.post( url, params={"grant_type": "client_credentials"}, @@ -63,13 +63,15 @@ class PontoInterface(models.AbstractModel): access_data.update(updated_data) return { "Accept": "application/json", - "Authorization": "Bearer %s" % access_data["access_token"], + "Authorization": "Bearer {access_token}".format( + access_token=access_data["access_token"] + ), } def _set_access_account(self, access_data, account_number): """Set ponto account for bank account in access_data.""" url = PONTO_ENDPOINT + "/accounts" - _logger.debug(_("GET request on %s"), url) + _logger.debug(_("GET request on %(url)s"), dict(url=url)) response = requests.get( url, params={"limit": 100}, @@ -86,8 +88,9 @@ class PontoInterface(models.AbstractModel): return # If we get here, we did not find Ponto account for bank account. raise UserError( - _("Ponto : wrong configuration, account %s not found in %s") - % (account_number, data) + _( + "Ponto : wrong configuration, account {account} not found in {data}" + ).format(account=account_number, data=data) ) def _get_transactions(self, access_data, last_identifier): @@ -116,8 +119,8 @@ class PontoInterface(models.AbstractModel): transactions = data.get("data", []) if not transactions: _logger.debug( - _("No transactions where found in data %s"), - data, + _("No transactions where found in data %(data)s"), + dict(data=data), ) else: _logger.debug( @@ -130,7 +133,12 @@ class PontoInterface(models.AbstractModel): """Interact with Ponto to get next page of data.""" headers = self._get_request_headers(access_data) _logger.debug( - _("GET request to %s with headers %s and params %s"), url, params, headers + _("GET request to %(url)s with headers %(headers)s and params %(params)s"), + dict( + url=url, + headers=headers, + params=params, + ), ) response = requests.get( url, @@ -142,10 +150,17 @@ class PontoInterface(models.AbstractModel): def _get_response_data(self, response): """Get response data for GET or POST request.""" - _logger.debug(_("HTTP answer code %s from Ponto"), response.status_code) + _logger.debug( + _("HTTP answer code %(response_code)s from Ponto"), + dict(response_code=response.status_code), + ) if response.status_code not in (200, 201): raise UserError( - _("Server returned status code %s: %s") - % (response.status_code, response.text) + _( + "Server returned status code {response_code}: {response_text}" + ).format( + response_code=response.status_code, + response_text=response.text, + ) ) return json.loads(response.text) diff --git a/account_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py b/account_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py index 5fd8f7ca..34157676 100644 --- a/account_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py +++ b/account_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py @@ -2,11 +2,11 @@ # Copyright 2022 Therp BV . # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). import logging -from datetime import date, datetime +from datetime import datetime from unittest import mock from odoo import _, fields -from odoo.tests import Form, common +from odoo.tests import common _logger = logging.getLogger(__name__) @@ -118,6 +118,59 @@ FOUR_TRANSACTIONS = [ EMPTY_TRANSACTIONS = [] +EARLY_TRANSACTIONS = [ + # First transaction in october 2019, month before other transactions. + { + "type": "transaction", + "relationships": { + "account": { + "links": {"related": "https://api.myponto.com/accounts/"}, + "data": { + "type": "account", + "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75", + }, + } + }, + "id": "1552c32f-e63f-4ce6-a974-f270e6cd5301", + "attributes": { + "valueDate": "2019-10-04T12:29:00.000Z", + "remittanceInformationType": "unstructured", + "remittanceInformation": "Arresto Momentum", + "executionDate": "2019-10-04T10:24:00.000Z", + "description": "Wire transfer after execution", + "currency": "EUR", + "counterpartReference": "BE10325927501996", + "counterpartName": "Some other customer", + "amount": 4.25, + }, + }, + # Second transaction in september 2019. + { + "type": "transaction", + "relationships": { + "account": { + "links": {"related": "https://api.myponto.com/accounts/"}, + "data": { + "type": "account", + "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75", + }, + } + }, + "id": "701ab965-21c4-46ca-b157-306c0646e002", + "attributes": { + "valueDate": "2019-09-18T01:00:00.000Z", + "remittanceInformationType": "unstructured", + "remittanceInformation": "Minima vitae totam!", + "executionDate": "2019-09-20T01:00:00.000Z", + "description": "Wire transfer", + "currency": "EUR", + "counterpartReference": "BE26089479973169", + "counterpartName": "Osinski Group", + "amount": 4.08, + }, + }, +] + transaction_amounts = [5.48, 5.83, 6.08, 8.95] @@ -152,13 +205,18 @@ class TestAccountStatementImportOnlinePonto(common.TransactionCase): "code": "BANK", "currency_id": self.currency_eur.id, "bank_statements_source": "online", - "online_bank_statement_provider": "ponto", "bank_account_id": self.bank_account.id, } ) - self.provider = self.journal.online_bank_statement_provider_id - # To get all the moves in a month at once - self.provider.statement_creation_mode = "monthly" + self.provider = self.OnlineBankStatementProvider.create( + { + "name": "Ponto Provider", + "service": "ponto", + "journal_id": self.journal.id, + # To get all the moves in a month at once + "statement_creation_mode": "monthly", + } + ) self.mock_login = lambda: mock.patch( _interface_class + "._login", @@ -181,34 +239,39 @@ class TestAccountStatementImportOnlinePonto(common.TransactionCase): EMPTY_TRANSACTIONS, ], ) + # return two times list of transactions, empty list on third call. + self.mock_get_transactions_multi = lambda: mock.patch( + _interface_class + "._get_transactions", + side_effect=[ + FOUR_TRANSACTIONS, + EARLY_TRANSACTIONS, + EMPTY_TRANSACTIONS, + ], + ) def test_balance_start(self): - st_form = Form(self.AccountBankStatement) - st_form.journal_id = self.journal - st_form.date = date(2019, 11, 1) - st_form.balance_end_real = 100 - with st_form.line_ids.new() as line_form: - line_form.payment_ref = "test move" - line_form.amount = 100 - initial_statement = st_form.save() - initial_statement.button_post() + """Test wether end balance of last statement, taken as start balance of new.""" + statement_date = datetime(2019, 11, 1) + data = self._get_statement_line_data(statement_date) + self.provider.statement_creation_mode = "daily" + self.provider._create_or_update_statement( + data, statement_date, datetime(2019, 11, 2) + ) with self.mock_login(), self.mock_set_access_account(), self.mock_get_transactions(): # noqa: B950 vals = { - "provider_ids": [(4, self.provider.id)], "date_since": datetime(2019, 11, 4), "date_until": datetime(2019, 11, 5), } wizard = self.AccountStatementPull.with_context( - active_model="account.journal", - active_id=self.journal.id, + active_model=self.provider._name, + active_id=self.provider.id, ).create(vals) - # For some reason the provider is not set in the create. - wizard.provider_ids = self.provider wizard.action_pull() statements = self.AccountBankStatement.search( - [("journal_id", "=", self.journal.id)] + [("journal_id", "=", self.journal.id)], order="name" ) - new_statement = statements - initial_statement + self.assertEqual(len(statements), 2) + new_statement = statements[1] self.assertEqual(len(new_statement.line_ids), 1) self.assertEqual(new_statement.balance_start, 100) self.assertEqual(new_statement.balance_end, 105.83) @@ -217,7 +280,7 @@ class TestAccountStatementImportOnlinePonto(common.TransactionCase): with self.mock_login(), self.mock_set_access_account(), self.mock_get_transactions(): # noqa: B950 # First base selection on execution date. self.provider.ponto_date_field = "execution_date" - statement = self._get_statement_from_wizard() + statement = self._get_statements_from_wizard() # Will get 1 statement self._check_line_count(statement.line_ids, expected_count=2) self._check_statement_amounts(statement, transaction_amounts[:2]) @@ -225,10 +288,27 @@ class TestAccountStatementImportOnlinePonto(common.TransactionCase): with self.mock_login(), self.mock_set_access_account(), self.mock_get_transactions(): # noqa: B950 # First base selection on execution date. self.provider.ponto_date_field = "value_date" - statement = self._get_statement_from_wizard() + statement = self._get_statements_from_wizard() # Will get 1 statement self._check_line_count(statement.line_ids, expected_count=3) self._check_statement_amounts(statement, transaction_amounts[:3]) + def test_ponto_get_transactions_multi(self): + with self.mock_login(), self.mock_set_access_account(), self.mock_get_transactions_multi(): # noqa: B950 + # First base selection on execution date. + self.provider.ponto_date_field = "execution_date" + # Expect statements for october and november. + statements = self._get_statements_from_wizard( + expected_statement_count=2, date_since=datetime(2019, 9, 25) + ) + self._check_line_count(statements[0].line_ids, expected_count=1) # october + self._check_line_count(statements[1].line_ids, expected_count=2) # november + self._check_statement_amounts(statements[0], [4.25]) + self._check_statement_amounts( + statements[1], + transaction_amounts[:2], + expected_balance_end=15.56, # Includes 4.25 from statement before. + ) + def test_ponto_scheduled(self): with self.mock_login(), self.mock_set_access_account(), self.mock_get_transactions(): # noqa: B950 # Scheduled should get all transaction, ignoring date_until. @@ -262,21 +342,21 @@ class TestAccountStatementImportOnlinePonto(common.TransactionCase): statements[1], transaction_amounts[3:], expected_balance_end=15.03 ) - def _get_statement_from_wizard(self): + def _get_statements_from_wizard(self, expected_statement_count=1, date_since=None): """Run wizard to pull data and return statement.""" + date_since = date_since if date_since else datetime(2019, 11, 3) vals = { - "provider_ids": [(4, self.provider.id)], - "date_since": datetime(2019, 11, 3), + "date_since": date_since, "date_until": datetime(2019, 11, 18), } wizard = self.AccountStatementPull.with_context( - active_model="account.journal", - active_id=self.journal.id, + active_model=self.provider._name, + active_id=self.provider.id, ).create(vals) - # For some reason the provider is not set in the create. - wizard.provider_ids = self.provider wizard.action_pull() - return self._get_statements_from_journal(expected_count=1) + return self._get_statements_from_journal( + expected_count=expected_statement_count + ) def _get_statements_from_journal(self, expected_count=0): """We only expect statements created by our tests.""" @@ -306,7 +386,24 @@ class TestAccountStatementImportOnlinePonto(common.TransactionCase): ): """Check wether amount in lines and end_balance as expected.""" sorted_amounts = sorted([round(line.amount, 2) for line in statement.line_ids]) - self.assertEqual(sorted_amounts, expected_amounts) + sorted_expected_amounts = sorted( + [round(amount, 2) for amount in expected_amounts] + ) + self.assertEqual(sorted_amounts, sorted_expected_amounts) if not expected_balance_end: expected_balance_end = sum(expected_amounts) - self.assertEqual(statement.balance_end, expected_balance_end) + self.assertEqual( + round(statement.balance_end, 2), round(expected_balance_end, 2) + ) + + def _get_statement_line_data(self, statement_date): + return [ + { + "payment_ref": "payment", + "amount": 100, + "date": statement_date, + "unique_import_id": str(statement_date), + "partner_name": "John Doe", + "account_number": "XX00 0000 0000 0000", + } + ], {} diff --git a/setup/account_statement_import_online_ponto/odoo/addons/account_statement_import_online_ponto b/setup/account_statement_import_online_ponto/odoo/addons/account_statement_import_online_ponto new file mode 120000 index 00000000..efde64a7 --- /dev/null +++ b/setup/account_statement_import_online_ponto/odoo/addons/account_statement_import_online_ponto @@ -0,0 +1 @@ +../../../../account_statement_import_online_ponto \ No newline at end of file diff --git a/setup/account_statement_import_online_ponto/setup.py b/setup/account_statement_import_online_ponto/setup.py new file mode 100644 index 00000000..28c57bb6 --- /dev/null +++ b/setup/account_statement_import_online_ponto/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +)