[IMP] *_online_ponto: forward port 12.0 improvements

Separate retrieval of data from ponto (buffer data) and
creation of statements.

account_statement_import_online_ponto/__manifest__.py

@@ -1,14 +1,17 @@
-# Copyright 2020 Florent de Labarre
+# Copyright 2020 Florent de Labarre.
-# Copyright 2020 Tecnativa - João Marques
+# Copyright 2020 Tecnativa - João Marques.
+# Copyright 2022-2023 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 {
     "name": "Online Bank Statements: MyPonto.com",
-    "version": "14.0.1.1.0",
+    "version": "14.0.2.0.0",
     "category": "Account",
     "website": "https://github.com/OCA/bank-statement-import",
-    "author": "Florent de Labarre, Odoo Community Association (OCA)",
+    "author": "Florent de Labarre, Therp BV, Odoo Community Association (OCA)",
     "license": "AGPL-3",
     "installable": True,
     "depends": ["account_statement_import_online"],
-    "data": ["view/online_bank_statement_provider.xml"],
+    "data": [
+        "views/online_bank_statement_provider.xml",
+    ],
 }

account_statement_import_online_ponto/models/__init__.py

@@ -1,3 +1,3 @@
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
+from . import ponto_interface
 from . import online_bank_statement_provider_ponto

account_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py

@@ -1,235 +1,96 @@
 # Copyright 2020 Florent de Labarre
+# Copyright 2022-2023 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
-import base64
 import json
+import logging
 import re
-import time
 from datetime import datetime
 
 import pytz
-import requests
-from dateutil.relativedelta import relativedelta
 
-from odoo import _, api, fields, models
+from odoo import _, api, models
-from odoo.exceptions import UserError
 
-from odoo.addons.base.models.res_bank import sanitize_account_number
+_logger = logging.getLogger(__name__)
-
-PONTO_ENDPOINT = "https://api.myponto.com"
 
 
 class OnlineBankStatementProviderPonto(models.Model):
     _inherit = "online.bank.statement.provider"
 
-    ponto_token = fields.Char(readonly=True)
-    ponto_token_expiration = fields.Datetime(readonly=True)
-    ponto_last_identifier = fields.Char(readonly=True)
-
-    def ponto_reset_last_identifier(self):
-        self.write({"ponto_last_identifier": False})
-
     @api.model
     def _get_available_services(self):
+        """Each provider model must register its service."""
         return super()._get_available_services() + [
             ("ponto", "MyPonto.com"),
         ]
 
     def _obtain_statement_data(self, date_since, date_until):
+        """Check wether called for ponto servide, otherwise pass the buck."""
         self.ensure_one()
-        if self.service != "ponto":
+        if self.service != "ponto":  # pragma: no cover
-            return super()._obtain_statement_data(date_since, date_until)
+            return super()._obtain_statement_data(
+                date_since,
+                date_until,
+            )
         return self._ponto_obtain_statement_data(date_since, date_until)
 
-    #########
-    # ponto #
-    #########
-
-    def _ponto_header_token(self):
-        self.ensure_one()
-        if self.username and self.password:
-            login = "{}:{}".format(self.username, self.password)
-            login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8")
-            return {
-                "Content-Type": "application/x-www-form-urlencoded",
-                "Accept": "application/json",
-                "Authorization": "Basic %s" % login,
-            }
-        raise UserError(_("Please fill login and key."))
-
-    def _ponto_header(self):
-        self.ensure_one()
-        if (
-            not self.ponto_token
-            or not self.ponto_token_expiration
-            or self.ponto_token_expiration <= fields.Datetime.now()
-        ):
-
-            url = PONTO_ENDPOINT + "/oauth2/token"
-            response = requests.post(
-                url,
-                params={"grant_type": "client_credentials"},
-                headers=self._ponto_header_token(),
-            )
-            if response.status_code == 200:
-                data = json.loads(response.text)
-                access_token = data.get("access_token", False)
-                if not access_token:
-                    raise UserError(_("Ponto : no token"))
-                else:
-                    self.sudo().ponto_token = access_token
-                    expiration_date = fields.Datetime.now() + relativedelta(
-                        seconds=data.get("expires_in", False)
-                    )
-                    self.sudo().ponto_token_expiration = expiration_date
-            else:
-                raise UserError(_("%s \n\n %s") % (response.status_code, response.text))
-        return {
-            "Accept": "application/json",
-            "Authorization": "Bearer %s" % self.ponto_token,
-        }
-
-    def _ponto_get_account_ids(self):
-        url = PONTO_ENDPOINT + "/accounts"
-        response = requests.get(
-            url, params={"limit": 100}, headers=self._ponto_header()
-        )
-        if response.status_code == 200:
-            data = json.loads(response.text)
-            res = {}
-            for account in data.get("data", []):
-                iban = sanitize_account_number(
-                    account.get("attributes", {}).get("reference", "")
-                )
-                res[iban] = account.get("id")
-            return res
-        raise UserError(_("%s \n\n %s") % (response.status_code, response.text))
-
-    def _ponto_synchronisation(self, account_id):
-        url = PONTO_ENDPOINT + "/synchronizations"
-        data = {
-            "data": {
-                "type": "synchronization",
-                "attributes": {
-                    "resourceType": "account",
-                    "resourceId": account_id,
-                    "subtype": "accountTransactions",
-                },
-            }
-        }
-        response = requests.post(url, headers=self._ponto_header(), json=data)
-        if response.status_code in (200, 201, 400):
-            data = json.loads(response.text)
-            sync_id = data.get("attributes", {}).get("resourceId", False)
-        else:
-            raise UserError(
-                _("Error during Create Synchronisation %s \n\n %s")
-                % (response.status_code, response.text)
-            )
-
-        # Check synchronisation
-        if not sync_id:
-            return
-        url = PONTO_ENDPOINT + "/synchronizations/" + sync_id
-        number = 0
-        while number == 100:
-            number += 1
-            response = requests.get(url, headers=self._ponto_header())
-            if response.status_code == 200:
-                data = json.loads(response.text)
-                status = data.get("status", {})
-                if status in ("success", "error"):
-                    return
-            time.sleep(4)
-
-    def _ponto_get_transaction(self, account_id, date_since, date_until):
-        page_url = PONTO_ENDPOINT + "/accounts/" + account_id + "/transactions"
-        params = {"limit": 100}
-        page_next = True
-        last_identifier = self.ponto_last_identifier
-        if last_identifier:
-            params["before"] = last_identifier
-            page_next = False
-        transaction_lines = []
-        latest_identifier = False
-        while page_url:
-            response = requests.get(
-                page_url, params=params, headers=self._ponto_header()
-            )
-            if response.status_code != 200:
-                raise UserError(
-                    _("Error during get transaction.\n\n%s \n\n %s")
-                    % (response.status_code, response.text)
-                )
-            if params.get("before"):
-                params.pop("before")
-            data = json.loads(response.text)
-            links = data.get("links", {})
-            if page_next:
-                page_url = links.get("next", False)
-            else:
-                page_url = links.get("prev", False)
-            transactions = data.get("data", [])
-            if transactions:
-                current_transactions = []
-                for transaction in transactions:
-                    date = self._ponto_date_from_string(
-                        transaction.get("attributes", {}).get("executionDate")
-                    )
-                    if date_since <= date < date_until:
-                        current_transactions.append(transaction)
-                if current_transactions:
-                    if not page_next or (page_next and not latest_identifier):
-                        latest_identifier = current_transactions[0].get("id")
-                    transaction_lines.extend(current_transactions)
-        if latest_identifier:
-            self.ponto_last_identifier = latest_identifier
-        return transaction_lines
-
-    def _ponto_date_from_string(self, date_str):
-        """Dates in Ponto are expressed in UTC, so we need to convert them
-        to supplied tz for proper classification.
-        """
-        dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
-        dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
-        return dt.replace(tzinfo=None)
-
     def _ponto_obtain_statement_data(self, date_since, date_until):
         """Translate information from Ponto to Odoo bank statement lines."""
         self.ensure_one()
-        account_ids = self._ponto_get_account_ids()
+        _logger.debug(
-        journal = self.journal_id
+            _("Ponto obtain statement data for journal %s from %s to %s"),
-        iban = self.account_number
+            self.journal_id.name,
-        account_id = account_ids.get(iban)
+            date_since,
-        if not account_id:
+            date_until,
-            raise UserError(
-                _("Ponto : wrong configuration, unknow account %s")
-                % journal.bank_account_id.acc_number
-            )
-        self._ponto_synchronisation(account_id)
-        transaction_lines = self._ponto_get_transaction(
-            account_id, date_since, date_until
         )
+        lines = self._ponto_retrieve_data(date_since)
         new_transactions = []
         sequence = 0
-        for transaction in transaction_lines:
+        for transaction in lines:
+            date = self._ponto_get_execution_datetime(transaction)
+            if date < date_since or date > date_until:
+                continue
             sequence += 1
             vals_line = self._ponto_get_transaction_vals(transaction, sequence)
             new_transactions.append(vals_line)
-        if new_transactions:
+        return new_transactions, {}
-            return new_transactions, {}
+
-        return
+    def _ponto_retrieve_data(self, date_since):
+        """Fill buffer with data from Ponto.
+
+        We will retrieve data from the latest transactions present in Ponto
+        backwards, until we find data that has an execution date before date_since.
+        """
+        lines = []
+        interface_model = self.env["ponto.interface"]
+        access_data = interface_model._login(self.username, self.password)
+        interface_model._set_access_account(access_data, self.account_number)
+        latest_identifier = False
+        transactions = interface_model._get_transactions(access_data, latest_identifier)
+        while transactions:
+            lines.extend(transactions)
+            latest_identifier = transactions[-1].get("id")
+            earliest_datetime = self._ponto_get_execution_datetime(transactions[-1])
+            if earliest_datetime < date_since:
+                break
+            transactions = interface_model._get_transactions(
+                access_data, latest_identifier
+            )
+        return lines
 
     def _ponto_get_transaction_vals(self, transaction, sequence):
         """Translate information from Ponto to statement line vals."""
         attributes = transaction.get("attributes", {})
         ref_list = [
             attributes.get(x)
-            for x in {"description", "counterpartName", "counterpartReference"}
+            for x in {
+                "description",
+                "counterpartName",
+                "counterpartReference",
+            }
             if attributes.get(x)
         ]
         ref = " ".join(ref_list)
-        date = self._ponto_date_from_string(attributes.get("executionDate"))
+        date = self._ponto_get_execution_datetime(transaction)
         vals_line = {
             "sequence": sequence,
             "date": date,
@@ -237,10 +98,28 @@ class OnlineBankStatementProviderPonto(models.Model):
             "payment_ref": attributes.get("remittanceInformation", ref),
             "unique_import_id": transaction["id"],
             "amount": attributes["amount"],
-            "raw_data": transaction,
+            "raw_data": json.dumps(transaction),
         }
         if attributes.get("counterpartReference"):
             vals_line["account_number"] = attributes["counterpartReference"]
         if attributes.get("counterpartName"):
             vals_line["partner_name"] = attributes["counterpartName"]
         return vals_line
+
+    def _ponto_get_execution_datetime(self, transaction):
+        """Get execution datetime for a transaction.
+
+        Odoo often names variables containing date and time just xxx_date or
+        date_xxx. We try to avoid this misleading naming by using datetime as
+        much for variables and fields of type datetime.
+        """
+        attributes = transaction.get("attributes", {})
+        return self._ponto_datetime_from_string(attributes.get("executionDate"))
+
+    def _ponto_datetime_from_string(self, date_str):
+        """Dates in Ponto are expressed in UTC, so we need to convert them
+        to supplied tz for proper classification.
+        """
+        dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
+        dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
+        return dt.replace(tzinfo=None)

account_statement_import_online_ponto/models/ponto_interface.py

@@ -0,0 +1,139 @@
+# Copyright 2020 Florent de Labarre
+# Copyright 2022 Therp BV <https://therp.nl>.
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+import base64
+import json
+import logging
+
+import requests
+from dateutil.relativedelta import relativedelta
+
+from odoo import _, fields, models
+from odoo.exceptions import UserError
+
+from odoo.addons.base.models.res_bank import sanitize_account_number
+
+_logger = logging.getLogger(__name__)
+
+PONTO_ENDPOINT = "https://api.myponto.com"
+
+
+class PontoInterface(models.AbstractModel):
+    _name = "ponto.interface"
+    _description = "Interface to all interactions with Ponto API"
+
+    def _login(self, username, password):
+        """Ponto login returns an access dictionary for further requests."""
+        url = PONTO_ENDPOINT + "/oauth2/token"
+        if not (username and password):
+            raise UserError(_("Please fill login and key."))
+        login = "%s:%s" % (username, password)
+        login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8")
+        login_headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Accept": "application/json",
+            "Authorization": "Basic %s" % login,
+        }
+        response = requests.post(
+            url,
+            params={"grant_type": "client_credentials"},
+            headers=login_headers,
+        )
+        data = self._get_response_data(response)
+        access_token = data.get("access_token", False)
+        if not access_token:
+            raise UserError(_("Ponto : no token"))
+        token_expiration = fields.Datetime.now() + relativedelta(
+            seconds=data.get("expires_in", False)
+        )
+        return {
+            "username": username,
+            "password": password,
+            "access_token": access_token,
+            "token_expiration": token_expiration,
+        }
+
+    def _get_request_headers(self, access_data):
+        """Get headers with authorization for further ponto requests."""
+        if access_data["token_expiration"] <= fields.Datetime.now():
+            updated_data = self._login(access_data["username"], access_data["password"])
+            access_data.update(updated_data)
+        return {
+            "Accept": "application/json",
+            "Authorization": "Bearer %s" % access_data["access_token"],
+        }
+
+    def _set_access_account(self, access_data, account_number):
+        """Set ponto account for bank account in access_data."""
+        url = PONTO_ENDPOINT + "/accounts"
+        response = requests.get(
+            url, params={"limit": 100}, headers=self._get_request_headers(access_data)
+        )
+        data = self._get_response_data(response)
+        for ponto_account in data.get("data", []):
+            ponto_iban = sanitize_account_number(
+                ponto_account.get("attributes", {}).get("reference", "")
+            )
+            if ponto_iban == account_number:
+                access_data["ponto_account"] = ponto_account.get("id")
+                return
+        # If we get here, we did not find Ponto account for bank account.
+        raise UserError(
+            _("Ponto : wrong configuration, account %s not found in %s")
+            % (account_number, data)
+        )
+
+    def _get_transactions(self, access_data, last_identifier):
+        """Get transactions from ponto, using last_identifier as pointer.
+
+        Note that Ponto has the transactions in descending order. The first
+        transaction, retrieved by not passing an identifier, is the latest
+        present in Ponto. If you read transactions 'after' a certain identifier
+        (Ponto id), you will get transactions with an earlier date.
+        """
+        url = (
+            PONTO_ENDPOINT
+            + "/accounts/"
+            + access_data["ponto_account"]
+            + "/transactions"
+        )
+        params = {"limit": 100}
+        if last_identifier:
+            params["after"] = last_identifier
+        data = self._get_request(access_data, url, params)
+        transactions = self._get_transactions_from_data(data)
+        return transactions
+
+    def _get_transactions_from_data(self, data):
+        """Get all transactions that are in the ponto response data."""
+        transactions = data.get("data", [])
+        if not transactions:
+            _logger.debug(
+                _("No transactions where found in data %s"),
+                data,
+            )
+        else:
+            _logger.debug(
+                _("%d transactions present in response data"),
+                len(transactions),
+            )
+        return transactions
+
+    def _get_request(self, access_data, url, params):
+        """Interact with Ponto to get next page of data."""
+        headers = self._get_request_headers(access_data)
+        _logger.debug(
+            _("Get request to %s, with headers %s and params %s"), url, params, headers
+        )
+        response = requests.get(url, params=params, headers=headers)
+        return self._get_response_data(response)
+
+    def _get_response_data(self, response):
+        """Get response data for GET or POST request."""
+        if response.status_code not in (200, 201):
+            raise UserError(
+                _("Server returned status code %s: %s")
+                % (response.status_code, response.text)
+            )
+        return json.loads(response.text)

account_statement_import_online_ponto/readme/CONTRIBUTORS.rst

@@ -3,7 +3,6 @@
 
   * Pedro M. Baeza
   * João Marques
-
+* `Therp BV <https://therp.nl>`__:
-* `Therp BV <https://therp.nl/>`__
 
   * Ronald Portier <ronald@therp.nl>

account_statement_import_online_ponto/tests/__init__.py

@@ -1,3 +1,4 @@
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
+from . import test_ponto_interface
 from . import test_account_statement_import_online_ponto

account_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py

@@ -1,4 +1,5 @@
 # Copyright 2020 Florent de Labarre
+# Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
 from datetime import date, datetime
@@ -8,14 +9,89 @@ from odoo import fields
 from odoo.tests import Form, common
 
 _module_ns = "odoo.addons.account_statement_import_online_ponto"
-_provider_class = (
+_interface_class = _module_ns + ".models.ponto_interface" + ".PontoInterface"
-    _module_ns
+
-    + ".models.online_bank_statement_provider_ponto"
+THREE_TRANSACTIONS = [
-    + ".OnlineBankStatementProviderPonto"
+    {
-)
+        "type": "transaction",
+        "relationships": {
+            "account": {
+                "links": {"related": "https://api.myponto.com/accounts/"},
+                "data": {
+                    "type": "account",
+                    "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                },
+            }
+        },
+        "id": "701ab965-21c4-46ca-b157-306c0646e0e2",
+        "attributes": {
+            "valueDate": "2019-11-18T00:00:00.000Z",
+            "remittanceInformationType": "unstructured",
+            "remittanceInformation": "Minima vitae totam!",
+            "executionDate": "2019-11-20T00:00:00.000Z",
+            "description": "Wire transfer",
+            "currency": "EUR",
+            "counterpartReference": "BE26089479973169",
+            "counterpartName": "Osinski Group",
+            "amount": 6.08,
+        },
+    },
+    {
+        "type": "transaction",
+        "relationships": {
+            "account": {
+                "links": {"related": "https://api.myponto.com/accounts/"},
+                "data": {
+                    "type": "account",
+                    "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                },
+            }
+        },
+        "id": "9ac50483-16dc-4a82-aa60-df56077405cd",
+        "attributes": {
+            "valueDate": "2019-11-04T00:00:00.000Z",
+            "remittanceInformationType": "unstructured",
+            "remittanceInformation": "Quia voluptatem blanditiis.",
+            "executionDate": "2019-11-06T00:00:00.000Z",
+            "description": "Wire transfer",
+            "currency": "EUR",
+            "counterpartReference": "BE97201830401438",
+            "counterpartName": "Stokes-Miller",
+            "amount": 5.48,
+        },
+    },
+    {
+        "type": "transaction",
+        "relationships": {
+            "account": {
+                "links": {"related": "https://api.myponto.com/accounts/"},
+                "data": {
+                    "type": "account",
+                    "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                },
+            }
+        },
+        "id": "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff",
+        "attributes": {
+            "valueDate": "2019-11-04T00:00:00.000Z",
+            "remittanceInformationType": "unstructured",
+            "remittanceInformation": "Laboriosam repelo?",
+            "executionDate": "2019-11-04T00:00:00.000Z",
+            "description": "Wire transfer",
+            "currency": "EUR",
+            "counterpartReference": "BE10325927501996",
+            "counterpartName": "Strosin-Veum",
+            "amount": 5.83,
+        },
+    },
+]
+
+EMPTY_TRANSACTIONS = []
 
 
-class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
+class TestAccountStatementImportOnlinePonto(common.TransactionCase):
+    post_install = True
+
     def setUp(self):
         super().setUp()
 
@@ -27,7 +103,7 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
         self.OnlineBankStatementProvider = self.env["online.bank.statement.provider"]
         self.AccountBankStatement = self.env["account.bank.statement"]
         self.AccountBankStatementLine = self.env["account.bank.statement.line"]
-        self.AccStatemenPull = self.env["online.bank.statement.pull.wizard"]
+        self.AccountStatementPull = self.env["online.bank.statement.pull.wizard"]
 
         self.bank_account = self.ResPartnerBank.create(
             {
@@ -48,98 +124,25 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
         )
         self.provider = self.journal.online_bank_statement_provider_id
 
-        self.mock_header = lambda: mock.patch(
+        self.mock_login = lambda: mock.patch(
-            _provider_class + "._ponto_header",
+            _interface_class + "._login",
             return_value={
-                "Accept": "application/json",
+                "username": "test_user",
-                "Authorization": "Bearer --TOKEN--",
+                "password": "very_secret",
+                "access_token": "abcd1234",
+                "token_expiration": datetime(2099, 12, 31, 23, 59, 59),
             },
         )
-
+        self.mock_set_access_account = lambda: mock.patch(
-        self.mock_account_ids = lambda: mock.patch(
+            _interface_class + "._set_access_account",
-            _provider_class + "._ponto_get_account_ids",
-            return_value={"FR0214508000302245362775K46": "id"},
-        )
-        self.mock_synchronisation = lambda: mock.patch(
-            _provider_class + "._ponto_synchronisation",
             return_value=None,
         )
-
+        # return list of transactions on first call, empty list on second call.
-        self.mock_transaction = lambda: mock.patch(
+        self.mock_get_transactions = lambda: mock.patch(
-            _provider_class + "._ponto_get_transaction",
+            _interface_class + "._get_transactions",
-            return_value=[
+            side_effect=[
-                {
+                THREE_TRANSACTIONS,
-                    "type": "transaction",
+                EMPTY_TRANSACTIONS,
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "701ab965-21c4-46ca-b157-306c0646e0e2",
-                    "attributes": {
-                        "valueDate": "2019-11-18T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Minima vitae totam!",
-                        "executionDate": "2019-11-20T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE26089479973169",
-                        "counterpartName": "Osinski Group",
-                        "amount": 6.08,
-                    },
-                },
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "9ac50483-16dc-4a82-aa60-df56077405cd",
-                    "attributes": {
-                        "valueDate": "2019-11-04T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Quia voluptatem blanditiis.",
-                        "executionDate": "2019-11-06T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE97201830401438",
-                        "counterpartName": "Stokes-Miller",
-                        "amount": 5.48,
-                    },
-                },
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff",
-                    "attributes": {
-                        "valueDate": "2019-11-04T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Laboriosam repelo?",
-                        "executionDate": "2019-11-04T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE10325927501996",
-                        "counterpartName": "Strosin-Veum",
-                        "amount": 5.83,
-                    },
-                },
             ],
         )
 
@@ -153,16 +156,18 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
             line_form.amount = 100
         initial_statement = st_form.save()
         initial_statement.button_post()
-        with self.mock_transaction(), self.mock_header(), self.mock_synchronisation(), self.mock_account_ids():  # noqa: B950
+        with self.mock_login(), self.mock_set_access_account(), self.mock_get_transactions():  # noqa: B950
             vals = {
-                "provider_ids": self.provider.ids,
+                "provider_ids": [(4, self.provider.id)],
                 "date_since": datetime(2019, 11, 4),
                 "date_until": datetime(2019, 11, 5),
             }
-            wizard = self.AccStatemenPull.with_context(
+            wizard = self.AccountStatementPull.with_context(
                 active_model="account.journal",
                 active_id=self.journal.id,
             ).create(vals)
+            # For some reason the provider is not set in the create.
+            wizard.provider_ids = self.provider
             wizard.action_pull()
             statements = self.AccountBankStatement.search(
                 [("journal_id", "=", self.journal.id)]
@@ -170,26 +175,33 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
             new_statement = statements - initial_statement
             self.assertEqual(len(new_statement.line_ids), 1)
             self.assertEqual(new_statement.balance_start, 100)
-            self.assertEqual(new_statement.balance_end_real, 105.83)
+            self.assertEqual(new_statement.balance_end, 105.83)
+            # Ponto does not give balance info in transactions.
+            # self.assertEqual(new_statement.balance_end_real, 105.83)
 
     def test_ponto(self):
-        with self.mock_transaction(), self.mock_header(), self.mock_synchronisation(), self.mock_account_ids():  # noqa: B950
+        with self.mock_login(), self.mock_set_access_account(), self.mock_get_transactions():  # noqa: B950
             vals = {
-                "provider_ids": self.provider.ids,
+                "provider_ids": [(4, self.provider.id)],
                 "date_since": datetime(2019, 11, 3),
                 "date_until": datetime(2019, 11, 17),
             }
-            wizard = self.AccStatemenPull.with_context(
+            wizard = self.AccountStatementPull.with_context(
                 active_model="account.journal",
                 active_id=self.journal.id,
             ).create(vals)
             # To get all the moves at once
             self.provider.statement_creation_mode = "monthly"
+            # For some reason the provider is not set in the create.
+            wizard.provider_ids = self.provider
             wizard.action_pull()
             statement = self.AccountBankStatement.search(
                 [("journal_id", "=", self.journal.id)]
             )
             self.assertEqual(len(statement), 1)
             self.assertEqual(len(statement.line_ids), 3)
-            self.assertEqual(statement.line_ids.mapped("amount"), [6.08, 5.48, 5.83])
+            sorted_amounts = sorted(statement.line_ids.mapped("amount"))
-            self.assertEqual(statement.balance_end_real, 17.39)
+            self.assertEqual(sorted_amounts, [5.48, 5.83, 6.08])
+            self.assertEqual(statement.balance_end, 17.39)
+            # Ponto does not give balance info in transactions.
+            # self.assertEqual(statement.balance_end_real, 17.39)

account_statement_import_online_ponto/tests/test_ponto_interface.py

@@ -0,0 +1,98 @@
+# Copyright 2022 Therp BV <https://therp.nl>.
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+import json
+from unittest.mock import MagicMock, patch
+
+from dateutil.relativedelta import relativedelta
+
+from odoo import fields
+from odoo.tests import common
+
+from .test_account_statement_import_online_ponto import THREE_TRANSACTIONS
+
+
+class TestPontoInterface(common.TransactionCase):
+    post_install = True
+
+    @patch("requests.post")
+    def test_login(self, requests_post):
+        """Check Ponto API login."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(
+            {
+                "access_token": "live_the_token",
+                "expires_in": 1799,
+                "scope": "ai",
+                "token_type": "bearer",
+            }
+        )
+        requests_post.return_value = mock_response
+        interface_model = self.env["ponto.interface"]
+        access_data = interface_model._login("uncle_john", "secret")
+        self.assertEqual(access_data["access_token"], "live_the_token")
+        self.assertIn("token_expiration", access_data)
+
+    @patch("requests.get")
+    def test_set_access_account(self, requests_get):
+        """Test getting account data for Ponto access."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(
+            {
+                "data": [
+                    {
+                        "id": "wrong_id",
+                        "attributes": {
+                            "reference": "NL66ABNA123456789",
+                        },
+                    },
+                    {
+                        "id": "2ad3df83-be01-47cf-a6be-cf0de5cb4c99",
+                        "attributes": {
+                            "reference": "NL66RABO123456789",
+                        },
+                    },
+                ],
+            }
+        )
+        requests_get.return_value = mock_response
+        # Start of actual test.
+        access_data = self._get_access_dict(include_account=False)
+        interface_model = self.env["ponto.interface"]
+        interface_model._set_access_account(access_data, "NL66RABO123456789")
+        self.assertIn("ponto_account", access_data)
+        self.assertEqual(
+            access_data["ponto_account"], "2ad3df83-be01-47cf-a6be-cf0de5cb4c99"
+        )
+
+    @patch("requests.get")
+    def test_get_transactions(self, requests_get):
+        """Test getting transactions from Ponto."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        # Key "data" will contain a list of transactions.
+        mock_response.text = json.dumps({"data": THREE_TRANSACTIONS})
+        requests_get.return_value = mock_response
+        # Start of actual test.
+        access_data = self._get_access_dict()
+        interface_model = self.env["ponto.interface"]
+        transactions = interface_model._get_transactions(access_data, False)
+        self.assertEqual(len(transactions), 3)
+        self.assertEqual(transactions[2]["id"], "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff")
+        self.assertEqual(
+            transactions[2]["attributes"]["counterpartReference"], "BE10325927501996"
+        )
+
+    def _get_access_dict(self, include_account=True):
+        """Get access dict that caches login/account information."""
+        token_expiration = fields.Datetime.now() + relativedelta(seconds=1800)
+        access_data = {
+            "username": "uncle_john",
+            "password": "secret",
+            "access_token": "live_the_token",
+            "token_expiration": token_expiration,
+        }
+        if include_account:
+            access_data["ponto_account"] = "2ad3df83-be01-47cf-a6be-cf0de5cb4c99"
+        return access_data

account_statement_import_online_ponto/views/online_bank_statement_provider.xml

@@ -12,13 +12,6 @@
                 <group name="ponto" attrs="{'invisible':[('service','!=','ponto')]}">
                     <field name="username" string="Login" />
                     <field name="password" string="Secret Key" />
-                    <field name="ponto_last_identifier" />
-                    <button
-                        name="ponto_reset_last_identifier"
-                        string="Reset Last identifier."
-                        type="object"
-                        attrs="{'invisible':[('ponto_last_identifier','=',False)]}"
-                    />
                 </group>
             </xpath>
         </field>

account_statement_import_online_ponto_ing/tests/__init__.py

@@ -1,3 +0,0 @@
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
-from . import test_account_statement_import_online_ponto

account_statement_import_online_ponto_ing/tests/test_account_statement_import_online_ponto.py

@@ -1,195 +0,0 @@
-# Copyright 2020 Florent de Labarre
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
-from datetime import date, datetime
-from unittest import mock
-
-from odoo import fields
-from odoo.tests import Form, common
-
-_module_ns = "odoo.addons.account_statement_import_online_ponto"
-_provider_class = (
-    _module_ns
-    + ".models.online_bank_statement_provider_ponto"
-    + ".OnlineBankStatementProviderPonto"
-)
-
-
-class TestAccountBankAccountStatementImportOnlinePonto(common.TransactionCase):
-    def setUp(self):
-        super().setUp()
-
-        self.now = fields.Datetime.now()
-        self.currency_eur = self.env.ref("base.EUR")
-        self.currency_usd = self.env.ref("base.USD")
-        self.AccountJournal = self.env["account.journal"]
-        self.ResPartnerBank = self.env["res.partner.bank"]
-        self.OnlineBankStatementProvider = self.env["online.bank.statement.provider"]
-        self.AccountBankStatement = self.env["account.bank.statement"]
-        self.AccountBankStatementLine = self.env["account.bank.statement.line"]
-        self.AccStatemenPull = self.env["online.bank.statement.pull.wizard"]
-
-        self.bank_account = self.ResPartnerBank.create(
-            {
-                "acc_number": "FR0214508000302245362775K46",
-                "partner_id": self.env.user.company_id.partner_id.id,
-            }
-        )
-        self.journal = self.AccountJournal.create(
-            {
-                "name": "Bank",
-                "type": "bank",
-                "code": "BANK",
-                "currency_id": self.currency_eur.id,
-                "bank_statements_source": "online",
-                "online_bank_statement_provider": "ponto",
-                "bank_account_id": self.bank_account.id,
-            }
-        )
-        self.provider = self.journal.online_bank_statement_provider_id
-
-        self.mock_header = lambda: mock.patch(
-            _provider_class + "._ponto_header",
-            return_value={
-                "Accept": "application/json",
-                "Authorization": "Bearer --TOKEN--",
-            },
-        )
-
-        self.mock_account_ids = lambda: mock.patch(
-            _provider_class + "._ponto_get_account_ids",
-            return_value={"FR0214508000302245362775K46": "id"},
-        )
-        self.mock_synchronisation = lambda: mock.patch(
-            _provider_class + "._ponto_synchronisation",
-            return_value=None,
-        )
-
-        self.mock_transaction = lambda: mock.patch(
-            _provider_class + "._ponto_get_transaction",
-            return_value=[
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "701ab965-21c4-46ca-b157-306c0646e0e2",
-                    "attributes": {
-                        "valueDate": "2019-11-18T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Minima vitae totam!",
-                        "executionDate": "2019-11-20T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE26089479973169",
-                        "counterpartName": "Osinski Group",
-                        "amount": 6.08,
-                    },
-                },
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "9ac50483-16dc-4a82-aa60-df56077405cd",
-                    "attributes": {
-                        "valueDate": "2019-11-04T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Quia voluptatem blanditiis.",
-                        "executionDate": "2019-11-06T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE97201830401438",
-                        "counterpartName": "Stokes-Miller",
-                        "amount": 5.48,
-                    },
-                },
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff",
-                    "attributes": {
-                        "valueDate": "2019-11-04T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Laboriosam repelo?",
-                        "executionDate": "2019-11-04T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE10325927501996",
-                        "counterpartName": "Strosin-Veum",
-                        "amount": 5.83,
-                    },
-                },
-            ],
-        )
-
-    def test_balance_start(self):
-        st_form = Form(self.AccountBankStatement)
-        st_form.journal_id = self.journal
-        st_form.date = date(2019, 11, 1)
-        st_form.balance_end_real = 100
-        with st_form.line_ids.new() as line_form:
-            line_form.payment_ref = "test move"
-            line_form.amount = 100
-        initial_statement = st_form.save()
-        initial_statement.button_post()
-        with self.mock_transaction(), self.mock_header(), self.mock_synchronisation(), self.mock_account_ids():  # noqa: B950
-            vals = {
-                "provider_ids": self.provider.ids,
-                "date_since": datetime(2019, 11, 4),
-                "date_until": datetime(2019, 11, 5),
-            }
-            wizard = self.AccStatemenPull.with_context(
-                active_model="account.journal",
-                active_id=self.journal.id,
-            ).create(vals)
-            wizard.action_pull()
-            statements = self.AccountBankStatement.search(
-                [("journal_id", "=", self.journal.id)]
-            )
-            new_statement = statements - initial_statement
-            self.assertEqual(len(new_statement.line_ids), 1)
-            self.assertEqual(new_statement.balance_start, 100)
-            self.assertEqual(new_statement.balance_end_real, 105.83)
-
-    def test_ponto(self):
-        with self.mock_transaction(), self.mock_header(), self.mock_synchronisation(), self.mock_account_ids():  # noqa: B950
-            vals = {
-                "provider_ids": self.provider.ids,
-                "date_since": datetime(2019, 11, 3),
-                "date_until": datetime(2019, 11, 17),
-            }
-            wizard = self.AccStatemenPull.with_context(
-                active_model="account.journal",
-                active_id=self.journal.id,
-            ).create(vals)
-            # To get all the moves at once
-            self.provider.statement_creation_mode = "monthly"
-            wizard.action_pull()
-            statement = self.AccountBankStatement.search(
-                [("journal_id", "=", self.journal.id)]
-            )
-            self.assertEqual(len(statement), 1)
-            self.assertEqual(len(statement.line_ids), 3)
-            self.assertEqual(statement.line_ids.mapped("amount"), [6.08, 5.48, 5.83])
-            self.assertEqual(statement.balance_end_real, 17.39)