From 7c74c154637c23a5d4e9705f03ad9e84cba9f19a Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 31 Aug 2022 12:34:31 +0200
Subject: [PATCH 01/15] [IMP] *_online_ponto: black

Blacked code for better comparison / easier backport from later
versions.
---
 .pre-commit-config.yaml                       |   1 -
 .../__manifest__.py                           |   4 +-
 .../online_bank_statement_provider_ponto.py   | 189 +++++++++-------
 ...ount_bank_statement_import_online_ponto.py | 203 ++++++++++--------
 4 files changed, 223 insertions(+), 174 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index ccc88915..49ed876e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -38,7 +38,6 @@ repos:
     rev: v3.4.1
     hooks:
       - id: flake8
-        language_version: python3.6
         name: flake8 excluding __init__.py
         exclude: __init__\.py
   - repo: https://github.com/pre-commit/mirrors-pylint
diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py
index dcfc8a28..1b0b8499 100644
--- a/account_bank_statement_import_online_ponto/__manifest__.py
+++ b/account_bank_statement_import_online_ponto/__manifest__.py
@@ -9,7 +9,5 @@
     "license": "AGPL-3",
     "installable": True,
     "depends": ["account_bank_statement_import_online"],
-    "data": [
-        "view/online_bank_statement_provider.xml"
-    ],
+    "data": ["view/online_bank_statement_provider.xml"],
 }
diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index 6802313d..ed5416e8 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -14,28 +14,28 @@ from odoo.exceptions import UserError
 from dateutil.relativedelta import relativedelta
 from odoo.addons.base.models.res_bank import sanitize_account_number
 
-PONTO_ENDPOINT = 'https://api.myponto.com'
+PONTO_ENDPOINT = "https://api.myponto.com"
 
 
 class OnlineBankStatementProviderPonto(models.Model):
-    _inherit = 'online.bank.statement.provider'
+    _inherit = "online.bank.statement.provider"
 
     ponto_token = fields.Char(readonly=True)
     ponto_token_expiration = fields.Datetime(readonly=True)
     ponto_last_identifier = fields.Char(readonly=True)
 
     def ponto_reset_last_identifier(self):
-        self.write({'ponto_last_identifier': False})
+        self.write({"ponto_last_identifier": False})
 
     @api.model
     def _get_available_services(self):
         return super()._get_available_services() + [
-            ('ponto', 'MyPonto.com'),
+            ("ponto", "MyPonto.com"),
         ]
 
     def _obtain_statement_data(self, date_since, date_until):
         self.ensure_one()
-        if self.service != 'ponto':
+        if self.service != "ponto":
             return super()._obtain_statement_data(
                 date_since,
                 date_until,
@@ -49,126 +49,145 @@ class OnlineBankStatementProviderPonto(models.Model):
     def _ponto_header_token(self):
         self.ensure_one()
         if self.username and self.password:
-            login = '%s:%s' % (self.username, self.password)
-            login = base64.b64encode(login.encode('UTF-8')).decode('UTF-8')
-            return {'Content-Type': 'application/x-www-form-urlencoded',
-                    'Accept': 'application/json',
-                    'Authorization': 'Basic %s' % login, }
-        raise UserError(_('Please fill login and key.'))
+            login = "%s:%s" % (self.username, self.password)
+            login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8")
+            return {
+                "Content-Type": "application/x-www-form-urlencoded",
+                "Accept": "application/json",
+                "Authorization": "Basic %s" % login,
+            }
+        raise UserError(_("Please fill login and key."))
 
     def _ponto_header(self):
         self.ensure_one()
-        if not self.ponto_token \
-                or not self.ponto_token_expiration \
-                or self.ponto_token_expiration <= fields.Datetime.now():
+        if (
+            not self.ponto_token
+            or not self.ponto_token_expiration
+            or self.ponto_token_expiration <= fields.Datetime.now()
+        ):
 
-            url = PONTO_ENDPOINT + '/oauth2/token'
-            response = requests.post(url, verify=False,
-                                     params={'grant_type': 'client_credentials'},
-                                     headers=self._ponto_header_token())
+            url = PONTO_ENDPOINT + "/oauth2/token"
+            response = requests.post(
+                url,
+                verify=False,
+                params={"grant_type": "client_credentials"},
+                headers=self._ponto_header_token(),
+            )
             if response.status_code == 200:
                 data = json.loads(response.text)
-                access_token = data.get('access_token', False)
+                access_token = data.get("access_token", False)
                 if not access_token:
-                    raise UserError(_('Ponto : no token'))
+                    raise UserError(_("Ponto : no token"))
                 else:
                     self.sudo().ponto_token = access_token
                     expiration_date = fields.Datetime.now() + relativedelta(
-                        seconds=data.get('expires_in', False))
+                        seconds=data.get("expires_in", False)
+                    )
                     self.sudo().ponto_token_expiration = expiration_date
             else:
-                raise UserError(_('%s \n\n %s') % (response.status_code, response.text))
-        return {'Accept': 'application/json',
-                'Authorization': 'Bearer %s' % self.ponto_token, }
+                raise UserError(_("%s \n\n %s") % (response.status_code, response.text))
+        return {
+            "Accept": "application/json",
+            "Authorization": "Bearer %s" % self.ponto_token,
+        }
 
     def _ponto_get_account_ids(self):
-        url = PONTO_ENDPOINT + '/accounts'
-        response = requests.get(url, verify=False, params={'limit': 100},
-                                headers=self._ponto_header())
+        url = PONTO_ENDPOINT + "/accounts"
+        response = requests.get(
+            url, verify=False, params={"limit": 100}, headers=self._ponto_header()
+        )
         if response.status_code == 200:
             data = json.loads(response.text)
             res = {}
-            for account in data.get('data', []):
+            for account in data.get("data", []):
                 iban = sanitize_account_number(
-                    account.get('attributes', {}).get('reference', ''))
-                res[iban] = account.get('id')
+                    account.get("attributes", {}).get("reference", "")
+                )
+                res[iban] = account.get("id")
             return res
-        raise UserError(_('%s \n\n %s') % (response.status_code, response.text))
+        raise UserError(_("%s \n\n %s") % (response.status_code, response.text))
 
     def _ponto_synchronisation(self, account_id):
-        url = PONTO_ENDPOINT + '/synchronizations'
-        data = {'data': {
-            'type': 'synchronization',
-            'attributes': {
-                'resourceType': 'account',
-                'resourceId': account_id,
-                'subtype': 'accountTransactions'
+        url = PONTO_ENDPOINT + "/synchronizations"
+        data = {
+            "data": {
+                "type": "synchronization",
+                "attributes": {
+                    "resourceType": "account",
+                    "resourceId": account_id,
+                    "subtype": "accountTransactions",
+                },
             }
-        }}
-        response = requests.post(url, verify=False,
-                                 headers=self._ponto_header(),
-                                 json=data)
+        }
+        response = requests.post(
+            url, verify=False, headers=self._ponto_header(), json=data
+        )
         if response.status_code in (200, 201, 400):
             data = json.loads(response.text)
-            sync_id = data.get('attributes', {}).get('resourceId', False)
+            sync_id = data.get("attributes", {}).get("resourceId", False)
         else:
-            raise UserError(_('Error during Create Synchronisation %s \n\n %s') % (
-                response.status_code, response.text))
+            raise UserError(
+                _("Error during Create Synchronisation %s \n\n %s")
+                % (response.status_code, response.text)
+            )
 
         # Check synchronisation
         if not sync_id:
             return
-        url = PONTO_ENDPOINT + '/synchronizations/' + sync_id
+        url = PONTO_ENDPOINT + "/synchronizations/" + sync_id
         number = 0
         while number == 100:
             number += 1
             response = requests.get(url, verify=False, headers=self._ponto_header())
             if response.status_code == 200:
                 data = json.loads(response.text)
-                status = data.get('status', {})
-                if status in ('success', 'error'):
+                status = data.get("status", {})
+                if status in ("success", "error"):
                     return
             time.sleep(4)
 
     def _ponto_get_transaction(self, account_id, date_since, date_until):
-        page_url = PONTO_ENDPOINT + '/accounts/' + account_id + '/transactions'
-        params = {'limit': 100}
+        page_url = PONTO_ENDPOINT + "/accounts/" + account_id + "/transactions"
+        params = {"limit": 100}
         page_next = True
         last_identifier = self.ponto_last_identifier
         if last_identifier:
-            params['before'] = last_identifier
+            params["before"] = last_identifier
             page_next = False
         transaction_lines = []
         latest_identifier = False
         while page_url:
-            response = requests.get(page_url, verify=False, params=params,
-                                    headers=self._ponto_header())
+            response = requests.get(
+                page_url, verify=False, params=params, headers=self._ponto_header()
+            )
             if response.status_code == 200:
-                if params.get('before'):
-                    params.pop('before')
+                if params.get("before"):
+                    params.pop("before")
                 data = json.loads(response.text)
-                links = data.get('links', {})
+                links = data.get("links", {})
                 if page_next:
-                    page_url = links.get('next', False)
+                    page_url = links.get("next", False)
                 else:
-                    page_url = links.get('prev', False)
-                transactions = data.get('data', [])
+                    page_url = links.get("prev", False)
+                transactions = data.get("data", [])
                 if transactions:
                     current_transactions = []
                     for transaction in transactions:
                         date = self._ponto_date_from_string(
-                            transaction.get('attributes', {}).get('executionDate'))
+                            transaction.get("attributes", {}).get("executionDate")
+                        )
                         if date_since <= date < date_until:
                             current_transactions.append(transaction)
 
                     if current_transactions:
                         if not page_next or (page_next and not latest_identifier):
-                            latest_identifier = current_transactions[0].get('id')
+                            latest_identifier = current_transactions[0].get("id")
                         transaction_lines.extend(current_transactions)
             else:
                 raise UserError(
-                    _('Error during get transaction.\n\n%s \n\n %s') % (
-                        response.status_code, response.text))
+                    _("Error during get transaction.\n\n%s \n\n %s")
+                    % (response.status_code, response.text)
+                )
         if latest_identifier:
             self.ponto_last_identifier = latest_identifier
         return transaction_lines
@@ -177,10 +196,8 @@ class OnlineBankStatementProviderPonto(models.Model):
         """Dates in Ponto are expressed in UTC, so we need to convert them
         to supplied tz for proper classification.
         """
-        dt = datetime.strptime(date_str, '%Y-%m-%dT%H:%M:%S.%fZ')
-        dt = dt.replace(tzinfo=pytz.utc).astimezone(
-            pytz.timezone(self.tz or 'utc')
-        )
+        dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
+        dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
         return dt.replace(tzinfo=None)
 
     def _ponto_obtain_statement_data(self, date_since, date_until):
@@ -191,30 +208,36 @@ class OnlineBankStatementProviderPonto(models.Model):
         account_id = account_ids.get(iban)
         if not account_id:
             raise UserError(
-                _('Ponto : wrong configuration, unknow account %s')
-                % journal.bank_account_id.acc_number)
+                _("Ponto : wrong configuration, unknow account %s")
+                % journal.bank_account_id.acc_number
+            )
         self._ponto_synchronisation(account_id)
         transaction_lines = self._ponto_get_transaction(
-            account_id, date_since, date_until)
+            account_id, date_since, date_until
+        )
         new_transactions = []
         sequence = 0
         for transaction in transaction_lines:
             sequence += 1
-            attributes = transaction.get('attributes', {})
-            ref_list = [attributes.get(x) for x in {
-                "description",
-                "counterpartName",
-                "counterpartReference",
-            } if attributes.get(x)]
+            attributes = transaction.get("attributes", {})
+            ref_list = [
+                attributes.get(x)
+                for x in {
+                    "description",
+                    "counterpartName",
+                    "counterpartReference",
+                }
+                if attributes.get(x)
+            ]
             ref = " ".join(ref_list)
-            date = self._ponto_date_from_string(attributes.get('executionDate'))
+            date = self._ponto_date_from_string(attributes.get("executionDate"))
             vals_line = {
-                'sequence': sequence,
-                'date': date,
-                'ref': re.sub(' +', ' ', ref) or '/',
-                'name': attributes.get('remittanceInformation') or ref,
-                'unique_import_id': transaction['id'],
-                'amount': attributes['amount'],
+                "sequence": sequence,
+                "date": date,
+                "ref": re.sub(" +", " ", ref) or "/",
+                "name": attributes.get("remittanceInformation") or ref,
+                "unique_import_id": transaction["id"],
+                "amount": attributes["amount"],
             }
             if attributes.get("counterpartReference"):
                 vals_line["account_number"] = attributes["counterpartReference"]
diff --git a/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py b/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
index 8b4353b0..f88df801 100644
--- a/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
+++ b/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
@@ -7,119 +7,148 @@ from unittest import mock
 from odoo import fields
 from odoo.tests import common
 
-_module_ns = 'odoo.addons.account_bank_statement_import_online_ponto'
+_module_ns = "odoo.addons.account_bank_statement_import_online_ponto"
 _provider_class = (
     _module_ns
-    + '.models.online_bank_statement_provider_ponto'
-    + '.OnlineBankStatementProviderPonto'
+    + ".models.online_bank_statement_provider_ponto"
+    + ".OnlineBankStatementProviderPonto"
 )
 
 
-class TestAccountBankAccountStatementImportOnlineQonto(
-    common.TransactionCase
-):
-
+class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
     def setUp(self):
         super().setUp()
 
         self.now = fields.Datetime.now()
-        self.currency_eur = self.env.ref('base.EUR')
-        self.currency_usd = self.env.ref('base.USD')
-        self.AccountJournal = self.env['account.journal']
-        self.ResPartnerBank = self.env['res.partner.bank']
-        self.OnlineBankStatementProvider = self.env[
-            'online.bank.statement.provider'
-        ]
-        self.AccountBankStatement = self.env['account.bank.statement']
-        self.AccountBankStatementLine = self.env['account.bank.statement.line']
+        self.currency_eur = self.env.ref("base.EUR")
+        self.currency_usd = self.env.ref("base.USD")
+        self.AccountJournal = self.env["account.journal"]
+        self.ResPartnerBank = self.env["res.partner.bank"]
+        self.OnlineBankStatementProvider = self.env["online.bank.statement.provider"]
+        self.AccountBankStatement = self.env["account.bank.statement"]
+        self.AccountBankStatementLine = self.env["account.bank.statement.line"]
 
         self.bank_account = self.ResPartnerBank.create(
-            {'acc_number': 'FR0214508000302245362775K46',
-             'partner_id': self.env.user.company_id.partner_id.id})
-        self.journal = self.AccountJournal.create({
-            'name': 'Bank',
-            'type': 'bank',
-            'code': 'BANK',
-            'currency_id': self.currency_eur.id,
-            'bank_statements_source': 'online',
-            'online_bank_statement_provider': 'ponto',
-            'bank_account_id': self.bank_account.id,
-        })
+            {
+                "acc_number": "FR0214508000302245362775K46",
+                "partner_id": self.env.user.company_id.partner_id.id,
+            }
+        )
+        self.journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_eur.id,
+                "bank_statements_source": "online",
+                "online_bank_statement_provider": "ponto",
+                "bank_account_id": self.bank_account.id,
+            }
+        )
         self.provider = self.journal.online_bank_statement_provider_id
 
         self.mock_header = lambda: mock.patch(
-            _provider_class + '._ponto_header',
-            return_value={'Accept': 'application/json',
-                          'Authorization': 'Bearer --TOKEN--'},
+            _provider_class + "._ponto_header",
+            return_value={
+                "Accept": "application/json",
+                "Authorization": "Bearer --TOKEN--",
+            },
         )
 
         self.mock_account_ids = lambda: mock.patch(
-            _provider_class + '._ponto_get_account_ids',
-            return_value={'FR0214508000302245362775K46': 'id'},
+            _provider_class + "._ponto_get_account_ids",
+            return_value={"FR0214508000302245362775K46": "id"},
         )
         self.mock_synchronisation = lambda: mock.patch(
-            _provider_class + '._ponto_synchronisation',
+            _provider_class + "._ponto_synchronisation",
             return_value=None,
         )
 
         self.mock_transaction = lambda: mock.patch(
-            _provider_class + '._ponto_get_transaction',
-            return_value=[{
-                'type': 'transaction',
-                'relationships': {'account': {
-                    'links': {
-                        'related': 'https://api.myponto.com/accounts/'},
-                    'data': {'type': 'account',
-                             'id': 'fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75'}}},
-                'id': '701ab965-21c4-46ca-b157-306c0646e0e2',
-                'attributes': {'valueDate': '2019-11-18T00:00:00.000Z',
-                               'remittanceInformationType': 'unstructured',
-                               'remittanceInformation': 'Minima vitae totam!',
-                               'executionDate': '2019-11-20T00:00:00.000Z',
-                               'description': 'Wire transfer',
-                               'currency': 'EUR',
-                               'counterpartReference': 'BE26089479973169',
-                               'counterpartName': 'Osinski Group',
-                               'amount': 6.08}},
-                {'type': 'transaction',
-                 'relationships': {
-                     'account': {'links': {
-                         'related': 'https://api.myponto.com/accounts/'},
-                         'data': {
-                             'type': 'account',
-                             'id': 'fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75'}}},
-                 'id': '9ac50483-16dc-4a82-aa60-df56077405cd',
-                 'attributes': {
-                     'valueDate': '2019-11-04T00:00:00.000Z',
-                     'remittanceInformationType': 'unstructured',
-                     'remittanceInformation': 'Quia voluptatem blanditiis.',
-                     'executionDate': '2019-11-06T00:00:00.000Z',
-                     'description': 'Wire transfer',
-                     'currency': 'EUR',
-                     'counterpartReference': 'BE97201830401438',
-                     'counterpartName': 'Stokes-Miller',
-                     'amount': 5.48}},
-                {'type': 'transaction', 'relationships': {'account': {'links': {
-                    'related': 'https://api.myponto.com/accounts/'},
-                    'data': {
-                        'type': 'account',
-                        'id': 'fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75'}}},
-                 'id': 'b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff',
-                 'attributes': {
-                     'valueDate': '2019-11-04T00:00:00.000Z',
-                     'remittanceInformationType': 'unstructured',
-                     'remittanceInformation': 'Laboriosam repelo?',
-                     'executionDate': '2019-11-04T00:00:00.000Z',
-                     'description': 'Wire transfer', 'currency': 'EUR',
-                     'counterpartReference': 'BE10325927501996',
-                     'counterpartName': 'Strosin-Veum', 'amount': 5.83}}],
+            _provider_class + "._ponto_get_transaction",
+            return_value=[
+                {
+                    "type": "transaction",
+                    "relationships": {
+                        "account": {
+                            "links": {"related": "https://api.myponto.com/accounts/"},
+                            "data": {
+                                "type": "account",
+                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                            },
+                        }
+                    },
+                    "id": "701ab965-21c4-46ca-b157-306c0646e0e2",
+                    "attributes": {
+                        "valueDate": "2019-11-18T00:00:00.000Z",
+                        "remittanceInformationType": "unstructured",
+                        "remittanceInformation": "Minima vitae totam!",
+                        "executionDate": "2019-11-20T00:00:00.000Z",
+                        "description": "Wire transfer",
+                        "currency": "EUR",
+                        "counterpartReference": "BE26089479973169",
+                        "counterpartName": "Osinski Group",
+                        "amount": 6.08,
+                    },
+                },
+                {
+                    "type": "transaction",
+                    "relationships": {
+                        "account": {
+                            "links": {"related": "https://api.myponto.com/accounts/"},
+                            "data": {
+                                "type": "account",
+                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                            },
+                        }
+                    },
+                    "id": "9ac50483-16dc-4a82-aa60-df56077405cd",
+                    "attributes": {
+                        "valueDate": "2019-11-04T00:00:00.000Z",
+                        "remittanceInformationType": "unstructured",
+                        "remittanceInformation": "Quia voluptatem blanditiis.",
+                        "executionDate": "2019-11-06T00:00:00.000Z",
+                        "description": "Wire transfer",
+                        "currency": "EUR",
+                        "counterpartReference": "BE97201830401438",
+                        "counterpartName": "Stokes-Miller",
+                        "amount": 5.48,
+                    },
+                },
+                {
+                    "type": "transaction",
+                    "relationships": {
+                        "account": {
+                            "links": {"related": "https://api.myponto.com/accounts/"},
+                            "data": {
+                                "type": "account",
+                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                            },
+                        }
+                    },
+                    "id": "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff",
+                    "attributes": {
+                        "valueDate": "2019-11-04T00:00:00.000Z",
+                        "remittanceInformationType": "unstructured",
+                        "remittanceInformation": "Laboriosam repelo?",
+                        "executionDate": "2019-11-04T00:00:00.000Z",
+                        "description": "Wire transfer",
+                        "currency": "EUR",
+                        "counterpartReference": "BE10325927501996",
+                        "counterpartName": "Strosin-Veum",
+                        "amount": 5.83,
+                    },
+                },
+            ],
         )
 
     def test_ponto(self):
-        with self.mock_transaction(), \
-                self.mock_header(),\
-                self.mock_synchronisation(), \
-                self.mock_account_ids():
+        with (
+            self.mock_transaction(),
+            self.mock_header(),
+            self.mock_synchronisation(),
+            self.mock_account_ids()
+        ):
             lines, statement_values = self.provider._obtain_statement_data(
                 datetime(2019, 11, 3),
                 datetime(2019, 11, 17),

From 4958e72cc41d8a1da7c1999eded52ec14f1f886c Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 31 Aug 2022 12:48:09 +0200
Subject: [PATCH 02/15] [FIX] *_online_ponto: Backport 9c1d6d0 to verify TLS
 connection

---
 .../models/online_bank_statement_provider_ponto.py       | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index ed5416e8..d107ef9e 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -69,7 +69,6 @@ class OnlineBankStatementProviderPonto(models.Model):
             url = PONTO_ENDPOINT + "/oauth2/token"
             response = requests.post(
                 url,
-                verify=False,
                 params={"grant_type": "client_credentials"},
                 headers=self._ponto_header_token(),
             )
@@ -94,7 +93,7 @@ class OnlineBankStatementProviderPonto(models.Model):
     def _ponto_get_account_ids(self):
         url = PONTO_ENDPOINT + "/accounts"
         response = requests.get(
-            url, verify=False, params={"limit": 100}, headers=self._ponto_header()
+            url, params={"limit": 100}, headers=self._ponto_header()
         )
         if response.status_code == 200:
             data = json.loads(response.text)
@@ -120,7 +119,7 @@ class OnlineBankStatementProviderPonto(models.Model):
             }
         }
         response = requests.post(
-            url, verify=False, headers=self._ponto_header(), json=data
+            url, headers=self._ponto_header(), json=data
         )
         if response.status_code in (200, 201, 400):
             data = json.loads(response.text)
@@ -138,7 +137,7 @@ class OnlineBankStatementProviderPonto(models.Model):
         number = 0
         while number == 100:
             number += 1
-            response = requests.get(url, verify=False, headers=self._ponto_header())
+            response = requests.get(url, headers=self._ponto_header())
             if response.status_code == 200:
                 data = json.loads(response.text)
                 status = data.get("status", {})
@@ -158,7 +157,7 @@ class OnlineBankStatementProviderPonto(models.Model):
         latest_identifier = False
         while page_url:
             response = requests.get(
-                page_url, verify=False, params=params, headers=self._ponto_header()
+                page_url, params=params, headers=self._ponto_header()
             )
             if response.status_code == 200:
                 if params.get("before"):

From daa143336e58f83f82fefea455314ce2dd0d7d1c Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 31 Aug 2022 12:51:28 +0200
Subject: [PATCH 03/15] [FIX] *_online_ponto: Backport 4a8f9ac better code
 organisation

---
 .../online_bank_statement_provider_ponto.py   | 47 +++++++++----------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index d107ef9e..f3c1bff2 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -159,34 +159,33 @@ class OnlineBankStatementProviderPonto(models.Model):
             response = requests.get(
                 page_url, params=params, headers=self._ponto_header()
             )
-            if response.status_code == 200:
-                if params.get("before"):
-                    params.pop("before")
-                data = json.loads(response.text)
-                links = data.get("links", {})
-                if page_next:
-                    page_url = links.get("next", False)
-                else:
-                    page_url = links.get("prev", False)
-                transactions = data.get("data", [])
-                if transactions:
-                    current_transactions = []
-                    for transaction in transactions:
-                        date = self._ponto_date_from_string(
-                            transaction.get("attributes", {}).get("executionDate")
-                        )
-                        if date_since <= date < date_until:
-                            current_transactions.append(transaction)
-
-                    if current_transactions:
-                        if not page_next or (page_next and not latest_identifier):
-                            latest_identifier = current_transactions[0].get("id")
-                        transaction_lines.extend(current_transactions)
-            else:
+            if response.status_code != 200:
                 raise UserError(
                     _("Error during get transaction.\n\n%s \n\n %s")
                     % (response.status_code, response.text)
                 )
+            if params.get("before"):
+                params.pop("before")
+            data = json.loads(response.text)
+            links = data.get("links", {})
+            if page_next:
+                page_url = links.get("next", False)
+            else:
+                page_url = links.get("prev", False)
+            transactions = data.get("data", [])
+            if transactions:
+                current_transactions = []
+                for transaction in transactions:
+                    date = self._ponto_date_from_string(
+                        transaction.get("attributes", {}).get("executionDate")
+                    )
+                    if date_since <= date < date_until:
+                        current_transactions.append(transaction)
+
+                if current_transactions:
+                    if not page_next or (page_next and not latest_identifier):
+                        latest_identifier = current_transactions[0].get("id")
+                    transaction_lines.extend(current_transactions)
         if latest_identifier:
             self.ponto_last_identifier = latest_identifier
         return transaction_lines

From fd071b3c98e92496f9a67365299cc656b4a00960 Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 31 Aug 2022 12:59:03 +0200
Subject: [PATCH 04/15] [FIX] *_online_ponto: Backport 40625b refactoring

---
 .../online_bank_statement_provider_ponto.py   | 54 ++++++++++---------
 1 file changed, 30 insertions(+), 24 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index f3c1bff2..9b9c6b53 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -199,6 +199,7 @@ class OnlineBankStatementProviderPonto(models.Model):
         return dt.replace(tzinfo=None)
 
     def _ponto_obtain_statement_data(self, date_since, date_until):
+        """Translate information from Ponto to Odoo bank statement lines."""
         self.ensure_one()
         account_ids = self._ponto_get_account_ids()
         journal = self.journal_id
@@ -217,31 +218,36 @@ class OnlineBankStatementProviderPonto(models.Model):
         sequence = 0
         for transaction in transaction_lines:
             sequence += 1
-            attributes = transaction.get("attributes", {})
-            ref_list = [
-                attributes.get(x)
-                for x in {
-                    "description",
-                    "counterpartName",
-                    "counterpartReference",
-                }
-                if attributes.get(x)
-            ]
-            ref = " ".join(ref_list)
-            date = self._ponto_date_from_string(attributes.get("executionDate"))
-            vals_line = {
-                "sequence": sequence,
-                "date": date,
-                "ref": re.sub(" +", " ", ref) or "/",
-                "name": attributes.get("remittanceInformation") or ref,
-                "unique_import_id": transaction["id"],
-                "amount": attributes["amount"],
-            }
-            if attributes.get("counterpartReference"):
-                vals_line["account_number"] = attributes["counterpartReference"]
-            if attributes.get("counterpartName"):
-                vals_line["partner_name"] = attributes["counterpartName"]
+            vals_line = self._ponto_get_transaction_vals(transaction, sequence)
             new_transactions.append(vals_line)
         if new_transactions:
             return new_transactions, {}
         return
+
+    def _ponto_get_transaction_vals(self, transaction, sequence):
+        """Translate information from Ponto to statement line vals."""
+        attributes = transaction.get("attributes", {})
+        ref_list = [
+            attributes.get(x)
+            for x in {
+                "description",
+                "counterpartName",
+                "counterpartReference",
+            }
+            if attributes.get(x)
+        ]
+        ref = " ".join(ref_list)
+        date = self._ponto_date_from_string(attributes.get("executionDate"))
+        vals_line = {
+            "sequence": sequence,
+            "date": date,
+            "ref": re.sub(" +", " ", ref) or "/",
+            "name": attributes.get("remittanceInformation") or ref,
+            "unique_import_id": transaction["id"],
+            "amount": attributes["amount"],
+        }
+        if attributes.get("counterpartReference"):
+            vals_line["account_number"] = attributes["counterpartReference"]
+        if attributes.get("counterpartName"):
+            vals_line["partner_name"] = attributes["counterpartName"]
+        return vals_line

From 46bb59aca7f345d6f906c055fc73d1ef22f344bc Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 31 Aug 2022 13:26:05 +0200
Subject: [PATCH 05/15] [IMP] *_online_ponto: extensive debugging

---
 .../online_bank_statement_provider_ponto.py   | 52 +++++++++++++++----
 1 file changed, 42 insertions(+), 10 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index 9b9c6b53..b264ad72 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -1,19 +1,23 @@
 # Copyright 2020 Florent de Labarre
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
-import requests
-import json
 import base64
-import time
+import json
+import logging
 import re
-import pytz
+import time
 from datetime import datetime
 
+import pytz
+import requests
+from dateutil.relativedelta import relativedelta
+
 from odoo import api, fields, models, _
 from odoo.exceptions import UserError
-from dateutil.relativedelta import relativedelta
 from odoo.addons.base.models.res_bank import sanitize_account_number
 
+_logger = logging.getLogger(__name__)
+
 PONTO_ENDPOINT = "https://api.myponto.com"
 
 
@@ -129,7 +133,6 @@ class OnlineBankStatementProviderPonto(models.Model):
                 _("Error during Create Synchronisation %s \n\n %s")
                 % (response.status_code, response.text)
             )
-
         # Check synchronisation
         if not sync_id:
             return
@@ -156,8 +159,15 @@ class OnlineBankStatementProviderPonto(models.Model):
         transaction_lines = []
         latest_identifier = False
         while page_url:
+            headers = self._ponto_header()
             response = requests.get(
-                page_url, params=params, headers=self._ponto_header()
+                page_url, params=params, headers=headers
+            )
+            _logger.debug(
+                _("Get request to %s, with headers %s and params %s"),
+                page_url,
+                params,
+                headers
             )
             if response.status_code != 200:
                 raise UserError(
@@ -173,7 +183,16 @@ class OnlineBankStatementProviderPonto(models.Model):
             else:
                 page_url = links.get("prev", False)
             transactions = data.get("data", [])
-            if transactions:
+            if not transactions:
+                _logger.debug(
+                    _("No transactions where found in response %s"),
+                    response.text,
+                )
+            else:
+                _logger.debug(
+                    _("%d transactions present in response data"),
+                    len(transactions),
+                )
                 current_transactions = []
                 for transaction in transactions:
                     date = self._ponto_date_from_string(
@@ -181,8 +200,15 @@ class OnlineBankStatementProviderPonto(models.Model):
                     )
                     if date_since <= date < date_until:
                         current_transactions.append(transaction)
-
-                if current_transactions:
+                if not current_transactions:
+                    _logger.debug(
+                        _("No lines selected from transactions")
+                    )
+                else:
+                    _logger.debug(
+                        _("%d lines selected from transactions"),
+                        len(current_transactions),
+                    )
                     if not page_next or (page_next and not latest_identifier):
                         latest_identifier = current_transactions[0].get("id")
                     transaction_lines.extend(current_transactions)
@@ -210,6 +236,12 @@ class OnlineBankStatementProviderPonto(models.Model):
                 _("Ponto : wrong configuration, unknow account %s")
                 % journal.bank_account_id.acc_number
             )
+        _logger.debug(
+            _("Ponto obtain statement data for journal %s from %s to %s"),
+            journal.name,
+            date_since,
+            date_until
+        )
         self._ponto_synchronisation(account_id)
         transaction_lines = self._ponto_get_transaction(
             account_id, date_since, date_until

From 26ed33bc7f071bc06a95b61ff1fd5fb242776d93 Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 31 Aug 2022 16:40:36 +0200
Subject: [PATCH 06/15] [IMP] *_online_ponto: more efficient ponto account
 retrieval

Also do not name variables account_id or account_ids when they do not
refer to ids in the Odoo sense, nor to either bank or ledger accounts.
---
 .../__manifest__.py                           |   8 +-
 .../models/__init__.py                        |   5 +-
 .../online_bank_statement_provider_ponto.py   | 261 ++++--------------
 .../models/ponto_buffer.py                    |  79 ++++++
 .../models/ponto_buffer_line.py               |  28 ++
 .../models/ponto_interface.py                 | 190 +++++++++++++
 6 files changed, 360 insertions(+), 211 deletions(-)
 create mode 100644 account_bank_statement_import_online_ponto/models/ponto_buffer.py
 create mode 100644 account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
 create mode 100644 account_bank_statement_import_online_ponto/models/ponto_interface.py

diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py
index 1b0b8499..919f0e0a 100644
--- a/account_bank_statement_import_online_ponto/__manifest__.py
+++ b/account_bank_statement_import_online_ponto/__manifest__.py
@@ -1,11 +1,15 @@
 # Copyright 2020 Florent de Labarre
+# Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 {
     "name": "Online Bank Statements: MyPonto.com",
-    "version": "12.0.1.1.1",
+    "version": "12.0.1.2.0",
     "category": "Account",
     "website": "https://github.com/OCA/bank-statement-import",
-    "author": "Florent de Labarre, Odoo Community Association (OCA)",
+    "author":
+        "Florent de Labarre"
+        ", Therp BV"
+        ", Odoo Community Association (OCA)",
     "license": "AGPL-3",
     "installable": True,
     "depends": ["account_bank_statement_import_online"],
diff --git a/account_bank_statement_import_online_ponto/models/__init__.py b/account_bank_statement_import_online_ponto/models/__init__.py
index cc57537b..4044c75c 100644
--- a/account_bank_statement_import_online_ponto/models/__init__.py
+++ b/account_bank_statement_import_online_ponto/models/__init__.py
@@ -1,4 +1,5 @@
-# Copyright 2020 Florent de Labarre
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
+from . import ponto_buffer
+from . import ponto_buffer_line
+from . import ponto_interface
 from . import online_bank_statement_provider_ponto
diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index b264ad72..30c65518 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -1,31 +1,21 @@
 # Copyright 2020 Florent de Labarre
+# Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
-import base64
+from datetime import datetime
 import json
+import pytz
+
 import logging
 import re
-import time
-from datetime import datetime
-
-import pytz
-import requests
-from dateutil.relativedelta import relativedelta
 
 from odoo import api, fields, models, _
-from odoo.exceptions import UserError
-from odoo.addons.base.models.res_bank import sanitize_account_number
 
 _logger = logging.getLogger(__name__)
 
-PONTO_ENDPOINT = "https://api.myponto.com"
-
 
 class OnlineBankStatementProviderPonto(models.Model):
     _inherit = "online.bank.statement.provider"
 
-    ponto_token = fields.Char(readonly=True)
-    ponto_token_expiration = fields.Datetime(readonly=True)
     ponto_last_identifier = fields.Char(readonly=True)
 
     def ponto_reset_last_identifier(self):
@@ -37,6 +27,35 @@ class OnlineBankStatementProviderPonto(models.Model):
             ("ponto", "MyPonto.com"),
         ]
 
+    @api.multi
+    def _pull(self, date_since, date_until):
+        """Override pull to first retrieve data from Ponto."""
+        self.ensure_one()
+        if self.service == "ponto":
+            self._ponto_retrieve_data()
+        super()._pull(date_since, date_until)
+
+    def _ponto_retrieve_data(self):
+        """Fill buffer with data from Ponto."""
+        interface_model = self.env["ponto.interface"]
+        buffer_model = self.env["ponto.buffer"]
+        access_data = interface_model._login(self.username, self.password)
+        interface_model._set_access_account(access_data, self.account_number)
+        interface_model._ponto_synchronisation(access_data)
+        latest_identifier = self.ponto_last_identifier
+        transactions = interface_model._get_transactions(
+            access_data,
+            latest_identifier
+        )
+        while transactions:
+            buffer_model.sudo()._store_transactions(self, transactions)
+            latest_identifier = transactions[-1].get("id")
+            transactions = interface_model._get_transactions(
+                access_data,
+                latest_identifier
+            )
+        self.ponto_last_identifier = latest_identifier
+
     def _obtain_statement_data(self, date_since, date_until):
         self.ensure_one()
         if self.service != "ponto":
@@ -46,211 +65,31 @@ class OnlineBankStatementProviderPonto(models.Model):
             )
         return self._ponto_obtain_statement_data(date_since, date_until)
 
-    #########
-    # ponto #
-    #########
-
-    def _ponto_header_token(self):
-        self.ensure_one()
-        if self.username and self.password:
-            login = "%s:%s" % (self.username, self.password)
-            login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8")
-            return {
-                "Content-Type": "application/x-www-form-urlencoded",
-                "Accept": "application/json",
-                "Authorization": "Basic %s" % login,
-            }
-        raise UserError(_("Please fill login and key."))
-
-    def _ponto_header(self):
-        self.ensure_one()
-        if (
-            not self.ponto_token
-            or not self.ponto_token_expiration
-            or self.ponto_token_expiration <= fields.Datetime.now()
-        ):
-
-            url = PONTO_ENDPOINT + "/oauth2/token"
-            response = requests.post(
-                url,
-                params={"grant_type": "client_credentials"},
-                headers=self._ponto_header_token(),
-            )
-            if response.status_code == 200:
-                data = json.loads(response.text)
-                access_token = data.get("access_token", False)
-                if not access_token:
-                    raise UserError(_("Ponto : no token"))
-                else:
-                    self.sudo().ponto_token = access_token
-                    expiration_date = fields.Datetime.now() + relativedelta(
-                        seconds=data.get("expires_in", False)
-                    )
-                    self.sudo().ponto_token_expiration = expiration_date
-            else:
-                raise UserError(_("%s \n\n %s") % (response.status_code, response.text))
-        return {
-            "Accept": "application/json",
-            "Authorization": "Bearer %s" % self.ponto_token,
-        }
-
-    def _ponto_get_account_ids(self):
-        url = PONTO_ENDPOINT + "/accounts"
-        response = requests.get(
-            url, params={"limit": 100}, headers=self._ponto_header()
-        )
-        if response.status_code == 200:
-            data = json.loads(response.text)
-            res = {}
-            for account in data.get("data", []):
-                iban = sanitize_account_number(
-                    account.get("attributes", {}).get("reference", "")
-                )
-                res[iban] = account.get("id")
-            return res
-        raise UserError(_("%s \n\n %s") % (response.status_code, response.text))
-
-    def _ponto_synchronisation(self, account_id):
-        url = PONTO_ENDPOINT + "/synchronizations"
-        data = {
-            "data": {
-                "type": "synchronization",
-                "attributes": {
-                    "resourceType": "account",
-                    "resourceId": account_id,
-                    "subtype": "accountTransactions",
-                },
-            }
-        }
-        response = requests.post(
-            url, headers=self._ponto_header(), json=data
-        )
-        if response.status_code in (200, 201, 400):
-            data = json.loads(response.text)
-            sync_id = data.get("attributes", {}).get("resourceId", False)
-        else:
-            raise UserError(
-                _("Error during Create Synchronisation %s \n\n %s")
-                % (response.status_code, response.text)
-            )
-        # Check synchronisation
-        if not sync_id:
-            return
-        url = PONTO_ENDPOINT + "/synchronizations/" + sync_id
-        number = 0
-        while number == 100:
-            number += 1
-            response = requests.get(url, headers=self._ponto_header())
-            if response.status_code == 200:
-                data = json.loads(response.text)
-                status = data.get("status", {})
-                if status in ("success", "error"):
-                    return
-            time.sleep(4)
-
-    def _ponto_get_transaction(self, account_id, date_since, date_until):
-        page_url = PONTO_ENDPOINT + "/accounts/" + account_id + "/transactions"
-        params = {"limit": 100}
-        page_next = True
-        last_identifier = self.ponto_last_identifier
-        if last_identifier:
-            params["before"] = last_identifier
-            page_next = False
-        transaction_lines = []
-        latest_identifier = False
-        while page_url:
-            headers = self._ponto_header()
-            response = requests.get(
-                page_url, params=params, headers=headers
-            )
-            _logger.debug(
-                _("Get request to %s, with headers %s and params %s"),
-                page_url,
-                params,
-                headers
-            )
-            if response.status_code != 200:
-                raise UserError(
-                    _("Error during get transaction.\n\n%s \n\n %s")
-                    % (response.status_code, response.text)
-                )
-            if params.get("before"):
-                params.pop("before")
-            data = json.loads(response.text)
-            links = data.get("links", {})
-            if page_next:
-                page_url = links.get("next", False)
-            else:
-                page_url = links.get("prev", False)
-            transactions = data.get("data", [])
-            if not transactions:
-                _logger.debug(
-                    _("No transactions where found in response %s"),
-                    response.text,
-                )
-            else:
-                _logger.debug(
-                    _("%d transactions present in response data"),
-                    len(transactions),
-                )
-                current_transactions = []
-                for transaction in transactions:
-                    date = self._ponto_date_from_string(
-                        transaction.get("attributes", {}).get("executionDate")
-                    )
-                    if date_since <= date < date_until:
-                        current_transactions.append(transaction)
-                if not current_transactions:
-                    _logger.debug(
-                        _("No lines selected from transactions")
-                    )
-                else:
-                    _logger.debug(
-                        _("%d lines selected from transactions"),
-                        len(current_transactions),
-                    )
-                    if not page_next or (page_next and not latest_identifier):
-                        latest_identifier = current_transactions[0].get("id")
-                    transaction_lines.extend(current_transactions)
-        if latest_identifier:
-            self.ponto_last_identifier = latest_identifier
-        return transaction_lines
-
-    def _ponto_date_from_string(self, date_str):
-        """Dates in Ponto are expressed in UTC, so we need to convert them
-        to supplied tz for proper classification.
-        """
-        dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
-        dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
-        return dt.replace(tzinfo=None)
-
     def _ponto_obtain_statement_data(self, date_since, date_until):
         """Translate information from Ponto to Odoo bank statement lines."""
         self.ensure_one()
-        account_ids = self._ponto_get_account_ids()
-        journal = self.journal_id
-        iban = self.account_number
-        account_id = account_ids.get(iban)
-        if not account_id:
-            raise UserError(
-                _("Ponto : wrong configuration, unknow account %s")
-                % journal.bank_account_id.acc_number
-            )
         _logger.debug(
             _("Ponto obtain statement data for journal %s from %s to %s"),
-            journal.name,
+            self.journal_id.name,
             date_since,
             date_until
         )
-        self._ponto_synchronisation(account_id)
-        transaction_lines = self._ponto_get_transaction(
-            account_id, date_since, date_until
+        line_model = self.env["ponto.buffer.line"]
+        lines = line_model.sudo().search(
+            [
+                ("buffer_id.provider_id", "=", self.id),
+                ("effective_date_time", ">=", date_since),
+                ("effective_date_time", "<=", date_until),
+            ]
         )
         new_transactions = []
         sequence = 0
-        for transaction in transaction_lines:
+        for transaction in lines:
             sequence += 1
-            vals_line = self._ponto_get_transaction_vals(transaction, sequence)
+            vals_line = self._ponto_get_transaction_vals(
+                json.loads(transaction.transaction_data),
+                sequence
+            )
             new_transactions.append(vals_line)
         if new_transactions:
             return new_transactions, {}
@@ -283,3 +122,11 @@ class OnlineBankStatementProviderPonto(models.Model):
         if attributes.get("counterpartName"):
             vals_line["partner_name"] = attributes["counterpartName"]
         return vals_line
+
+    def _ponto_date_from_string(self, date_str):
+        """Dates in Ponto are expressed in UTC, so we need to convert them
+        to supplied tz for proper classification.
+        """
+        dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
+        dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
+        return dt.replace(tzinfo=None)
diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer.py b/account_bank_statement_import_online_ponto/models/ponto_buffer.py
new file mode 100644
index 00000000..5acffa3d
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/models/ponto_buffer.py
@@ -0,0 +1,79 @@
+# Copyright 2022 Therp BV <https://therp.nl>.
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+"""Define model to hold data retrieved from Ponto."""
+import json
+import logging
+
+from odoo import _, fields, models
+
+_logger = logging.getLogger(__name__)
+
+
+class PontoBuffer(models.Model):
+    """Define model to hold data retrieved from Ponto."""
+    _name = "ponto.buffer"
+    _description = "Group transactions retrieved from Ponto."
+
+    provider_id = fields.Many2one(
+        comodel_name="online.bank.statement.provider",
+        required=True,
+        readonly=True,
+    )
+    effective_date = fields.Date(readonly=True, required=True)
+    buffer_line_ids = fields.One2many(
+        comodel_name="ponto.buffer.line",
+        inverse_name="buffer_id",
+        readonly=True,
+        ondelete="cascade",
+    )
+
+    def _store_transactions(self, provider, transactions):
+        """Store transactions retrieved from Ponto in buffer, preventing duplicates."""
+        # Start by sorting all transactions per date.
+        transactions_per_date = {}
+        for transaction in transactions:
+            ponto_execution_date = transaction.get(
+                "attributes", {}
+            ).get("executionDate")
+            effective_date_time = provider._ponto_date_from_string(ponto_execution_date)
+            transaction["effective_date_time"] = effective_date_time.isoformat()
+            key = effective_date_time.isoformat()[0:10]
+            if key not in transactions_per_date:
+                transactions_per_date[key] = []  # Initialize transaction array.
+            transactions_per_date[key].append(transaction)
+        # Now store the transactions, but not when already present.
+        for key, date_transactions in transactions_per_date.items():
+            _logger.debug(
+                _("For date %s we retrieved %d transactions"),
+                key,
+                len(date_transactions)
+            )
+            ponto_buffer = self.search(
+                [
+                    ("provider_id", "=", provider.id),
+                    ("effective_date", "=", key),
+                ],
+                limit=1
+            ) or self.create(
+                {
+                    "provider_id": provider.id,
+                    "effective_date": key,
+                }
+            )
+            already_present = ponto_buffer.buffer_line_ids.mapped("ponto_id")
+            new_lines = [
+                (
+                    0,
+                    0,
+                    {
+                        "buffer_id": ponto_buffer.id,
+                        "ponto_id": t.get("id"),
+                        "effective_date_time": t.get("effective_date_time"),
+                        "transaction_data": json.dumps(t),
+                    },
+                )
+                for t in date_transactions
+                if t.get("id") not in already_present
+            ]
+            if new_lines:
+                ponto_buffer.write({"buffer_line_ids": new_lines})
diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
new file mode 100644
index 00000000..54527476
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
@@ -0,0 +1,28 @@
+# Copyright 2022 Therp BV <https://therp.nl>.
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+"""Define model to hold transactions retrieved from Ponto."""
+from odoo import fields, models
+
+
+class PontoBuffer(models.Model):
+    """Define model to hold transactions retrieved from Ponto."""
+    _name = "ponto.buffer.line"
+    _description = "Hold transactions retrieved from Ponto."
+
+    buffer_id = fields.Many2one(
+        comodel_name="ponto.buffer",
+        required=True,
+        readonly=True,
+    )
+    ponto_id = fields.Char(
+        required=True,
+        readonly=True,
+    )
+    effective_date_time = fields.Datetime(
+        required=True,
+        readonly=True,
+    )
+    transaction_data = fields.Char(
+        required=True,
+        readonly=True,
+    )
diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py
new file mode 100644
index 00000000..2c979d86
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py
@@ -0,0 +1,190 @@
+# Copyright 2020 Florent de Labarre
+# Copyright 2022 Therp BV <https://therp.nl>.
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+
+import base64
+import json
+import logging
+import time
+
+import requests
+from dateutil.relativedelta import relativedelta
+
+from odoo import fields, models, _
+from odoo.exceptions import UserError
+from odoo.addons.base.models.res_bank import sanitize_account_number
+
+_logger = logging.getLogger(__name__)
+
+PONTO_ENDPOINT = "https://api.myponto.com"
+
+
+class PontoInterface(models.AbstractModel):
+    _name = "ponto.interface"
+    _description = "Interface to all interactions with Ponto API"
+
+    def _login(self, username, password):
+        """Ponto login returns an access dictionary for further requests."""
+        url = PONTO_ENDPOINT + "/oauth2/token"
+        if not(username and password):
+            raise UserError(_("Please fill login and key."))
+        login = "%s:%s" % (username, password)
+        login = base64.b64encode(login.encode("UTF-8")).decode("UTF-8")
+        login_headers = {
+            "Content-Type": "application/x-www-form-urlencoded",
+            "Accept": "application/json",
+            "Authorization": "Basic %s" % login,
+        }
+        response = requests.post(
+            url,
+            params={"grant_type": "client_credentials"},
+            headers=login_headers,
+        )
+        data = self._get_response_data(response)
+        access_token = data.get("access_token", False)
+        if not access_token:
+            raise UserError(_("Ponto : no token"))
+        token_expiration = fields.Datetime.now() + relativedelta(
+            seconds=data.get("expires_in", False)
+        )
+        return {
+            "username": username,
+            "password": password,
+            "access_token": access_token,
+            "token_expiration": token_expiration,
+        }
+
+    def _get_request_headers(self, access_data):
+        """Get headers with authorization for further ponto requests."""
+        if access_data["token_expiration"] <= fields.Datetime.now():
+            updated_data = self._login(access_data["username"], access_data["password"])
+            access_data.update(updated_data)
+        return {
+            "Accept": "application/json",
+            "Authorization": "Bearer %s" % access_data["access_token"],
+        }
+
+    def _set_access_account(self, access_data, account_number):
+        """Set ponto account for bank account in access_data."""
+        url = PONTO_ENDPOINT + "/accounts"
+        response = requests.get(
+            url, params={"limit": 100}, headers=self._get_request_headers(access_data)
+        )
+        data = self._get_response_data(response)
+        for ponto_account in data.get("data", []):
+            ponto_iban = sanitize_account_number(
+                ponto_account.get("attributes", {}).get("reference", "")
+            )
+            if ponto_iban == account_number:
+                access_data["ponto_account"] = ponto_account.get("id")
+                return
+        # If we get here, we did not find Ponto account for bank account.
+        raise UserError(
+            _("Ponto : wrong configuration, account %s not found in %s")
+            % (account_number, data)
+        )
+
+    def _ponto_synchronisation(self, access_data):
+        """Make sure Ponto has retrieved latest data from financial institution."""
+        url = PONTO_ENDPOINT + "/synchronizations"
+        # TODO: According to spec we should provide an IP number in the data.
+        # See: https://documentation.myponto.com/1/api/curl#create-synchronization
+        payload = {
+            "data": {
+                "type": "synchronization",
+                "attributes": {
+                    "resourceType": "account",
+                    "resourceId": access_data["ponto_account"],
+                    "subtype": "accountTransactions",
+                },
+            }
+        }
+        response = requests.post(
+            url, headers=self._get_request_headers(access_data), json=payload
+        )
+        if response.status_code == 400:
+            # Probably synchronization recently done already.
+            _logger.debug(
+                _("Syncronization request rejected: %s"),
+                response.text
+            )
+            return
+        data = self._get_response_data(response)
+        sync_id = data.get("attributes", {}).get("resourceId", False)
+        # Check synchronisation
+        if not sync_id:
+            return
+        # Poll synchronization during 400 seconds for completion.
+        url = PONTO_ENDPOINT + "/synchronizations/" + sync_id
+        number = 0
+        while number <= 10:
+            number += 1
+            response = requests.get(url, headers=self._get_request_headers(access_data))
+            if response.status_code == 200:
+                data = json.loads(response.text)
+                status = data.get("status", {})
+                if status in ("success", "error"):
+                    if status == "error":
+                        _logger.debug(
+                            _("Syncronization was succesfully completed")
+                        )
+                    else:
+                        _logger.debug(
+                            _("Syncronization had an error: %s"),
+                            response.text
+                        )
+                    return
+            time.sleep(40)
+
+    def _get_transactions(self, access_data, last_identifier):
+        """Get transactions from ponto, using last_identifier as pointer."""
+        url = (
+            PONTO_ENDPOINT
+            + "/accounts/"
+            + access_data["ponto_account"]
+            + "/transactions"
+        )
+        params = {"limit": 100}
+        if last_identifier:
+            params["after"] = last_identifier
+        data = self._get_request(access_data, url, params)
+        transactions = self._get_transactions_from_data(data)
+        return transactions
+
+    def _get_transactions_from_data(self, data):
+        """Get all transactions that are in the ponto response data."""
+        transactions = data.get("data", [])
+        if not transactions:
+            _logger.debug(
+                _("No transactions where found in data %s"),
+                data,
+            )
+        else:
+            _logger.debug(
+                _("%d transactions present in response data"),
+                len(transactions),
+            )
+        return transactions
+
+    def _get_request(self, access_data, url, params):
+        """Interact with Ponto to get next page of data."""
+        headers = self._get_request_headers(access_data)
+        _logger.debug(
+            _("Get request to %s, with headers %s and params %s"),
+            url,
+            params,
+            headers
+        )
+        response = requests.get(
+            url, params=params, headers=headers
+        )
+        return self._get_response_data(response)
+
+    def _get_response_data(self, response):
+        """Get response data for GET or POST request."""
+        if response.status_code not in (200, 201):
+            raise UserError(
+                _("Server returned status code %s: %s")
+                % (response.status_code, response.text)
+            )
+        return json.loads(response.text)

From 530e7e540d5b0a8dbbfe3199e57f99af0fd6ac4b Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Sat, 3 Sep 2022 15:20:40 +0200
Subject: [PATCH 07/15] [TST] *_online_ponto: Backport tests from 14.0

---
 ...ount_bank_statement_import_online_ponto.py | 64 ++++++++++++++++---
 1 file changed, 56 insertions(+), 8 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py b/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
index f88df801..ede7446f 100644
--- a/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
+++ b/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
@@ -1,11 +1,11 @@
 # Copyright 2020 Florent de Labarre
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
-from datetime import datetime
+from datetime import date, datetime
 from unittest import mock
 
 from odoo import fields
-from odoo.tests import common
+from odoo.tests import Form, common
 
 _module_ns = "odoo.addons.account_bank_statement_import_online_ponto"
 _provider_class = (
@@ -27,6 +27,7 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
         self.OnlineBankStatementProvider = self.env["online.bank.statement.provider"]
         self.AccountBankStatement = self.env["account.bank.statement"]
         self.AccountBankStatementLine = self.env["account.bank.statement.line"]
+        self.AccStatemenPull = self.env["online.bank.statement.pull.wizard"]
 
         self.bank_account = self.ResPartnerBank.create(
             {
@@ -142,16 +143,63 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
             ],
         )
 
+    def test_balance_start(self):
+        st_form = Form(self.AccountBankStatement)
+        st_form.journal_id = self.journal
+        st_form.date = date(2019, 11, 1)
+        st_form.balance_end_real = 100
+        with st_form.line_ids.new() as line_form:
+            line_form.payment_ref = "test move"
+            line_form.amount = 100
+        initial_statement = st_form.save()
+        initial_statement.button_post()
+        with (
+            self.mock_transaction(),
+            self.mock_header(),
+            self.mock_synchronisation(),
+            self.mock_account_ids()
+        ):  # noqa: B950
+            vals = {
+                "provider_ids": self.provider.ids,
+                "date_since": datetime(2019, 11, 4),
+                "date_until": datetime(2019, 11, 5),
+            }
+            wizard = self.AccStatemenPull.with_context(
+                active_model="account.journal",
+                active_id=self.journal.id,
+            ).create(vals)
+            wizard.action_pull()
+            statements = self.AccountBankStatement.search(
+                [("journal_id", "=", self.journal.id)]
+            )
+            new_statement = statements - initial_statement
+            self.assertEqual(len(new_statement.line_ids), 1)
+            self.assertEqual(new_statement.balance_start, 100)
+            self.assertEqual(new_statement.balance_end_real, 105.83)
+
     def test_ponto(self):
         with (
             self.mock_transaction(),
             self.mock_header(),
             self.mock_synchronisation(),
             self.mock_account_ids()
-        ):
-            lines, statement_values = self.provider._obtain_statement_data(
-                datetime(2019, 11, 3),
-                datetime(2019, 11, 17),
+        ):  # noqa: B950
+            vals = {
+                "provider_ids": self.provider.ids,
+                "date_since": datetime(2019, 11, 3),
+                "date_until": datetime(2019, 11, 17),
+            }
+            wizard = self.AccStatemenPull.with_context(
+                active_model="account.journal",
+                active_id=self.journal.id,
+            ).create(vals)
+            # To get all the moves at once
+            self.provider.statement_creation_mode = "monthly"
+            wizard.action_pull()
+            statement = self.AccountBankStatement.search(
+                [("journal_id", "=", self.journal.id)]
             )
-
-        self.assertEqual(len(lines), 3)
+            self.assertEqual(len(statement), 1)
+            self.assertEqual(len(statement.line_ids), 3)
+            self.assertEqual(statement.line_ids.mapped("amount"), [6.08, 5.48, 5.83])
+            self.assertEqual(statement.balance_end_real, 17.39)

From 96f0f836bdcf76c4e4c4e517e57be7ccddc1126e Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Mon, 5 Sep 2022 09:48:49 +0200
Subject: [PATCH 08/15] [FIX] *_online_ponto: adapt test to new interface

---
 .../tests/__init__.py                         |  2 +-
 ..._account_statement_import_online_ponto.py} | 55 ++++++++++---------
 2 files changed, 30 insertions(+), 27 deletions(-)
 rename account_bank_statement_import_online_ponto/tests/{test_account_bank_statement_import_online_ponto.py => test_account_statement_import_online_ponto.py} (86%)

diff --git a/account_bank_statement_import_online_ponto/tests/__init__.py b/account_bank_statement_import_online_ponto/tests/__init__.py
index 8e1252ec..d1f2895e 100644
--- a/account_bank_statement_import_online_ponto/tests/__init__.py
+++ b/account_bank_statement_import_online_ponto/tests/__init__.py
@@ -1,3 +1,3 @@
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
-from . import test_account_bank_statement_import_online_ponto
+from . import test_account_statement_import_online_ponto
diff --git a/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
similarity index 86%
rename from account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
rename to account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
index ede7446f..5fc137b2 100644
--- a/account_bank_statement_import_online_ponto/tests/test_account_bank_statement_import_online_ponto.py
+++ b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
@@ -1,4 +1,5 @@
 # Copyright 2020 Florent de Labarre
+# Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
 from datetime import date, datetime
@@ -8,14 +9,16 @@ from odoo import fields
 from odoo.tests import Form, common
 
 _module_ns = "odoo.addons.account_bank_statement_import_online_ponto"
-_provider_class = (
+_interface_class = (
     _module_ns
-    + ".models.online_bank_statement_provider_ponto"
-    + ".OnlineBankStatementProviderPonto"
+    + ".models.ponto_interface"
+    + ".PontoInterface"
 )
 
 
-class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
+class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
+    post_install = True
+
     def setUp(self):
         super().setUp()
 
@@ -48,25 +51,25 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
         )
         self.provider = self.journal.online_bank_statement_provider_id
 
-        self.mock_header = lambda: mock.patch(
-            _provider_class + "._ponto_header",
+        self.mock_login = lambda: mock.patch(
+            _interface_class + "._login",
             return_value={
-                "Accept": "application/json",
-                "Authorization": "Bearer --TOKEN--",
+                "username": "test_user",
+                "password": "very_secret",
+                "access_token": "abcd1234",
+                "token_expiration": datetime(2099, 12, 31, 23, 59, 59),
             },
         )
-
-        self.mock_account_ids = lambda: mock.patch(
-            _provider_class + "._ponto_get_account_ids",
-            return_value={"FR0214508000302245362775K46": "id"},
-        )
-        self.mock_synchronisation = lambda: mock.patch(
-            _provider_class + "._ponto_synchronisation",
+        self.mock_set_access_account = lambda: mock.patch(
+            _interface_class + "._set_access_account",
             return_value=None,
         )
-
-        self.mock_transaction = lambda: mock.patch(
-            _provider_class + "._ponto_get_transaction",
+        self.mock_synchronisation = lambda: mock.patch(
+            _interface_class + "._ponto_synchronisation",
+            return_value=None,
+        )
+        self.mock_get_transactions = lambda: mock.patch(
+            _interface_class + "._get_transactions",
             return_value=[
                 {
                     "type": "transaction",
@@ -149,15 +152,15 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
         st_form.date = date(2019, 11, 1)
         st_form.balance_end_real = 100
         with st_form.line_ids.new() as line_form:
-            line_form.payment_ref = "test move"
+            line_form.name = "test move"
             line_form.amount = 100
         initial_statement = st_form.save()
-        initial_statement.button_post()
+        initial_statement.button_confirm_bank()  # button_post in 14.0.
         with (
-            self.mock_transaction(),
-            self.mock_header(),
+            self.mock_login(),
             self.mock_synchronisation(),
-            self.mock_account_ids()
+            self.mock_set_access_account(),
+            self.mock_get_transactions()
         ):  # noqa: B950
             vals = {
                 "provider_ids": self.provider.ids,
@@ -179,10 +182,10 @@ class TestAccountBankAccountStatementImportOnlineQonto(common.TransactionCase):
 
     def test_ponto(self):
         with (
-            self.mock_transaction(),
-            self.mock_header(),
+            self.mock_login(),
             self.mock_synchronisation(),
-            self.mock_account_ids()
+            self.mock_set_access_account(),
+            self.mock_get_transactions()
         ):  # noqa: B950
             vals = {
                 "provider_ids": self.provider.ids,

From 1085ab17e306724a20506d79d6526c5dee69ee10 Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Mon, 5 Sep 2022 11:26:04 +0200
Subject: [PATCH 09/15] [FIX] *_online_ponto: add security

---
 account_bank_statement_import_online_ponto/__manifest__.py   | 5 ++++-
 .../readme/CONTRIBUTORS.rst                                  | 3 +++
 .../security/ir.model.access.csv                             | 3 +++
 3 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100644 account_bank_statement_import_online_ponto/security/ir.model.access.csv

diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py
index 919f0e0a..0a5b371d 100644
--- a/account_bank_statement_import_online_ponto/__manifest__.py
+++ b/account_bank_statement_import_online_ponto/__manifest__.py
@@ -13,5 +13,8 @@
     "license": "AGPL-3",
     "installable": True,
     "depends": ["account_bank_statement_import_online"],
-    "data": ["view/online_bank_statement_provider.xml"],
+    "data": [
+        "security/ir.model.access.csv",
+        "view/online_bank_statement_provider.xml",
+    ],
 }
diff --git a/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst b/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst
index 077a3ecf..5544a1f7 100644
--- a/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst
+++ b/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst
@@ -2,3 +2,6 @@
 * `Tecnativa <https://www.tecnativa.com>`__:
 
   * Pedro M. Baeza
+* `Therp BV <https://therp.nl>`__:
+
+  * Ronald Portier
diff --git a/account_bank_statement_import_online_ponto/security/ir.model.access.csv b/account_bank_statement_import_online_ponto/security/ir.model.access.csv
new file mode 100644
index 00000000..4950f516
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/security/ir.model.access.csv
@@ -0,0 +1,3 @@
+id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink
+access_ponto_buffer,access_ponto_buffer,model_ponto_buffer,base.group_user,1,0,0,0
+access_ponto_buffer_line,access_ponto_buffer_line,model_ponto_buffer_line,base.group_user,1,0,0,0

From a33edfb75fafe824349639fd927a2da1c63ea664 Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Mon, 5 Sep 2022 15:15:05 +0200
Subject: [PATCH 10/15] [FIX] *_online_ponto: tests should run OK now

---
 .../online_bank_statement_provider_ponto.py   |  1 -
 .../readme/CONTRIBUTORS.rst                   |  3 -
 ...t_account_statement_import_online_ponto.py | 62 ++++++++++++-------
 3 files changed, 39 insertions(+), 27 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index 30c65518..7f25be7c 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -30,7 +30,6 @@ class OnlineBankStatementProviderPonto(models.Model):
     @api.multi
     def _pull(self, date_since, date_until):
         """Override pull to first retrieve data from Ponto."""
-        self.ensure_one()
         if self.service == "ponto":
             self._ponto_retrieve_data()
         super()._pull(date_since, date_until)
diff --git a/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst b/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst
index 5544a1f7..077a3ecf 100644
--- a/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst
+++ b/account_bank_statement_import_online_ponto/readme/CONTRIBUTORS.rst
@@ -2,6 +2,3 @@
 * `Tecnativa <https://www.tecnativa.com>`__:
 
   * Pedro M. Baeza
-* `Therp BV <https://therp.nl>`__:
-
-  * Ronald Portier
diff --git a/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
index 5fc137b2..3cebf21f 100644
--- a/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
+++ b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
@@ -28,9 +28,10 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
         self.AccountJournal = self.env["account.journal"]
         self.ResPartnerBank = self.env["res.partner.bank"]
         self.OnlineBankStatementProvider = self.env["online.bank.statement.provider"]
+        self.AccountAccount = self.env["account.account"]
         self.AccountBankStatement = self.env["account.bank.statement"]
         self.AccountBankStatementLine = self.env["account.bank.statement.line"]
-        self.AccStatemenPull = self.env["online.bank.statement.pull.wizard"]
+        self.AccountStatementPull = self.env["online.bank.statement.pull.wizard"]
 
         self.bank_account = self.ResPartnerBank.create(
             {
@@ -49,6 +50,14 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
                 "bank_account_id": self.bank_account.id,
             }
         )
+        self.receivable_account = self.AccountAccount.create(
+            {
+                "code": "1325",
+                "name": "Test receivable account",
+                "user_type_id": self.env.ref("account.data_account_type_payable").id,
+                "reconcile": True,
+            }
+        )
         self.provider = self.journal.online_bank_statement_provider_id
 
         self.mock_login = lambda: mock.patch(
@@ -68,9 +77,10 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
             _interface_class + "._ponto_synchronisation",
             return_value=None,
         )
+        # return list of transactions on first call, empty list on second call.
         self.mock_get_transactions = lambda: mock.patch(
             _interface_class + "._get_transactions",
-            return_value=[
+            side_effect=[[
                 {
                     "type": "transaction",
                     "relationships": {
@@ -143,7 +153,7 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
                         "amount": 5.83,
                     },
                 },
-            ],
+            ], [], ]
         )
 
     def test_balance_start(self):
@@ -155,22 +165,23 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
             line_form.name = "test move"
             line_form.amount = 100
         initial_statement = st_form.save()
-        initial_statement.button_confirm_bank()  # button_post in 14.0.
-        with (
-            self.mock_login(),
-            self.mock_synchronisation(),
-            self.mock_set_access_account(),
-            self.mock_get_transactions()
-        ):  # noqa: B950
+        initial_statement.line_ids[0].account_id = self.receivable_account
+        initial_statement.button_confirm_bank()
+        with self.mock_login(), \
+            self.mock_synchronisation(), \
+            self.mock_set_access_account(),  \
+            self.mock_get_transactions():  # noqa: B950
             vals = {
-                "provider_ids": self.provider.ids,
+                "provider_ids": [(4, self.provider.id)],
                 "date_since": datetime(2019, 11, 4),
                 "date_until": datetime(2019, 11, 5),
             }
-            wizard = self.AccStatemenPull.with_context(
+            wizard = self.AccountStatementPull.with_context(
                 active_model="account.journal",
                 active_id=self.journal.id,
             ).create(vals)
+            # For some reason the provider is not set in the create.
+            wizard.provider_ids = self.provider
             wizard.action_pull()
             statements = self.AccountBankStatement.search(
                 [("journal_id", "=", self.journal.id)]
@@ -178,31 +189,36 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
             new_statement = statements - initial_statement
             self.assertEqual(len(new_statement.line_ids), 1)
             self.assertEqual(new_statement.balance_start, 100)
-            self.assertEqual(new_statement.balance_end_real, 105.83)
+            self.assertEqual(new_statement.balance_end, 105.83)
+            # Ponto does not give balance info in transactions.
+            # self.assertEqual(new_statement.balance_end_real, 105.83)
 
     def test_ponto(self):
-        with (
-            self.mock_login(),
-            self.mock_synchronisation(),
-            self.mock_set_access_account(),
-            self.mock_get_transactions()
-        ):  # noqa: B950
+        with self.mock_login(), \
+            self.mock_synchronisation(), \
+            self.mock_set_access_account(), \
+            self.mock_get_transactions():  # noqa: B950
             vals = {
-                "provider_ids": self.provider.ids,
+                "provider_ids": [(4, self.provider.id)],
                 "date_since": datetime(2019, 11, 3),
                 "date_until": datetime(2019, 11, 17),
             }
-            wizard = self.AccStatemenPull.with_context(
+            wizard = self.AccountStatementPull.with_context(
                 active_model="account.journal",
                 active_id=self.journal.id,
             ).create(vals)
             # To get all the moves at once
             self.provider.statement_creation_mode = "monthly"
+            # For some reason the provider is not set in the create.
+            wizard.provider_ids = self.provider
             wizard.action_pull()
             statement = self.AccountBankStatement.search(
                 [("journal_id", "=", self.journal.id)]
             )
             self.assertEqual(len(statement), 1)
             self.assertEqual(len(statement.line_ids), 3)
-            self.assertEqual(statement.line_ids.mapped("amount"), [6.08, 5.48, 5.83])
-            self.assertEqual(statement.balance_end_real, 17.39)
+            sorted_amounts = sorted(statement.line_ids.mapped("amount"))
+            self.assertEqual(sorted_amounts, [5.48, 5.83, 6.08])
+            self.assertEqual(statement.balance_end, 17.39)
+            # Ponto does not give balance info in transactions.
+            # self.assertEqual(statement.balance_end_real, 17.39)

From 0920960d1e87e3eb47faa9b2de8c8faf8ed42e60 Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Tue, 13 Sep 2022 22:46:34 +0200
Subject: [PATCH 11/15] [IMP] *_online_ponto: Transactions are read backwards
 from Ponto.

The streamlined logic means we no longer have to store (or clear) the
last identifier retrieved from Ponto. We will always read from the latest data,
backward until we hit an execution date that is before the date we are interested in.
---
 .../__manifest__.py                           |  5 +-
 .../data/ir_cron.xml                          | 20 ++++++
 .../online_bank_statement_provider_ponto.py   | 63 +++++++++++++++----
 .../models/ponto_buffer.py                    |  6 +-
 .../models/ponto_buffer_line.py               |  1 +
 .../models/ponto_interface.py                 |  8 ++-
 .../online_bank_statement_provider.xml        | 17 +++--
 7 files changed, 95 insertions(+), 25 deletions(-)
 create mode 100644 account_bank_statement_import_online_ponto/data/ir_cron.xml
 rename account_bank_statement_import_online_ponto/{view => views}/online_bank_statement_provider.xml (52%)

diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py
index 0a5b371d..76d197cd 100644
--- a/account_bank_statement_import_online_ponto/__manifest__.py
+++ b/account_bank_statement_import_online_ponto/__manifest__.py
@@ -1,4 +1,4 @@
-# Copyright 2020 Florent de Labarre
+# Copyright 2020 Florent de Labarre.
 # Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 {
@@ -14,7 +14,8 @@
     "installable": True,
     "depends": ["account_bank_statement_import_online"],
     "data": [
+        "data/ir_cron.xml",
         "security/ir.model.access.csv",
-        "view/online_bank_statement_provider.xml",
+        "views/online_bank_statement_provider.xml",
     ],
 }
diff --git a/account_bank_statement_import_online_ponto/data/ir_cron.xml b/account_bank_statement_import_online_ponto/data/ir_cron.xml
new file mode 100644
index 00000000..0e1b3c25
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/data/ir_cron.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" ?>
+<!--
+    Copyright 2022 Therp BV (https://therp.nl)
+    License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+-->
+<odoo noupdate="1">
+
+    <record model="ir.cron" id="ir_cron_purge_ponto_buffer">
+        <field name="name">Remove old data from ponto buffers</field>
+        <field name="interval_number">1</field>
+        <field name="interval_type">days</field>
+        <field name="numbercall">-1</field>
+        <field name="state">code</field>
+        <field name="nextcall">2019-01-01 00:20:00</field>
+        <field name="doall" eval="False"/>
+        <field name="model_id" ref="account_bank_statement_import_online_ponto.model_online_bank_statement_provider"/>
+        <field name="code">model._ponto_buffer_purge()</field>
+    </record>
+
+</odoo>
diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index 7f25be7c..17413361 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -1,7 +1,8 @@
 # Copyright 2020 Florent de Labarre
 # Copyright 2022 Therp BV <https://therp.nl>.
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-from datetime import datetime
+from datetime import date, datetime
+from dateutil.relativedelta import relativedelta
 import json
 import pytz
 
@@ -16,10 +17,12 @@ _logger = logging.getLogger(__name__)
 class OnlineBankStatementProviderPonto(models.Model):
     _inherit = "online.bank.statement.provider"
 
-    ponto_last_identifier = fields.Char(readonly=True)
-
-    def ponto_reset_last_identifier(self):
-        self.write({"ponto_last_identifier": False})
+    ponto_buffer_retain_days = fields.Integer(
+        string="Number of days to keep Ponto Buffers",
+        default=61,
+        help="By default buffers will be kept for 61 days.\n"
+        "Set this to 0 to keep buffers indefinitely.",
+    )
 
     @api.model
     def _get_available_services(self):
@@ -31,17 +34,21 @@ class OnlineBankStatementProviderPonto(models.Model):
     def _pull(self, date_since, date_until):
         """Override pull to first retrieve data from Ponto."""
         if self.service == "ponto":
-            self._ponto_retrieve_data()
+            self._ponto_retrieve_data(date_since)
         super()._pull(date_since, date_until)
 
-    def _ponto_retrieve_data(self):
-        """Fill buffer with data from Ponto."""
+    def _ponto_retrieve_data(self, date_since):
+        """Fill buffer with data from Ponto.
+
+        We will retrieve data from the latest transactions present in Ponto
+        backwards, until we find data that has an execution date before date_since.
+        """
         interface_model = self.env["ponto.interface"]
         buffer_model = self.env["ponto.buffer"]
         access_data = interface_model._login(self.username, self.password)
         interface_model._set_access_account(access_data, self.account_number)
         interface_model._ponto_synchronisation(access_data)
-        latest_identifier = self.ponto_last_identifier
+        latest_identifier = False
         transactions = interface_model._get_transactions(
             access_data,
             latest_identifier
@@ -49,6 +56,9 @@ class OnlineBankStatementProviderPonto(models.Model):
         while transactions:
             buffer_model.sudo()._store_transactions(self, transactions)
             latest_identifier = transactions[-1].get("id")
+            earliest_datetime = self._ponto_get_execution_datetime(transactions[-1])
+            if earliest_datetime < date_since:
+                break
             transactions = interface_model._get_transactions(
                 access_data,
                 latest_identifier
@@ -107,7 +117,7 @@ class OnlineBankStatementProviderPonto(models.Model):
             if attributes.get(x)
         ]
         ref = " ".join(ref_list)
-        date = self._ponto_date_from_string(attributes.get("executionDate"))
+        date = self._ponto_get_execution_datetime(transaction)
         vals_line = {
             "sequence": sequence,
             "date": date,
@@ -122,10 +132,41 @@ class OnlineBankStatementProviderPonto(models.Model):
             vals_line["partner_name"] = attributes["counterpartName"]
         return vals_line
 
-    def _ponto_date_from_string(self, date_str):
+    def _ponto_get_execution_datetime(self, transaction):
+        """Get execution datetime for a transaction.
+
+        Odoo often names variables containing date and time just xxx_date or
+        date_xxx. We try to avoid this misleading naming by using datetime as
+        much for variables and fields of type datetime.
+        """
+        attributes = transaction.get("attributes", {})
+        return self._ponto_datetime_from_string(attributes.get("executionDate"))
+
+    def _ponto_datetime_from_string(self, date_str):
         """Dates in Ponto are expressed in UTC, so we need to convert them
         to supplied tz for proper classification.
         """
         dt = datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%S.%fZ")
         dt = dt.replace(tzinfo=pytz.utc).astimezone(pytz.timezone(self.tz or "utc"))
         return dt.replace(tzinfo=None)
+
+    def _ponto_buffer_purge(self):
+        """Remove buffers from Ponto no longer needed to import statements."""
+        _logger.info("Scheduled purge of old ponto buffers...")
+        today = date.today()
+        buffer_model = self.env["ponto.buffer"]
+        providers = self.search([
+            ("active", "=", True),
+        ])
+        for provider in providers:
+            if not provider.ponto_buffer_retain_days:
+                continue
+            cutoff_date = today - relativedelta(days=provider.ponto_buffer_retain_days)
+            old_buffers = buffer_model.search(
+                [
+                    ("provider_id", "=", provider.id),
+                    ("effective_date", "<", cutoff_date),
+                ]
+            )
+            old_buffers.unlink()
+        _logger.info("Scheduled purge of old ponto buffers complete.")
diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer.py b/account_bank_statement_import_online_ponto/models/ponto_buffer.py
index 5acffa3d..25157300 100644
--- a/account_bank_statement_import_online_ponto/models/ponto_buffer.py
+++ b/account_bank_statement_import_online_ponto/models/ponto_buffer.py
@@ -24,7 +24,6 @@ class PontoBuffer(models.Model):
         comodel_name="ponto.buffer.line",
         inverse_name="buffer_id",
         readonly=True,
-        ondelete="cascade",
     )
 
     def _store_transactions(self, provider, transactions):
@@ -32,10 +31,7 @@ class PontoBuffer(models.Model):
         # Start by sorting all transactions per date.
         transactions_per_date = {}
         for transaction in transactions:
-            ponto_execution_date = transaction.get(
-                "attributes", {}
-            ).get("executionDate")
-            effective_date_time = provider._ponto_date_from_string(ponto_execution_date)
+            effective_date_time = provider._ponto_get_execution_datetime(transaction)
             transaction["effective_date_time"] = effective_date_time.isoformat()
             key = effective_date_time.isoformat()[0:10]
             if key not in transactions_per_date:
diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
index 54527476..fcf0a953 100644
--- a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
+++ b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
@@ -13,6 +13,7 @@ class PontoBuffer(models.Model):
         comodel_name="ponto.buffer",
         required=True,
         readonly=True,
+        ondelete="cascade",
     )
     ponto_id = fields.Char(
         required=True,
diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py
index 2c979d86..c84626c0 100644
--- a/account_bank_statement_import_online_ponto/models/ponto_interface.py
+++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py
@@ -137,7 +137,13 @@ class PontoInterface(models.AbstractModel):
             time.sleep(40)
 
     def _get_transactions(self, access_data, last_identifier):
-        """Get transactions from ponto, using last_identifier as pointer."""
+        """Get transactions from ponto, using last_identifier as pointer.
+
+        Note that Ponto has the transactions in descending order. The first
+        transaction, retrieved by not passing an identifier, is the latest
+        present in Ponto. If you read transactions 'after' a certain identifier
+        (Ponto id), you will get transactions with an earlier date.
+        """
         url = (
             PONTO_ENDPOINT
             + "/accounts/"
diff --git a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml
similarity index 52%
rename from account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml
rename to account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml
index e856cf56..93e98e87 100644
--- a/account_bank_statement_import_online_ponto/view/online_bank_statement_provider.xml
+++ b/account_bank_statement_import_online_ponto/views/online_bank_statement_provider.xml
@@ -6,12 +6,17 @@
         <field name="inherit_id" ref="account_bank_statement_import_online.online_bank_statement_provider_form"/>
         <field name="arch" type="xml">
             <xpath expr="//page[@name='configuration']" position="inside">
-                <group name="qonto" attrs="{'invisible':[('service','!=','ponto')]}">
-                    <field name="username" string="Login"/>
-                    <field name="password" string="Secret Key"/>
-                    <field name="ponto_last_identifier"/>
-                    <button name="ponto_reset_last_identifier" string="Reset Last identifier." type="object"
-                            attrs="{'invisible':[('ponto_last_identifier','=',False)]}"/>
+                <group
+                    name="ponto_configuration"
+                    attrs="{'invisible':[('service','!=','ponto')]}"
+                    colspan="6"
+                    col="3"
+                >
+                    <group colspan="2" col="2" name="ponto_login">
+                        <field name="username" string="Login"/>
+                        <field name="password" string="Secret Key"/>
+                        <field name="ponto_buffer_retain_days"/>
+                    </group>
                 </group>
             </xpath>
         </field>

From 0f742a137eb7fa4c6fdb290cbe3e7f6efd0023ab Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 14 Sep 2022 10:52:17 +0200
Subject: [PATCH 12/15] [IMP] *_online_ponto: Add UI to show ponto buffers

---
 .../__manifest__.py                           |  3 ++
 .../models/ponto_buffer.py                    |  2 ++
 .../models/ponto_buffer_line.py               |  2 ++
 .../views/ir_actions_act_window.xml           | 11 ++++++
 .../views/ir_ui_menu.xml                      | 13 +++++++
 .../views/ponto_buffer.xml                    | 35 +++++++++++++++++++
 6 files changed, 66 insertions(+)
 create mode 100644 account_bank_statement_import_online_ponto/views/ir_actions_act_window.xml
 create mode 100644 account_bank_statement_import_online_ponto/views/ir_ui_menu.xml
 create mode 100644 account_bank_statement_import_online_ponto/views/ponto_buffer.xml

diff --git a/account_bank_statement_import_online_ponto/__manifest__.py b/account_bank_statement_import_online_ponto/__manifest__.py
index 76d197cd..c6ede0b6 100644
--- a/account_bank_statement_import_online_ponto/__manifest__.py
+++ b/account_bank_statement_import_online_ponto/__manifest__.py
@@ -17,5 +17,8 @@
         "data/ir_cron.xml",
         "security/ir.model.access.csv",
         "views/online_bank_statement_provider.xml",
+        "views/ponto_buffer.xml",
+        "views/ir_actions_act_window.xml",
+        "views/ir_ui_menu.xml",
     ],
 }
diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer.py b/account_bank_statement_import_online_ponto/models/ponto_buffer.py
index 25157300..f18b92e8 100644
--- a/account_bank_statement_import_online_ponto/models/ponto_buffer.py
+++ b/account_bank_statement_import_online_ponto/models/ponto_buffer.py
@@ -13,6 +13,8 @@ class PontoBuffer(models.Model):
     """Define model to hold data retrieved from Ponto."""
     _name = "ponto.buffer"
     _description = "Group transactions retrieved from Ponto."
+    _order = "effective_date desc"
+    _rec_name = "effective_date"
 
     provider_id = fields.Many2one(
         comodel_name="online.bank.statement.provider",
diff --git a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
index fcf0a953..b0f8ef86 100644
--- a/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
+++ b/account_bank_statement_import_online_ponto/models/ponto_buffer_line.py
@@ -8,6 +8,8 @@ class PontoBuffer(models.Model):
     """Define model to hold transactions retrieved from Ponto."""
     _name = "ponto.buffer.line"
     _description = "Hold transactions retrieved from Ponto."
+    _order = "effective_date_time desc"
+    _rec_name = "effective_date_time"
 
     buffer_id = fields.Many2one(
         comodel_name="ponto.buffer",
diff --git a/account_bank_statement_import_online_ponto/views/ir_actions_act_window.xml b/account_bank_statement_import_online_ponto/views/ir_actions_act_window.xml
new file mode 100644
index 00000000..d0e1355a
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/views/ir_actions_act_window.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+    <record id="action_ponto_buffer" model="ir.actions.act_window">
+        <field name="name">Ponto Buffer</field>
+        <field name="res_model">ponto.buffer</field>
+        <field name="view_type">form</field>
+        <field name="view_mode">tree,form</field>
+        <field eval="False" name="view_id"/>
+    </record>
+</odoo>
+
diff --git a/account_bank_statement_import_online_ponto/views/ir_ui_menu.xml b/account_bank_statement_import_online_ponto/views/ir_ui_menu.xml
new file mode 100644
index 00000000..73a9e936
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/views/ir_ui_menu.xml
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="utf-8"?>
+<odoo>
+
+    <menuitem
+        id="menu_ponto_buffer"
+        name="Show Ponto Buffer"
+        parent="account.account_reports_management_menu"
+        sequence="90"
+        action="action_ponto_buffer"
+        groups="account.group_account_user"
+    />
+
+</odoo>
diff --git a/account_bank_statement_import_online_ponto/views/ponto_buffer.xml b/account_bank_statement_import_online_ponto/views/ponto_buffer.xml
new file mode 100644
index 00000000..4d64d11e
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/views/ponto_buffer.xml
@@ -0,0 +1,35 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<odoo>
+    <record model="ir.ui.view" id="ponto_buffer_tree">
+        <field name="name">ponto.buffer.tree</field>
+        <field name="model">ponto.buffer</field>
+        <field name="arch" type="xml">
+            <tree>
+                <field name="provider_id" />
+                <field name="effective_date" />
+            </tree>
+        </field>
+    </record>
+
+    <record model="ir.ui.view" id="ponto_buffer_form">
+        <field name="name">ponto.buffer.form</field>
+        <field name="model">ponto.buffer</field>
+        <field name="arch" type="xml">
+            <form>
+                <group name="ponto_buffer_main">
+                    <field name="provider_id" />
+                    <field name="effective_date" />
+                </group>
+                <group name="ponto_buffer_lines" string="Buffer Lines">
+                    <field name="buffer_line_ids" mode="tree">
+                        <tree>
+                            <field name="effective_date_time" />
+                            <field name="ponto_id" />
+                            <field name="transaction_data" />
+                        </tree>
+                    </field>
+                </group>
+            </form>
+        </field>
+    </record>
+</odoo>

From 0cda9d172711caabc44738bf0fb4cdd793c904ec Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Wed, 14 Sep 2022 16:21:37 +0200
Subject: [PATCH 13/15] [IMP] *_online_ponto: test ponto_interface and buffer
 purge.

---
 .../online_bank_statement_provider_ponto.py   |   5 +-
 .../tests/__init__.py                         |   1 +
 ...t_account_statement_import_online_ponto.py | 169 ++++++++++--------
 .../tests/test_ponto_interface.py             | 124 +++++++++++++
 4 files changed, 223 insertions(+), 76 deletions(-)
 create mode 100644 account_bank_statement_import_online_ponto/tests/test_ponto_interface.py

diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index 17413361..355de12b 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -63,11 +63,10 @@ class OnlineBankStatementProviderPonto(models.Model):
                 access_data,
                 latest_identifier
             )
-        self.ponto_last_identifier = latest_identifier
 
     def _obtain_statement_data(self, date_since, date_until):
         self.ensure_one()
-        if self.service != "ponto":
+        if self.service != "ponto":  # pragma: no cover
             return super()._obtain_statement_data(
                 date_since,
                 date_until,
@@ -159,6 +158,8 @@ class OnlineBankStatementProviderPonto(models.Model):
             ("active", "=", True),
         ])
         for provider in providers:
+            if provider.service != "ponto":
+                continue
             if not provider.ponto_buffer_retain_days:
                 continue
             cutoff_date = today - relativedelta(days=provider.ponto_buffer_retain_days)
diff --git a/account_bank_statement_import_online_ponto/tests/__init__.py b/account_bank_statement_import_online_ponto/tests/__init__.py
index d1f2895e..7f083075 100644
--- a/account_bank_statement_import_online_ponto/tests/__init__.py
+++ b/account_bank_statement_import_online_ponto/tests/__init__.py
@@ -1,3 +1,4 @@
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
+from . import test_ponto_interface
 from . import test_account_statement_import_online_ponto
diff --git a/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
index 3cebf21f..d27893cd 100644
--- a/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
+++ b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
@@ -15,6 +15,83 @@ _interface_class = (
     + ".PontoInterface"
 )
 
+THREE_TRANSACTIONS = [
+    {
+        "type": "transaction",
+        "relationships": {
+            "account": {
+                "links": {"related": "https://api.myponto.com/accounts/"},
+                "data": {
+                    "type": "account",
+                    "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                },
+            }
+        },
+        "id": "701ab965-21c4-46ca-b157-306c0646e0e2",
+        "attributes": {
+            "valueDate": "2019-11-18T00:00:00.000Z",
+            "remittanceInformationType": "unstructured",
+            "remittanceInformation": "Minima vitae totam!",
+            "executionDate": "2019-11-20T00:00:00.000Z",
+            "description": "Wire transfer",
+            "currency": "EUR",
+            "counterpartReference": "BE26089479973169",
+            "counterpartName": "Osinski Group",
+            "amount": 6.08,
+        },
+    },
+    {
+        "type": "transaction",
+        "relationships": {
+            "account": {
+                "links": {"related": "https://api.myponto.com/accounts/"},
+                "data": {
+                    "type": "account",
+                    "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                },
+            }
+        },
+        "id": "9ac50483-16dc-4a82-aa60-df56077405cd",
+        "attributes": {
+            "valueDate": "2019-11-04T00:00:00.000Z",
+            "remittanceInformationType": "unstructured",
+            "remittanceInformation": "Quia voluptatem blanditiis.",
+            "executionDate": "2019-11-06T00:00:00.000Z",
+            "description": "Wire transfer",
+            "currency": "EUR",
+            "counterpartReference": "BE97201830401438",
+            "counterpartName": "Stokes-Miller",
+            "amount": 5.48,
+        },
+    },
+    {
+        "type": "transaction",
+        "relationships": {
+            "account": {
+                "links": {"related": "https://api.myponto.com/accounts/"},
+                "data": {
+                    "type": "account",
+                    "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
+                },
+            }
+        },
+        "id": "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff",
+        "attributes": {
+            "valueDate": "2019-11-04T00:00:00.000Z",
+            "remittanceInformationType": "unstructured",
+            "remittanceInformation": "Laboriosam repelo?",
+            "executionDate": "2019-11-04T00:00:00.000Z",
+            "description": "Wire transfer",
+            "currency": "EUR",
+            "counterpartReference": "BE10325927501996",
+            "counterpartName": "Strosin-Veum",
+            "amount": 5.83,
+        },
+    },
+]
+
+EMPTY_TRANSACTIONS = []
+
 
 class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
     post_install = True
@@ -80,80 +157,7 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
         # return list of transactions on first call, empty list on second call.
         self.mock_get_transactions = lambda: mock.patch(
             _interface_class + "._get_transactions",
-            side_effect=[[
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "701ab965-21c4-46ca-b157-306c0646e0e2",
-                    "attributes": {
-                        "valueDate": "2019-11-18T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Minima vitae totam!",
-                        "executionDate": "2019-11-20T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE26089479973169",
-                        "counterpartName": "Osinski Group",
-                        "amount": 6.08,
-                    },
-                },
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "9ac50483-16dc-4a82-aa60-df56077405cd",
-                    "attributes": {
-                        "valueDate": "2019-11-04T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Quia voluptatem blanditiis.",
-                        "executionDate": "2019-11-06T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE97201830401438",
-                        "counterpartName": "Stokes-Miller",
-                        "amount": 5.48,
-                    },
-                },
-                {
-                    "type": "transaction",
-                    "relationships": {
-                        "account": {
-                            "links": {"related": "https://api.myponto.com/accounts/"},
-                            "data": {
-                                "type": "account",
-                                "id": "fd3d5b1d-fca9-4310-a5c8-76f2a9dc7c75",
-                            },
-                        }
-                    },
-                    "id": "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff",
-                    "attributes": {
-                        "valueDate": "2019-11-04T00:00:00.000Z",
-                        "remittanceInformationType": "unstructured",
-                        "remittanceInformation": "Laboriosam repelo?",
-                        "executionDate": "2019-11-04T00:00:00.000Z",
-                        "description": "Wire transfer",
-                        "currency": "EUR",
-                        "counterpartReference": "BE10325927501996",
-                        "counterpartName": "Strosin-Veum",
-                        "amount": 5.83,
-                    },
-                },
-            ], [], ]
+            side_effect=[THREE_TRANSACTIONS, EMPTY_TRANSACTIONS, ]
         )
 
     def test_balance_start(self):
@@ -222,3 +226,20 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
             self.assertEqual(statement.balance_end, 17.39)
             # Ponto does not give balance info in transactions.
             # self.assertEqual(statement.balance_end_real, 17.39)
+
+    def test_ponto_buffer_purge(self):
+        """Create some old buffer records and test purging them."""
+        self.provider.write(
+            {
+                "ponto_buffer_retain_days": 31,
+                "active": True,
+            }
+        )
+        buffer_model = self.env["ponto.buffer"]
+        buffer_model.sudo()._store_transactions(self.provider, THREE_TRANSACTIONS)
+        buffers = buffer_model.search([("provider_id", "=", self.provider.id)])
+        # As all transactions have a different date, they will be in separate buffers.
+        self.assertEqual(len(buffers), 3)
+        self.provider._ponto_buffer_purge()
+        buffers = buffer_model.search([("provider_id", "=", self.provider.id)])
+        self.assertFalse(bool(buffers))
diff --git a/account_bank_statement_import_online_ponto/tests/test_ponto_interface.py b/account_bank_statement_import_online_ponto/tests/test_ponto_interface.py
new file mode 100644
index 00000000..17bcf662
--- /dev/null
+++ b/account_bank_statement_import_online_ponto/tests/test_ponto_interface.py
@@ -0,0 +1,124 @@
+# Copyright 2022 Therp BV <https://therp.nl>.
+# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
+from dateutil.relativedelta import relativedelta
+import json
+
+from unittest.mock import MagicMock, patch
+
+from odoo import fields
+from odoo.tests import common
+
+from .test_account_statement_import_online_ponto import THREE_TRANSACTIONS
+
+
+class TestPontoInterface(common.TransactionCase):
+    post_install = True
+
+    @patch("requests.post")
+    def test_login(self, requests_post):
+        """Check Ponto API login."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(
+            {
+                "access_token": "live_the_token",
+                "expires_in": 1799,
+                "scope": "ai",
+                "token_type": "bearer",
+            }
+        )
+        requests_post.return_value = mock_response
+        interface_model = self.env["ponto.interface"]
+        access_data = interface_model._login("uncle_john", "secret")
+        self.assertEqual(access_data["access_token"], "live_the_token")
+        self.assertIn("token_expiration", access_data)
+
+    @patch("requests.get")
+    def test_set_access_account(self, requests_get):
+        """Test getting account data for Ponto access."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps(
+            {
+                "data": [
+                    {
+                        "id": "wrong_id",
+                        "attributes": {
+                            "reference": "NL66ABNA123456789",
+                        },
+                    },
+                    {
+                        "id": "2ad3df83-be01-47cf-a6be-cf0de5cb4c99",
+                        "attributes": {
+                            "reference": "NL66RABO123456789",
+                        },
+                    },
+                ],
+            }
+        )
+        requests_get.return_value = mock_response
+        # Start of actual test.
+        access_data = self._get_access_dict(include_account=False)
+        interface_model = self.env["ponto.interface"]
+        interface_model._set_access_account(access_data, "NL66RABO123456789")
+        self.assertIn("ponto_account", access_data)
+        self.assertEqual(
+            access_data["ponto_account"],
+            "2ad3df83-be01-47cf-a6be-cf0de5cb4c99"
+        )
+
+    @patch("requests.post")
+    def test_ponto_synchronisation(self, requests_post):
+        """Test requesting Ponto synchronization."""
+        mock_response = MagicMock()
+        mock_response.status_code = 400
+        mock_response.text = json.dumps(
+            {
+                "errors": [
+                    {
+                        "code": "accountRecentlySynchronized",
+                        "detail":
+                            "This type of synchronization was already created recently"
+                            " for the account. Try again later or on the Dashboard.",
+                        "meta": {}
+                    }
+                ]
+            }
+        )
+        requests_post.return_value = mock_response
+        # Start of actual test (succeeds if no Exceptions occur).
+        access_data = self._get_access_dict()
+        interface_model = self.env["ponto.interface"]
+        interface_model._ponto_synchronisation(access_data)
+
+    @patch("requests.get")
+    def test_get_transactions(self, requests_get):
+        """Test getting transactions from Ponto."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        # Key "data" will contain a list of transactions.
+        mock_response.text = json.dumps({"data": THREE_TRANSACTIONS})
+        requests_get.return_value = mock_response
+        # Start of actual test.
+        access_data = self._get_access_dict()
+        interface_model = self.env["ponto.interface"]
+        transactions = interface_model._get_transactions(access_data, False)
+        self.assertEqual(len(transactions), 3)
+        self.assertEqual(transactions[2]["id"], "b21a6c65-1c52-4ba6-8cbc-127d2b2d85ff")
+        self.assertEqual(
+            transactions[2]["attributes"]["counterpartReference"],
+            "BE10325927501996"
+        )
+
+    def _get_access_dict(self, include_account=True):
+        """Get access dict that caches login/account information."""
+        token_expiration = fields.Datetime.now() + relativedelta(seconds=1800)
+        access_data = {
+            "username": "uncle_john",
+            "password": "secret",
+            "access_token": "live_the_token",
+            "token_expiration": token_expiration,
+        }
+        if include_account:
+            access_data["ponto_account"] = "2ad3df83-be01-47cf-a6be-cf0de5cb4c99"
+        return access_data

From 9d1a2f39959146386a4fcea8e3a27c5945b31376 Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Thu, 15 Sep 2022 00:47:29 +0200
Subject: [PATCH 14/15] [IMP] *_online_ponto: increase test coverage.

---
 .../online_bank_statement_provider_ponto.py   |  4 +-
 .../models/ponto_interface.py                 | 46 +++++++++++--------
 ...t_account_statement_import_online_ponto.py | 27 +++++++----
 .../tests/test_ponto_interface.py             | 26 +++++++++++
 4 files changed, 73 insertions(+), 30 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
index 355de12b..28658fe0 100644
--- a/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
+++ b/account_bank_statement_import_online_ponto/models/online_bank_statement_provider_ponto.py
@@ -99,9 +99,7 @@ class OnlineBankStatementProviderPonto(models.Model):
                 sequence
             )
             new_transactions.append(vals_line)
-        if new_transactions:
-            return new_transactions, {}
-        return
+        return new_transactions, {}
 
     def _ponto_get_transaction_vals(self, transaction, sequence):
         """Translate information from Ponto to statement line vals."""
diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py
index c84626c0..d9f0c70d 100644
--- a/account_bank_statement_import_online_ponto/models/ponto_interface.py
+++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py
@@ -105,37 +105,45 @@ class PontoInterface(models.AbstractModel):
         if response.status_code == 400:
             # Probably synchronization recently done already.
             _logger.debug(
-                _("Syncronization request rejected: %s"),
+                _("Synchronization request rejected: %s"),
                 response.text
             )
             return
         data = self._get_response_data(response)
         sync_id = data.get("attributes", {}).get("resourceId", False)
-        # Check synchronisation
         if not sync_id:
-            return
+            raise UserError(
+                _("Ponto : no resourceId in synchronization data %s") % data
+            )
         # Poll synchronization during 400 seconds for completion.
         url = PONTO_ENDPOINT + "/synchronizations/" + sync_id
         number = 0
-        while number <= 10:
+        while number < 10:
             number += 1
-            response = requests.get(url, headers=self._get_request_headers(access_data))
-            if response.status_code == 200:
-                data = json.loads(response.text)
-                status = data.get("status", {})
-                if status in ("success", "error"):
-                    if status == "error":
-                        _logger.debug(
-                            _("Syncronization was succesfully completed")
-                        )
-                    else:
-                        _logger.debug(
-                            _("Syncronization had an error: %s"),
-                            response.text
-                        )
-                    return
+            if self._synchronization_done(access_data, url):
+                break
             time.sleep(40)
 
+    def _synchronization_done(self, access_data, url):
+        """Check wether requested synchronization done."""
+        response = requests.get(url, headers=self._get_request_headers(access_data))
+        if response.status_code != 200:
+            return False
+        data = json.loads(response.text)
+        status = data.get("status", {})
+        if status not in ("success", "error"):
+            return False
+        if status == "error":
+            _logger.debug(
+                _("Synchronization was succesfully completed")
+            )
+        else:
+            _logger.debug(
+                _("Synchronization had an error: %s"),
+                response.text
+            )
+        return True
+
     def _get_transactions(self, access_data, last_identifier):
         """Get transactions from ponto, using last_identifier as pointer.
 
diff --git a/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
index d27893cd..3f66213a 100644
--- a/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
+++ b/account_bank_statement_import_online_ponto/tests/test_account_statement_import_online_ponto.py
@@ -229,17 +229,28 @@ class TestBankAccountStatementImportOnlinePonto(common.TransactionCase):
 
     def test_ponto_buffer_purge(self):
         """Create some old buffer records and test purging them."""
-        self.provider.write(
-            {
-                "ponto_buffer_retain_days": 31,
-                "active": True,
-            }
-        )
         buffer_model = self.env["ponto.buffer"]
         buffer_model.sudo()._store_transactions(self.provider, THREE_TRANSACTIONS)
-        buffers = buffer_model.search([("provider_id", "=", self.provider.id)])
         # As all transactions have a different date, they will be in separate buffers.
+        buffers = buffer_model.search([("provider_id", "=", self.provider.id)])
         self.assertEqual(len(buffers), 3)
+        # Non ponto providers should not affect buffers.
+        self._expect_purge_result(service="dummy", retain_days=15, expected_length=3)
+        # If retain date not filled, buffers should not be purged.
+        self._expect_purge_result(expected_length=3)
+        # If retain date filled, buffers should be purged.
+        self._expect_purge_result(retain_days=15)
+
+    def _expect_purge_result(self, service="ponto", retain_days=0, expected_length=0):
+        """Check result for purge in different scenario's."""
+        buffer_model = self.env["ponto.buffer"]
+        self.provider.write(
+            {
+                "active": True,
+                "ponto_buffer_retain_days": retain_days,
+                "service": service,
+            }
+        )
         self.provider._ponto_buffer_purge()
         buffers = buffer_model.search([("provider_id", "=", self.provider.id)])
-        self.assertFalse(bool(buffers))
+        self.assertEqual(len(buffers), expected_length)
diff --git a/account_bank_statement_import_online_ponto/tests/test_ponto_interface.py b/account_bank_statement_import_online_ponto/tests/test_ponto_interface.py
index 17bcf662..d56a6caf 100644
--- a/account_bank_statement_import_online_ponto/tests/test_ponto_interface.py
+++ b/account_bank_statement_import_online_ponto/tests/test_ponto_interface.py
@@ -91,6 +91,32 @@ class TestPontoInterface(common.TransactionCase):
         interface_model = self.env["ponto.interface"]
         interface_model._ponto_synchronisation(access_data)
 
+    @patch("requests.get")
+    def test_synchronization_done(self, requests_get):
+        """Test getting account data for Ponto access."""
+        mock_response = MagicMock()
+        mock_response.status_code = 200
+        mock_response.text = json.dumps({"status": "success"})
+        requests_get.return_value = mock_response
+        # Succesfull sync.
+        self._check_synchronization_done(True)
+        # Error in sync.
+        mock_response.text = json.dumps({"status": "error"})
+        self._check_synchronization_done(True)
+        # Unexpected error in sync.
+        mock_response.status_code = 404
+        self._check_synchronization_done(False)
+
+    def _check_synchronization_done(self, expected_result):
+        """Check result for synchronization with current mock."""
+        interface_model = self.env["ponto.interface"]
+        access_data = self._get_access_dict()
+        synchronization_done = interface_model._synchronization_done(
+            access_data,
+            "https//does.not.matter.com/synchronization"
+        )
+        self.assertEqual(synchronization_done, expected_result)
+
     @patch("requests.get")
     def test_get_transactions(self, requests_get):
         """Test getting transactions from Ponto."""

From 6cce2257f5dc99b8d8fc2534061fa9cffb2b03dd Mon Sep 17 00:00:00 2001
From: "Ronald Portier (Therp BV)" <ronald@therp.nl>
Date: Thu, 15 Sep 2022 09:57:52 +0200
Subject: [PATCH 15/15] [FIX] *_online_ponto: Correct retrieval of sync_id

---
 .../models/ponto_interface.py                               | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/account_bank_statement_import_online_ponto/models/ponto_interface.py b/account_bank_statement_import_online_ponto/models/ponto_interface.py
index d9f0c70d..7a1df283 100644
--- a/account_bank_statement_import_online_ponto/models/ponto_interface.py
+++ b/account_bank_statement_import_online_ponto/models/ponto_interface.py
@@ -110,10 +110,12 @@ class PontoInterface(models.AbstractModel):
             )
             return
         data = self._get_response_data(response)
-        sync_id = data.get("attributes", {}).get("resourceId", False)
+        # The resourceId in data["attributes"] is the account,
+        # we need the synchronization.
+        sync_id = data.get("data", {}).get("id", False)
         if not sync_id:
             raise UserError(
-                _("Ponto : no resourceId in synchronization data %s") % data
+                _("Ponto : no synchronization id in data %s") % data
             )
         # Poll synchronization during 400 seconds for completion.
         url = PONTO_ENDPOINT + "/synchronizations/" + sync_id