Merge PR #691 into 15.0

Signed-off-by alexey-pelykh
2025-01-20 12:37:43 +02:00 · 2024-05-07 15:01:31 +00:00
parent 756e4e3bd9 1716514e04
commit 204ec79a21
9 changed files with 277 additions and 137 deletions
--- a/account_statement_import_txt_xlsx/README.rst
+++ b/account_statement_import_txt_xlsx/README.rst
@@ -101,6 +101,8 @@ Contributors

  * Alexey Pelykh <alexey.pelykh@corphub.eu>

+* Sebastiano Picchi <sebastiano.picchi@pytech.it>
+
 Maintainers
 ~~~~~~~~~~~

--- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py
+++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py
@@ -2,7 +2,8 @@
 # Copyright 2020 CorporateHub (https://corporatehub.eu)
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).

-from odoo import api, fields, models
+from odoo import _, api, fields, models
+from odoo.exceptions import ValidationError


 class AccountStatementImportSheetMapping(models.Model):
@@ -60,6 +61,18 @@ class AccountStatementImportSheetMapping(models.Model):
        help="When this occurs please indicate the column number in the Columns section "
        "instead of the column name, considering that the first column is 0",
    )
+    skip_empty_lines = fields.Boolean(
+        default=False,
+        help="Allows to skip empty lines",
+    )
+    offset_column = fields.Integer(
+        default=0,
+        help="Horizontal spaces to ignore before starting to parse",
+    )
+    offset_row = fields.Integer(
+        default=0,
+        help="Vertical spaces to ignore before starting to parse",
+    )
    timestamp_column = fields.Char(required=True)
    currency_column = fields.Char(
        help=(
@@ -151,6 +164,12 @@ class AccountStatementImportSheetMapping(models.Model):
        elif "comma" == self.float_thousands_sep == self.float_decimal_sep:
            self.float_thousands_sep = "dot"

+    @api.constrains("offset_column", "offset_row")
+    def _check_columns(self):
+        for mapping in self:
+            if mapping.offset_column < 0 or mapping.offset_row < 0:
+                raise ValidationError(_("Offsets cannot be negative"))
+
    def _get_float_separators(self):
        self.ensure_one()
        separators = {
--- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py
+++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py
@@ -171,9 +171,14 @@ class AccountStatementImportSheetParser(models.TransientModel):
        header = False
        if not mapping.no_header:
            if isinstance(csv_or_xlsx, tuple):
-                header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
+                header = [
+                    str(value)
+                    for value in csv_or_xlsx[1].row_values(mapping.offset_row)
+                ]
            else:
                header = [value.strip() for value in next(csv_or_xlsx)]
+        if mapping.offset_column:
+            header = header[mapping.offset_column :]
        for column_name in self._get_column_names():
            columns[column_name] = self._get_column_indexes(
                header, column_name, mapping
@@ -195,9 +200,134 @@ class AccountStatementImportSheetParser(models.TransientModel):
            return " ".join(content_l)
        return content_l[0]

+    def _parse_row(self, mapping, currency_code, values, columns):  # noqa: C901
+        timestamp = self._get_values_from_column(values, columns, "timestamp_column")
+        currency = (
+            self._get_values_from_column(values, columns, "currency_column")
+            if columns["currency_column"]
+            else currency_code
+        )
+
+        def _decimal(column_name):
+            if columns[column_name]:
+                return self._parse_decimal(
+                    self._get_values_from_column(values, columns, column_name),
+                    mapping,
+                )
+
+        amount = _decimal("amount_column")
+        if not amount:
+            amount = abs(_decimal("amount_debit_column") or 0)
+        if not amount:
+            amount = -abs(_decimal("amount_credit_column") or 0)
+
+        balance = (
+            self._get_values_from_column(values, columns, "balance_column")
+            if columns["balance_column"]
+            else None
+        )
+        original_currency = (
+            self._get_values_from_column(values, columns, "original_currency_column")
+            if columns["original_currency_column"]
+            else None
+        )
+        original_amount = (
+            self._get_values_from_column(values, columns, "original_amount_column")
+            if columns["original_amount_column"]
+            else None
+        )
+        debit_credit = (
+            self._get_values_from_column(values, columns, "debit_credit_column")
+            if columns["debit_credit_column"]
+            else None
+        )
+        transaction_id = (
+            self._get_values_from_column(values, columns, "transaction_id_column")
+            if columns["transaction_id_column"]
+            else None
+        )
+        description = (
+            self._get_values_from_column(values, columns, "description_column")
+            if columns["description_column"]
+            else None
+        )
+        notes = (
+            self._get_values_from_column(values, columns, "notes_column")
+            if columns["notes_column"]
+            else None
+        )
+        reference = (
+            self._get_values_from_column(values, columns, "reference_column")
+            if columns["reference_column"]
+            else None
+        )
+        partner_name = (
+            self._get_values_from_column(values, columns, "partner_name_column")
+            if columns["partner_name_column"]
+            else None
+        )
+        bank_name = (
+            self._get_values_from_column(values, columns, "bank_name_column")
+            if columns["bank_name_column"]
+            else None
+        )
+        bank_account = (
+            self._get_values_from_column(values, columns, "bank_account_column")
+            if columns["bank_account_column"]
+            else None
+        )
+
+        if currency != currency_code:
+            return {}
+
+        if isinstance(timestamp, str):
+            timestamp = datetime.strptime(timestamp, mapping.timestamp_format)
+
+        if balance:
+            balance = self._parse_decimal(balance, mapping)
+        else:
+            balance = None
+
+        if debit_credit:
+            amount = amount.copy_abs()
+            if debit_credit == mapping.debit_value:
+                amount = -amount
+
+        if original_amount:
+            original_amount = self._parse_decimal(original_amount, mapping).copy_sign(
+                amount
+            )
+        else:
+            original_amount = 0.0
+
+        line = {
+            "timestamp": timestamp,
+            "amount": amount,
+            "currency": currency,
+            "original_amount": original_amount,
+            "original_currency": original_currency,
+        }
+        if balance is not None:
+            line["balance"] = balance
+        if transaction_id is not None:
+            line["transaction_id"] = transaction_id
+        if description is not None:
+            line["description"] = description
+        if notes is not None:
+            line["notes"] = notes
+        if reference is not None:
+            line["reference"] = reference
+        if partner_name is not None:
+            line["partner_name"] = partner_name
+        if bank_name is not None:
+            line["bank_name"] = bank_name
+        if bank_account is not None:
+            line["bank_account"] = bank_account
+        return line
+
    def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C901
        if isinstance(csv_or_xlsx, tuple):
-            rows = range(1, csv_or_xlsx[1].nrows)
+            rows = range(mapping.offset_row + 1, csv_or_xlsx[1].nrows)
        else:
            rows = csv_or_xlsx

@@ -207,7 +337,7 @@ class AccountStatementImportSheetParser(models.TransientModel):
                book = csv_or_xlsx[0]
                sheet = csv_or_xlsx[1]
                values = []
-                for col_index in range(sheet.row_len(row)):
+                for col_index in range(mapping.offset_column, sheet.row_len(row)):
                    cell_type = sheet.cell_type(row, col_index)
                    cell_value = sheet.cell_value(row, col_index)
                    if cell_type == xlrd.XL_CELL_DATE:
@@ -215,134 +345,11 @@ class AccountStatementImportSheetParser(models.TransientModel):
                    values.append(cell_value)
            else:
                values = list(row)
-
-            timestamp = self._get_values_from_column(
-                values, columns, "timestamp_column"
-            )
-            currency = (
-                self._get_values_from_column(values, columns, "currency_column")
-                if columns["currency_column"]
-                else currency_code
-            )
-
-            def _decimal(column_name):
-                if columns[column_name]:
-                    return self._parse_decimal(
-                        self._get_values_from_column(values, columns, column_name),
-                        mapping,
-                    )
-
-            amount = _decimal("amount_column")
-            if not amount:
-                amount = abs(_decimal("amount_debit_column") or 0)
-            if not amount:
-                amount = -abs(_decimal("amount_credit_column") or 0)
-
-            balance = (
-                self._get_values_from_column(values, columns, "balance_column")
-                if columns["balance_column"]
-                else None
-            )
-            original_currency = (
-                self._get_values_from_column(
-                    values, columns, "original_currency_column"
-                )
-                if columns["original_currency_column"]
-                else None
-            )
-            original_amount = (
-                self._get_values_from_column(values, columns, "original_amount_column")
-                if columns["original_amount_column"]
-                else None
-            )
-            debit_credit = (
-                self._get_values_from_column(values, columns, "debit_credit_column")
-                if columns["debit_credit_column"]
-                else None
-            )
-            transaction_id = (
-                self._get_values_from_column(values, columns, "transaction_id_column")
-                if columns["transaction_id_column"]
-                else None
-            )
-            description = (
-                self._get_values_from_column(values, columns, "description_column")
-                if columns["description_column"]
-                else None
-            )
-            notes = (
-                self._get_values_from_column(values, columns, "notes_column")
-                if columns["notes_column"]
-                else None
-            )
-            reference = (
-                self._get_values_from_column(values, columns, "reference_column")
-                if columns["reference_column"]
-                else None
-            )
-            partner_name = (
-                self._get_values_from_column(values, columns, "partner_name_column")
-                if columns["partner_name_column"]
-                else None
-            )
-            bank_name = (
-                self._get_values_from_column(values, columns, "bank_name_column")
-                if columns["bank_name_column"]
-                else None
-            )
-            bank_account = (
-                self._get_values_from_column(values, columns, "bank_account_column")
-                if columns["bank_account_column"]
-                else None
-            )
-
-            if currency != currency_code:
+            if mapping.skip_empty_lines and not any(values):
                continue
-
-            if isinstance(timestamp, str):
-                timestamp = datetime.strptime(timestamp, mapping.timestamp_format)
-
-            if balance:
-                balance = self._parse_decimal(balance, mapping)
-            else:
-                balance = None
-
-            if debit_credit:
-                amount = amount.copy_abs()
-                if debit_credit == mapping.debit_value:
-                    amount = -amount
-
-            if original_amount:
-                original_amount = self._parse_decimal(
-                    original_amount, mapping
-                ).copy_sign(amount)
-            else:
-                original_amount = 0.0
-
-            line = {
-                "timestamp": timestamp,
-                "amount": amount,
-                "currency": currency,
-                "original_amount": original_amount,
-                "original_currency": original_currency,
-            }
-            if balance is not None:
-                line["balance"] = balance
-            if transaction_id is not None:
-                line["transaction_id"] = transaction_id
-            if description is not None:
-                line["description"] = description
-            if notes is not None:
-                line["notes"] = notes
-            if reference is not None:
-                line["reference"] = reference
-            if partner_name is not None:
-                line["partner_name"] = partner_name
-            if bank_name is not None:
-                line["bank_name"] = bank_name
-            if bank_account is not None:
-                line["bank_account"] = bank_account
-            lines.append(line)
+            line = self._parse_row(mapping, currency_code, values, columns)
+            if line:
+                lines.append(line)
        return lines

    @api.model
--- a/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst
+++ b/account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst
@@ -13,3 +13,5 @@
 * `CorporateHub <https://corporatehub.eu/>`__

  * Alexey Pelykh <alexey.pelykh@corphub.eu>
+
+* Sebastiano Picchi <sebastiano.picchi@pytech.it>
--- a/account_statement_import_txt_xlsx/static/description/index.html
+++ b/account_statement_import_txt_xlsx/static/description/index.html
@@ -1,4 +1,3 @@
-<?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 <head>
@@ -9,10 +8,11 @@

 /*
 :Author: David Goodger (goodger@python.org)
-:Id: $Id: html4css1.css 8954 2022-01-20 10:10:25Z milde $
+:Id: $Id: html4css1.css 9511 2024-01-13 09:50:07Z milde $
 :Copyright: This stylesheet has been placed in the public domain.

 Default cascading style sheet for the HTML output of Docutils.
+Despite the name, some widely supported CSS2 features are used.

 See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to
 customize this style sheet.
@@ -275,7 +275,7 @@ pre.literal-block, pre.doctest-block, pre.math, pre.code {
  margin-left: 2em ;
  margin-right: 2em }

-pre.code .ln { color: grey; } /* line numbers */
+pre.code .ln { color: gray; } /* line numbers */
 pre.code, code { background-color: #eeeeee }
 pre.code .comment, code .comment { color: #5C6576 }
 pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }
@@ -301,7 +301,7 @@ span.option {
 span.pre {
  white-space: pre }

-span.problematic {
+span.problematic, pre.problematic {
  color: red }

 span.section-subtitle {
@@ -454,12 +454,15 @@ If you spotted it first, help us to smash it by providing a detailed and welcome
 <li>Alexey Pelykh &lt;<a class="reference external" href="mailto:alexey.pelykh&#64;corphub.eu">alexey.pelykh&#64;corphub.eu</a>&gt;</li>
 </ul>
 </li>
+<li>Sebastiano Picchi &lt;<a class="reference external" href="mailto:sebastiano.picchi&#64;pytech.it">sebastiano.picchi&#64;pytech.it</a>&gt;</li>
 </ul>
 </div>
 <div class="section" id="maintainers">
 <h2><a class="toc-backref" href="#toc-entry-9">Maintainers</a></h2>
 <p>This module is maintained by the OCA.</p>
-<a class="reference external image-reference" href="https://odoo-community.org"><img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" /></a>
+<a class="reference external image-reference" href="https://odoo-community.org">
+<img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" />
+</a>
 <p>OCA, or the Odoo Community Association, is a nonprofit organization whose
 mission is to support the collaborative development of Odoo features and
 promote its widespread use.</p>
--- a/account_statement_import_txt_xlsx/tests/fixtures/empty_lines_statement.csv
+++ b/account_statement_import_txt_xlsx/tests/fixtures/empty_lines_statement.csv
@@ -0,0 +1,5 @@
+"Date","Label","Currency","Amount","Amount Currency","Partner Name","Bank Account"
+"02/25/2018","AAAOOO 1","EUR","-33.50","0.0","John Doe","123456789"
+"02/26/2018","AAAOOO 2","EUR","1,525.00","1,000.00","Azure Interior",""
+,,,,,,
+"02/27/2018","AAAOOO 3","EUR","800.00","800.00","Azure Interior","123456789"
--- a/account_statement_import_txt_xlsx/tests/fixtures/sample_statement_offsets.xlsx
+++ b/account_statement_import_txt_xlsx/tests/fixtures/sample_statement_offsets.xlsx
--- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py
+++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py
@@ -1,7 +1,7 @@
 # Copyright 2019 ForgeFlow, S.L.
 # Copyright 2020 CorporateHub (https://corporatehub.eu)
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
+import decimal
 from base64 import b64encode
 from os import path

@@ -458,3 +458,96 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase):
        self.assertEqual(statement.balance_start, 10.0)
        self.assertEqual(statement.balance_end_real, 1510.0)
        self.assertEqual(statement.balance_end, 1510.0)
+
+    def test_offsets(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/sample_statement_offsets.xlsx"
+        data = self._data_file(file_name)
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        with self.assertRaises(ValueError):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        statement_map_offsets = self.sample_statement_map.copy(
+            {
+                "offset_column": 1,
+                "offset_row": 2,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_offsets.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        # import wdb; wdb.set_trace()
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 2)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 1491.5)
+        self.assertEqual(statement.balance_end, 1491.5)
+
+    def test_skip_empty_lines(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/empty_lines_statement.csv"
+        data = self._data_file(file_name, "utf-8")
+
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        with self.assertRaises(decimal.InvalidOperation):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        statement_map_empty_line = self.sample_statement_map.copy(
+            {
+                "skip_empty_lines": True,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_empty_line.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 3)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 2291.5)
+        self.assertEqual(statement.balance_end, 2291.5)
--- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml
+++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml
@@ -50,6 +50,15 @@
                                    class="fa fa-info-circle"
                                /> indicate the column number in the Columns section. The first column is 0.
                            </div>
+                            <field name="skip_empty_lines" />
+                            <field
+                                name="offset_column"
+                                attrs="{'invisible': [('no_header', '=', True)]}"
+                            />
+                            <field
+                                name="offset_row"
+                                attrs="{'invisible': [('no_header', '=', True)]}"
+                            />
                        </group>
                        <group
                            attrs="{'invisible': [('debit_credit_column', '=', False)]}"