[IMP]account_statement_import_txt_xlsx: additional mapping controls

account_statement_import_txt_xlsx/README.rst

@@ -101,6 +101,8 @@ Contributors
 
   * Alexey Pelykh <alexey.pelykh@corphub.eu>
 
+* Sebastiano Picchi <sebastiano.picchi@pytech.it>
+
 Maintainers
 ~~~~~~~~~~~
 

account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py

@@ -2,7 +2,8 @@
 # Copyright 2020 CorporateHub (https://corporatehub.eu)
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
 
-from odoo import api, fields, models
+from odoo import _, api, fields, models
+from odoo.exceptions import ValidationError
 
 
 class AccountStatementImportSheetMapping(models.Model):
@@ -61,6 +62,21 @@ class AccountStatementImportSheetMapping(models.Model):
         help="When this occurs please indicate the column number in the Columns section "
         "instead of the column name, considering that the first column is 0",
     )
+    skip_empty_lines = fields.Boolean(
+        "Skip Empty Lines",
+        default=False,
+        help="Allows to skip empty lines",
+    )
+    offset_column = fields.Integer(
+        "Offset Column",
+        default=0,
+        help="Horizontal spaces to ignore before starting to parse",
+    )
+    offset_row = fields.Integer(
+        "Offset Row",
+        default=0,
+        help="Vertical spaces to ignore before starting to parse",
+    )
     timestamp_column = fields.Char(string="Timestamp column", required=True)
     currency_column = fields.Char(
         string="Currency column",
@@ -169,6 +185,12 @@ class AccountStatementImportSheetMapping(models.Model):
         elif "comma" == self.float_thousands_sep == self.float_decimal_sep:
             self.float_thousands_sep = "dot"
 
+    @api.constrains("offset_column", "offset_row")
+    def _check_columns(self):
+        for mapping in self:
+            if mapping.offset_column < 0 or mapping.offset_row < 0:
+                raise ValidationError(_("Offsets cannot be negative"))
+
     def _get_float_separators(self):
         self.ensure_one()
         separators = {

account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py

@@ -170,9 +170,14 @@ class AccountStatementImportSheetParser(models.TransientModel):
         header = False
         if not mapping.no_header:
             if isinstance(csv_or_xlsx, tuple):
-                header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
+                header = [
+                    str(value)
+                    for value in csv_or_xlsx[1].row_values(mapping.offset_row)
+                ]
             else:
                 header = [value.strip() for value in next(csv_or_xlsx)]
+        if mapping.offset_column:
+            header = header[mapping.offset_column :]
         for column_name in self._get_column_names():
             columns[column_name] = self._get_column_indexes(
                 header, column_name, mapping
@@ -321,7 +326,7 @@ class AccountStatementImportSheetParser(models.TransientModel):
 
     def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns):  # noqa: C901
         if isinstance(csv_or_xlsx, tuple):
-            rows = range(1, csv_or_xlsx[1].nrows)
+            rows = range(mapping.offset_row + 1, csv_or_xlsx[1].nrows)
         else:
             rows = csv_or_xlsx
 
@@ -331,7 +336,7 @@ class AccountStatementImportSheetParser(models.TransientModel):
                 book = csv_or_xlsx[0]
                 sheet = csv_or_xlsx[1]
                 values = []
-                for col_index in range(sheet.row_len(row)):
+                for col_index in range(mapping.offset_column, sheet.row_len(row)):
                     cell_type = sheet.cell_type(row, col_index)
                     cell_value = sheet.cell_value(row, col_index)
                     if cell_type == xlrd.XL_CELL_DATE:
@@ -339,6 +344,8 @@ class AccountStatementImportSheetParser(models.TransientModel):
                     values.append(cell_value)
             else:
                 values = list(row)
+            if mapping.skip_empty_lines and not any(values):
+                continue
             line = self._parse_row(mapping, currency_code, values, columns)
             if line:
                 lines.append(line)

account_statement_import_txt_xlsx/readme/CONTRIBUTORS.rst

@@ -13,3 +13,5 @@
 * `CorporateHub <https://corporatehub.eu/>`__
 
   * Alexey Pelykh <alexey.pelykh@corphub.eu>
+
+* Sebastiano Picchi <sebastiano.picchi@pytech.it>

account_statement_import_txt_xlsx/static/description/index.html

@@ -1,4 +1,3 @@
-<?xml version="1.0" encoding="utf-8"?>
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
 <head>
@@ -9,10 +8,11 @@
 
 /*
 :Author: David Goodger (goodger@python.org)
-:Id: $Id: html4css1.css 8954 2022-01-20 10:10:25Z milde $
+:Id: $Id: html4css1.css 9511 2024-01-13 09:50:07Z milde $
 :Copyright: This stylesheet has been placed in the public domain.
 
 Default cascading style sheet for the HTML output of Docutils.
+Despite the name, some widely supported CSS2 features are used.
 
 See https://docutils.sourceforge.io/docs/howto/html-stylesheets.html for how to
 customize this style sheet.
@@ -275,7 +275,7 @@ pre.literal-block, pre.doctest-block, pre.math, pre.code {
   margin-left: 2em ;
   margin-right: 2em }
 
-pre.code .ln { color: grey; } /* line numbers */
+pre.code .ln { color: gray; } /* line numbers */
 pre.code, code { background-color: #eeeeee }
 pre.code .comment, code .comment { color: #5C6576 }
 pre.code .keyword, code .keyword { color: #3B0D06; font-weight: bold }
@@ -301,7 +301,7 @@ span.option {
 span.pre {
   white-space: pre }
 
-span.problematic {
+span.problematic, pre.problematic {
   color: red }
 
 span.section-subtitle {
@@ -454,12 +454,15 @@ If you spotted it first, help us to smash it by providing a detailed and welcome
 <li>Alexey Pelykh &lt;<a class="reference external" href="mailto:alexey.pelykh&#64;corphub.eu">alexey.pelykh&#64;corphub.eu</a>&gt;</li>
 </ul>
 </li>
+<li>Sebastiano Picchi &lt;<a class="reference external" href="mailto:sebastiano.picchi&#64;pytech.it">sebastiano.picchi&#64;pytech.it</a>&gt;</li>
 </ul>
 </div>
 <div class="section" id="maintainers">
 <h2><a class="toc-backref" href="#toc-entry-9">Maintainers</a></h2>
 <p>This module is maintained by the OCA.</p>
-<a class="reference external image-reference" href="https://odoo-community.org"><img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" /></a>
+<a class="reference external image-reference" href="https://odoo-community.org">
+<img alt="Odoo Community Association" src="https://odoo-community.org/logo.png" />
+</a>
 <p>OCA, or the Odoo Community Association, is a nonprofit organization whose
 mission is to support the collaborative development of Odoo features and
 promote its widespread use.</p>

account_statement_import_txt_xlsx/tests/fixtures/empty_lines_statement.csv

@@ -0,0 +1,5 @@
+"Date","Label","Currency","Amount","Amount Currency","Partner Name","Bank Account"
+"02/25/2018","AAAOOO 1","EUR","-33.50","0.0","John Doe","123456789"
+"02/26/2018","AAAOOO 2","EUR","1,525.00","1,000.00","Azure Interior",""
+,,,,,,
+"02/27/2018","AAAOOO 3","EUR","800.00","800.00","Azure Interior","123456789"

account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py

@@ -1,7 +1,7 @@
 # Copyright 2019 ForgeFlow, S.L.
 # Copyright 2020 CorporateHub (https://corporatehub.eu)
 # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl).
-
+import decimal
 from base64 import b64encode
 from os import path
 
@@ -448,3 +448,96 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase):
         self.assertEqual(statement.balance_start, 10.0)
         self.assertEqual(statement.balance_end_real, 1510.0)
         self.assertEqual(statement.balance_end, 1510.0)
+
+    def test_offsets(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/sample_statement_offsets.xlsx"
+        data = self._data_file(file_name)
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        with self.assertRaises(ValueError):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        statement_map_offsets = self.sample_statement_map.copy(
+            {
+                "offset_column": 1,
+                "offset_row": 2,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_offsets.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        # import wdb; wdb.set_trace()
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 2)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 1491.5)
+        self.assertEqual(statement.balance_end, 1491.5)
+
+    def test_skip_empty_lines(self):
+        journal = self.AccountJournal.create(
+            {
+                "name": "Bank",
+                "type": "bank",
+                "code": "BANK",
+                "currency_id": self.currency_usd.id,
+                "suspense_account_id": self.suspense_account.id,
+            }
+        )
+        file_name = "fixtures/empty_lines_statement.csv"
+        data = self._data_file(file_name, "utf-8")
+
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": self.sample_statement_map.id,
+            }
+        )
+        with self.assertRaises(decimal.InvalidOperation):
+            wizard.with_context(
+                account_statement_import_txt_xlsx_test=True
+            ).import_file_button()
+        statement_map_empty_line = self.sample_statement_map.copy(
+            {
+                "skip_empty_lines": True,
+            }
+        )
+        wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+            {
+                "statement_filename": file_name,
+                "statement_file": data,
+                "sheet_mapping_id": statement_map_empty_line.id,
+            }
+        )
+        wizard.with_context(
+            account_statement_import_txt_xlsx_test=True
+        ).import_file_button()
+        statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+        self.assertEqual(len(statement), 1)
+        self.assertEqual(len(statement.line_ids), 3)
+        self.assertEqual(statement.balance_start, 0.0)
+        self.assertEqual(statement.balance_end_real, 2291.5)
+        self.assertEqual(statement.balance_end, 2291.5)

account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml

@@ -50,6 +50,15 @@
                                     class="fa fa-info-circle"
                                 /> indicate the column number in the Columns section. The first column is 0.
                             </div>
+                            <field name="skip_empty_lines" />
+                            <field
+                                name="offset_column"
+                                attrs="{'invisible': [('no_header', '=', True)]}"
+                            />
+                            <field
+                                name="offset_row"
+                                attrs="{'invisible': [('no_header', '=', True)]}"
+                            />
                         </group>
                         <group
                             attrs="{'invisible': [('debit_credit_column', '=', False)]}"