From 516df6c2bfaa2fce6c7080b425e1a37ac73c2d31 Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Fri, 16 Dec 2022 11:09:18 +0100 Subject: [PATCH 1/3] [IMP] account_bank_statement_import_txt_xlsx: Allow mapping with reference to column numbers, and concatenation A new field 'File does not contain header line' is added in the Statement Sheet Mappings. If you set to True, then you can map the columns by indicating in each field of the 'Columns' section the column number in the file. We also allow to concatenate multiple columns in the file to a single column to a single field of the statement line. You have to indicate the names of the columns separated by comma. --- .../__manifest__.py | 2 +- .../models/account_bank_statement_import.py | 4 +- ...unt_bank_statement_import_sheet_mapping.py | 5 + ...ount_bank_statement_import_sheet_parser.py | 191 ++++++++++-------- ...nt_bank_statement_import_sheet_mapping.xml | 51 +++-- requirements.txt | 1 + 6 files changed, 154 insertions(+), 100 deletions(-) diff --git a/account_bank_statement_import_txt_xlsx/__manifest__.py b/account_bank_statement_import_txt_xlsx/__manifest__.py index 99185276..5e574882 100644 --- a/account_bank_statement_import_txt_xlsx/__manifest__.py +++ b/account_bank_statement_import_txt_xlsx/__manifest__.py @@ -17,7 +17,7 @@ "multi_step_wizard", "web_widget_dropdown_dynamic", ], - "external_dependencies": {"python": ["xlrd"]}, + "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ "security/ir.model.access.csv", "data/map_data.xml", diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py index 19df908e..a86aa844 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py @@ -28,7 +28,9 @@ class AccountBankStatementImport(models.TransientModel): self.ensure_one() try: Parser = self.env["account.bank.statement.import.sheet.parser"] - return Parser.parse(data_file, self.sheet_mapping_id) + return Parser.parse( + data_file, self.sheet_mapping_id, self.attachment_ids[:1].name + ) except BaseException: if self.env.context.get("account_bank_statement_import_txt_xlsx_test"): raise diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py index 4e90b0df..5a927b1e 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py @@ -54,6 +54,11 @@ class AccountBankStatementImportSheetMapping(models.Model): ) quotechar = fields.Char(string="Text qualifier", size=1, default='"') timestamp_format = fields.Char(string="Timestamp Format", required=True) + no_header = fields.Boolean( + "File does not contain header line", + help="When this occurs please indicate the column number in the Columns section " + "instead of the column name, considering that the first column is 0", + ) timestamp_column = fields.Char(string="Timestamp column", required=True) currency_column = fields.Char( string="Currency column", diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py index c26300b4..b67f06cd 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py @@ -7,8 +7,10 @@ import logging from datetime import datetime from decimal import Decimal from io import StringIO +from os import path from odoo import _, api, models +from odoo.exceptions import UserError _logger = logging.getLogger(__name__) @@ -19,6 +21,14 @@ try: except (ImportError, IOError) as err: # pragma: no cover _logger.error(err) +try: + import chardet +except ImportError: + _logger.warning( + "chardet library not found, please install it " + "from http://pypi.python.org/pypi/chardet" + ) + class AccountBankStatementImportSheetParser(models.TransientModel): _name = "account.bank.statement.import.sheet.parser" @@ -42,7 +52,7 @@ class AccountBankStatementImportSheetParser(models.TransientModel): return list(next(csv_data)) @api.model - def parse(self, data_file, mapping): + def parse(self, data_file, mapping, filename): journal = self.env["account.journal"].browse(self.env.context.get("journal_id")) currency_code = (journal.currency_id or journal.company_id.currency_id).name account_number = journal.bank_account_id.acc_number @@ -66,6 +76,7 @@ class AccountBankStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), + "name": _("%s: %s") % (journal.code, path.basename(filename),), } ) @@ -78,6 +89,43 @@ class AccountBankStatementImportSheetParser(models.TransientModel): return currency_code, account_number, [data] + def _get_column_indexes(self, header, column_name, mapping): + column_indexes = [] + if mapping[column_name] and "," in mapping[column_name]: + # We have to concatenate the values + column_names_or_indexes = mapping[column_name].split(",") + else: + column_names_or_indexes = [mapping[column_name]] + for column_name_or_index in column_names_or_indexes: + if mapping.no_header: + column_index = ( + column_name_or_index and int(column_name_or_index) or None + ) + if column_index: + column_indexes.append(column_index) + else: + if column_name_or_index: + column_indexes.append(header.index(column_name_or_index)) + return column_indexes + + def _get_column_names(self): + return [ + "timestamp_column", + "currency_column", + "amount_column", + "balance_column", + "original_currency_column", + "original_amount_column", + "debit_credit_column", + "transaction_id_column", + "description_column", + "notes_column", + "reference_column", + "partner_name_column", + "bank_name_column", + "bank_account_column", + ] + def _parse_lines(self, mapping, data_file, currency_code): columns = dict() try: @@ -98,69 +146,40 @@ class AccountBankStatementImportSheetParser(models.TransientModel): csv_options["delimiter"] = csv_delimiter if mapping.quotechar: csv_options["quotechar"] = mapping.quotechar - csv_or_xlsx = reader( - StringIO(data_file.decode(mapping.file_encoding or "utf-8")), - **csv_options - ) - + try: + decoded_file = data_file.decode(mapping.file_encoding or "utf-8") + except UnicodeDecodeError: + # Try auto guessing the format + detected_encoding = chardet.detect(data_file).get("encoding", False) + if not detected_encoding: + raise UserError( + _("No valid encoding was found for the attached file") + ) + decoded_file = data_file.decode(detected_encoding) + csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) if isinstance(csv_or_xlsx, tuple): header = [str(value) for value in csv_or_xlsx[1].row_values(0)] else: header = [value.strip() for value in next(csv_or_xlsx)] - columns["timestamp_column"] = header.index(mapping.timestamp_column) - columns["currency_column"] = ( - header.index(mapping.currency_column) if mapping.currency_column else None - ) - columns["amount_column"] = header.index(mapping.amount_column) - columns["balance_column"] = ( - header.index(mapping.balance_column) if mapping.balance_column else None - ) - columns["original_currency_column"] = ( - header.index(mapping.original_currency_column) - if mapping.original_currency_column - else None - ) - columns["original_amount_column"] = ( - header.index(mapping.original_amount_column) - if mapping.original_amount_column - else None - ) - columns["debit_credit_column"] = ( - header.index(mapping.debit_credit_column) - if mapping.debit_credit_column - else None - ) - columns["transaction_id_column"] = ( - header.index(mapping.transaction_id_column) - if mapping.transaction_id_column - else None - ) - columns["description_column"] = ( - header.index(mapping.description_column) - if mapping.description_column - else None - ) - columns["notes_column"] = ( - header.index(mapping.notes_column) if mapping.notes_column else None - ) - columns["reference_column"] = ( - header.index(mapping.reference_column) if mapping.reference_column else None - ) - columns["partner_name_column"] = ( - header.index(mapping.partner_name_column) - if mapping.partner_name_column - else None - ) - columns["bank_name_column"] = ( - header.index(mapping.bank_name_column) if mapping.bank_name_column else None - ) - columns["bank_account_column"] = ( - header.index(mapping.bank_account_column) - if mapping.bank_account_column - else None - ) + for column_name in self._get_column_names(): + columns[column_name] = self._get_column_indexes( + header, column_name, mapping + ) return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + def _get_values_from_column(self, values, columns, column_name): + indexes = columns[column_name] + content_l = [] + max_index = len(values) - 1 + for index in indexes: + if isinstance(index, int) and index <= max_index: + content_l.append(values[index]) + else: + content_l.append(values[index]) + if all(isinstance(content, str) for content in content_l): + return " ".join(content_l) + return content_l[0] + def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 if isinstance(csv_or_xlsx, tuple): rows = range(1, csv_or_xlsx[1].nrows) @@ -182,66 +201,70 @@ class AccountBankStatementImportSheetParser(models.TransientModel): else: values = list(row) - timestamp = values[columns["timestamp_column"]] + timestamp = self._get_values_from_column( + values, columns, "timestamp_column" + ) currency = ( - values[columns["currency_column"]] - if columns["currency_column"] is not None + self._get_values_from_column(values, columns, "currency_column") + if columns["currency_column"] else currency_code ) - amount = values[columns["amount_column"]] + amount = self._get_values_from_column(values, columns, "amount_column") balance = ( - values[columns["balance_column"]] - if columns["balance_column"] is not None + self._get_values_from_column(values, columns, "balance_column") + if columns["balance_column"] else None ) original_currency = ( - values[columns["original_currency_column"]] - if columns["original_currency_column"] is not None + self._get_values_from_column( + values, columns, "original_currency_column" + ) + if columns["original_currency_column"] else None ) original_amount = ( - values[columns["original_amount_column"]] - if columns["original_amount_column"] is not None + self._get_values_from_column(values, columns, "original_amount_column") + if columns["original_amount_column"] else None ) debit_credit = ( - values[columns["debit_credit_column"]] - if columns["debit_credit_column"] is not None + self._get_values_from_column(values, columns, "debit_credit_column") + if columns["debit_credit_column"] else None ) transaction_id = ( - values[columns["transaction_id_column"]] - if columns["transaction_id_column"] is not None + self._get_values_from_column(values, columns, "transaction_id_column") + if columns["transaction_id_column"] else None ) description = ( - values[columns["description_column"]] - if columns["description_column"] is not None + self._get_values_from_column(values, columns, "description_column") + if columns["description_column"] else None ) notes = ( - values[columns["notes_column"]] - if columns["notes_column"] is not None + self._get_values_from_column(values, columns, "notes_column") + if columns["notes_column"] else None ) reference = ( - values[columns["reference_column"]] - if columns["reference_column"] is not None + self._get_values_from_column(values, columns, "reference_column") + if columns["reference_column"] else None ) partner_name = ( - values[columns["partner_name_column"]] - if columns["partner_name_column"] is not None + self._get_values_from_column(values, columns, "partner_name_column") + if columns["partner_name_column"] else None ) bank_name = ( - values[columns["bank_name_column"]] - if columns["bank_name_column"] is not None + self._get_values_from_column(values, columns, "bank_name_column") + if columns["bank_name_column"] else None ) bank_account = ( - values[columns["bank_account_column"]] - if columns["bank_account_column"] is not None + self._get_values_from_column(values, columns, "bank_account_column") + if columns["bank_account_column"] else None ) diff --git a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml index 5caa5b12..299eac72 100644 --- a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml +++ b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml @@ -39,6 +39,18 @@ + + + + @@ -53,20 +65,31 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index d925fe08..75851065 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ # generated from manifests external_dependencies +chardet cryptography ofxparse xlrd From bfa2e3fdb09efb6ccda7a7d6cf573c5919a8f7ce Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Sun, 18 Dec 2022 14:29:39 +0100 Subject: [PATCH 2/3] [IMP] account_bank_statement_import_txt_xlsx: add tests --- ...ount_bank_statement_import_sheet_parser.py | 30 +++++++---- .../fixtures/original_currency_no_header.csv | 1 + ..._account_bank_statement_import_txt_xlsx.py | 50 +++++++++++++++++++ 3 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 account_bank_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py index b67f06cd..debd0f3a 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py @@ -97,11 +97,15 @@ class AccountBankStatementImportSheetParser(models.TransientModel): else: column_names_or_indexes = [mapping[column_name]] for column_name_or_index in column_names_or_indexes: + if not column_name_or_index: + continue + column_index = None if mapping.no_header: - column_index = ( - column_name_or_index and int(column_name_or_index) or None - ) - if column_index: + try: + column_index = int(column_name_or_index) + except Exception: + pass + if column_index is not None: column_indexes.append(column_index) else: if column_name_or_index: @@ -157,10 +161,12 @@ class AccountBankStatementImportSheetParser(models.TransientModel): ) decoded_file = data_file.decode(detected_encoding) csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) - if isinstance(csv_or_xlsx, tuple): - header = [str(value) for value in csv_or_xlsx[1].row_values(0)] - else: - header = [value.strip() for value in next(csv_or_xlsx)] + header = False + if not mapping.no_header: + if isinstance(csv_or_xlsx, tuple): + header = [str(value) for value in csv_or_xlsx[1].row_values(0)] + else: + header = [value.strip() for value in next(csv_or_xlsx)] for column_name in self._get_column_names(): columns[column_name] = self._get_column_indexes( header, column_name, mapping @@ -172,10 +178,12 @@ class AccountBankStatementImportSheetParser(models.TransientModel): content_l = [] max_index = len(values) - 1 for index in indexes: - if isinstance(index, int) and index <= max_index: - content_l.append(values[index]) + if isinstance(index, int): + if index <= max_index: + content_l.append(values[index]) else: - content_l.append(values[index]) + if index in values: + content_l.append(values[index]) if all(isinstance(content, str) for content in content_l): return " ".join(content_l) return content_l[0] diff --git a/account_bank_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv b/account_bank_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv new file mode 100644 index 00000000..4e91582f --- /dev/null +++ b/account_bank_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv @@ -0,0 +1 @@ +"12/15/2018","Your payment","EUR","1,525.00","-1,000.00","Azure Interior","","INV0001" diff --git a/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py b/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py index fe009b06..a474ff69 100644 --- a/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py +++ b/account_bank_statement_import_txt_xlsx/tests/test_account_bank_statement_import_txt_xlsx.py @@ -223,6 +223,56 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): self.assertEqual(line.currency_id, self.currency_eur) self.assertEqual(line.amount_currency, 1000.0) + def test_original_currency_no_header(self): + no_header_statement_map = self.AccountBankStatementImportSheetMapping.create( + { + "name": "Sample Statement", + "float_thousands_sep": "comma", + "float_decimal_sep": "dot", + "delimiter": "comma", + "quotechar": '"', + "timestamp_format": "%m/%d/%Y", + "no_header": True, + "timestamp_column": "0", + "amount_column": "3", + "original_currency_column": "2", + "original_amount_column": "4", + "description_column": "1,7", + "partner_name_column": "5", + "bank_account_column": "6", + } + ) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + } + ) + data = self._data_file("fixtures/original_currency_no_header.csv", "utf-8") + wizard = self.AccountBankStatementImport.with_context( + journal_id=journal.id + ).create( + { + "attachment_ids": [ + (0, 0, {"name": "fixtures/original_currency.csv", "datas": data}) + ], + "sheet_mapping_id": no_header_statement_map.id, + } + ) + wizard.with_context( + account_bank_statement_import_txt_xlsx_test=True + ).import_file() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 1) + + line = statement.line_ids + self.assertEqual(line.currency_id, self.currency_eur) + self.assertEqual(line.amount_currency, 1000.0) + self.assertEqual(line.name, "Your payment INV0001") + def test_original_currency_empty(self): journal = self.AccountJournal.create( { From 6baec37ede86f5b57202e5a28d813c6a58671c91 Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Wed, 22 Feb 2023 17:44:24 +0100 Subject: [PATCH 3/3] [IMP] calculate the final balance if not provided in the import file. --- .../models/account_bank_statement_import.py | 13 +++++++++++++ .../account_bank_statement_import_sheet_parser.py | 3 +-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py index a86aa844..28bf2418 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py @@ -36,3 +36,16 @@ class AccountBankStatementImport(models.TransientModel): raise _logger.warning("Sheet parser error", exc_info=True) return super()._parse_file(data_file) + + def _create_bank_statements(self, stmts_vals): + """ Set balance_end_real if not already provided by the file.""" + + statement_line_ids, notifications = super()._create_bank_statements(stmts_vals) + statements = self.env["account.bank.statement"].search( + [("line_ids", "in", statement_line_ids)] + ) + for statement in statements: + if not statement.balance_end_real: + amount = sum(statement.line_ids.mapped("amount")) + statement.balance_end_real = statement.balance_start + amount + return statement_line_ids, notifications diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py index debd0f3a..4b42d0f9 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py @@ -66,6 +66,7 @@ class AccountBankStatementImportSheetParser(models.TransientModel): last_line = lines[-1] data = { "date": first_line["timestamp"].date(), + "name": _("%s: %s") % (journal.code, path.basename(filename),), } if mapping.balance_column: @@ -76,10 +77,8 @@ class AccountBankStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), - "name": _("%s: %s") % (journal.code, path.basename(filename),), } ) - transactions = list( itertools.chain.from_iterable( map(lambda line: self._convert_line_to_transactions(line), lines)