From 516df6c2bfaa2fce6c7080b425e1a37ac73c2d31 Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Fri, 16 Dec 2022 11:09:18 +0100 Subject: [PATCH] [IMP] account_bank_statement_import_txt_xlsx: Allow mapping with reference to column numbers, and concatenation A new field 'File does not contain header line' is added in the Statement Sheet Mappings. If you set to True, then you can map the columns by indicating in each field of the 'Columns' section the column number in the file. We also allow to concatenate multiple columns in the file to a single column to a single field of the statement line. You have to indicate the names of the columns separated by comma. --- .../__manifest__.py | 2 +- .../models/account_bank_statement_import.py | 4 +- ...unt_bank_statement_import_sheet_mapping.py | 5 + ...ount_bank_statement_import_sheet_parser.py | 191 ++++++++++-------- ...nt_bank_statement_import_sheet_mapping.xml | 51 +++-- requirements.txt | 1 + 6 files changed, 154 insertions(+), 100 deletions(-) diff --git a/account_bank_statement_import_txt_xlsx/__manifest__.py b/account_bank_statement_import_txt_xlsx/__manifest__.py index 99185276..5e574882 100644 --- a/account_bank_statement_import_txt_xlsx/__manifest__.py +++ b/account_bank_statement_import_txt_xlsx/__manifest__.py @@ -17,7 +17,7 @@ "multi_step_wizard", "web_widget_dropdown_dynamic", ], - "external_dependencies": {"python": ["xlrd"]}, + "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ "security/ir.model.access.csv", "data/map_data.xml", diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py index 19df908e..a86aa844 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import.py @@ -28,7 +28,9 @@ class AccountBankStatementImport(models.TransientModel): self.ensure_one() try: Parser = self.env["account.bank.statement.import.sheet.parser"] - return Parser.parse(data_file, self.sheet_mapping_id) + return Parser.parse( + data_file, self.sheet_mapping_id, self.attachment_ids[:1].name + ) except BaseException: if self.env.context.get("account_bank_statement_import_txt_xlsx_test"): raise diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py index 4e90b0df..5a927b1e 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_mapping.py @@ -54,6 +54,11 @@ class AccountBankStatementImportSheetMapping(models.Model): ) quotechar = fields.Char(string="Text qualifier", size=1, default='"') timestamp_format = fields.Char(string="Timestamp Format", required=True) + no_header = fields.Boolean( + "File does not contain header line", + help="When this occurs please indicate the column number in the Columns section " + "instead of the column name, considering that the first column is 0", + ) timestamp_column = fields.Char(string="Timestamp column", required=True) currency_column = fields.Char( string="Currency column", diff --git a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py index c26300b4..b67f06cd 100644 --- a/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py +++ b/account_bank_statement_import_txt_xlsx/models/account_bank_statement_import_sheet_parser.py @@ -7,8 +7,10 @@ import logging from datetime import datetime from decimal import Decimal from io import StringIO +from os import path from odoo import _, api, models +from odoo.exceptions import UserError _logger = logging.getLogger(__name__) @@ -19,6 +21,14 @@ try: except (ImportError, IOError) as err: # pragma: no cover _logger.error(err) +try: + import chardet +except ImportError: + _logger.warning( + "chardet library not found, please install it " + "from http://pypi.python.org/pypi/chardet" + ) + class AccountBankStatementImportSheetParser(models.TransientModel): _name = "account.bank.statement.import.sheet.parser" @@ -42,7 +52,7 @@ class AccountBankStatementImportSheetParser(models.TransientModel): return list(next(csv_data)) @api.model - def parse(self, data_file, mapping): + def parse(self, data_file, mapping, filename): journal = self.env["account.journal"].browse(self.env.context.get("journal_id")) currency_code = (journal.currency_id or journal.company_id.currency_id).name account_number = journal.bank_account_id.acc_number @@ -66,6 +76,7 @@ class AccountBankStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), + "name": _("%s: %s") % (journal.code, path.basename(filename),), } ) @@ -78,6 +89,43 @@ class AccountBankStatementImportSheetParser(models.TransientModel): return currency_code, account_number, [data] + def _get_column_indexes(self, header, column_name, mapping): + column_indexes = [] + if mapping[column_name] and "," in mapping[column_name]: + # We have to concatenate the values + column_names_or_indexes = mapping[column_name].split(",") + else: + column_names_or_indexes = [mapping[column_name]] + for column_name_or_index in column_names_or_indexes: + if mapping.no_header: + column_index = ( + column_name_or_index and int(column_name_or_index) or None + ) + if column_index: + column_indexes.append(column_index) + else: + if column_name_or_index: + column_indexes.append(header.index(column_name_or_index)) + return column_indexes + + def _get_column_names(self): + return [ + "timestamp_column", + "currency_column", + "amount_column", + "balance_column", + "original_currency_column", + "original_amount_column", + "debit_credit_column", + "transaction_id_column", + "description_column", + "notes_column", + "reference_column", + "partner_name_column", + "bank_name_column", + "bank_account_column", + ] + def _parse_lines(self, mapping, data_file, currency_code): columns = dict() try: @@ -98,69 +146,40 @@ class AccountBankStatementImportSheetParser(models.TransientModel): csv_options["delimiter"] = csv_delimiter if mapping.quotechar: csv_options["quotechar"] = mapping.quotechar - csv_or_xlsx = reader( - StringIO(data_file.decode(mapping.file_encoding or "utf-8")), - **csv_options - ) - + try: + decoded_file = data_file.decode(mapping.file_encoding or "utf-8") + except UnicodeDecodeError: + # Try auto guessing the format + detected_encoding = chardet.detect(data_file).get("encoding", False) + if not detected_encoding: + raise UserError( + _("No valid encoding was found for the attached file") + ) + decoded_file = data_file.decode(detected_encoding) + csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) if isinstance(csv_or_xlsx, tuple): header = [str(value) for value in csv_or_xlsx[1].row_values(0)] else: header = [value.strip() for value in next(csv_or_xlsx)] - columns["timestamp_column"] = header.index(mapping.timestamp_column) - columns["currency_column"] = ( - header.index(mapping.currency_column) if mapping.currency_column else None - ) - columns["amount_column"] = header.index(mapping.amount_column) - columns["balance_column"] = ( - header.index(mapping.balance_column) if mapping.balance_column else None - ) - columns["original_currency_column"] = ( - header.index(mapping.original_currency_column) - if mapping.original_currency_column - else None - ) - columns["original_amount_column"] = ( - header.index(mapping.original_amount_column) - if mapping.original_amount_column - else None - ) - columns["debit_credit_column"] = ( - header.index(mapping.debit_credit_column) - if mapping.debit_credit_column - else None - ) - columns["transaction_id_column"] = ( - header.index(mapping.transaction_id_column) - if mapping.transaction_id_column - else None - ) - columns["description_column"] = ( - header.index(mapping.description_column) - if mapping.description_column - else None - ) - columns["notes_column"] = ( - header.index(mapping.notes_column) if mapping.notes_column else None - ) - columns["reference_column"] = ( - header.index(mapping.reference_column) if mapping.reference_column else None - ) - columns["partner_name_column"] = ( - header.index(mapping.partner_name_column) - if mapping.partner_name_column - else None - ) - columns["bank_name_column"] = ( - header.index(mapping.bank_name_column) if mapping.bank_name_column else None - ) - columns["bank_account_column"] = ( - header.index(mapping.bank_account_column) - if mapping.bank_account_column - else None - ) + for column_name in self._get_column_names(): + columns[column_name] = self._get_column_indexes( + header, column_name, mapping + ) return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + def _get_values_from_column(self, values, columns, column_name): + indexes = columns[column_name] + content_l = [] + max_index = len(values) - 1 + for index in indexes: + if isinstance(index, int) and index <= max_index: + content_l.append(values[index]) + else: + content_l.append(values[index]) + if all(isinstance(content, str) for content in content_l): + return " ".join(content_l) + return content_l[0] + def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 if isinstance(csv_or_xlsx, tuple): rows = range(1, csv_or_xlsx[1].nrows) @@ -182,66 +201,70 @@ class AccountBankStatementImportSheetParser(models.TransientModel): else: values = list(row) - timestamp = values[columns["timestamp_column"]] + timestamp = self._get_values_from_column( + values, columns, "timestamp_column" + ) currency = ( - values[columns["currency_column"]] - if columns["currency_column"] is not None + self._get_values_from_column(values, columns, "currency_column") + if columns["currency_column"] else currency_code ) - amount = values[columns["amount_column"]] + amount = self._get_values_from_column(values, columns, "amount_column") balance = ( - values[columns["balance_column"]] - if columns["balance_column"] is not None + self._get_values_from_column(values, columns, "balance_column") + if columns["balance_column"] else None ) original_currency = ( - values[columns["original_currency_column"]] - if columns["original_currency_column"] is not None + self._get_values_from_column( + values, columns, "original_currency_column" + ) + if columns["original_currency_column"] else None ) original_amount = ( - values[columns["original_amount_column"]] - if columns["original_amount_column"] is not None + self._get_values_from_column(values, columns, "original_amount_column") + if columns["original_amount_column"] else None ) debit_credit = ( - values[columns["debit_credit_column"]] - if columns["debit_credit_column"] is not None + self._get_values_from_column(values, columns, "debit_credit_column") + if columns["debit_credit_column"] else None ) transaction_id = ( - values[columns["transaction_id_column"]] - if columns["transaction_id_column"] is not None + self._get_values_from_column(values, columns, "transaction_id_column") + if columns["transaction_id_column"] else None ) description = ( - values[columns["description_column"]] - if columns["description_column"] is not None + self._get_values_from_column(values, columns, "description_column") + if columns["description_column"] else None ) notes = ( - values[columns["notes_column"]] - if columns["notes_column"] is not None + self._get_values_from_column(values, columns, "notes_column") + if columns["notes_column"] else None ) reference = ( - values[columns["reference_column"]] - if columns["reference_column"] is not None + self._get_values_from_column(values, columns, "reference_column") + if columns["reference_column"] else None ) partner_name = ( - values[columns["partner_name_column"]] - if columns["partner_name_column"] is not None + self._get_values_from_column(values, columns, "partner_name_column") + if columns["partner_name_column"] else None ) bank_name = ( - values[columns["bank_name_column"]] - if columns["bank_name_column"] is not None + self._get_values_from_column(values, columns, "bank_name_column") + if columns["bank_name_column"] else None ) bank_account = ( - values[columns["bank_account_column"]] - if columns["bank_account_column"] is not None + self._get_values_from_column(values, columns, "bank_account_column") + if columns["bank_account_column"] else None ) diff --git a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml index 5caa5b12..299eac72 100644 --- a/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml +++ b/account_bank_statement_import_txt_xlsx/views/account_bank_statement_import_sheet_mapping.xml @@ -39,6 +39,18 @@ + + + + @@ -53,20 +65,31 @@ - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index d925fe08..75851065 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ # generated from manifests external_dependencies +chardet cryptography ofxparse xlrd