From 0a0d55d4db9247a39ae4625d4fbc9dab4983300e Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Fri, 16 Dec 2022 11:09:18 +0100 Subject: [PATCH] [IMP] account_statement_import_txt_xlsx: Allow mapping with reference to column numbers, and concatenation A new field 'File does not contain header line' is added in the Statement Sheet Mappings. If you set to True, then you can map the columns by indicating in each field of the 'Columns' section the column number in the file. We also allow to concatenate multiple columns in the file to a single column to a single field of the statement line. You have to indicate the names of the columns separated by comma. --- .../__manifest__.py | 2 +- .../models/account_statement_import.py | 4 +- .../account_statement_import_sheet_mapping.py | 5 + .../account_statement_import_sheet_parser.py | 212 ++++++++++-------- ...account_statement_import_sheet_mapping.xml | 55 +++-- requirements.txt | 1 + 6 files changed, 165 insertions(+), 114 deletions(-) diff --git a/account_statement_import_txt_xlsx/__manifest__.py b/account_statement_import_txt_xlsx/__manifest__.py index bbf9d439..6403a216 100644 --- a/account_statement_import_txt_xlsx/__manifest__.py +++ b/account_statement_import_txt_xlsx/__manifest__.py @@ -17,7 +17,7 @@ "multi_step_wizard", "web_widget_dropdown_dynamic", ], - "external_dependencies": {"python": ["xlrd"]}, + "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ "security/ir.model.access.csv", "data/map_data.xml", diff --git a/account_statement_import_txt_xlsx/models/account_statement_import.py b/account_statement_import_txt_xlsx/models/account_statement_import.py index be961fb1..0ef50771 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import.py @@ -28,7 +28,9 @@ class AccountStatementImport(models.TransientModel): self.ensure_one() try: Parser = self.env["account.statement.import.sheet.parser"] - return Parser.parse(data_file, self.sheet_mapping_id) + return Parser.parse( + data_file, self.sheet_mapping_id, self.statement_filename + ) except BaseException: if self.env.context.get("account_statement_import_txt_xlsx_test"): raise diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index c3295b62..58a7820c 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -56,6 +56,11 @@ class AccountStatementImportSheetMapping(models.Model): ) quotechar = fields.Char(string="Text qualifier", size=1, default='"') timestamp_format = fields.Char(string="Timestamp Format", required=True) + no_header = fields.Boolean( + "File does not contain header line", + help="When this occurs please indicate the column number in the Columns section " + "instead of the column name, considering that the first column is 0", + ) timestamp_column = fields.Char(string="Timestamp column", required=True) currency_column = fields.Char( string="Currency column", diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 044d81f2..19b59f8c 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -7,8 +7,10 @@ import logging from datetime import datetime from decimal import Decimal from io import StringIO +from os import path from odoo import _, api, models +from odoo.exceptions import UserError _logger = logging.getLogger(__name__) @@ -20,6 +22,14 @@ try: except (ImportError, IOError) as err: # pragma: no cover _logger.error(err) +try: + import chardet +except ImportError: + _logger.warning( + "chardet library not found, please install it " + "from http://pypi.python.org/pypi/chardet" + ) + class AccountStatementImportSheetParser(models.TransientModel): _name = "account.statement.import.sheet.parser" @@ -43,7 +53,7 @@ class AccountStatementImportSheetParser(models.TransientModel): return list(next(csv_data)) @api.model - def parse(self, data_file, mapping): + def parse(self, data_file, mapping, filename): journal = self.env["account.journal"].browse(self.env.context.get("journal_id")) currency_code = (journal.currency_id or journal.company_id.currency_id).name account_number = journal.bank_account_id.acc_number @@ -67,6 +77,11 @@ class AccountStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), + "name": _("%s: %s") + % ( + journal.code, + path.basename(filename), + ), } ) @@ -79,6 +94,45 @@ class AccountStatementImportSheetParser(models.TransientModel): return currency_code, account_number, [data] + def _get_column_indexes(self, header, column_name, mapping): + column_indexes = [] + if mapping[column_name] and "," in mapping[column_name]: + # We have to concatenate the values + column_names_or_indexes = mapping[column_name].split(",") + else: + column_names_or_indexes = [mapping[column_name]] + for column_name_or_index in column_names_or_indexes: + if mapping.no_header: + column_index = ( + column_name_or_index and int(column_name_or_index) or None + ) + if column_index: + column_indexes.append(column_index) + else: + if column_name_or_index: + column_indexes.append(header.index(column_name_or_index)) + return column_indexes + + def _get_column_names(self): + return [ + "timestamp_column", + "currency_column", + "amount_column", + "amount_debit_column", + "amount_credit_column", + "balance_column", + "original_currency_column", + "original_amount_column", + "debit_credit_column", + "transaction_id_column", + "description_column", + "notes_column", + "reference_column", + "partner_name_column", + "bank_name_column", + "bank_account_column", + ] + def _parse_lines(self, mapping, data_file, currency_code): columns = dict() try: @@ -99,81 +153,40 @@ class AccountStatementImportSheetParser(models.TransientModel): csv_options["delimiter"] = csv_delimiter if mapping.quotechar: csv_options["quotechar"] = mapping.quotechar - csv_or_xlsx = reader( - StringIO(data_file.decode(mapping.file_encoding or "utf-8")), - **csv_options - ) - + try: + decoded_file = data_file.decode(mapping.file_encoding or "utf-8") + except UnicodeDecodeError: + # Try auto guessing the format + detected_encoding = chardet.detect(data_file).get("encoding", False) + if not detected_encoding: + raise UserError( + _("No valid encoding was found for the attached file") + ) + decoded_file = data_file.decode(detected_encoding) + csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) if isinstance(csv_or_xlsx, tuple): header = [str(value) for value in csv_or_xlsx[1].row_values(0)] else: header = [value.strip() for value in next(csv_or_xlsx)] - columns["timestamp_column"] = header.index(mapping.timestamp_column) - columns["currency_column"] = ( - header.index(mapping.currency_column) if mapping.currency_column else None - ) - columns["amount_column"] = ( - header.index(mapping.amount_column) if mapping.amount_column else None - ) - columns["amount_debit_column"] = ( - header.index(mapping.amount_debit_column) - if mapping.amount_debit_column - else None - ) - columns["amount_credit_column"] = ( - header.index(mapping.amount_credit_column) - if mapping.amount_credit_column - else None - ) - columns["balance_column"] = ( - header.index(mapping.balance_column) if mapping.balance_column else None - ) - columns["original_currency_column"] = ( - header.index(mapping.original_currency_column) - if mapping.original_currency_column - else None - ) - columns["original_amount_column"] = ( - header.index(mapping.original_amount_column) - if mapping.original_amount_column - else None - ) - columns["debit_credit_column"] = ( - header.index(mapping.debit_credit_column) - if mapping.debit_credit_column - else None - ) - columns["transaction_id_column"] = ( - header.index(mapping.transaction_id_column) - if mapping.transaction_id_column - else None - ) - columns["description_column"] = ( - header.index(mapping.description_column) - if mapping.description_column - else None - ) - columns["notes_column"] = ( - header.index(mapping.notes_column) if mapping.notes_column else None - ) - columns["reference_column"] = ( - header.index(mapping.reference_column) if mapping.reference_column else None - ) - columns["partner_name_column"] = ( - header.index(mapping.partner_name_column) - if mapping.partner_name_column - else None - ) - columns["bank_name_column"] = ( - header.index(mapping.bank_name_column) if mapping.bank_name_column else None - ) - columns["bank_account_column"] = ( - header.index(mapping.bank_account_column) - if mapping.bank_account_column - else None - ) + for column_name in self._get_column_names(): + columns[column_name] = self._get_column_indexes( + header, column_name, mapping + ) return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + def _get_values_from_column(self, values, columns, column_name): + indexes = columns[column_name] + content_l = [] + max_index = len(values) - 1 + for index in indexes: + if isinstance(index, int) and index <= max_index: + content_l.append(values[index]) + else: + content_l.append(values[index]) + if all(isinstance(content, str) for content in content_l): + return " ".join(content_l) + return content_l[0] + def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 if isinstance(csv_or_xlsx, tuple): rows = range(1, csv_or_xlsx[1].nrows) @@ -195,16 +208,21 @@ class AccountStatementImportSheetParser(models.TransientModel): else: values = list(row) - timestamp = values[columns["timestamp_column"]] + timestamp = self._get_values_from_column( + values, columns, "timestamp_column" + ) currency = ( - values[columns["currency_column"]] - if columns["currency_column"] is not None + self._get_values_from_column(values, columns, "currency_column") + if columns["currency_column"] else currency_code ) def _decimal(column_name): if columns[column_name]: - return self._parse_decimal(values[columns[column_name]], mapping) + return self._parse_decimal( + self._get_values_from_column(values, columns, column_name), + mapping, + ) amount = _decimal("amount_column") if not amount: @@ -213,58 +231,60 @@ class AccountStatementImportSheetParser(models.TransientModel): amount = -abs(_decimal("amount_credit_column") or 0) balance = ( - values[columns["balance_column"]] - if columns["balance_column"] is not None + self._get_values_from_column(values, columns, "balance_column") + if columns["balance_column"] else None ) original_currency = ( - values[columns["original_currency_column"]] - if columns["original_currency_column"] is not None + self._get_values_from_column( + values, columns, "original_currency_column" + ) + if columns["original_currency_column"] else None ) original_amount = ( - values[columns["original_amount_column"]] - if columns["original_amount_column"] is not None + self._get_values_from_column(values, columns, "original_amount_column") + if columns["original_amount_column"] else None ) debit_credit = ( - values[columns["debit_credit_column"]] - if columns["debit_credit_column"] is not None + self._get_values_from_column(values, columns, "debit_credit_column") + if columns["debit_credit_column"] else None ) transaction_id = ( - values[columns["transaction_id_column"]] - if columns["transaction_id_column"] is not None + self._get_values_from_column(values, columns, "transaction_id_column") + if columns["transaction_id_column"] else None ) description = ( - values[columns["description_column"]] - if columns["description_column"] is not None + self._get_values_from_column(values, columns, "description_column") + if columns["description_column"] else None ) notes = ( - values[columns["notes_column"]] - if columns["notes_column"] is not None + self._get_values_from_column(values, columns, "notes_column") + if columns["notes_column"] else None ) reference = ( - values[columns["reference_column"]] - if columns["reference_column"] is not None + self._get_values_from_column(values, columns, "reference_column") + if columns["reference_column"] else None ) partner_name = ( - values[columns["partner_name_column"]] - if columns["partner_name_column"] is not None + self._get_values_from_column(values, columns, "partner_name_column") + if columns["partner_name_column"] else None ) bank_name = ( - values[columns["bank_name_column"]] - if columns["bank_name_column"] is not None + self._get_values_from_column(values, columns, "bank_name_column") + if columns["bank_name_column"] else None ) bank_account = ( - values[columns["bank_account_column"]] - if columns["bank_account_column"] is not None + self._get_values_from_column(values, columns, "bank_account_column") + if columns["bank_account_column"] else None ) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 32409bb0..62d617dd 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -39,6 +39,18 @@ + + + + @@ -53,22 +65,33 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index 6fff9265..d3433a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # generated from manifests external_dependencies +chardet ofxparse xlrd