From a11ca1ec8aae70ce1e5d7af1dbd5073ff5d271c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miquel=20Ra=C3=AFch?= Date: Wed, 22 Feb 2023 18:30:26 +0100 Subject: [PATCH 1/4] [FIX] account_statement_import_online: not require odoo_test_helper --- account_statement_import_online/__manifest__.py | 1 - requirements.txt | 1 - test-requirements.txt | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 test-requirements.txt diff --git a/account_statement_import_online/__manifest__.py b/account_statement_import_online/__manifest__.py index 15505f31..17ffdf72 100644 --- a/account_statement_import_online/__manifest__.py +++ b/account_statement_import_online/__manifest__.py @@ -12,7 +12,6 @@ "license": "AGPL-3", "category": "Accounting", "summary": "Online bank statements update", - "external_dependencies": {"python": ["odoo_test_helper"]}, "depends": [ "account_statement_import_base", "web_widget_dropdown_dynamic", diff --git a/requirements.txt b/requirements.txt index 962f3242..6fff9265 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ # generated from manifests external_dependencies -odoo_test_helper ofxparse xlrd diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..66bc2cba --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +odoo_test_helper From 0a0d55d4db9247a39ae4625d4fbc9dab4983300e Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Fri, 16 Dec 2022 11:09:18 +0100 Subject: [PATCH 2/4] [IMP] account_statement_import_txt_xlsx: Allow mapping with reference to column numbers, and concatenation A new field 'File does not contain header line' is added in the Statement Sheet Mappings. If you set to True, then you can map the columns by indicating in each field of the 'Columns' section the column number in the file. We also allow to concatenate multiple columns in the file to a single column to a single field of the statement line. You have to indicate the names of the columns separated by comma. --- .../__manifest__.py | 2 +- .../models/account_statement_import.py | 4 +- .../account_statement_import_sheet_mapping.py | 5 + .../account_statement_import_sheet_parser.py | 212 ++++++++++-------- ...account_statement_import_sheet_mapping.xml | 55 +++-- requirements.txt | 1 + 6 files changed, 165 insertions(+), 114 deletions(-) diff --git a/account_statement_import_txt_xlsx/__manifest__.py b/account_statement_import_txt_xlsx/__manifest__.py index bbf9d439..6403a216 100644 --- a/account_statement_import_txt_xlsx/__manifest__.py +++ b/account_statement_import_txt_xlsx/__manifest__.py @@ -17,7 +17,7 @@ "multi_step_wizard", "web_widget_dropdown_dynamic", ], - "external_dependencies": {"python": ["xlrd"]}, + "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ "security/ir.model.access.csv", "data/map_data.xml", diff --git a/account_statement_import_txt_xlsx/models/account_statement_import.py b/account_statement_import_txt_xlsx/models/account_statement_import.py index be961fb1..0ef50771 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import.py @@ -28,7 +28,9 @@ class AccountStatementImport(models.TransientModel): self.ensure_one() try: Parser = self.env["account.statement.import.sheet.parser"] - return Parser.parse(data_file, self.sheet_mapping_id) + return Parser.parse( + data_file, self.sheet_mapping_id, self.statement_filename + ) except BaseException: if self.env.context.get("account_statement_import_txt_xlsx_test"): raise diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index c3295b62..58a7820c 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -56,6 +56,11 @@ class AccountStatementImportSheetMapping(models.Model): ) quotechar = fields.Char(string="Text qualifier", size=1, default='"') timestamp_format = fields.Char(string="Timestamp Format", required=True) + no_header = fields.Boolean( + "File does not contain header line", + help="When this occurs please indicate the column number in the Columns section " + "instead of the column name, considering that the first column is 0", + ) timestamp_column = fields.Char(string="Timestamp column", required=True) currency_column = fields.Char( string="Currency column", diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 044d81f2..19b59f8c 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -7,8 +7,10 @@ import logging from datetime import datetime from decimal import Decimal from io import StringIO +from os import path from odoo import _, api, models +from odoo.exceptions import UserError _logger = logging.getLogger(__name__) @@ -20,6 +22,14 @@ try: except (ImportError, IOError) as err: # pragma: no cover _logger.error(err) +try: + import chardet +except ImportError: + _logger.warning( + "chardet library not found, please install it " + "from http://pypi.python.org/pypi/chardet" + ) + class AccountStatementImportSheetParser(models.TransientModel): _name = "account.statement.import.sheet.parser" @@ -43,7 +53,7 @@ class AccountStatementImportSheetParser(models.TransientModel): return list(next(csv_data)) @api.model - def parse(self, data_file, mapping): + def parse(self, data_file, mapping, filename): journal = self.env["account.journal"].browse(self.env.context.get("journal_id")) currency_code = (journal.currency_id or journal.company_id.currency_id).name account_number = journal.bank_account_id.acc_number @@ -67,6 +77,11 @@ class AccountStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), + "name": _("%s: %s") + % ( + journal.code, + path.basename(filename), + ), } ) @@ -79,6 +94,45 @@ class AccountStatementImportSheetParser(models.TransientModel): return currency_code, account_number, [data] + def _get_column_indexes(self, header, column_name, mapping): + column_indexes = [] + if mapping[column_name] and "," in mapping[column_name]: + # We have to concatenate the values + column_names_or_indexes = mapping[column_name].split(",") + else: + column_names_or_indexes = [mapping[column_name]] + for column_name_or_index in column_names_or_indexes: + if mapping.no_header: + column_index = ( + column_name_or_index and int(column_name_or_index) or None + ) + if column_index: + column_indexes.append(column_index) + else: + if column_name_or_index: + column_indexes.append(header.index(column_name_or_index)) + return column_indexes + + def _get_column_names(self): + return [ + "timestamp_column", + "currency_column", + "amount_column", + "amount_debit_column", + "amount_credit_column", + "balance_column", + "original_currency_column", + "original_amount_column", + "debit_credit_column", + "transaction_id_column", + "description_column", + "notes_column", + "reference_column", + "partner_name_column", + "bank_name_column", + "bank_account_column", + ] + def _parse_lines(self, mapping, data_file, currency_code): columns = dict() try: @@ -99,81 +153,40 @@ class AccountStatementImportSheetParser(models.TransientModel): csv_options["delimiter"] = csv_delimiter if mapping.quotechar: csv_options["quotechar"] = mapping.quotechar - csv_or_xlsx = reader( - StringIO(data_file.decode(mapping.file_encoding or "utf-8")), - **csv_options - ) - + try: + decoded_file = data_file.decode(mapping.file_encoding or "utf-8") + except UnicodeDecodeError: + # Try auto guessing the format + detected_encoding = chardet.detect(data_file).get("encoding", False) + if not detected_encoding: + raise UserError( + _("No valid encoding was found for the attached file") + ) + decoded_file = data_file.decode(detected_encoding) + csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) if isinstance(csv_or_xlsx, tuple): header = [str(value) for value in csv_or_xlsx[1].row_values(0)] else: header = [value.strip() for value in next(csv_or_xlsx)] - columns["timestamp_column"] = header.index(mapping.timestamp_column) - columns["currency_column"] = ( - header.index(mapping.currency_column) if mapping.currency_column else None - ) - columns["amount_column"] = ( - header.index(mapping.amount_column) if mapping.amount_column else None - ) - columns["amount_debit_column"] = ( - header.index(mapping.amount_debit_column) - if mapping.amount_debit_column - else None - ) - columns["amount_credit_column"] = ( - header.index(mapping.amount_credit_column) - if mapping.amount_credit_column - else None - ) - columns["balance_column"] = ( - header.index(mapping.balance_column) if mapping.balance_column else None - ) - columns["original_currency_column"] = ( - header.index(mapping.original_currency_column) - if mapping.original_currency_column - else None - ) - columns["original_amount_column"] = ( - header.index(mapping.original_amount_column) - if mapping.original_amount_column - else None - ) - columns["debit_credit_column"] = ( - header.index(mapping.debit_credit_column) - if mapping.debit_credit_column - else None - ) - columns["transaction_id_column"] = ( - header.index(mapping.transaction_id_column) - if mapping.transaction_id_column - else None - ) - columns["description_column"] = ( - header.index(mapping.description_column) - if mapping.description_column - else None - ) - columns["notes_column"] = ( - header.index(mapping.notes_column) if mapping.notes_column else None - ) - columns["reference_column"] = ( - header.index(mapping.reference_column) if mapping.reference_column else None - ) - columns["partner_name_column"] = ( - header.index(mapping.partner_name_column) - if mapping.partner_name_column - else None - ) - columns["bank_name_column"] = ( - header.index(mapping.bank_name_column) if mapping.bank_name_column else None - ) - columns["bank_account_column"] = ( - header.index(mapping.bank_account_column) - if mapping.bank_account_column - else None - ) + for column_name in self._get_column_names(): + columns[column_name] = self._get_column_indexes( + header, column_name, mapping + ) return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + def _get_values_from_column(self, values, columns, column_name): + indexes = columns[column_name] + content_l = [] + max_index = len(values) - 1 + for index in indexes: + if isinstance(index, int) and index <= max_index: + content_l.append(values[index]) + else: + content_l.append(values[index]) + if all(isinstance(content, str) for content in content_l): + return " ".join(content_l) + return content_l[0] + def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 if isinstance(csv_or_xlsx, tuple): rows = range(1, csv_or_xlsx[1].nrows) @@ -195,16 +208,21 @@ class AccountStatementImportSheetParser(models.TransientModel): else: values = list(row) - timestamp = values[columns["timestamp_column"]] + timestamp = self._get_values_from_column( + values, columns, "timestamp_column" + ) currency = ( - values[columns["currency_column"]] - if columns["currency_column"] is not None + self._get_values_from_column(values, columns, "currency_column") + if columns["currency_column"] else currency_code ) def _decimal(column_name): if columns[column_name]: - return self._parse_decimal(values[columns[column_name]], mapping) + return self._parse_decimal( + self._get_values_from_column(values, columns, column_name), + mapping, + ) amount = _decimal("amount_column") if not amount: @@ -213,58 +231,60 @@ class AccountStatementImportSheetParser(models.TransientModel): amount = -abs(_decimal("amount_credit_column") or 0) balance = ( - values[columns["balance_column"]] - if columns["balance_column"] is not None + self._get_values_from_column(values, columns, "balance_column") + if columns["balance_column"] else None ) original_currency = ( - values[columns["original_currency_column"]] - if columns["original_currency_column"] is not None + self._get_values_from_column( + values, columns, "original_currency_column" + ) + if columns["original_currency_column"] else None ) original_amount = ( - values[columns["original_amount_column"]] - if columns["original_amount_column"] is not None + self._get_values_from_column(values, columns, "original_amount_column") + if columns["original_amount_column"] else None ) debit_credit = ( - values[columns["debit_credit_column"]] - if columns["debit_credit_column"] is not None + self._get_values_from_column(values, columns, "debit_credit_column") + if columns["debit_credit_column"] else None ) transaction_id = ( - values[columns["transaction_id_column"]] - if columns["transaction_id_column"] is not None + self._get_values_from_column(values, columns, "transaction_id_column") + if columns["transaction_id_column"] else None ) description = ( - values[columns["description_column"]] - if columns["description_column"] is not None + self._get_values_from_column(values, columns, "description_column") + if columns["description_column"] else None ) notes = ( - values[columns["notes_column"]] - if columns["notes_column"] is not None + self._get_values_from_column(values, columns, "notes_column") + if columns["notes_column"] else None ) reference = ( - values[columns["reference_column"]] - if columns["reference_column"] is not None + self._get_values_from_column(values, columns, "reference_column") + if columns["reference_column"] else None ) partner_name = ( - values[columns["partner_name_column"]] - if columns["partner_name_column"] is not None + self._get_values_from_column(values, columns, "partner_name_column") + if columns["partner_name_column"] else None ) bank_name = ( - values[columns["bank_name_column"]] - if columns["bank_name_column"] is not None + self._get_values_from_column(values, columns, "bank_name_column") + if columns["bank_name_column"] else None ) bank_account = ( - values[columns["bank_account_column"]] - if columns["bank_account_column"] is not None + self._get_values_from_column(values, columns, "bank_account_column") + if columns["bank_account_column"] else None ) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 32409bb0..62d617dd 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -39,6 +39,18 @@ + + + + @@ -53,22 +65,33 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index 6fff9265..d3433a96 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ # generated from manifests external_dependencies +chardet ofxparse xlrd From 84c1823fa1e962b20bcf105e912e50d6ffbedfc2 Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Sun, 18 Dec 2022 14:29:39 +0100 Subject: [PATCH 3/4] [IMP] account_statement_import_txt_xlsx: add tests --- .../account_statement_import_sheet_parser.py | 30 +++++++----- .../fixtures/original_currency_no_header.csv | 1 + .../test_account_statement_import_txt_xlsx.py | 49 +++++++++++++++++++ 3 files changed, 69 insertions(+), 11 deletions(-) create mode 100644 account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 19b59f8c..4d480378 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -102,11 +102,15 @@ class AccountStatementImportSheetParser(models.TransientModel): else: column_names_or_indexes = [mapping[column_name]] for column_name_or_index in column_names_or_indexes: + if not column_name_or_index: + continue + column_index = None if mapping.no_header: - column_index = ( - column_name_or_index and int(column_name_or_index) or None - ) - if column_index: + try: + column_index = int(column_name_or_index) + except Exception: + pass + if column_index is not None: column_indexes.append(column_index) else: if column_name_or_index: @@ -164,10 +168,12 @@ class AccountStatementImportSheetParser(models.TransientModel): ) decoded_file = data_file.decode(detected_encoding) csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) - if isinstance(csv_or_xlsx, tuple): - header = [str(value) for value in csv_or_xlsx[1].row_values(0)] - else: - header = [value.strip() for value in next(csv_or_xlsx)] + header = False + if not mapping.no_header: + if isinstance(csv_or_xlsx, tuple): + header = [str(value) for value in csv_or_xlsx[1].row_values(0)] + else: + header = [value.strip() for value in next(csv_or_xlsx)] for column_name in self._get_column_names(): columns[column_name] = self._get_column_indexes( header, column_name, mapping @@ -179,10 +185,12 @@ class AccountStatementImportSheetParser(models.TransientModel): content_l = [] max_index = len(values) - 1 for index in indexes: - if isinstance(index, int) and index <= max_index: - content_l.append(values[index]) + if isinstance(index, int): + if index <= max_index: + content_l.append(values[index]) else: - content_l.append(values[index]) + if index in values: + content_l.append(values[index]) if all(isinstance(content, str) for content in content_l): return " ".join(content_l) return content_l[0] diff --git a/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv b/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv new file mode 100644 index 00000000..4e91582f --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv @@ -0,0 +1 @@ +"12/15/2018","Your payment","EUR","1,525.00","-1,000.00","Azure Interior","","INV0001" diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py index f9586125..ca8b6f11 100644 --- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py @@ -224,6 +224,55 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): self.assertEqual(line.foreign_currency_id, self.currency_eur) self.assertEqual(line.amount_currency, 1000.0) + def test_original_currency_no_header(self): + no_header_statement_map = self.AccountStatementImportSheetMapping.create( + { + "name": "Sample Statement", + "float_thousands_sep": "comma", + "float_decimal_sep": "dot", + "delimiter": "comma", + "quotechar": '"', + "timestamp_format": "%m/%d/%Y", + "no_header": True, + "timestamp_column": "0", + "amount_column": "3", + "original_currency_column": "2", + "original_amount_column": "4", + "description_column": "1,7", + "partner_name_column": "5", + "bank_account_column": "6", + } + ) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/original_currency_no_header.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/original_currency.csv", + "statement_file": data, + "sheet_mapping_id": no_header_statement_map.id, + } + ) + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 1) + + line = statement.line_ids + self.assertEqual(line.currency_id, self.currency_usd) + self.assertEqual(line.foreign_currency_id, self.currency_eur) + self.assertEqual(line.amount_currency, 1000.0) + self.assertEqual(line.payment_ref, "Your payment INV0001") + def test_original_currency_empty(self): journal = self.AccountJournal.create( { From 07e47a83b84fd5e6e57df3d6fb9156eee39ba6c4 Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Wed, 22 Feb 2023 17:44:24 +0100 Subject: [PATCH 4/4] [IMP] account_statement_import_txt_xlsx: Calculate final balance if not provided in the import file --- .../models/account_statement_import.py | 10 ++++++++++ .../models/account_statement_import_sheet_parser.py | 11 +++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/account_statement_import_txt_xlsx/models/account_statement_import.py b/account_statement_import_txt_xlsx/models/account_statement_import.py index 0ef50771..6f74228d 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import.py @@ -36,3 +36,13 @@ class AccountStatementImport(models.TransientModel): raise _logger.warning("Sheet parser error", exc_info=True) return super()._parse_file(data_file) + + def _create_bank_statements(self, stmts_vals, result): + """Set balance_end_real if not already provided by the file.""" + res = super()._create_bank_statements(stmts_vals, result) + statements = self.env["account.bank.statement"].browse(result["statement_ids"]) + for statement in statements: + if not statement.balance_end_real: + amount = sum(statement.line_ids.mapped("amount")) + statement.balance_end_real = statement.balance_start + amount + return res diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 4d480378..b629da86 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -67,6 +67,11 @@ class AccountStatementImportSheetParser(models.TransientModel): last_line = lines[-1] data = { "date": first_line["timestamp"].date(), + "name": _("%s: %s") + % ( + journal.code, + path.basename(filename), + ), } if mapping.balance_column: @@ -77,14 +82,8 @@ class AccountStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), - "name": _("%s: %s") - % ( - journal.code, - path.basename(filename), - ), } ) - transactions = list( itertools.chain.from_iterable( map(lambda line: self._convert_line_to_transactions(line), lines)