From 9df67995671b2f47b91d591e47ebf38af3115f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miquel=20Ra=C3=AFch?= Date: Wed, 22 Feb 2023 18:30:26 +0100 Subject: [PATCH 1/7] [FIX] account_statement_import_online: not require odoo_test_helper --- account_statement_import_online/__manifest__.py | 1 - requirements.txt | 1 - test-requirements.txt | 1 + 3 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 test-requirements.txt diff --git a/account_statement_import_online/__manifest__.py b/account_statement_import_online/__manifest__.py index ab9a6d2b..eb80f843 100644 --- a/account_statement_import_online/__manifest__.py +++ b/account_statement_import_online/__manifest__.py @@ -11,7 +11,6 @@ "license": "AGPL-3", "category": "Accounting", "summary": "Online bank statements update", - "external_dependencies": {"python": ["odoo_test_helper"]}, "depends": [ "account", "account_statement_import", diff --git a/requirements.txt b/requirements.txt index 4318b1c6..542d351c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,2 @@ # generated from manifests external_dependencies -odoo_test_helper xlrd diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 00000000..66bc2cba --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1 @@ +odoo_test_helper From 60b4066b8aac143a0a1597a8bf20bfde5a3371d7 Mon Sep 17 00:00:00 2001 From: NuriaMForgeFlow Date: Wed, 11 Aug 2021 16:13:55 +0200 Subject: [PATCH 2/7] [FIX] account_bank_statement_import_txt_xlsx: import sheet notes --- .../models/account_statement_import_sheet_parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 61571545..31233a09 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -359,9 +359,11 @@ class AccountStatementImportSheetParser(models.TransientModel): if transaction_id: note += _("Transaction ID: %s; ") % (transaction_id,) if note and notes: - note = "{}\n{}".format(note, note.strip()) + note = "{}\n{}".format(notes, note.strip()) elif note: note = note.strip() + elif notes: + note = notes if note: transaction["narration"] = note From cdb5dcfb7a8240dce87ad37b5b2bed03f6ac7032 Mon Sep 17 00:00:00 2001 From: Henrik Norlin Date: Sun, 3 Apr 2022 15:06:07 +0200 Subject: [PATCH 3/7] [IMP] account_statement_import_txt_xlsx: wizard: 2 amount columns, for IN and OUT [IMP] account_statement_import_txt_xlsx: wizard: amount2_reverse boolean field, with tests --- .../account_statement_import_sheet_mapping.py | 8 +++ .../account_statement_import_sheet_parser.py | 12 +++- .../tests/fixtures/amount2.csv | 3 + .../test_account_statement_import_txt_xlsx.py | 55 +++++++++++++++++++ ...account_statement_import_sheet_mapping.xml | 5 ++ ...t_statement_import_sheet_mapping_wizard.py | 8 +++ ..._statement_import_sheet_mapping_wizard.xml | 6 ++ 7 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 account_statement_import_txt_xlsx/tests/fixtures/amount2.csv diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index 70d11b5d..5a14e293 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -66,6 +66,14 @@ class AccountStatementImportSheetMapping(models.Model): required=True, help="Amount of transaction in journal's currency", ) + amount2_column = fields.Char( + string="Amount2 column", + help="Some statements have two amount columns, for IN and OUT", + ) + amount2_reverse = fields.Boolean( + string="Amount2 reverse +/-", + help="If there are positive numbers for money going OUT, reverse +/-", + ) balance_column = fields.Char( help="Balance after transaction in journal's currency", ) diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 31233a09..df2bbffc 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -113,6 +113,9 @@ class AccountStatementImportSheetParser(models.TransientModel): header.index(mapping.currency_column) if mapping.currency_column else None ) columns["amount_column"] = header.index(mapping.amount_column) + columns["amount2_column"] = ( + header.index(mapping.amount2_column) if mapping.amount2_column else None + ) columns["balance_column"] = ( header.index(mapping.balance_column) if mapping.balance_column else None ) @@ -189,7 +192,13 @@ class AccountStatementImportSheetParser(models.TransientModel): if columns["currency_column"] is not None else currency_code ) - amount = values[columns["amount_column"]] + factor = -1 if mapping["amount2_reverse"] else 1 + if values[columns["amount_column"]] in (False, "", "0.00"): + amount = values[columns["amount2_column"]] + amount = factor * self._parse_decimal(amount, mapping) + else: + amount = values[columns.get("amount_column")] + amount = self._parse_decimal(amount, mapping) balance = ( values[columns["balance_column"]] if columns["balance_column"] is not None @@ -252,7 +261,6 @@ class AccountStatementImportSheetParser(models.TransientModel): if isinstance(timestamp, str): timestamp = datetime.strptime(timestamp, mapping.timestamp_format) - amount = self._parse_decimal(amount, mapping) if balance: balance = self._parse_decimal(balance, mapping) else: diff --git a/account_statement_import_txt_xlsx/tests/fixtures/amount2.csv b/account_statement_import_txt_xlsx/tests/fixtures/amount2.csv new file mode 100644 index 00000000..9bff8eec --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/amount2.csv @@ -0,0 +1,3 @@ +"Date","Label","Amount","Amount2","Balance","Partner Name","Bank Account" +"12/15/2018","Your best supplier","0.00","33.50","-23.50","John Doe","123456789" +"12/15/2018","Your payment","1,533.50","0.00","1,510.00","Azure Interior","" diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py index 0ffecd93..0a28dad9 100644 --- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py @@ -31,6 +31,15 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): self.AccountStatementImportSheetMappingWizard = self.env[ "account.statement.import.sheet.mapping.wizard" ] + self.suspense_account = self.env["account.account"].create( + { + "code": "987654", + "name": "Suspense Account", + "user_type_id": self.env.ref( + "account.data_account_type_current_assets" + ).id, + } + ) def _data_file(self, filename, encoding=None): mode = "rt" if encoding else "rb" @@ -47,6 +56,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) data = self._data_file("fixtures/sample_statement_en.csv", "utf-8") @@ -71,6 +81,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) data = self._data_file("fixtures/empty_statement_en.csv", "utf-8") @@ -95,6 +106,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) data = self._data_file("fixtures/sample_statement_en.xlsx") @@ -119,6 +131,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) data = self._data_file("fixtures/empty_statement_en.xlsx") @@ -189,6 +202,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) data = self._data_file("fixtures/original_currency.csv", "utf-8") @@ -219,6 +233,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) data = self._data_file("fixtures/original_currency_empty.csv", "utf-8") @@ -247,6 +262,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) statement_map = self.sample_statement_map.copy( @@ -282,6 +298,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) statement_map = self.sample_statement_map.copy( @@ -316,6 +333,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): "type": "bank", "code": "BANK", "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, } ) statement_map = self.sample_statement_map.copy( @@ -345,3 +363,40 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): self.assertEqual(statement.balance_start, 10.0) self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) + + def test_amount2(self): + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + statement_map = self.sample_statement_map.copy( + { + "amount2_column": "Amount2", + "amount2_reverse": True, + "balance_column": "Balance", + "original_currency_column": None, + "original_amount_column": None, + } + ) + data = self._data_file("fixtures/amount2.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/amount2.csv", + "statement_file": data, + "sheet_mapping_id": statement_map.id, + } + ) + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 2) + self.assertEqual(statement.balance_start, 10.0) + self.assertEqual(statement.balance_end_real, 1510.0) + self.assertEqual(statement.balance_end, 1510.0) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 8f9718b5..9981063b 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -38,6 +38,10 @@ + + diff --git a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py index 47f4eeab..3b84ba94 100644 --- a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py +++ b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py @@ -42,6 +42,14 @@ class AccountStatementImportSheetMappingWizard(models.TransientModel): amount_column = fields.Char( help="Amount of transaction in journal's currency", ) + amount2_column = fields.Char( + string="Amount2 column", + help="Some statements have two amount columns, for IN and OUT", + ) + amount2_reverse = fields.Boolean( + string="Amount2 reverse +/-", + help="If there are positive numbers for money going OUT, reverse +/-", + ) balance_column = fields.Char( help="Balance after transaction in journal's currency", ) diff --git a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml index 51097816..9ebadc38 100644 --- a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml +++ b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml @@ -52,6 +52,12 @@ context="{'header': header}" attrs="{'required': [('state', '=', 'final')]}" /> + Date: Tue, 20 Dec 2022 11:06:19 +0100 Subject: [PATCH 4/7] [IMP] account_statement_import_txt_xlsx: amount_debit_column, amount_credit_column --- .../migrations/14.0.1.0.0/pre-migrate.py | 40 ------------------ .../migrations/15.0.1.1.0/pre-migrate.py | 42 +++++++++++++++++++ .../account_statement_import_sheet_mapping.py | 26 ++++++++---- .../account_statement_import_sheet_parser.py | 33 ++++++++++----- .../tests/fixtures/amount2.csv | 3 -- .../tests/fixtures/debit_credit_amount.csv | 5 +++ .../test_account_statement_import_txt_xlsx.py | 13 +++--- ...account_statement_import_sheet_mapping.xml | 7 +--- ...t_statement_import_sheet_mapping_wizard.py | 12 +++--- ..._statement_import_sheet_mapping_wizard.xml | 9 +++- 10 files changed, 111 insertions(+), 79 deletions(-) delete mode 100644 account_statement_import_txt_xlsx/migrations/14.0.1.0.0/pre-migrate.py create mode 100644 account_statement_import_txt_xlsx/migrations/15.0.1.1.0/pre-migrate.py delete mode 100644 account_statement_import_txt_xlsx/tests/fixtures/amount2.csv create mode 100644 account_statement_import_txt_xlsx/tests/fixtures/debit_credit_amount.csv diff --git a/account_statement_import_txt_xlsx/migrations/14.0.1.0.0/pre-migrate.py b/account_statement_import_txt_xlsx/migrations/14.0.1.0.0/pre-migrate.py deleted file mode 100644 index 6bef441e..00000000 --- a/account_statement_import_txt_xlsx/migrations/14.0.1.0.0/pre-migrate.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2021 Tecnativa - Carlos Roca -# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). - -from openupgradelib import openupgrade - -_model_renames = [ - ( - "account.bank.statement.import.sheet.mapping", - "account.statement.import.sheet.mapping", - ), - ( - "account.bank.statement.import.sheet.parser", - "account.statement.import.sheet.parser", - ), - ( - "account.bank.statement.import.sheet.mapping.wizard", - "account.statement.import.sheet.mapping.wizard", - ), -] - -_table_renames = [ - ( - "account_bank_statement_import_sheet_mapping", - "account_statement_import_sheet_mapping", - ), - ( - "account_bank_statement_import_sheet_parser", - "account_statement_import_sheet_parser", - ), - ( - "account_bank_statement_import_sheet_mapping_wizard", - "account_statement_import_sheet_mapping_wizard", - ), -] - - -@openupgrade.migrate() -def migrate(env, version): - openupgrade.rename_models(env.cr, _model_renames) - openupgrade.rename_tables(env.cr, _table_renames) diff --git a/account_statement_import_txt_xlsx/migrations/15.0.1.1.0/pre-migrate.py b/account_statement_import_txt_xlsx/migrations/15.0.1.1.0/pre-migrate.py new file mode 100644 index 00000000..e610de6f --- /dev/null +++ b/account_statement_import_txt_xlsx/migrations/15.0.1.1.0/pre-migrate.py @@ -0,0 +1,42 @@ +# Copyright 2022 AppsToGROW - Henrik Norlin +# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl). + +from openupgradelib import openupgrade + +_fields_to_add = [ + ( + "amount_debit_column", + "account.statement.import.sheet.mapping", + "account_statement_import_sheet_mapping", + "char", + "varchar", + "account_statement_import_txt_xlsx", + ), + ( + "amount_credit_column", + "account.statement.import.sheet.mapping", + "account_statement_import_sheet_mapping", + "char", + "varchar", + "account_statement_import_txt_xlsx", + ), +] + + +def add_fields_and_drop_not_null(env): + cr = env.cr + sql_debit_exists = """SELECT count(id) FROM ir_model_fields + WHERE name = 'amount_debit_column' + AND model = 'account.statement.import.sheet.mapping';""" + cr.execute(sql_debit_exists) + if cr.fetchone()[0] > 0: + openupgrade.add_fields(env, _fields_to_add) + cr.execute( + """ALTER TABLE account_statement_import_sheet_mapping + ALTER COLUMN amount_column DROP NOT NULL;""" + ) + + +@openupgrade.migrate() +def migrate(env, version): + add_fields_and_drop_not_null(env) diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index 5a14e293..cd75449d 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -63,16 +63,15 @@ class AccountStatementImportSheetMapping(models.Model): ), ) amount_column = fields.Char( - required=True, help="Amount of transaction in journal's currency", ) - amount2_column = fields.Char( - string="Amount2 column", - help="Some statements have two amount columns, for IN and OUT", + amount_debit_column = fields.Char( + string="Debit amount column", + help="Debit amount of transaction in journal's currency", ) - amount2_reverse = fields.Boolean( - string="Amount2 reverse +/-", - help="If there are positive numbers for money going OUT, reverse +/-", + amount_credit_column = fields.Char( + string="Credit amount column", + help="Credit amount of transaction in journal's currency", ) balance_column = fields.Char( help="Balance after transaction in journal's currency", @@ -120,6 +119,19 @@ class AccountStatementImportSheetMapping(models.Model): help="Partner's bank account", ) + _sql_constraints = [ + ( + "check_amount_columns", + ( + "CHECK(" + "amount_column IS NULL " + "OR (amount_debit_column IS NULL AND amount_credit_column IS NULL)" + ")" + ), + "Use amount_column OR (amount_debit_column AND amount_credit_column).", + ), + ] + @api.onchange("float_thousands_sep") def onchange_thousands_separator(self): if "dot" == self.float_thousands_sep == self.float_decimal_sep: diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index df2bbffc..dd76c88f 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -112,9 +112,18 @@ class AccountStatementImportSheetParser(models.TransientModel): columns["currency_column"] = ( header.index(mapping.currency_column) if mapping.currency_column else None ) - columns["amount_column"] = header.index(mapping.amount_column) - columns["amount2_column"] = ( - header.index(mapping.amount2_column) if mapping.amount2_column else None + columns["amount_column"] = ( + header.index(mapping.amount_column) if mapping.amount_column else None + ) + columns["amount_debit_column"] = ( + header.index(mapping.amount_debit_column) + if mapping.amount_debit_column + else None + ) + columns["amount_credit_column"] = ( + header.index(mapping.amount_credit_column) + if mapping.amount_credit_column + else None ) columns["balance_column"] = ( header.index(mapping.balance_column) if mapping.balance_column else None @@ -192,13 +201,17 @@ class AccountStatementImportSheetParser(models.TransientModel): if columns["currency_column"] is not None else currency_code ) - factor = -1 if mapping["amount2_reverse"] else 1 - if values[columns["amount_column"]] in (False, "", "0.00"): - amount = values[columns["amount2_column"]] - amount = factor * self._parse_decimal(amount, mapping) - else: - amount = values[columns.get("amount_column")] - amount = self._parse_decimal(amount, mapping) + + def _decimal(column_name): + if columns[column_name]: + return self._parse_decimal(values[columns[column_name]], mapping) + + amount = _decimal("amount_column") + if not amount: + amount = abs(_decimal("amount_debit_column") or 0) + if not amount: + amount = -abs(_decimal("amount_credit_column") or 0) + balance = ( values[columns["balance_column"]] if columns["balance_column"] is not None diff --git a/account_statement_import_txt_xlsx/tests/fixtures/amount2.csv b/account_statement_import_txt_xlsx/tests/fixtures/amount2.csv deleted file mode 100644 index 9bff8eec..00000000 --- a/account_statement_import_txt_xlsx/tests/fixtures/amount2.csv +++ /dev/null @@ -1,3 +0,0 @@ -"Date","Label","Amount","Amount2","Balance","Partner Name","Bank Account" -"12/15/2018","Your best supplier","0.00","33.50","-23.50","John Doe","123456789" -"12/15/2018","Your payment","1,533.50","0.00","1,510.00","Azure Interior","" diff --git a/account_statement_import_txt_xlsx/tests/fixtures/debit_credit_amount.csv b/account_statement_import_txt_xlsx/tests/fixtures/debit_credit_amount.csv new file mode 100644 index 00000000..f7e8e75a --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/debit_credit_amount.csv @@ -0,0 +1,5 @@ +"Date","Label","Debit","Credit","Balance","Partner Name","Bank Account" +"12/15/2018","Credit 20.00","0.00","20.00","-10.00","John Doe","123456789" +"12/15/2018","Credit 13.50","0.00","-13.50","-23.50","John Doe","123456789" +"12/15/2018","Debit 33.50","-33.50","0.00","10.00","Azure Interior","" +"12/15/2018","Debit 1500","1,500.00","0.00","1,510.00","Azure Interior","" diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py index 0a28dad9..90de8c68 100644 --- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py @@ -364,7 +364,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) - def test_amount2(self): + def test_debit_credit_amount(self): journal = self.AccountJournal.create( { "name": "Bank", @@ -376,17 +376,18 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): ) statement_map = self.sample_statement_map.copy( { - "amount2_column": "Amount2", - "amount2_reverse": True, + "amount_debit_column": "Debit", + "amount_credit_column": "Credit", "balance_column": "Balance", + "amount_column": None, "original_currency_column": None, "original_amount_column": None, } ) - data = self._data_file("fixtures/amount2.csv", "utf-8") + data = self._data_file("fixtures/debit_credit_amount.csv", "utf-8") wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( { - "statement_filename": "fixtures/amount2.csv", + "statement_filename": "fixtures/debit_credit_amount.csv", "statement_file": data, "sheet_mapping_id": statement_map.id, } @@ -396,7 +397,7 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): ).import_file_button() statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) self.assertEqual(len(statement), 1) - self.assertEqual(len(statement.line_ids), 2) + self.assertEqual(len(statement.line_ids), 4) self.assertEqual(statement.balance_start, 10.0) self.assertEqual(statement.balance_end_real, 1510.0) self.assertEqual(statement.balance_end, 1510.0) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 9981063b..32409bb0 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -38,10 +38,6 @@ - - + + diff --git a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py index 3b84ba94..d1314edb 100644 --- a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py +++ b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.py @@ -42,13 +42,13 @@ class AccountStatementImportSheetMappingWizard(models.TransientModel): amount_column = fields.Char( help="Amount of transaction in journal's currency", ) - amount2_column = fields.Char( - string="Amount2 column", - help="Some statements have two amount columns, for IN and OUT", + amount_debit_column = fields.Char( + string="Debit amount column", + help="Debit amount of transaction in journal's currency", ) - amount2_reverse = fields.Boolean( - string="Amount2 reverse +/-", - help="If there are positive numbers for money going OUT, reverse +/-", + amount_credit_column = fields.Boolean( + string="Credit amount column", + help="Credit amount of transaction in journal's currency", ) balance_column = fields.Char( help="Balance after transaction in journal's currency", diff --git a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml index 9ebadc38..dc5021a7 100644 --- a/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml +++ b/account_statement_import_txt_xlsx/wizards/account_statement_import_sheet_mapping_wizard.xml @@ -50,10 +50,15 @@ widget="dynamic_dropdown" values="statement_columns" context="{'header': header}" - attrs="{'required': [('state', '=', 'final')]}" /> + Date: Fri, 16 Dec 2022 11:09:18 +0100 Subject: [PATCH 5/7] [IMP] account_statement_import_txt_xlsx: Allow mapping with reference to column numbers, and concatenation A new field 'File does not contain header line' is added in the Statement Sheet Mappings. If you set to True, then you can map the columns by indicating in each field of the 'Columns' section the column number in the file. We also allow to concatenate multiple columns in the file to a single column to a single field of the statement line. You have to indicate the names of the columns separated by comma. --- .../__manifest__.py | 2 +- .../models/account_statement_import.py | 4 +- .../account_statement_import_sheet_mapping.py | 5 + .../account_statement_import_sheet_parser.py | 212 ++++++++++-------- ...account_statement_import_sheet_mapping.xml | 55 +++-- requirements.txt | 1 + 6 files changed, 165 insertions(+), 114 deletions(-) diff --git a/account_statement_import_txt_xlsx/__manifest__.py b/account_statement_import_txt_xlsx/__manifest__.py index 3dbd1590..2b158e8a 100644 --- a/account_statement_import_txt_xlsx/__manifest__.py +++ b/account_statement_import_txt_xlsx/__manifest__.py @@ -17,7 +17,7 @@ "multi_step_wizard", "web_widget_dropdown_dynamic", ], - "external_dependencies": {"python": ["xlrd"]}, + "external_dependencies": {"python": ["xlrd", "chardet"]}, "data": [ "security/ir.model.access.csv", "data/map_data.xml", diff --git a/account_statement_import_txt_xlsx/models/account_statement_import.py b/account_statement_import_txt_xlsx/models/account_statement_import.py index be961fb1..0ef50771 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import.py @@ -28,7 +28,9 @@ class AccountStatementImport(models.TransientModel): self.ensure_one() try: Parser = self.env["account.statement.import.sheet.parser"] - return Parser.parse(data_file, self.sheet_mapping_id) + return Parser.parse( + data_file, self.sheet_mapping_id, self.statement_filename + ) except BaseException: if self.env.context.get("account_statement_import_txt_xlsx_test"): raise diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py index cd75449d..caefa7ac 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py @@ -55,6 +55,11 @@ class AccountStatementImportSheetMapping(models.Model): ) quotechar = fields.Char(string="Text qualifier", size=1, default='"') timestamp_format = fields.Char(required=True) + no_header = fields.Boolean( + string="File does not contain header line", + help="When this occurs please indicate the column number in the Columns section " + "instead of the column name, considering that the first column is 0", + ) timestamp_column = fields.Char(required=True) currency_column = fields.Char( help=( diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index dd76c88f..719a815f 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -7,8 +7,10 @@ import logging from datetime import datetime from decimal import Decimal from io import StringIO +from os import path from odoo import _, api, models +from odoo.exceptions import UserError _logger = logging.getLogger(__name__) @@ -20,6 +22,14 @@ try: except (ImportError, IOError) as err: # pragma: no cover _logger.error(err) +try: + import chardet +except ImportError: + _logger.warning( + "chardet library not found, please install it " + "from http://pypi.python.org/pypi/chardet" + ) + class AccountStatementImportSheetParser(models.TransientModel): _name = "account.statement.import.sheet.parser" @@ -43,7 +53,7 @@ class AccountStatementImportSheetParser(models.TransientModel): return list(next(csv_data)) @api.model - def parse(self, data_file, mapping): + def parse(self, data_file, mapping, filename): journal = self.env["account.journal"].browse(self.env.context.get("journal_id")) currency_code = (journal.currency_id or journal.company_id.currency_id).name account_number = journal.bank_account_id.acc_number @@ -67,6 +77,11 @@ class AccountStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), + "name": _("%s: %s") + % ( + journal.code, + path.basename(filename), + ), } ) @@ -79,6 +94,45 @@ class AccountStatementImportSheetParser(models.TransientModel): return currency_code, account_number, [data] + def _get_column_indexes(self, header, column_name, mapping): + column_indexes = [] + if mapping[column_name] and "," in mapping[column_name]: + # We have to concatenate the values + column_names_or_indexes = mapping[column_name].split(",") + else: + column_names_or_indexes = [mapping[column_name]] + for column_name_or_index in column_names_or_indexes: + if mapping.no_header: + column_index = ( + column_name_or_index and int(column_name_or_index) or None + ) + if column_index: + column_indexes.append(column_index) + else: + if column_name_or_index: + column_indexes.append(header.index(column_name_or_index)) + return column_indexes + + def _get_column_names(self): + return [ + "timestamp_column", + "currency_column", + "amount_column", + "amount_debit_column", + "amount_credit_column", + "balance_column", + "original_currency_column", + "original_amount_column", + "debit_credit_column", + "transaction_id_column", + "description_column", + "notes_column", + "reference_column", + "partner_name_column", + "bank_name_column", + "bank_account_column", + ] + def _parse_lines(self, mapping, data_file, currency_code): columns = dict() try: @@ -99,81 +153,40 @@ class AccountStatementImportSheetParser(models.TransientModel): csv_options["delimiter"] = csv_delimiter if mapping.quotechar: csv_options["quotechar"] = mapping.quotechar - csv_or_xlsx = reader( - StringIO(data_file.decode(mapping.file_encoding or "utf-8")), - **csv_options - ) - + try: + decoded_file = data_file.decode(mapping.file_encoding or "utf-8") + except UnicodeDecodeError: + # Try auto guessing the format + detected_encoding = chardet.detect(data_file).get("encoding", False) + if not detected_encoding: + raise UserError( + _("No valid encoding was found for the attached file") + ) from None + decoded_file = data_file.decode(detected_encoding) + csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) if isinstance(csv_or_xlsx, tuple): header = [str(value) for value in csv_or_xlsx[1].row_values(0)] else: header = [value.strip() for value in next(csv_or_xlsx)] - columns["timestamp_column"] = header.index(mapping.timestamp_column) - columns["currency_column"] = ( - header.index(mapping.currency_column) if mapping.currency_column else None - ) - columns["amount_column"] = ( - header.index(mapping.amount_column) if mapping.amount_column else None - ) - columns["amount_debit_column"] = ( - header.index(mapping.amount_debit_column) - if mapping.amount_debit_column - else None - ) - columns["amount_credit_column"] = ( - header.index(mapping.amount_credit_column) - if mapping.amount_credit_column - else None - ) - columns["balance_column"] = ( - header.index(mapping.balance_column) if mapping.balance_column else None - ) - columns["original_currency_column"] = ( - header.index(mapping.original_currency_column) - if mapping.original_currency_column - else None - ) - columns["original_amount_column"] = ( - header.index(mapping.original_amount_column) - if mapping.original_amount_column - else None - ) - columns["debit_credit_column"] = ( - header.index(mapping.debit_credit_column) - if mapping.debit_credit_column - else None - ) - columns["transaction_id_column"] = ( - header.index(mapping.transaction_id_column) - if mapping.transaction_id_column - else None - ) - columns["description_column"] = ( - header.index(mapping.description_column) - if mapping.description_column - else None - ) - columns["notes_column"] = ( - header.index(mapping.notes_column) if mapping.notes_column else None - ) - columns["reference_column"] = ( - header.index(mapping.reference_column) if mapping.reference_column else None - ) - columns["partner_name_column"] = ( - header.index(mapping.partner_name_column) - if mapping.partner_name_column - else None - ) - columns["bank_name_column"] = ( - header.index(mapping.bank_name_column) if mapping.bank_name_column else None - ) - columns["bank_account_column"] = ( - header.index(mapping.bank_account_column) - if mapping.bank_account_column - else None - ) + for column_name in self._get_column_names(): + columns[column_name] = self._get_column_indexes( + header, column_name, mapping + ) return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns) + def _get_values_from_column(self, values, columns, column_name): + indexes = columns[column_name] + content_l = [] + max_index = len(values) - 1 + for index in indexes: + if isinstance(index, int) and index <= max_index: + content_l.append(values[index]) + else: + content_l.append(values[index]) + if all(isinstance(content, str) for content in content_l): + return " ".join(content_l) + return content_l[0] + def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901 if isinstance(csv_or_xlsx, tuple): rows = range(1, csv_or_xlsx[1].nrows) @@ -195,16 +208,21 @@ class AccountStatementImportSheetParser(models.TransientModel): else: values = list(row) - timestamp = values[columns["timestamp_column"]] + timestamp = self._get_values_from_column( + values, columns, "timestamp_column" + ) currency = ( - values[columns["currency_column"]] - if columns["currency_column"] is not None + self._get_values_from_column(values, columns, "currency_column") + if columns["currency_column"] else currency_code ) def _decimal(column_name): if columns[column_name]: - return self._parse_decimal(values[columns[column_name]], mapping) + return self._parse_decimal( + self._get_values_from_column(values, columns, column_name), + mapping, + ) amount = _decimal("amount_column") if not amount: @@ -213,58 +231,60 @@ class AccountStatementImportSheetParser(models.TransientModel): amount = -abs(_decimal("amount_credit_column") or 0) balance = ( - values[columns["balance_column"]] - if columns["balance_column"] is not None + self._get_values_from_column(values, columns, "balance_column") + if columns["balance_column"] else None ) original_currency = ( - values[columns["original_currency_column"]] - if columns["original_currency_column"] is not None + self._get_values_from_column( + values, columns, "original_currency_column" + ) + if columns["original_currency_column"] else None ) original_amount = ( - values[columns["original_amount_column"]] - if columns["original_amount_column"] is not None + self._get_values_from_column(values, columns, "original_amount_column") + if columns["original_amount_column"] else None ) debit_credit = ( - values[columns["debit_credit_column"]] - if columns["debit_credit_column"] is not None + self._get_values_from_column(values, columns, "debit_credit_column") + if columns["debit_credit_column"] else None ) transaction_id = ( - values[columns["transaction_id_column"]] - if columns["transaction_id_column"] is not None + self._get_values_from_column(values, columns, "transaction_id_column") + if columns["transaction_id_column"] else None ) description = ( - values[columns["description_column"]] - if columns["description_column"] is not None + self._get_values_from_column(values, columns, "description_column") + if columns["description_column"] else None ) notes = ( - values[columns["notes_column"]] - if columns["notes_column"] is not None + self._get_values_from_column(values, columns, "notes_column") + if columns["notes_column"] else None ) reference = ( - values[columns["reference_column"]] - if columns["reference_column"] is not None + self._get_values_from_column(values, columns, "reference_column") + if columns["reference_column"] else None ) partner_name = ( - values[columns["partner_name_column"]] - if columns["partner_name_column"] is not None + self._get_values_from_column(values, columns, "partner_name_column") + if columns["partner_name_column"] else None ) bank_name = ( - values[columns["bank_name_column"]] - if columns["bank_name_column"] is not None + self._get_values_from_column(values, columns, "bank_name_column") + if columns["bank_name_column"] else None ) bank_account = ( - values[columns["bank_account_column"]] - if columns["bank_account_column"] is not None + self._get_values_from_column(values, columns, "bank_account_column") + if columns["bank_account_column"] else None ) diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml index 32409bb0..62d617dd 100644 --- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml +++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml @@ -39,6 +39,18 @@ + + + + @@ -53,22 +65,33 @@ - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/requirements.txt b/requirements.txt index 542d351c..5a2df807 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ # generated from manifests external_dependencies +chardet xlrd From ceee58af90ae03149b203f94de26c098de30fa31 Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Sun, 18 Dec 2022 14:29:39 +0100 Subject: [PATCH 6/7] [IMP] account_statement_import_txt_xlsx: add tests --- .../account_statement_import_sheet_parser.py | 31 +++++++----- .../fixtures/original_currency_no_header.csv | 1 + .../test_account_statement_import_txt_xlsx.py | 49 +++++++++++++++++++ 3 files changed, 70 insertions(+), 11 deletions(-) create mode 100644 account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 719a815f..2f9ba8af 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -102,11 +102,16 @@ class AccountStatementImportSheetParser(models.TransientModel): else: column_names_or_indexes = [mapping[column_name]] for column_name_or_index in column_names_or_indexes: + if not column_name_or_index: + continue + column_index = None if mapping.no_header: - column_index = ( - column_name_or_index and int(column_name_or_index) or None - ) - if column_index: + try: + column_index = int(column_name_or_index) + # pylint: disable=except-pass + except Exception: + pass + if column_index is not None: column_indexes.append(column_index) else: if column_name_or_index: @@ -164,10 +169,12 @@ class AccountStatementImportSheetParser(models.TransientModel): ) from None decoded_file = data_file.decode(detected_encoding) csv_or_xlsx = reader(StringIO(decoded_file), **csv_options) - if isinstance(csv_or_xlsx, tuple): - header = [str(value) for value in csv_or_xlsx[1].row_values(0)] - else: - header = [value.strip() for value in next(csv_or_xlsx)] + header = False + if not mapping.no_header: + if isinstance(csv_or_xlsx, tuple): + header = [str(value) for value in csv_or_xlsx[1].row_values(0)] + else: + header = [value.strip() for value in next(csv_or_xlsx)] for column_name in self._get_column_names(): columns[column_name] = self._get_column_indexes( header, column_name, mapping @@ -179,10 +186,12 @@ class AccountStatementImportSheetParser(models.TransientModel): content_l = [] max_index = len(values) - 1 for index in indexes: - if isinstance(index, int) and index <= max_index: - content_l.append(values[index]) + if isinstance(index, int): + if index <= max_index: + content_l.append(values[index]) else: - content_l.append(values[index]) + if index in values: + content_l.append(values[index]) if all(isinstance(content, str) for content in content_l): return " ".join(content_l) return content_l[0] diff --git a/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv b/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv new file mode 100644 index 00000000..4e91582f --- /dev/null +++ b/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv @@ -0,0 +1 @@ +"12/15/2018","Your payment","EUR","1,525.00","-1,000.00","Azure Interior","","INV0001" diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py index 90de8c68..71595b87 100644 --- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py +++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py @@ -226,6 +226,55 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase): self.assertEqual(line.foreign_currency_id, self.currency_eur) self.assertEqual(line.amount_currency, 1000.0) + def test_original_currency_no_header(self): + no_header_statement_map = self.AccountStatementImportSheetMapping.create( + { + "name": "Sample Statement", + "float_thousands_sep": "comma", + "float_decimal_sep": "dot", + "delimiter": "comma", + "quotechar": '"', + "timestamp_format": "%m/%d/%Y", + "no_header": True, + "timestamp_column": "0", + "amount_column": "3", + "original_currency_column": "2", + "original_amount_column": "4", + "description_column": "1,7", + "partner_name_column": "5", + "bank_account_column": "6", + } + ) + journal = self.AccountJournal.create( + { + "name": "Bank", + "type": "bank", + "code": "BANK", + "currency_id": self.currency_usd.id, + "suspense_account_id": self.suspense_account.id, + } + ) + data = self._data_file("fixtures/original_currency_no_header.csv", "utf-8") + wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create( + { + "statement_filename": "fixtures/original_currency.csv", + "statement_file": data, + "sheet_mapping_id": no_header_statement_map.id, + } + ) + wizard.with_context( + account_statement_import_txt_xlsx_test=True + ).import_file_button() + statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)]) + self.assertEqual(len(statement), 1) + self.assertEqual(len(statement.line_ids), 1) + + line = statement.line_ids + self.assertEqual(line.currency_id, self.currency_usd) + self.assertEqual(line.foreign_currency_id, self.currency_eur) + self.assertEqual(line.amount_currency, 1000.0) + self.assertEqual(line.payment_ref, "Your payment INV0001") + def test_original_currency_empty(self): journal = self.AccountJournal.create( { From dbe5c35ed80b8296b9b41796c4a45cbc1a600b5c Mon Sep 17 00:00:00 2001 From: Jordi Ballester Alomar Date: Wed, 22 Feb 2023 17:44:24 +0100 Subject: [PATCH 7/7] [IMP] account_statement_import_txt_xlsx: Calculate final balance if not provided in the import file --- .../models/account_statement_import.py | 10 ++++++++++ .../models/account_statement_import_sheet_parser.py | 11 +++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/account_statement_import_txt_xlsx/models/account_statement_import.py b/account_statement_import_txt_xlsx/models/account_statement_import.py index 0ef50771..6f74228d 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import.py @@ -36,3 +36,13 @@ class AccountStatementImport(models.TransientModel): raise _logger.warning("Sheet parser error", exc_info=True) return super()._parse_file(data_file) + + def _create_bank_statements(self, stmts_vals, result): + """Set balance_end_real if not already provided by the file.""" + res = super()._create_bank_statements(stmts_vals, result) + statements = self.env["account.bank.statement"].browse(result["statement_ids"]) + for statement in statements: + if not statement.balance_end_real: + amount = sum(statement.line_ids.mapped("amount")) + statement.balance_end_real = statement.balance_start + amount + return res diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py index 2f9ba8af..deee5fd0 100644 --- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py +++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py @@ -67,6 +67,11 @@ class AccountStatementImportSheetParser(models.TransientModel): last_line = lines[-1] data = { "date": first_line["timestamp"].date(), + "name": _("%(code)s: %(filename)s") + % { + "code": journal.code, + "filename": path.basename(filename), + }, } if mapping.balance_column: @@ -77,14 +82,8 @@ class AccountStatementImportSheetParser(models.TransientModel): { "balance_start": float(balance_start), "balance_end_real": float(balance_end), - "name": _("%s: %s") - % ( - journal.code, - path.basename(filename), - ), } ) - transactions = list( itertools.chain.from_iterable( map(lambda line: self._convert_line_to_transactions(line), lines)