diff --git a/account_statement_import_online/__manifest__.py b/account_statement_import_online/__manifest__.py
index 15505f31..17ffdf72 100644
--- a/account_statement_import_online/__manifest__.py
+++ b/account_statement_import_online/__manifest__.py
@@ -12,7 +12,6 @@
"license": "AGPL-3",
"category": "Accounting",
"summary": "Online bank statements update",
- "external_dependencies": {"python": ["odoo_test_helper"]},
"depends": [
"account_statement_import_base",
"web_widget_dropdown_dynamic",
diff --git a/account_statement_import_txt_xlsx/__manifest__.py b/account_statement_import_txt_xlsx/__manifest__.py
index bbf9d439..6403a216 100644
--- a/account_statement_import_txt_xlsx/__manifest__.py
+++ b/account_statement_import_txt_xlsx/__manifest__.py
@@ -17,7 +17,7 @@
"multi_step_wizard",
"web_widget_dropdown_dynamic",
],
- "external_dependencies": {"python": ["xlrd"]},
+ "external_dependencies": {"python": ["xlrd", "chardet"]},
"data": [
"security/ir.model.access.csv",
"data/map_data.xml",
diff --git a/account_statement_import_txt_xlsx/models/account_statement_import.py b/account_statement_import_txt_xlsx/models/account_statement_import.py
index be961fb1..6f74228d 100644
--- a/account_statement_import_txt_xlsx/models/account_statement_import.py
+++ b/account_statement_import_txt_xlsx/models/account_statement_import.py
@@ -28,9 +28,21 @@ class AccountStatementImport(models.TransientModel):
self.ensure_one()
try:
Parser = self.env["account.statement.import.sheet.parser"]
- return Parser.parse(data_file, self.sheet_mapping_id)
+ return Parser.parse(
+ data_file, self.sheet_mapping_id, self.statement_filename
+ )
except BaseException:
if self.env.context.get("account_statement_import_txt_xlsx_test"):
raise
_logger.warning("Sheet parser error", exc_info=True)
return super()._parse_file(data_file)
+
+ def _create_bank_statements(self, stmts_vals, result):
+ """Set balance_end_real if not already provided by the file."""
+ res = super()._create_bank_statements(stmts_vals, result)
+ statements = self.env["account.bank.statement"].browse(result["statement_ids"])
+ for statement in statements:
+ if not statement.balance_end_real:
+ amount = sum(statement.line_ids.mapped("amount"))
+ statement.balance_end_real = statement.balance_start + amount
+ return res
diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py
index c3295b62..58a7820c 100644
--- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py
+++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py
@@ -56,6 +56,11 @@ class AccountStatementImportSheetMapping(models.Model):
)
quotechar = fields.Char(string="Text qualifier", size=1, default='"')
timestamp_format = fields.Char(string="Timestamp Format", required=True)
+ no_header = fields.Boolean(
+ "File does not contain header line",
+ help="When this occurs please indicate the column number in the Columns section "
+ "instead of the column name, considering that the first column is 0",
+ )
timestamp_column = fields.Char(string="Timestamp column", required=True)
currency_column = fields.Char(
string="Currency column",
diff --git a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py
index 044d81f2..b629da86 100644
--- a/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py
+++ b/account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py
@@ -7,8 +7,10 @@ import logging
from datetime import datetime
from decimal import Decimal
from io import StringIO
+from os import path
from odoo import _, api, models
+from odoo.exceptions import UserError
_logger = logging.getLogger(__name__)
@@ -20,6 +22,14 @@ try:
except (ImportError, IOError) as err: # pragma: no cover
_logger.error(err)
+try:
+ import chardet
+except ImportError:
+ _logger.warning(
+ "chardet library not found, please install it "
+ "from http://pypi.python.org/pypi/chardet"
+ )
+
class AccountStatementImportSheetParser(models.TransientModel):
_name = "account.statement.import.sheet.parser"
@@ -43,7 +53,7 @@ class AccountStatementImportSheetParser(models.TransientModel):
return list(next(csv_data))
@api.model
- def parse(self, data_file, mapping):
+ def parse(self, data_file, mapping, filename):
journal = self.env["account.journal"].browse(self.env.context.get("journal_id"))
currency_code = (journal.currency_id or journal.company_id.currency_id).name
account_number = journal.bank_account_id.acc_number
@@ -57,6 +67,11 @@ class AccountStatementImportSheetParser(models.TransientModel):
last_line = lines[-1]
data = {
"date": first_line["timestamp"].date(),
+ "name": _("%s: %s")
+ % (
+ journal.code,
+ path.basename(filename),
+ ),
}
if mapping.balance_column:
@@ -69,7 +84,6 @@ class AccountStatementImportSheetParser(models.TransientModel):
"balance_end_real": float(balance_end),
}
)
-
transactions = list(
itertools.chain.from_iterable(
map(lambda line: self._convert_line_to_transactions(line), lines)
@@ -79,6 +93,49 @@ class AccountStatementImportSheetParser(models.TransientModel):
return currency_code, account_number, [data]
+ def _get_column_indexes(self, header, column_name, mapping):
+ column_indexes = []
+ if mapping[column_name] and "," in mapping[column_name]:
+ # We have to concatenate the values
+ column_names_or_indexes = mapping[column_name].split(",")
+ else:
+ column_names_or_indexes = [mapping[column_name]]
+ for column_name_or_index in column_names_or_indexes:
+ if not column_name_or_index:
+ continue
+ column_index = None
+ if mapping.no_header:
+ try:
+ column_index = int(column_name_or_index)
+ except Exception:
+ pass
+ if column_index is not None:
+ column_indexes.append(column_index)
+ else:
+ if column_name_or_index:
+ column_indexes.append(header.index(column_name_or_index))
+ return column_indexes
+
+ def _get_column_names(self):
+ return [
+ "timestamp_column",
+ "currency_column",
+ "amount_column",
+ "amount_debit_column",
+ "amount_credit_column",
+ "balance_column",
+ "original_currency_column",
+ "original_amount_column",
+ "debit_credit_column",
+ "transaction_id_column",
+ "description_column",
+ "notes_column",
+ "reference_column",
+ "partner_name_column",
+ "bank_name_column",
+ "bank_account_column",
+ ]
+
def _parse_lines(self, mapping, data_file, currency_code):
columns = dict()
try:
@@ -99,81 +156,44 @@ class AccountStatementImportSheetParser(models.TransientModel):
csv_options["delimiter"] = csv_delimiter
if mapping.quotechar:
csv_options["quotechar"] = mapping.quotechar
- csv_or_xlsx = reader(
- StringIO(data_file.decode(mapping.file_encoding or "utf-8")),
- **csv_options
+ try:
+ decoded_file = data_file.decode(mapping.file_encoding or "utf-8")
+ except UnicodeDecodeError:
+ # Try auto guessing the format
+ detected_encoding = chardet.detect(data_file).get("encoding", False)
+ if not detected_encoding:
+ raise UserError(
+ _("No valid encoding was found for the attached file")
+ )
+ decoded_file = data_file.decode(detected_encoding)
+ csv_or_xlsx = reader(StringIO(decoded_file), **csv_options)
+ header = False
+ if not mapping.no_header:
+ if isinstance(csv_or_xlsx, tuple):
+ header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
+ else:
+ header = [value.strip() for value in next(csv_or_xlsx)]
+ for column_name in self._get_column_names():
+ columns[column_name] = self._get_column_indexes(
+ header, column_name, mapping
)
-
- if isinstance(csv_or_xlsx, tuple):
- header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
- else:
- header = [value.strip() for value in next(csv_or_xlsx)]
- columns["timestamp_column"] = header.index(mapping.timestamp_column)
- columns["currency_column"] = (
- header.index(mapping.currency_column) if mapping.currency_column else None
- )
- columns["amount_column"] = (
- header.index(mapping.amount_column) if mapping.amount_column else None
- )
- columns["amount_debit_column"] = (
- header.index(mapping.amount_debit_column)
- if mapping.amount_debit_column
- else None
- )
- columns["amount_credit_column"] = (
- header.index(mapping.amount_credit_column)
- if mapping.amount_credit_column
- else None
- )
- columns["balance_column"] = (
- header.index(mapping.balance_column) if mapping.balance_column else None
- )
- columns["original_currency_column"] = (
- header.index(mapping.original_currency_column)
- if mapping.original_currency_column
- else None
- )
- columns["original_amount_column"] = (
- header.index(mapping.original_amount_column)
- if mapping.original_amount_column
- else None
- )
- columns["debit_credit_column"] = (
- header.index(mapping.debit_credit_column)
- if mapping.debit_credit_column
- else None
- )
- columns["transaction_id_column"] = (
- header.index(mapping.transaction_id_column)
- if mapping.transaction_id_column
- else None
- )
- columns["description_column"] = (
- header.index(mapping.description_column)
- if mapping.description_column
- else None
- )
- columns["notes_column"] = (
- header.index(mapping.notes_column) if mapping.notes_column else None
- )
- columns["reference_column"] = (
- header.index(mapping.reference_column) if mapping.reference_column else None
- )
- columns["partner_name_column"] = (
- header.index(mapping.partner_name_column)
- if mapping.partner_name_column
- else None
- )
- columns["bank_name_column"] = (
- header.index(mapping.bank_name_column) if mapping.bank_name_column else None
- )
- columns["bank_account_column"] = (
- header.index(mapping.bank_account_column)
- if mapping.bank_account_column
- else None
- )
return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns)
+ def _get_values_from_column(self, values, columns, column_name):
+ indexes = columns[column_name]
+ content_l = []
+ max_index = len(values) - 1
+ for index in indexes:
+ if isinstance(index, int):
+ if index <= max_index:
+ content_l.append(values[index])
+ else:
+ if index in values:
+ content_l.append(values[index])
+ if all(isinstance(content, str) for content in content_l):
+ return " ".join(content_l)
+ return content_l[0]
+
def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901
if isinstance(csv_or_xlsx, tuple):
rows = range(1, csv_or_xlsx[1].nrows)
@@ -195,16 +215,21 @@ class AccountStatementImportSheetParser(models.TransientModel):
else:
values = list(row)
- timestamp = values[columns["timestamp_column"]]
+ timestamp = self._get_values_from_column(
+ values, columns, "timestamp_column"
+ )
currency = (
- values[columns["currency_column"]]
- if columns["currency_column"] is not None
+ self._get_values_from_column(values, columns, "currency_column")
+ if columns["currency_column"]
else currency_code
)
def _decimal(column_name):
if columns[column_name]:
- return self._parse_decimal(values[columns[column_name]], mapping)
+ return self._parse_decimal(
+ self._get_values_from_column(values, columns, column_name),
+ mapping,
+ )
amount = _decimal("amount_column")
if not amount:
@@ -213,58 +238,60 @@ class AccountStatementImportSheetParser(models.TransientModel):
amount = -abs(_decimal("amount_credit_column") or 0)
balance = (
- values[columns["balance_column"]]
- if columns["balance_column"] is not None
+ self._get_values_from_column(values, columns, "balance_column")
+ if columns["balance_column"]
else None
)
original_currency = (
- values[columns["original_currency_column"]]
- if columns["original_currency_column"] is not None
+ self._get_values_from_column(
+ values, columns, "original_currency_column"
+ )
+ if columns["original_currency_column"]
else None
)
original_amount = (
- values[columns["original_amount_column"]]
- if columns["original_amount_column"] is not None
+ self._get_values_from_column(values, columns, "original_amount_column")
+ if columns["original_amount_column"]
else None
)
debit_credit = (
- values[columns["debit_credit_column"]]
- if columns["debit_credit_column"] is not None
+ self._get_values_from_column(values, columns, "debit_credit_column")
+ if columns["debit_credit_column"]
else None
)
transaction_id = (
- values[columns["transaction_id_column"]]
- if columns["transaction_id_column"] is not None
+ self._get_values_from_column(values, columns, "transaction_id_column")
+ if columns["transaction_id_column"]
else None
)
description = (
- values[columns["description_column"]]
- if columns["description_column"] is not None
+ self._get_values_from_column(values, columns, "description_column")
+ if columns["description_column"]
else None
)
notes = (
- values[columns["notes_column"]]
- if columns["notes_column"] is not None
+ self._get_values_from_column(values, columns, "notes_column")
+ if columns["notes_column"]
else None
)
reference = (
- values[columns["reference_column"]]
- if columns["reference_column"] is not None
+ self._get_values_from_column(values, columns, "reference_column")
+ if columns["reference_column"]
else None
)
partner_name = (
- values[columns["partner_name_column"]]
- if columns["partner_name_column"] is not None
+ self._get_values_from_column(values, columns, "partner_name_column")
+ if columns["partner_name_column"]
else None
)
bank_name = (
- values[columns["bank_name_column"]]
- if columns["bank_name_column"] is not None
+ self._get_values_from_column(values, columns, "bank_name_column")
+ if columns["bank_name_column"]
else None
)
bank_account = (
- values[columns["bank_account_column"]]
- if columns["bank_account_column"] is not None
+ self._get_values_from_column(values, columns, "bank_account_column")
+ if columns["bank_account_column"]
else None
)
diff --git a/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv b/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv
new file mode 100644
index 00000000..4e91582f
--- /dev/null
+++ b/account_statement_import_txt_xlsx/tests/fixtures/original_currency_no_header.csv
@@ -0,0 +1 @@
+"12/15/2018","Your payment","EUR","1,525.00","-1,000.00","Azure Interior","","INV0001"
diff --git a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py
index f9586125..ca8b6f11 100644
--- a/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py
+++ b/account_statement_import_txt_xlsx/tests/test_account_statement_import_txt_xlsx.py
@@ -224,6 +224,55 @@ class TestAccountBankStatementImportTxtXlsx(common.TransactionCase):
self.assertEqual(line.foreign_currency_id, self.currency_eur)
self.assertEqual(line.amount_currency, 1000.0)
+ def test_original_currency_no_header(self):
+ no_header_statement_map = self.AccountStatementImportSheetMapping.create(
+ {
+ "name": "Sample Statement",
+ "float_thousands_sep": "comma",
+ "float_decimal_sep": "dot",
+ "delimiter": "comma",
+ "quotechar": '"',
+ "timestamp_format": "%m/%d/%Y",
+ "no_header": True,
+ "timestamp_column": "0",
+ "amount_column": "3",
+ "original_currency_column": "2",
+ "original_amount_column": "4",
+ "description_column": "1,7",
+ "partner_name_column": "5",
+ "bank_account_column": "6",
+ }
+ )
+ journal = self.AccountJournal.create(
+ {
+ "name": "Bank",
+ "type": "bank",
+ "code": "BANK",
+ "currency_id": self.currency_usd.id,
+ "suspense_account_id": self.suspense_account.id,
+ }
+ )
+ data = self._data_file("fixtures/original_currency_no_header.csv", "utf-8")
+ wizard = self.AccountStatementImport.with_context(journal_id=journal.id).create(
+ {
+ "statement_filename": "fixtures/original_currency.csv",
+ "statement_file": data,
+ "sheet_mapping_id": no_header_statement_map.id,
+ }
+ )
+ wizard.with_context(
+ account_statement_import_txt_xlsx_test=True
+ ).import_file_button()
+ statement = self.AccountBankStatement.search([("journal_id", "=", journal.id)])
+ self.assertEqual(len(statement), 1)
+ self.assertEqual(len(statement.line_ids), 1)
+
+ line = statement.line_ids
+ self.assertEqual(line.currency_id, self.currency_usd)
+ self.assertEqual(line.foreign_currency_id, self.currency_eur)
+ self.assertEqual(line.amount_currency, 1000.0)
+ self.assertEqual(line.payment_ref, "Your payment INV0001")
+
def test_original_currency_empty(self):
journal = self.AccountJournal.create(
{
diff --git a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml
index 32409bb0..62d617dd 100644
--- a/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml
+++ b/account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml
@@ -39,6 +39,18 @@
+
+
+
+ indicate the column number in the Columns section. The first column is 0.
+
+
@@ -53,22 +65,33 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+ Add the column names or column number (when the file has no header).
+ You can concatenate multiple columns in the file into the same field, indicating the
+ column names or numbers separated by comma.
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/requirements.txt b/requirements.txt
index 962f3242..d3433a96 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
# generated from manifests external_dependencies
-odoo_test_helper
+chardet
ofxparse
xlrd
diff --git a/test-requirements.txt b/test-requirements.txt
new file mode 100644
index 00000000..66bc2cba
--- /dev/null
+++ b/test-requirements.txt
@@ -0,0 +1 @@
+odoo_test_helper