audrius/bank-statement-import
1
0
Fork 1
mirror of https://github.com/OCA/bank-statement-import.git synced 2025-01-20 12:37:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

[IMP] account_statement_import_txt_xlsx: Allow mapping with reference to column numbers, and concatenation

Browse Source 
A new field 'File does not contain header line' is added in the Statement Sheet Mappings. If you set to True,
then you can map the columns by indicating in each field of the 'Columns' section the column number in the file.

We also allow to concatenate multiple columns in the file to a single column to a single field of the statement line.
You have to indicate the names of the columns separated by comma.
This commit is contained in:
Jordi Ballester Alomar
2022-12-16 11:09:18 +01:00
committed by Miquel Raïch
parent a11ca1ec8a
commit 0a0d55d4db
6 changed files with 165 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
account_statement_import_txt_xlsx/__manifest__.py
View File

@@ -17,7 +17,7 @@
"multi_step_wizard",
"web_widget_dropdown_dynamic",
],
"external_dependencies": {"python": ["xlrd"]},
"external_dependencies": {"python": ["xlrd", "chardet"]},
"data": [
"security/ir.model.access.csv",
"data/map_data.xml",

4
account_statement_import_txt_xlsx/models/account_statement_import.py
View File

@@ -28,7 +28,9 @@ class AccountStatementImport(models.TransientModel):
self.ensure_one()
try:
Parser = self.env["account.statement.import.sheet.parser"]
return Parser.parse(data_file, self.sheet_mapping_id)
return Parser.parse(
data_file, self.sheet_mapping_id, self.statement_filename
)
except BaseException:
if self.env.context.get("account_statement_import_txt_xlsx_test"):
raise

5
account_statement_import_txt_xlsx/models/account_statement_import_sheet_mapping.py
View File

@@ -56,6 +56,11 @@ class AccountStatementImportSheetMapping(models.Model):
)
quotechar = fields.Char(string="Text qualifier", size=1, default='"')
timestamp_format = fields.Char(string="Timestamp Format", required=True)
no_header = fields.Boolean(
"File does not contain header line",
help="When this occurs please indicate the column number in the Columns section "
"instead of the column name, considering that the first column is 0",
)
timestamp_column = fields.Char(string="Timestamp column", required=True)
currency_column = fields.Char(
string="Currency column",

208
account_statement_import_txt_xlsx/models/account_statement_import_sheet_parser.py
View File

@@ -7,8 +7,10 @@ import logging
from datetime import datetime
from decimal import Decimal
from io import StringIO
from os import path
from odoo import _, api, models
from odoo.exceptions import UserError
_logger = logging.getLogger(__name__)
@@ -20,6 +22,14 @@ try:
except (ImportError, IOError) as err: # pragma: no cover
_logger.error(err)
try:
import chardet
except ImportError:
_logger.warning(
"chardet library not found, please install it "
"from http://pypi.python.org/pypi/chardet"
)
class AccountStatementImportSheetParser(models.TransientModel):
_name = "account.statement.import.sheet.parser"
@@ -43,7 +53,7 @@ class AccountStatementImportSheetParser(models.TransientModel):
return list(next(csv_data))
@api.model
def parse(self, data_file, mapping):
def parse(self, data_file, mapping, filename):
journal = self.env["account.journal"].browse(self.env.context.get("journal_id"))
currency_code = (journal.currency_id or journal.company_id.currency_id).name
account_number = journal.bank_account_id.acc_number
@@ -67,6 +77,11 @@ class AccountStatementImportSheetParser(models.TransientModel):
{
"balance_start": float(balance_start),
"balance_end_real": float(balance_end),
"name": _("%s: %s")
% (
journal.code,
path.basename(filename),
),
}
)
@@ -79,6 +94,45 @@ class AccountStatementImportSheetParser(models.TransientModel):
return currency_code, account_number, [data]
def _get_column_indexes(self, header, column_name, mapping):
column_indexes = []
if mapping[column_name] and "," in mapping[column_name]:
# We have to concatenate the values
column_names_or_indexes = mapping[column_name].split(",")
else:
column_names_or_indexes = [mapping[column_name]]
for column_name_or_index in column_names_or_indexes:
if mapping.no_header:
column_index = (
column_name_or_index and int(column_name_or_index) or None
)
if column_index:
column_indexes.append(column_index)
else:
if column_name_or_index:
column_indexes.append(header.index(column_name_or_index))
return column_indexes
def _get_column_names(self):
return [
"timestamp_column",
"currency_column",
"amount_column",
"amount_debit_column",
"amount_credit_column",
"balance_column",
"original_currency_column",
"original_amount_column",
"debit_credit_column",
"transaction_id_column",
"description_column",
"notes_column",
"reference_column",
"partner_name_column",
"bank_name_column",
"bank_account_column",
]
def _parse_lines(self, mapping, data_file, currency_code):
columns = dict()
try:
@@ -99,81 +153,40 @@ class AccountStatementImportSheetParser(models.TransientModel):
csv_options["delimiter"] = csv_delimiter
if mapping.quotechar:
csv_options["quotechar"] = mapping.quotechar
csv_or_xlsx = reader(
StringIO(data_file.decode(mapping.file_encoding or "utf-8")),
**csv_options
try:
decoded_file = data_file.decode(mapping.file_encoding or "utf-8")
except UnicodeDecodeError:
# Try auto guessing the format
detected_encoding = chardet.detect(data_file).get("encoding", False)
if not detected_encoding:
raise UserError(
_("No valid encoding was found for the attached file")
)
decoded_file = data_file.decode(detected_encoding)
csv_or_xlsx = reader(StringIO(decoded_file), **csv_options)
if isinstance(csv_or_xlsx, tuple):
header = [str(value) for value in csv_or_xlsx[1].row_values(0)]
else:
header = [value.strip() for value in next(csv_or_xlsx)]
columns["timestamp_column"] = header.index(mapping.timestamp_column)
columns["currency_column"] = (
header.index(mapping.currency_column) if mapping.currency_column else None
)
columns["amount_column"] = (
header.index(mapping.amount_column) if mapping.amount_column else None
)
columns["amount_debit_column"] = (
header.index(mapping.amount_debit_column)
if mapping.amount_debit_column
else None
)
columns["amount_credit_column"] = (
header.index(mapping.amount_credit_column)
if mapping.amount_credit_column
else None
)
columns["balance_column"] = (
header.index(mapping.balance_column) if mapping.balance_column else None
)
columns["original_currency_column"] = (
header.index(mapping.original_currency_column)
if mapping.original_currency_column
else None
)
columns["original_amount_column"] = (
header.index(mapping.original_amount_column)
if mapping.original_amount_column
else None
)
columns["debit_credit_column"] = (
header.index(mapping.debit_credit_column)
if mapping.debit_credit_column
else None
)
columns["transaction_id_column"] = (
header.index(mapping.transaction_id_column)
if mapping.transaction_id_column
else None
)
columns["description_column"] = (
header.index(mapping.description_column)
if mapping.description_column
else None
)
columns["notes_column"] = (
header.index(mapping.notes_column) if mapping.notes_column else None
)
columns["reference_column"] = (
header.index(mapping.reference_column) if mapping.reference_column else None
)
columns["partner_name_column"] = (
header.index(mapping.partner_name_column)
if mapping.partner_name_column
else None
)
columns["bank_name_column"] = (
header.index(mapping.bank_name_column) if mapping.bank_name_column else None
)
columns["bank_account_column"] = (
header.index(mapping.bank_account_column)
if mapping.bank_account_column
else None
for column_name in self._get_column_names():
columns[column_name] = self._get_column_indexes(
header, column_name, mapping
)
return self._parse_rows(mapping, currency_code, csv_or_xlsx, columns)
def _get_values_from_column(self, values, columns, column_name):
indexes = columns[column_name]
content_l = []
max_index = len(values) - 1
for index in indexes:
if isinstance(index, int) and index <= max_index:
content_l.append(values[index])
else:
content_l.append(values[index])
if all(isinstance(content, str) for content in content_l):
return " ".join(content_l)
return content_l[0]
def _parse_rows(self, mapping, currency_code, csv_or_xlsx, columns): # noqa: C901
if isinstance(csv_or_xlsx, tuple):
rows = range(1, csv_or_xlsx[1].nrows)
@@ -195,16 +208,21 @@ class AccountStatementImportSheetParser(models.TransientModel):
else:
values = list(row)
timestamp = values[columns["timestamp_column"]]
timestamp = self._get_values_from_column(
values, columns, "timestamp_column"
)
currency = (
values[columns["currency_column"]]
if columns["currency_column"] is not None
self._get_values_from_column(values, columns, "currency_column")
if columns["currency_column"]
else currency_code
)
def _decimal(column_name):
if columns[column_name]:
return self._parse_decimal(values[columns[column_name]], mapping)
return self._parse_decimal(
self._get_values_from_column(values, columns, column_name),
mapping,
)
amount = _decimal("amount_column")
if not amount:
@@ -213,58 +231,60 @@ class AccountStatementImportSheetParser(models.TransientModel):
amount = -abs(_decimal("amount_credit_column") or 0)
balance = (
values[columns["balance_column"]]
if columns["balance_column"] is not None
self._get_values_from_column(values, columns, "balance_column")
if columns["balance_column"]
else None
)
original_currency = (
values[columns["original_currency_column"]]
if columns["original_currency_column"] is not None
self._get_values_from_column(
values, columns, "original_currency_column"
)
if columns["original_currency_column"]
else None
)
original_amount = (
values[columns["original_amount_column"]]
if columns["original_amount_column"] is not None
self._get_values_from_column(values, columns, "original_amount_column")
if columns["original_amount_column"]
else None
)
debit_credit = (
values[columns["debit_credit_column"]]
if columns["debit_credit_column"] is not None
self._get_values_from_column(values, columns, "debit_credit_column")
if columns["debit_credit_column"]
else None
)
transaction_id = (
values[columns["transaction_id_column"]]
if columns["transaction_id_column"] is not None
self._get_values_from_column(values, columns, "transaction_id_column")
if columns["transaction_id_column"]
else None
)
description = (
values[columns["description_column"]]
if columns["description_column"] is not None
self._get_values_from_column(values, columns, "description_column")
if columns["description_column"]
else None
)
notes = (
values[columns["notes_column"]]
if columns["notes_column"] is not None
self._get_values_from_column(values, columns, "notes_column")
if columns["notes_column"]
else None
)
reference = (
values[columns["reference_column"]]
if columns["reference_column"] is not None
self._get_values_from_column(values, columns, "reference_column")
if columns["reference_column"]
else None
)
partner_name = (
values[columns["partner_name_column"]]
if columns["partner_name_column"] is not None
self._get_values_from_column(values, columns, "partner_name_column")
if columns["partner_name_column"]
else None
)
bank_name = (
values[columns["bank_name_column"]]
if columns["bank_name_column"] is not None
self._get_values_from_column(values, columns, "bank_name_column")
if columns["bank_name_column"]
else None
)
bank_account = (
values[columns["bank_account_column"]]
if columns["bank_account_column"] is not None
self._get_values_from_column(values, columns, "bank_account_column")
if columns["bank_account_column"]
else None
)

23
account_statement_import_txt_xlsx/views/account_statement_import_sheet_mapping.xml
View File

@@ -39,6 +39,18 @@
<group>
<field name="timestamp_format" />
</group>
<group>
<field name="no_header" />
<div
class="alert alert-warning"
role="alert"
attrs="{'invisible': [('no_header', '=', False)]}"
>
<span
class="fa fa-info-circle"
/> indicate the column number in the Columns section. The first column is 0.
</div>
</group>
<group
attrs="{'invisible': [('debit_credit_column', '=', False)]}"
>
@@ -53,6 +65,16 @@
</group>
</group>
<group string="Columns">
<group colspan="4" col="2">
<div class="alert alert-info" role="alert">
<span
class="fa fa-info-circle"
/> Add the column names or column number (when the file has no header).
You can concatenate multiple columns in the file into the same field, indicating the
column names or numbers separated by comma.
</div>
</group>
<group>
<field name="timestamp_column" />
<field name="currency_column" />
<field name="amount_column" />
@@ -70,6 +92,7 @@
<field name="bank_name_column" />
<field name="bank_account_column" />
</group>
</group>
</sheet>
</form>
</field>

1
requirements.txt
View File

@@ -1,3 +1,4 @@
# generated from manifests external_dependencies
chardet
ofxparse
xlrd