audrius/bank-statement-import
1
0
Fork 1
mirror of https://github.com/OCA/bank-statement-import.git synced 2025-01-20 12:37:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

[11.0] [MIG] account_bank_statement_import_mt940_base.

Browse Source 
Update version to 11.

Get balance start even if currency code is already set.

Add SNS test, add header_lines parameter in parse file.

Update code, add pre process datafile to change from different format to only one, statements as dict starting with {4:..}, split file to import multiple statements at once, the check of date of statement import not implemented, be carefully not to import twice the same statements.

Update flake.

Add wrong file.
This commit is contained in:
Fekete Mihai
2017-12-11 01:12:28 -08:00
parent a9fc5c1295
commit fe73418dff
6 changed files with 245 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
account_bank_statement_import_mt940_base/README.rst
View File

@@ -1,5 +1,5 @@
.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
:target: http://www.gnu.org/licenses/agpl
.. image:: https://img.shields.io/badge/license-AGPL--3-blue.png
:target: https://www.gnu.org/licenses/agpl
:alt: License: AGPL-3
====================
@@ -36,6 +36,7 @@ Contributors
* Stefan Rijnhart <srijnhart@therp.nl>
* Ronald Portier <rportier@therp.nl>
* Andrea Stirpe <a.stirpe@onestein.nl>
* Fekete Mihai <mihai.fekete@forbiom.eu>
Maintainer
----------

21
account_bank_statement_import_mt940_base/__manifest__.py
View File

@@ -1,25 +1,8 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (C) 2013-2015 Therp BV <http://therp.nl>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
{
'name': 'MT940 Bank Statements Import',
'version': '10.0.1.0.0',
'version': '11.0.1.0.0',
'license': 'AGPL-3',
'author': 'Odoo Community Association (OCA), Therp BV',
'website': 'https://github.com/OCA/bank-statement-import',

75
account_bank_statement_import_mt940_base/mt940.py
View File

@@ -1,23 +1,7 @@
# -*- coding: utf-8 -*-
"""Generic parser for MT940 files, base for customized versions per bank."""
##############################################################################
#
# Copyright (C) 2014-2015 Therp BV <http://therp.nl>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
"""Generic parser for MT940 files, base for customized versions per bank."""
import re
import logging
from datetime import datetime
@@ -138,17 +122,47 @@ class MT940(object):
(line[:12], self.mt940_type)
)
def parse(self, data):
def is_mt940_statement(self, line):
"""determine if line is the start of a statement"""
if not bool(line.startswith('{4:')):
raise ValueError(
'The pre processed match %s does not seem to be a'
' valid %s MT940 format bank statement. Every statement'
' should start be a dict starting with {4:..' % line
)
def pre_process_data(self, data):
matches = []
self.is_mt940(line=data)
data = data.replace(
'-}', '}').replace('}{', '}\r\n{').replace('\r\n', '\n')
if data.startswith(':940:'):
for statement in data.replace(':940:', '').split(':20:'):
match = '{4:\n:20:' + statement + '}'
matches.append(match)
else:
tag_re = re.compile(
r'(\{4:[^{}]+\})',
re.MULTILINE)
matches = tag_re.findall(data)
return matches
def parse(self, data, header_lines=None):
"""Parse mt940 bank statement file contents."""
self.is_mt940(data)
iterator = data.replace('\r\n', '\n').split('\n').__iter__()
data = data.decode()
matches = self.pre_process_data(data)
for match in matches:
self.is_mt940_statement(line=match)
iterator = '\n'.join(
match.split('\n')[1:-1]).split('\n').__iter__()
line = None
record_line = ''
try:
while True:
if not self.current_statement:
self.handle_header(line, iterator)
line = iterator.next()
self.handle_header(line, iterator,
header_lines=header_lines)
line = next(iterator)
if not self.is_tag(line) and not self.is_footer(line):
record_line = self.add_record_line(line, record_line)
continue
@@ -181,10 +195,12 @@ class MT940(object):
"""determine if a line has a tag"""
return line and bool(re.match(self.tag_regex, line))
def handle_header(self, dummy_line, iterator):
def handle_header(self, dummy_line, iterator, header_lines=None):
"""skip header lines, create current statement"""
for dummy_i in range(self.header_lines):
iterator.next()
if not header_lines:
header_lines = self.header_lines
for dummy_i in range(header_lines):
next(iterator)
self.current_statement = {
'name': None,
'date': None,
@@ -233,6 +249,9 @@ class MT940(object):
data[0],
data[10:]
)
if not self.current_statement['date']:
self.current_statement['date'] = datetime.strptime(data[1:7],
'%y%m%d')
def handle_tag_61(self, data):
"""get transaction values"""
@@ -271,7 +290,7 @@ class MT940(object):
statement_name = self.current_statement['name'] or ''
test_empty_id = re.sub(r'[\s0]', '', statement_name)
is_account_number = statement_name.startswith(self.account_number)
if ((not test_empty_id) or is_account_number):
if not test_empty_id or is_account_number:
self.current_statement['name'] = '%s-%s' % (
self.account_number,
self.current_statement['date'].strftime('%Y-%m-%d'),

78
account_bank_statement_import_mt940_base/test_files/test-sns.940 Executable file
View File

@@ -0,0 +1,78 @@
{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
:20:0000000000
:25:NL05SNSB0908244436
:28C:361/1
:60F:C171227EUR3026,96
:61:1712271227D713,13NOVBNL49RABO0166416932
gerrits glas en schilderwerk
:86:NL49RABO0166416932 gerrits glas en schilderwerk
Factuur 17227/248/20
:61:1712271227D10,18NBEA//228ohu/972795
:86:
Jumbo Wijchen B.V. >WIJCHEN 27.12.2017 13U38 KV005 4XZJ4Z M
CC:5411 Contactloze betaling
:61:1712271227D13,52NINCNL94INGB0000869000
vitens nv
:86:NL94INGB0000869000 vitens nv
Europese incasso door:VITENS NV NL-Factuurnr 072304597540 VNKlant
nr 00000000000 BTW 0,77PC 6605 DW 6223, DEC-Incassant ID: NL84ZZZ0
50695810000-Kenmerk Machtiging: bla
:61:1712271227D25,61NBEA//229hro/195867
:86:
Albert Heijn 1370 >WIJCHEN 27.12.2017 19U13 KV006 70X708 M
CC:5411
:62F:C171227EUR2264,52
-}{5:}
{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
:20:0000000000
:25:NL05SNSB0908244436
:28C:362/1
:60F:C171228EUR2264,52
:61:1712281228D10,95NINCNL40RABO0127859497
antagonist b.v.
:86:NL40RABO0127859497 antagonist b.v.
Europese incasso door:ANTAGONIST B.V. NL-DDWPN954156 AI529942Cweb
share.nl-Incassant ID: NL65ZZZ091364410000-Kenmerk Machtiging: bla
:61:1712281228C0,00NDIV
:86:
Mobiel betalen is vanaf nu niet meer mogelijk voor B
ERG M A VAN DEN met Lenovo P2
:62F:C171228EUR2253,57
-}{5:}
{1:F01SNSBNL2AXXXX0000000000}{2:O940SNSBNL2AXXXXN}{3:}{4:
:20:0000000000
:25:NL05SNSB0908244436
:28C:363/1
:60F:C171229EUR2253,57
:61:1712291229D907,29NINCNL19ABNA0427093546
florius
:86:NL19ABNA0427093546 florius
Europese incasso door:FLORIUS NL-Verschuldigde bedragen PERIODE 1
2-2017-Incassant ID: NL42ZZZ080242850000-Kenmerk Machtiging: bla
:61:1712291229D35,00NINCNL28DEUT0265186439
stichting derdengelden bucka
:86:NL28DEUT0265186439 stichting derdengelden bucka
Europese incasso door:STICHTING DERDENGELDEN BUCKAROO-T-Mobile Th
uis B.V.: Rekening dec 2017. Bekijk je rekening op thuismy.t-mobi
le.nl-Incassant ID: NL39ZZZ302317620000-Kenmerk Machtiging: bla
-}{5:}

11
account_bank_statement_import_mt940_base/test_files/test-wrong-file.940 Normal file
View File

@@ -0,0 +1,11 @@
:9401:
:20:940S140102
:25:NL34RABO0142623393 EUR
:28C:0
:60F:C131231EUR000000004433,52
:61:140102C000000000400,00N541NONREF
NL66RABO0160878799
:86:/ORDP//NAME/R. SMITH/ADDR/Green market 74 3311BE Sheepcity Nederl
and NL/REMI/Test money paid by other partner:
/ISDT/2014-01-02
:62F:C140102EUR000000004833,52

89
account_bank_statement_import_mt940_base/tests/test_import_bank_statement.py
View File

@@ -1,4 +1,3 @@
# -*- coding: utf-8 -*-
# Copyright 2017 Onestein (<http://www.onestein.eu>)
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html).
@@ -48,13 +47,38 @@ class TestImport(TransactionCase):
'bank_account_id': bank2.id,
'currency_id': self.env.ref('base.EUR').id,
})
bank3 = self.env['res.partner.bank'].create({
'acc_number': 'NL05SNSB0908244436',
'partner_id': self.env.ref('base.main_partner').id,
'company_id': self.env.ref('base.main_company').id,
'bank_id': self.env.ref('base.res_bank_1').id,
})
self.env['account.journal'].create({
'name': 'Bank Journal - (test3 mt940)',
'code': 'TBNK3MT940',
'type': 'bank',
'bank_account_id': bank3.id,
'currency_id': self.env.ref('base.EUR').id,
})
self.data =\
"/BENM//NAME/Cost/REMI/Period 01-10-2013 t/m 31-12-2013/ISDT/20"
self.codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', 'ISDT', 'REMI']
def test_wrong_file_import(self):
"""Test wrong file import."""
testfile = get_module_resource(
'account_bank_statement_import_mt940_base',
'test_files',
'test-wrong-file.940',
)
parser = MT940()
datafile = open(testfile, 'rb').read()
with self.assertRaises(ValueError):
parser.parse(datafile, header_lines=1)
def test_statement_import(self):
"""Test correct creation of single statement."""
"""Test correct creation of single statement ING."""
def _prepare_statement_lines(statements):
transact = self.transactions[0]
@@ -72,7 +96,7 @@ class TestImport(TransactionCase):
)
parser = MT940()
datafile = open(testfile, 'rb').read()
statements = parser.parse(datafile)
statements = parser.parse(datafile, header_lines=1)
_prepare_statement_lines(statements)
@@ -89,8 +113,7 @@ class TestImport(TransactionCase):
transact = self.transactions[0]
for statement in self.env['account.bank.statement'].browse(
action['context']['statement_ids']
):
action['context']['statement_ids']):
for line in statement.line_ids:
self.assertTrue(
line.bank_account_id.acc_number ==
@@ -121,7 +144,7 @@ class TestImport(TransactionCase):
handle_common_subfields(transaction, subfields)
def test_statement_import2(self):
"""Test correct creation of single statement."""
"""Test correct creation of single statement RABO."""
def _prepare_statement_lines(statements):
transact = self.transactions[0]
@@ -141,7 +164,7 @@ class TestImport(TransactionCase):
parser.header_regex = '^:940:' # Start of header
parser.header_lines = 1 # Number of lines to skip
datafile = open(testfile, 'rb').read()
statements = parser.parse(datafile)
statements = parser.parse(datafile, header_lines=1)
_prepare_statement_lines(statements)
@@ -155,11 +178,11 @@ class TestImport(TransactionCase):
action = self.env['account.bank.statement.import'].create({
'data_file': base64.b64encode(datafile),
}).import_file()
# The file contains 4 statements, but only 2 with transactions
self.assertTrue(len(action['context']['statement_ids']) == 2)
transact = self.transactions[0]
for statement in self.env['account.bank.statement'].browse(
action['context']['statement_ids']
):
action['context']['statement_ids']):
for line in statement.line_ids:
self.assertTrue(
line.bank_account_id.acc_number ==
@@ -168,3 +191,49 @@ class TestImport(TransactionCase):
self.assertTrue(line.date)
self.assertTrue(line.name == transact['name'])
self.assertTrue(line.ref == transact['ref'])
def test_statement_import3(self):
"""Test correct creation of multiple statements SNS."""
def _prepare_statement_lines(statements):
transact = self.transactions[0]
for st_vals in statements[2]:
for line_vals in st_vals['transactions']:
line_vals['amount'] = transact['amount']
line_vals['name'] = transact['name']
line_vals['account_number'] = transact['account_number']
line_vals['ref'] = transact['ref']
testfile = get_module_resource(
'account_bank_statement_import_mt940_base',
'test_files',
'test-sns.940',
)
parser = MT940()
datafile = open(testfile, 'rb').read()
statements = parser.parse(datafile, header_lines=1)
_prepare_statement_lines(statements)
path_addon = 'odoo.addons.account_bank_statement_import.'
path_file = 'account_bank_statement_import.'
path_class = 'AccountBankStatementImport.'
method = path_addon + path_file + path_class + '_parse_file'
with patch(method) as my_mock:
my_mock.return_value = statements
action = self.env['account.bank.statement.import'].create({
'data_file': base64.b64encode(datafile),
}).import_file()
self.assertTrue(len(action['context']['statement_ids']) == 3)
transact = self.transactions[-1]
for statement in self.env['account.bank.statement'].browse(
action['context']['statement_ids'][-1]):
for line in statement.line_ids:
self.assertTrue(
line.bank_account_id.acc_number ==
transact['account_number'])
self.assertTrue(line.amount == transact['amount'])
self.assertTrue(line.date == statement.date)
self.assertTrue(line.name == transact['name'])
self.assertTrue(line.ref == transact['ref'])