[ENH] Add enhanced mt940 import.

bank_statement_parse_mt940/__init__.py

@@ -0,0 +1 @@
+from . import mt940

bank_statement_parse_mt940/__openerp__.py

@@ -0,0 +1,40 @@
+##############################################################################
+#
+#    Copyright (C) 2013 Therp BV <http://therp.nl>
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+{
+    'name': 'MT940 Bank Statements Import',
+    'version': '1.1',
+    'license': 'AGPL-3',
+    'author': 'Therp BV',
+    'website': 'https://github.com/OCA/bank-statement-import',
+    'category': 'Banking addons',
+    'description': '''
+This addon provides a generic parser for MT940 files. Given that MT940 is a
+non-open non-standard of pure evil in the way that every bank cooks up its own
+interpretation of it, this addon alone won't help you much. It is rather
+intended to be used by other addons to implement the dialect specific to a
+certain bank.
+See bank_statement_parse_nl_ing_mt940 for an example on how to use it.
+''',
+    'depends': [
+        'bank_statement_parse'
+    ],
+    'data': [],
+    'installable': True
+}

bank_statement_parse_mt940/mt940.py

@@ -0,0 +1,257 @@
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+"""Generic parser for MT940 files, base for customized versions per bank."""
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2014 Therp BV (<http://therp.nl>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import re
+import logging
+from datetime import datetime
+
+from openerp.addons.bank_statement_parse import parserlib
+
+
+def str2amount(sign, amount_str):
+    """Convert sign (C or D) and amount in string to signed amount (float)."""
+    factor = (1 if sign == 'C' else -1)
+    return factor * float(amount_str.replace(',', '.'))
+
+
+def get_subfields(data, codewords):
+    """Return dictionary with value array for each codeword in data.
+
+    For instance:
+    data =
+        /BENM//NAME/Kosten/REMI/Periode 01-10-2013 t/m 31-12-2013/ISDT/20
+    codewords = ['BENM', 'ADDR', 'NAME', 'CNTP', ISDT', 'REMI']
+    Then return subfields = {
+        'BENM': [],
+        'NAME': ['Kosten'],
+        'REMI': ['Periode 01-10-2013 t', 'm 31-12-2013'],
+        'ISDT': ['20'],
+    }
+    """
+    subfields = {}
+    current_codeword = None
+    for word in data.split('/'):
+        if not word and not current_codeword:
+            continue
+        if word in codewords:
+            current_codeword = word
+            subfields[current_codeword] = []
+            continue
+        if current_codeword in subfields:
+            subfields[current_codeword].append(word)
+    return subfields
+
+
+def get_counterpart(transaction, subfield):
+    """Get counterpart from transaction.
+
+    Counterpart is often stored in subfield of tag 86. The subfield
+    can be BENM, ORDP, CNTP"""
+    if not subfield:
+        return  # subfield is empty
+    if len(subfield) >= 1 and subfield[0]:
+        transaction.remote_account = subfield[0]
+    if len(subfield) >= 2 and subfield[1]:
+        transaction.remote_bank_bic = subfield[1]
+    if len(subfield) >= 3 and subfield[2]:
+        transaction.remote_owner = subfield[2]
+    if len(subfield) >= 4 and subfield[3]:
+        transaction.remote_owner_city = subfield[3]
+
+
+def handle_common_subfields(transaction, subfields):
+    """Deal with common functionality for tag 86 subfields."""
+    # Get counterpart from CNTP, BENM or ORDP subfields:
+    for counterpart_field in ['CNTP', 'BENM', 'ORDP']:
+        if counterpart_field in subfields:
+            get_counterpart(transaction, subfields[counterpart_field])
+    # REMI: Remitter information (text entered by other party on trans.):
+    if 'REMI' in subfields:
+        transaction.message = (
+            '/'.join(x for x in subfields['REMI'] if x))
+    # Get transaction reference subfield (might vary):
+    if transaction.eref in subfields:
+        transaction.eref = ''.join(
+            subfields[transaction.eref])
+
+
+class MT940(object):
+    """Inherit this class in your account_banking.parsers.models.parser,
+    define functions to handle the tags you need to handle and adjust static
+    variables as needed.
+
+    At least, you should override handle_tag_61 and handle_tag_86. Don't forget
+    to call super.
+    handle_tag_* functions receive the remainder of the the line (that is,
+    without ':XX:') and are supposed to write into self.current_transaction"""
+
+    header_lines = 3
+    """One file can contain multiple statements, each with its own poorly
+    documented header. For now, the best thing to do seems to skip that"""
+
+    header_regex = '^{1:[0-9A-Z]{25,25}}'
+    'The file is considered a valid MT940 file when it contains this line'
+
+    footer_regex = '^-XXX$'
+    'The line that denotes end of message, we need to create a new statement'
+
+    tag_regex = '^:[0-9]{2}[A-Z]*:'
+    'The beginning of a record, should be anchored to beginning of the line'
+
+    def __init__(self):
+        self.current_statement = None
+        self.current_transaction = None
+        self.statements = []
+
+    def create_transaction(self):
+        """Create and return BankTransaction object."""
+        transaction = parserlib.BankTransaction()
+        return transaction
+
+    def is_mt940(self, line):
+        """determine if a line is the header of a statement"""
+        if not bool(re.match(self.header_regex, line)):
+            raise ValueError(
+                'This does not seem to be a MT940 format bank statement.')
+
+    def parse(self, data):
+        """Parse mt940 bank statement file contents."""
+        self.is_mt940(data)
+        iterator = data.replace('\r\n', '\n').split('\n').__iter__()
+        line = None
+        record_line = ''
+        try:
+            while True:
+                if not self.current_statement:
+                    self.handle_header(line, iterator)
+                line = iterator.next()
+                if not self.is_tag(line) and not self.is_footer(line):
+                    record_line = self.append_continuation_line(
+                        record_line, line)
+                    continue
+                if record_line:
+                    self.handle_record(record_line)
+                if self.is_footer(line):
+                    self.handle_footer(line, iterator)
+                    record_line = ''
+                    continue
+                record_line = line
+        except StopIteration:
+            pass
+        if self.current_statement:
+            if record_line:
+                self.handle_record(record_line)
+                record_line = ''
+            self.statements.append(self.current_statement)
+            self.current_statement = None
+        return self.statements
+
+    def append_continuation_line(self, line, continuation_line):
+        """append a continuation line for a multiline record.
+        Override and do data cleanups as necessary."""
+        return line + continuation_line
+
+    def create_statement(self):
+        """create a BankStatement."""
+        return parserlib.BankStatement()
+
+    def is_footer(self, line):
+        """determine if a line is the footer of a statement"""
+        return line and bool(re.match(self.footer_regex, line))
+
+    def is_tag(self, line):
+        """determine if a line has a tag"""
+        return line and bool(re.match(self.tag_regex, line))
+
+    def handle_header(self, line, iterator):
+        """skip header lines, create current statement"""
+        for dummy_i in range(self.header_lines):
+            iterator.next()
+        self.current_statement = self.create_statement()
+
+    def handle_footer(self, line, iterator):
+        """add current statement to list, reset state"""
+        self.statements.append(self.current_statement)
+        self.current_statement = None
+
+    def handle_record(self, line):
+        """find a function to handle the record represented by line"""
+        tag_match = re.match(self.tag_regex, line)
+        tag = tag_match.group(0).strip(':')
+        if not hasattr(self, 'handle_tag_%s' % tag):
+            logging.error('Unknown tag %s', tag)
+            logging.error(line)
+            return
+        handler = getattr(self, 'handle_tag_%s' % tag)
+        handler(line[tag_match.end():])
+
+    def handle_tag_20(self, data):
+        """Contains unique ? message ID"""
+        pass
+
+    def handle_tag_25(self, data):
+        """Handle tag 25: local bank account information."""
+        data = data.replace('EUR', '').replace('.', '').strip()
+        self.current_statement.local_account = data
+
+    def handle_tag_28C(self, data):
+        """get sequence number _within_this_batch_ - this alone
+        doesn't provide a unique id!"""
+        self.current_statement.statement_id = data
+
+    def handle_tag_60F(self, data):
+        """get start balance and currency"""
+        self.current_statement.local_currency = data[7:10]
+        self.current_statement.date = datetime.strptime(data[1:7], '%y%m%d')
+        self.current_statement.start_balance = str2amount(data[0], data[10:])
+        self.current_statement.statement_id = '%s/%s' % (
+            self.current_statement.date.strftime('%Y-%m-%d'),
+            self.current_statement.statement_id,
+        )
+
+    def handle_tag_62F(self, data):
+        """get ending balance"""
+        self.current_statement.end_balance = str2amount(data[0], data[10:])
+
+    def handle_tag_64(self, data):
+        """get current balance in currency"""
+        pass
+
+    def handle_tag_65(self, data):
+        """get future balance in currency"""
+        pass
+
+    def handle_tag_61(self, data):
+        """get transaction values"""
+        transaction = self.create_transaction()
+        self.current_statement.transactions.append(transaction)
+        self.current_transaction = transaction
+        transaction.execution_date = datetime.strptime(data[:6], '%y%m%d')
+        transaction.value_date = datetime.strptime(data[:6], '%y%m%d')
+        #  ...and the rest already is highly bank dependent
+
+    def handle_tag_86(self, data):
+        """details for previous transaction, here most differences between
+        banks occur"""
+        pass
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

bank_statement_parse_nl_ing_mt940/README.rst

@@ -0,0 +1,25 @@
+Import MT940 IBAN ING Bank Statements
+=====================================
+
+This module allows you to import the MT940 IBAN files from the Dutch ING bank
+in Odoo as bank statements.
+The specifications are published at:
+    https://www.ing.nl/media/ING_ming_mt940s_24_juli_tcm162-46356.pdf
+and were last updated august 2014.
+
+Installation
+============
+
+This module is available:
+* for Odoo version 8: in the OCA project bank-statement-import: 
+  https://github.com/OCA/bank-statement-import
+
+Configuration
+=============
+
+In the menu Accounting > Configuration > Accounts > Setup your Bank Accounts,
+make sure that you have your ING bank account with the following parameters:
+
+* Bank Account Type: Normal Bank Account
+* Account Number: the bank account number also appearing in the statements
+* Account Journal: the journal associated to your bank account

bank_statement_parse_nl_ing_mt940/__init__.py

@@ -0,0 +1 @@
+from . import account_bank_statement_import

bank_statement_parse_nl_ing_mt940/__openerp__.py

@@ -0,0 +1,32 @@
+##############################################################################
+#
+#    Copyright (C) 2013 Therp BV (<http://therp.nl>)
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+{
+    'name': 'MT940 IBAN ING Format Bank Statements Import',
+    'version': '0.3',
+    'license': 'AGPL-3',
+    'author': 'Therp BV',
+    'website': 'https://github.com/OCA/banking',
+    'category': 'Banking addons',
+    'depends': [
+        'bank_statement_parse_mt940'
+    ],
+    'data': [],
+    'installable': True
+}

bank_statement_parse_nl_ing_mt940/account_bank_statement_import.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""Parse a MT940 IBAN ING file."""
+##############################################################################
+#
+#    Copyright (C) 2013 Therp BV (<http://therp.nl>)
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import logging
+from openerp import models
+from openerp.addons.bank_statement_parse.parserlib import convert_statements
+from .mt940 import MT940Parser as Parser
+
+
+_logger = logging.getLogger(__name__)
+
+
+class AccountBankStatementImport(models.TransientModel):
+    """Add parsing of mt940 files to bank statement import."""
+    _inherit = 'account.bank.statement.import'
+
+    def _parse_file(self, cr, uid, data_file, context=None):
+        """Parse a MT940 IBAN ING file."""
+        parser = Parser()
+        try:
+            _logger.debug("Try parsing with MT940 IBAN ING.")
+            return convert_statements(parser.parse(data_file))
+        except ValueError:
+            # Returning super will call next candidate:
+            _logger.debug("Statement file was not a MT940 IBAN ING file.")
+            return super(AccountBankStatementImport, self)._parse_file(
+                cr, uid, data_file, context=context)
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

bank_statement_parse_nl_ing_mt940/mt940.py

@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+"""Implement BankStatementParser for MT940 IBAN ING files."""
+##############################################################################
+#
+#    Copyright (C) 2013 Therp BV (<http://therp.nl>)
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import re
+from openerp.addons.bank_statement_parse_mt940.mt940 import (
+    MT940, str2amount, get_subfields, handle_common_subfields)
+
+
+class MT940Parser(MT940):
+    """Parser for ing MT940 bank statement import files."""
+
+    name = 'ING MT940 (structured)'
+    country_code = 'NL'
+    code = 'INT_MT940_STRUC'
+    footer_regex = '^-}$|^-XXX$'
+
+    tag_61_regex = re.compile(
+        r'^(?P<date>\d{6})(?P<line_date>\d{0,4})'
+        r'(?P<sign>[CD])(?P<amount>\d+,\d{2})N(?P<type>.{3})'
+        r'(?P<reference>\w{1,50})'
+    )
+
+    def handle_tag_61(self, data):
+        """get transaction values"""
+        super(MT940Parser, self).handle_tag_61(data)
+        re_61 = self.tag_61_regex.match(data)
+        if not re_61:
+            raise ValueError("Cannot parse %s" % data)
+        parsed_data = re_61.groupdict()
+        self.current_transaction.transferred_amount = (
+            str2amount(parsed_data['sign'], parsed_data['amount']))
+        self.current_transaction.eref = parsed_data['reference']
+
+    def handle_tag_86(self, data):
+        """Parse 86 tag containing reference data."""
+        if not self.current_transaction:
+            return
+        codewords = ['RTRN', 'BENM', 'ORDP', 'CSID', 'BUSP', 'MARF', 'EREF',
+                     'PREF', 'REMI', 'ID', 'PURP', 'ULTB', 'ULTD',
+                     'CREF', 'IREF', 'CNTP', 'ULTC', 'EXCH', 'CHGS']
+        subfields = get_subfields(data, codewords)
+        transaction = self.current_transaction
+        # If we have no subfields, set message to whole of data passed:
+        if not subfields:
+            transaction.message = data
+        else:
+            handle_common_subfields(transaction, subfields)
+        # Prevent handling tag 86 later for non transaction details:
+        self.current_transaction = None
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

bank_statement_parse_nl_ing_mt940/test_files/test-ing.940

@@ -0,0 +1,62 @@
+{1:F01INGBNL2ABXXX0000000000}
+{2:I940INGBNL2AXXXN}
+{4:
+:20:P140220000000001
+:25:NL77ABNA0574908765EUR
+:28C:0000
+0
+:60F:C140219EUR662,23
+:61:1402200220C1,56NTRFEREF//00000000001
+005
+/TRCD/00100/
+:86:/EREF/EV12341REP1231456T1234//CNTP/NL32INGB0000012345/INGBNL2
+A/ING BANK NV INZAKE WEB///REMI/USTD//EV10001REP1000000T1000/
+:61:1402200220D1,57NTRFPREF//00000000001006
+/TRCD/00200/
+:86:/PREF/M000000003333333//REMI/USTD//TOTAAL 1 VZ/
+:61:1402200220C1,57NRTIEREF//00000000001007
+/TRCD/00190/
+:86:/RTRN/MS03//EREF/20120123456789//CNTP/NL32INGB0000012345/INGB
+NL2A/J.Janssen///REMI/USTD//Factuurnr 123456 Klantnr 00123/
+:61:1402200220D1,14NDDTEREF//00000000001009
+/TRCD/010
+16
+/
+:86:/EREF/EV123R
+EP123412T1234//MARF/MND
+-
+EV01//CSID/NL32ZZZ9999999
+91234//CNTP/NL32INGB0000012345/INGBNL2A/ING Bank N.V. inzake WeB/
+//REMI/USTD//EV123REP123412T1234/
+:61:1402200220C1,45NDDTPREF//00000000001008
+/TRCD/01000/
+:86:/PREF/M000000001111111/
+/CSID/
+NL32ZZZ999999991234
+/
+/REMI/USTD//
+TOTAAL 1 POSTEN/
+:61:1402200220D12,75NRTIEREF//00000000001010
+/TRCD/01315/
+:86:/RTRN/MS03//EREF/20120501P0123478//MARF/MND
+-
+120123//CSID/NL32
+ZZZ999999991234//CNTP/NL32INGB0000012345/INGBNL2A/J.Janssen///REM
+I/USTD//CO
+NTRIBUTIE FEB 2014/
+:61:1402200220C32,00NTRF9001123412341234//00000000001011
+/TRCD/00108/
+:86:/EREF/15814016000676480//CNTP/NL32INGB0000012345/INGBNL2A/J.J
+anssen///REMI/STRD/CUR/9001123412341234/
+:61:1402200220D119,00NTRF1070123412341234//00000000001012
+/
+TRCD/00108/
+:86:/EREF/15614016000384600//CNTP/NL32INGB0000012345/INGBNL2A/ING
+BANK NV///REMI/STRD/CUR/1070123412341234/
+:62F:C140220EUR564,35
+:64:C140220EUR564,35
+:65:C140221EUR564,35
+:65:C140224EUR564,35
+:86:/SUM/4/4/134,46/36,58/
+-
+}

bank_statement_parse_nl_ing_mt940/tests/__init__.py

@@ -0,0 +1,25 @@
+# -*- encoding: utf-8 -*-
+"""Test import of bank statement for MT940 ING."""
+##############################################################################
+#
+#    Copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    All other contributions are (C) by their respective contributors
+#
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import test_import_bank_statement

bank_statement_parse_nl_ing_mt940/tests/test_import_bank_statement.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+"""Run test to import MT940 IBAN ING import."""
+##############################################################################
+#
+#    Copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    All other contributions are (C) by their respective contributors
+#
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from openerp.tests.common import TransactionCase
+from openerp.modules.module import get_module_resource
+
+
+class TestStatementFile(TransactionCase):
+    """Run test to import MT940 ING import."""
+
+    def test_statement_import(self):
+        """Test correct creation of single statement."""
+        import_model = self.registry('account.bank.statement.import')
+        statement_model = self.registry('account.bank.statement')
+        cr, uid = self.cr, self.uid
+        statement_path = get_module_resource(
+            'bank_statement_parse_nl_ing_mt940',
+            'test_files',
+            'test-ing.940'
+        )
+        statement_file = open(
+            statement_path, 'rb').read().encode('base64')
+        bank_statement_id = import_model.create(
+            cr, uid,
+            dict(
+                data_file=statement_file,
+            )
+        )
+        import_model.import_file(cr, uid, [bank_statement_id])
+        ids = statement_model.search(
+            cr, uid, [('name', '=', '2014-02-19/00000')])
+        self.assertTrue(ids, 'Statement not found after parse.')
+        statement_id = ids[0]
+        statement_obj = statement_model.browse(
+            cr, uid, statement_id)
+        self.assertTrue(
+            abs(statement_obj.balance_start - 662.23) < 0.00001,
+            'Start balance %f not equal to 662.23' %
+            statement_obj.balance_start
+        )
+        self.assertTrue(
+            abs(statement_obj.balance_end_real - 564.35) < 0.00001,
+            'Real end balance %f not equal to 564.35' %
+            statement_obj.balance_end_real
+        )
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: