[ENH] Support both Rabo and ING MT940.

bank_statement_parse_mt940/mt940.py

@@ -99,25 +99,17 @@ class MT940(object):
     define functions to handle the tags you need to handle and adjust static
     variables as needed.
 
-    At least, you should override handle_tag_61 and handle_tag_86. Don't forget
-    to call super.
-    handle_tag_* functions receive the remainder of the the line (that is,
-    without ':XX:') and are supposed to write into self.current_transaction"""
-
-    header_lines = 3
-    """One file can contain multiple statements, each with its own poorly
-    documented header. For now, the best thing to do seems to skip that"""
-
-    header_regex = '^{1:[0-9A-Z]{25,25}}'
-    'The file is considered a valid MT940 file when it contains this line'
-
-    footer_regex = '^-XXX$'
-    'The line that denotes end of message, we need to create a new statement'
-
-    tag_regex = '^:[0-9]{2}[A-Z]*:'
-    'The beginning of a record, should be anchored to beginning of the line'
+    At least, you should override handle_tag_61 and handle_tag_86.
+    """
 
     def __init__(self):
+        """Initialize parser - override at least header_regex.
+        
+        This in fact uses the ING syntax, override in others."""
+        self.header_lines = 3  # Number of lines to skip
+        self.header_regex = '^{1:[0-9A-Z]{25,25}}'  # Start of relevant data
+        self.footer_regex = '^-}$|^-XXX$'  # Stop processing on seeing this
+        self.tag_regex = '^:[0-9]{2}[A-Z]*:'  # Start of new tag
         self.current_statement = None
         self.current_transaction = None
         self.statements = []

bank_statement_parse_nl_ing_mt940/mt940.py

@@ -27,11 +27,6 @@ from openerp.addons.bank_statement_parse_mt940.mt940 import (
 class MT940Parser(MT940):
     """Parser for ing MT940 bank statement import files."""
 
-    name = 'ING MT940 (structured)'
-    country_code = 'NL'
-    code = 'INT_MT940_STRUC'
-    footer_regex = '^-}$|^-XXX$'
-
     tag_61_regex = re.compile(
         r'^(?P<date>\d{6})(?P<line_date>\d{0,4})'
         r'(?P<sign>[CD])(?P<amount>\d+,\d{2})N(?P<type>.{3})'

bank_statement_parse_nl_rabo_mt940/__init__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2014 Therp BV (<http://therp.nl>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+from . import account_bank_statement_import

bank_statement_parse_nl_rabo_mt940/__openerp__.py

@@ -0,0 +1,41 @@
+# -*- coding: utf-8 -*-
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    This module copyright (C) 2014 Therp BV (<http://therp.nl>).
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+{
+    "name": "MT940 import for dutch Rabobank",
+    "version": "1.1",
+    "author": "Therp BV,Odoo Community Association (OCA)",
+    "complexity": "normal",
+    "description": """
+This addon imports the structured MT940 format as offered by the dutch
+    Rabobank.
+    """,
+    "category": "Account Banking",
+    "depends": [
+        'bank_statement_parse_mt940'
+    ],
+    "data": [],
+    "js": [],
+    "css": [],
+    "qweb": [],
+    "auto_install": False,
+    "installable": True,
+    "application": False,
+}

bank_statement_parse_nl_rabo_mt940/account_bank_statement_import.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+"""Parse a MT940 RABO file."""
+##############################################################################
+#
+#    Copyright (C) 2013 Therp BV <http://therp.nl>
+#    All Rights Reserved
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as published
+#    by the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import logging
+from openerp import models
+from openerp.addons.bank_statement_parse.parserlib import convert_statements
+from .mt940 import MT940Parser as Parser
+
+
+_logger = logging.getLogger(__name__)
+
+
+class AccountBankStatementImport(models.TransientModel):
+    """Add parsing of RABO mt940 files to bank statement import."""
+    _inherit = 'account.bank.statement.import'
+
+    def _parse_file(self, cr, uid, data_file, context=None):
+        """Parse a MT940 RABO file."""
+        parser = Parser()
+        try:
+            _logger.debug("Try parsing with MT940 RABO.")
+            return convert_statements(parser.parse(data_file))
+        except ValueError:
+            # Returning super will call next candidate:
+            _logger.debug("Statement file was not a MT940 RABO file.")
+            return super(AccountBankStatementImport, self)._parse_file(
+                cr, uid, data_file, context=context)
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

bank_statement_parse_nl_rabo_mt940/mt940.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+"""Implement parser for MT940 files - Rabobank dialect."""
+##############################################################################
+#
+#    OpenERP, Open Source Management Solution
+#    Copyright (C) 2015 Therp BV <http://therp.nl>.
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+##############################################################################
+import re
+from string import printable
+from datetime import datetime
+from openerp.addons.bank_statement_parse_mt940.mt940 import (
+    MT940, str2amount, get_subfields, handle_common_subfields)
+
+
+class MT940Parser(MT940):
+    """Implement parser for MT940 files - Rabobank dialect."""
+
+    tag_61_regex = re.compile(
+        r'^(?P<date>\d{6})(?P<sign>[CD])(?P<amount>\d+,\d{2})N(?P<type>.{3})'
+        r'(?P<reference>\w{1,16})')
+
+    def __init__(self):
+        """Initialize parser - override at least header_regex."""
+        super(MT940Parser, self).__init__()
+        self.header_lines = 1  # Number of lines to skip
+
+        # Do not user $ for end of string below: line contains much
+        # more data than just the first line.
+        self.header_regex = '^:940:'  # Start of relevant data
+
+    def parse(self, data):
+        """Filter Unprintable characters from file data.
+
+        The file contents of the Rabobank tend to contain unprintable
+        characters that prevent proper parsing. These will be removed.
+        """
+        data = ''.join([x for x in data if x in printable])
+        return super(MT940Parser, self).parse(data)
+
+    def handle_tag_60F(self, data):
+        """get start balance and currency"""
+        # For the moment only first 60F record
+        # The alternative would be to split the file and start a new
+        # statement for each 20: tag encountered.
+        stmt = self.current_statement
+        if not stmt.local_currency:
+            stmt.local_currency = data[7:10]
+            stmt.date = datetime.strptime(data[1:7], '%y%m%d')
+            stmt.start_balance = str2amount(data[0], data[10:])
+            stmt.statement_id = '%s-%s' % (
+                self.current_statement.date.strftime('%Y-%m-%d'),
+                self.current_statement.statement_id)
+
+    def handle_tag_61(self, data):
+        """Handle tag 61: transaction data."""
+        super(MT940Parser, self).handle_tag_61(data)
+        parsed_data = self.tag_61_regex.match(data).groupdict()
+        self.current_transaction.transferred_amount = (
+            str2amount(parsed_data['sign'], parsed_data['amount']))
+        self.current_transaction.eref = parsed_data['reference']
+
+    def handle_tag_86(self, data):
+        """Handle tag 86: transaction details"""
+        if not self.current_transaction:
+            return
+        codewords = ['RTRN', 'BENM', 'ORDP', 'CSID', 'BUSP', 'MARF', 'EREF',
+                     'PREF', 'REMI', 'ID', 'PURP', 'ULTB', 'ULTD',
+                     'CREF', 'IREF', 'NAME', 'ADDR', 'ULTC', 'EXCH', 'CHGS']
+        subfields = get_subfields(data, codewords)
+        transaction = self.current_transaction
+        # If we have no subfields, set message to whole of data passed:
+        if not subfields:
+            transaction.message = data
+        else:
+            handle_common_subfields(transaction, subfields)
+            # Use subfields for transaction details:
+            if 'NAME' in subfields:
+                transaction.remote_owner = ' '.join(subfields['NAME'])
+            if 'ADDR' in subfields:
+                # Do NOT join address fields, array is expected on other code!
+                transaction.remote_owner_address = subfields['ADDR']
+        # Prevent handling tag 86 later for non transaction details:
+        self.current_transaction = None
+
+# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: