From b8af67d9ebc9660e726ada3fefa2799d60f49aa0 Mon Sep 17 00:00:00 2001 From: OpenERP instance user Date: Sun, 11 Dec 2011 17:09:20 +0100 Subject: [PATCH] [ADD] Parser for ING NL statements --- account_banking_nl_ing/__init__.py | 4 + account_banking_nl_ing/__openerp__.py | 57 +++++++ account_banking_nl_ing/__terp__.py | 57 +++++++ account_banking_nl_ing/ing.py | 218 ++++++++++++++++++++++++++ 4 files changed, 336 insertions(+) create mode 100644 account_banking_nl_ing/__init__.py create mode 100644 account_banking_nl_ing/__openerp__.py create mode 100644 account_banking_nl_ing/__terp__.py create mode 100644 account_banking_nl_ing/ing.py diff --git a/account_banking_nl_ing/__init__.py b/account_banking_nl_ing/__init__.py new file mode 100644 index 000000000..12fdf6088 --- /dev/null +++ b/account_banking_nl_ing/__init__.py @@ -0,0 +1,4 @@ +# -*- encoding: utf-8 -*- +import ing + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/account_banking_nl_ing/__openerp__.py b/account_banking_nl_ing/__openerp__.py new file mode 100644 index 000000000..4d06e970c --- /dev/null +++ b/account_banking_nl_ing/__openerp__.py @@ -0,0 +1,57 @@ +############################################################################## +# +# Copyright (C) 2009 - 2011 EduSense BV () +# and Therp BV () +# All Rights Reserved +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract EduSense BV +# or Therp BV +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################## +{ + 'name': 'ING (NL) Bank Statements Import', + 'version': '0.1', + 'license': 'GPL-3', + 'author': 'Therp BV / EduSense BV', + 'website': 'http://therp.nl', + 'category': 'Banking addons', + 'depends': ['account_banking'], + 'init_xml': [], + 'update_xml': [ + ], + 'demo_xml': [], + 'description': ''' +Module to import Dutch ING bank format transation files (CSV format). + +As the ING bank does not provide detailed specification concerning possible +values and their meaning for the fields in the CSV file format, the statements +are parsed according to an educated guess based on incomplete information. +You can contact the banking-addons developers through their launchpad page and +help improve the performance of this import filter on +https://launchpad.net/banking-addons. + +Note that imported bank transfers are organized in statements covering periods +of one week, even if the imported files cover a different period. + +This modules contains no logic, just an import filter for account_banking. + ''', + 'active': False, + 'installable': True, +} diff --git a/account_banking_nl_ing/__terp__.py b/account_banking_nl_ing/__terp__.py new file mode 100644 index 000000000..4d06e970c --- /dev/null +++ b/account_banking_nl_ing/__terp__.py @@ -0,0 +1,57 @@ +############################################################################## +# +# Copyright (C) 2009 - 2011 EduSense BV () +# and Therp BV () +# All Rights Reserved +# +# WARNING: This program as such is intended to be used by professional +# programmers who take the whole responsability of assessing all potential +# consequences resulting from its eventual inadequacies and bugs +# End users who are looking for a ready-to-use solution with commercial +# garantees and support are strongly adviced to contract EduSense BV +# or Therp BV +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################## +{ + 'name': 'ING (NL) Bank Statements Import', + 'version': '0.1', + 'license': 'GPL-3', + 'author': 'Therp BV / EduSense BV', + 'website': 'http://therp.nl', + 'category': 'Banking addons', + 'depends': ['account_banking'], + 'init_xml': [], + 'update_xml': [ + ], + 'demo_xml': [], + 'description': ''' +Module to import Dutch ING bank format transation files (CSV format). + +As the ING bank does not provide detailed specification concerning possible +values and their meaning for the fields in the CSV file format, the statements +are parsed according to an educated guess based on incomplete information. +You can contact the banking-addons developers through their launchpad page and +help improve the performance of this import filter on +https://launchpad.net/banking-addons. + +Note that imported bank transfers are organized in statements covering periods +of one week, even if the imported files cover a different period. + +This modules contains no logic, just an import filter for account_banking. + ''', + 'active': False, + 'installable': True, +} diff --git a/account_banking_nl_ing/ing.py b/account_banking_nl_ing/ing.py new file mode 100644 index 000000000..73b5720bb --- /dev/null +++ b/account_banking_nl_ing/ing.py @@ -0,0 +1,218 @@ +# -*- encoding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2011 Therp BV (). +# +# Based on the multibank module by EduSense BV +# Copyright (C) 2009 EduSense BV (), +# +# All Rights Reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################## + +from account_banking.parsers import models +from account_banking.parsers.convert import str2date +from account_banking.sepa import postalcode +from tools.translate import _ + +import re +import csv + +__all__ = ['parser'] + +bt = models.mem_bank_transaction + +""" +First line states the legend +"Datum","Naam / Omschrijving","Rekening","Tegenrekening","Code","Af Bij","Bedrag (EUR)","MutatieSoort","Mededelingen + +""" + +class transaction_message(object): + ''' + A auxiliary class to validate and coerce read values + ''' + attrnames = [ + 'date', 'remote_owner', 'local_account', 'remote_account', + 'transfer_type', 'debcred', 'transferred_amount', + 'transfer_type_verbose', 'message' + +# 'date', 'local_account', 'transferred_amount', 'debcred', +# 'remote_owner', 'remote_account', 'transfer_type', 'reference', + ] + + ref_expr = re.compile('REF[\*:]([0-9A-Z-z_-]+)') + + def __init__(self, values, subno): + ''' + Initialize own dict with attributes and coerce values to right type + ''' + if len(self.attrnames) != len(values): + raise ValueError( + _('Invalid transaction line: expected %d columns, found %d') + % (len(self.attrnames), len(values))) + self.__dict__.update(dict(zip(self.attrnames, values))) + # for lack of a standardized locale function to parse amounts + self.transferred_amount = float( + re.sub(',', '.', self.transferred_amount)) + if self.debcred == 'Af': + self.transferred_amount = -self.transferred_amount + self.execution_date = self.effective_date = str2date(self.date, '%Y%m%d') + # Set statement_id based on week number + self.statement_id = self.effective_date.strftime('%Yw%W') + self.id = str(subno).zfill(4) + # Normalize basic account numbers + self.remote_account = self.remote_account.replace('.', '').zfill(10) + self.local_account = self.local_account.replace('.', '').zfill(10) + +class transaction(models.mem_bank_transaction): + ''' + Implementation of transaction communication class for account_banking. + ''' + attrnames = ['local_account', 'remote_account', + 'remote_owner', 'transferred_amount', + 'execution_date', 'effective_date', 'transfer_type', + 'reference', 'id', + ] + + """ + Presumably the same transaction types occur in the MT940 format of ING. + From www.ing.nl/Images/MT940_Technische_handleiding_tcm7-69020.pdf + + """ + type_map = { + + 'AC': bt.ORDER, # Acceptgiro + 'BA': bt.PAYMENT_TERMNINAL, # Betaalautomaattransactie + 'CH': bt.ORDER, # Cheque + 'DV': bt.ORDER, # Diversen + 'FL': bt.BANK_TERMINAL, # Filiaalboeking, concernboeking + 'GF': bt.ORDER, # Telefonisch bankieren + 'GM': bt.BANK_TERMINAL, # Geldautomaat + 'GT': bt.ORDER, # Internetbankieren + 'IC': bt.DIRECT_DEBIT, # Incasso + 'OV': bt.ORDER, # Overschrijving + 'PK': bt.BANK_TERMINAL, # Opname kantoor + 'PO': bt.ORDER, # Periodieke overschrijving + 'ST': bt.BANK_TERMINAL, # Storting (eigen rekening of derde) + 'VZ': bt.ORDER, # Verzamelbetaling + } + + def __init__(self, line, *args, **kwargs): + ''' + Initialize own dict with read values. + ''' + super(transaction, self).__init__(*args, **kwargs) + # Copy attributes from auxiliary class to self. + for attr in self.attrnames: + setattr(self, attr, getattr(line, attr)) + self.message = '' + # Decompose structured messages + self.parse_message() + if (self.transfer_type == 'OV' and + not self.remote_account and + not self.remote_owner): + self.transfer_type = 'KN' + + def is_valid(self): + if not self.error_message: + if not self.transferred_amount: + self.error_message = "No transferred amount" + elif not self.execution_date: + self.error_message = "No execution date" + elif not self.remote_account and self.transfer_type not in [ + 'BA', 'FL', 'GM', 'IC', 'PK', 'ST' + ]: + self.error_message = ( + "No remote account for transaction type %s" % + self.transfer_type) + return not self.error_message + + def parse_message(self): + ''' + Parse structured message parts into appropriate attributes. + No processing done here for Triodos, maybe later. + ''' + if self.transfer_type == 'DV': + res = self.ref_expr.search(self.remote_owner) + if res: + self.reference = res.group(1) + self.remote_owner = False + else: + res = self.ref_expr.search(self.message) + if res: + self.reference = res.group(1) + +class statement(models.mem_bank_statement): + ''' + Implementation of bank_statement communication class of account_banking + ''' + def __init__(self, msg, *args, **kwargs): + ''' + Set decent start values based on first transaction read + ''' + super(statement, self).__init__(*args, **kwargs) + self.id = msg.statement_id + self.local_account = msg.local_account + self.date = str2date(msg.date, '%d-%m-%Y') + self.start_balance = self.end_balance = 0 # msg.start_balance + self.import_transaction(msg) + + def import_transaction(self, msg): + ''' + Import a transaction and keep some house holding in the mean time. + ''' + trans = transaction(msg) + self.end_balance += trans.transferred_amount + self.transactions.append(trans) + +class parser(models.parser): + code = 'ING' + country_code = 'NL' + name = _('ING Bank') + doc = _('''\ +The Dutch ING format is basicly a MS Excel CSV format. It is specifically +distinct from the Dutch multibank format. Transactions are not tied to Bank +Statements. +''') + + def parse(self, data): + result = [] + stmnt = None + dialect = csv.excel() + dialect.quotechar = '"' + dialect.delimiter = ';' + lines = data.split('\n') + # Transaction lines are not numbered, so keep a tracer + subno = 0 + for line in csv.reader(lines, dialect=dialect): + # Skip empty (last) lines + if not line: + continue + subno += 1 + msg = transaction_message(line, subno) + if stmnt and stmnt.id != msg.statement_id: + result.append(stmnt) + stmnt = None + subno = 0 + if not stmnt: + stmnt = statement(msg) + else: + stmnt.import_transaction(msg) + result.append(stmnt) + return result + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: