diff --git a/account_banking_uk_hsbc/__init__.py b/account_banking_uk_hsbc/__init__.py index 2ecd93336..b9c3e7a5e 100644 --- a/account_banking_uk_hsbc/__init__.py +++ b/account_banking_uk_hsbc/__init__.py @@ -21,4 +21,5 @@ import account_banking_uk_hsbc import wizard +import hsbc_mt940 # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/account_banking_uk_hsbc/hsbc_mt940.py b/account_banking_uk_hsbc/hsbc_mt940.py new file mode 100644 index 000000000..fe60a8ccb --- /dev/null +++ b/account_banking_uk_hsbc/hsbc_mt940.py @@ -0,0 +1,157 @@ +# -*- encoding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2011 credativ Ltd (). +# All Rights Reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +# Import of HSBC data in Swift MT940 format +# + +from account_banking.parsers import models +from account_banking.parsers.convert import str2date +from tools.translate import _ +from mt940_parser import HSBCParser +import re + +bt = models.mem_bank_transaction + +def record2float(record, value): + if record['creditmarker'][-1] == 'C': + return float(record[value]) + return -float(record[value]) + +class transaction(models.mem_bank_transaction): + + mapping = { + 'execution_date' : 'valuedate', + 'effective_date' : 'bookingdate', + 'local_currency' : 'currency', + 'transfer_type' : 'bookingcode', + 'reference' : 'custrefno', + 'message' : 'furtherinfo' + } + + type_map = { + 'TRF': bt.ORDER, + } + + def __init__(self, record, *args, **kwargs): + ''' + Transaction creation + ''' + super(transaction, self).__init__(*args, **kwargs) + for key, value in self.mapping.iteritems(): + if record.has_key(value): + setattr(self, key, record[value]) + + self.transferred_amount = record2float(record, 'amount') + + #print record.get('bookingcode') + if not self.is_valid(): + print "Invalid: %s" % record + def is_valid(self): + ''' + We don't have remote_account so override base + ''' + return (self.execution_date + and self.transferred_amount and True) or False + +class statement(models.mem_bank_statement): + ''' + Bank statement imported data + ''' + + def import_record(self, record): + def _transmission_number(): + self.id = record['transref'] + def _account_number(): + # The wizard doesn't check for sort code + self.local_account = record['sortcode'] + ' ' + record['accnum'].zfill(8) + def _statement_number(): + self.id = '-'.join([self.id, self.local_account, record['statementnr']]) + def _opening_balance(): + self.start_balance = record2float(record,'startingbalance') + self.currency_code = record['currencycode'] + def _closing_balance(): + self.end_balance = record2float(record, 'endingbalance') + self.date = record['bookingdate'] + def _transaction_new(): + self.transactions.append(transaction(record)) + def _transaction_info(): + self.transaction_info(record) + def _not_used(): + print "Didn't use record: %s" % (record,) + + rectypes = { + '20' : _transmission_number, + '25' : _account_number, + '28' : _statement_number, + '28C': _statement_number, + '60F': _opening_balance, + '62F': _closing_balance, + #'64' : _forward_available, + #'62M': _interim_balance, + '61' : _transaction_new, + '86' : _transaction_info, + } + + rectypes.get(record['recordid'], _not_used)() + + def transaction_info(self, record): + ''' + Add extra information to transaction + ''' + # Additional information for previous transaction + if len(self.transactions) < 1: + raise_error('Received additional information for non existent transaction', record) + + transaction = self.transactions[-1] + + transaction.id = ','.join([record[k] for k in ['infoline{0}'.format(i) for i in range(1,5)] if record.has_key(k)]) + +def raise_error(message, line): + raise osv.except_osv(_('Import error'), + 'Error in import:%s\n\n%s' % (message, line)) + +class parser_hsbc_mt940(models.parser): + code = 'HSBC-MT940' + name = _('HSBC Swift MT940 statement export') + country_code = 'GB' + doc = _('''\ + This format is available through + the HSBC web interface. + ''') + + def parse(self, data): + result = [] + statement_list = [st.splitlines() for st in re.split('\n(?=:20:)', data)] + + for statement_lines in statement_list: + stmnt = statement() + records = HSBCParser().parse(statement_lines) + [stmnt.import_record(r) for r in records if r is not None] + + + if stmnt.is_valid(): + result.append(stmnt) + else: + print "Invalid Statement:" + print records[0] + + return result + +# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/account_banking_uk_hsbc/mt940_parser.py b/account_banking_uk_hsbc/mt940_parser.py new file mode 100644 index 000000000..bad3984bc --- /dev/null +++ b/account_banking_uk_hsbc/mt940_parser.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +############################################################################## +# +# Copyright (C) 2011 credativ Ltd (). +# All Rights Reserved +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## + +""" +Parser for HSBC UK MT940 format files +Based on fi_patu's parser +""" +import re +from datetime import datetime + +class HSBCParser(object): + + def __init__( self ): + recparse = dict() + + # MT940 header + recparse["20"] = ":(?P20):(?P.{1,16})" + recparse["25"] = ":(?P25):(?P\d{6})(?P\d{1,29})" + recparse["28"] = ":(?P28C?):(?P.{1,8})" + + # Opening balance 60F + recparse["60F"] = ":(?P60F):(?P[CD])" \ + + "(?P\d{6})(?P.{3})" \ + + "(?P[\d,]{1,15})" + + # Transaction + recparse["61"] = ":(?P61):" \ + + "(?P\d{6})(?P\d{4})?" \ + + "(?PR?[CD])" \ + + "(?P[A-Z])?" \ + + "(?P[\d,]{1,15})" \ + + "(?P[A-Z][A-Z0-9]{3})" \ + + "(?P[A-Za-z0-9 _-]{1,16})" \ + + "(?://)" \ + + "(?P[A-Za-z0-9 _]{1,16})?" \ + + "(?:\n(?P[A-Za-z0-9 _.]))?" + + # Further info + recparse["86"] = ":(?P86):" \ + + "(?P.{1,80})?" \ + + "(?:\n(?P.{1,80}))?" \ + + "(?:\n(?P.{1,80}))?" \ + + "(?:\n(?P.{1,80}))?" \ + + "(?:\n(?P.{1,80}))?" + + # Forward available balance (64) / Closing balance (62F) / Interim balance (62M) + recparse["64"] = ":(?P64|62[FM]):" \ + + "(?P[CD])" \ + + "(?P\d{6})(?P.{3})" \ + + "(?P[\d,]{1,15})" + + for record in recparse: + recparse[record] = re.compile(recparse[record]) + self.recparse = recparse + + + def parse_record(self, line): + """ + Parse record using regexps and apply post processing + """ + for matcher in self.recparse: + matchobj = self.recparse[matcher].match(line) + if matchobj: + break + if not matchobj: + print " **** failed to match line '%s'" % (line) + return + # Strip strings + matchdict = matchobj.groupdict() + + # Remove members set to None + matchdict=dict([(k,v) for k,v in matchdict.iteritems() if v]) + + matchkeys = set(matchdict.keys()) + needstrip = set(["transref", "accnum", "statementnr", "custrefno", + "bankref", "furtherinfo", "infoline1", "infoline2", "infoline3", + "infoline4", "infoline5", "startingbalance", "endingbalance"]) + for field in matchkeys & needstrip: + matchdict[field] = matchdict[field].strip() + + # Convert to float. Comma is decimal separator + needsfloat = set(["startingbalance", "endingbalance", "amount"]) + for field in matchkeys & needsfloat: + matchdict[field] = float(matchdict[field].replace(',','.')) + + # Convert date fields + needdate = set(["prevstmtdate", "valuedate", "bookingdate"]) + for field in matchkeys & needdate: + datestring = matchdict[field] + + post_check = False + if len(datestring) == 4 and field=="bookingdate" and matchdict.has_key("valuedate"): + # Get year from valuedate + datestring = matchdict['valuedate'].strftime('%y') + datestring + post_check = True + try: + matchdict[field] = datetime.strptime(datestring,'%y%m%d') + if post_check and matchdict[field] > matchdict["valuedate"]: + matchdict[field]=matchdict[field].replace(year=matchdict[field].year-1) + except ValueError: + matchdict[field] = None + + return matchdict + + def parse(self, data): + records = [] + # Some records are multiline + for line in data: + if len(line) <= 1: + continue + if line[0] == ':' and len(line) > 1: + records.append(line[:-1]) + else: + records[-1] = records[-1] + '\n' + line[:-1] + + output = [] + for rec in records: + output.append(self.parse_record(rec)) + + return output + +def parse_file(filename): + hsbcfile = open(filename, "r") + p = HSBCParser().parse(hsbcfile.readlines()) + +def main(): + """The main function, currently just calls a dummy filename + + :returns: description + """ + parse_file("testfile") + +if __name__ == '__main__': + main()