diff --git a/account_banking/parsers/models.py b/account_banking/parsers/models.py index 3ed4511b7..a0579af74 100644 --- a/account_banking/parsers/models.py +++ b/account_banking/parsers/models.py @@ -65,10 +65,116 @@ class mem_bank_transaction(object): ''' # Lock attributes to enable parsers to trigger non-conformity faults __slots__ = [ - 'id', 'local_account', 'local_currency', 'execution_date', - 'effective_date', 'remote_owner', 'remote_account', - 'remote_currency', 'transferred_amount', 'transfer_type', - 'reference', 'message', 'statement_id', + + 'id', + # Message id + + 'statement_id', + # The bank statement this message was reported on + + 'transfer_type', + # Action type that initiated this message + + 'reference', + # A reference to this message for communication + + 'local_account', + # The account this message was meant for + + 'local_currency', + # The currency the bank used to process the transferred amount + + 'execution_date', + # The requested execution date of the action - order date if you like + + 'effective_date', + # The real execution date of the action + + 'remote_account', + # The account of the other party + + 'remote_currency', + # The currency used by the other party + + 'exchange_rate', + # The exchange rate used for conversion of local_currency and + # remote_currency + + 'transferred_amount', + # The actual amount transferred - + # negative means sent, positive means received + # Most banks use the local_currency to express this amount, but there + # may be exceptions I'm unaware of. + + 'message', + # A direct message from the initiating party to the receiving party + # A lot of banks abuse this to stuff all kinds of structured + # information in this message. It is the task of the parser to split + # this up into the appropriate attributes. + + 'remote_owner', + # The name of the other party + + 'remote_owner_address', + # The other parties address lines - the only attribute that is a list + + 'remote_owner_city', + # The other parties city name belonging to the previous + + 'remote_owner_postalcode', + # The other parties postal code belonging to the address + + 'remote_owner_country_code', + # The other parties two letter ISO country code belonging to the previous + + 'remote_owner_custno', + # The other parties customer number + + # For identification of private other parties, the following attributes + # are available and self explaining. Most banks allow only one per + # message. + 'remote_owner_ssn', + 'remote_owner_tax_id', + 'remote_owner_employer_id', + 'remote_owner_passport_no', + 'remote_owner_idcard_no', + 'remote_owner_driverslicense_no', + + # Other private party information fields. Not all banks use it, but + # at least SEPA messaging allowes it. + 'remote_owner_birthdate', + 'remote_owner_cityofbirth', + 'remote_owner_countryofbirth', + 'remote_owner_provinceofbirth', + + # For the identification of remote banks, the following attributes are + # available and self explaining. Most banks allow only one per + # message. + 'remote_bank_bic', + 'remote_bank_bei', + 'remote_bank_ibei', + 'remote_bank_eangln', + 'remote_bank_chips_uid', + 'remote_bank_duns', + 'remote_bank_tax_id', + + # For other identification purposes: both of the next attributes must + # be filled. + 'remote_bank_proprietary_id', + 'remote_bank_proprietary_id_issuer', + + # The following attributes are for allowing banks to communicate about + # specific transactions. The transferred_amount must include these + # costs. + # Please make sure that the costs are signed for the right direction. + 'provision_costs', + 'provision_costs_currency', + 'provision_costs_description', + + # An error message for interaction with the user + # Only used when mem_transaction.valid returns False. + 'error_message', + ] # transfer_type's to be used by the business logic. @@ -128,20 +234,13 @@ class mem_bank_transaction(object): } def __init__(self, *args, **kwargs): + ''' + Initialize values + ''' super(mem_bank_transaction, self).__init__(*args, **kwargs) - self.id = '' - self.local_account = '' - self.local_currency = '' - self.execution_date = '' - self.effective_date = '' - self.remote_account = '' - self.remote_owner = '' - self.remote_currency = '' - self.transferred_amount = '' - self.transfer_type = '' - self.reference = '' - self.message = '' - self.statement_id = '' + for attr in self.__slots__: + setattr(self, attr, None) + self.remote_owner_address = [] def copy(self): ''' @@ -218,6 +317,9 @@ class parser(object): name -> the name of the parser, shown to the user and translatable. code -> the identifier you care to give it. Not translatable + country_code -> the two letter ISO code of the country this parser is + built for: used to recreate country when new partners + are auto created doc -> the description of the identifier. Shown to the user. Translatable. @@ -226,6 +328,7 @@ class parser(object): __metaclass__ = parser_type name = None code = None + country_code = None doc = __doc__ def parse(self, data): diff --git a/account_banking/sepa/online.py b/account_banking/sepa/online.py index ed8582552..fe50bd717 100644 --- a/account_banking/sepa/online.py +++ b/account_banking/sepa/online.py @@ -235,8 +235,8 @@ def bank_info(bic): # We need to split it into two fields... if not address.Zip_Code: if address.Location: - address.Zip_Code, address.Location = \ - postalcode.split(full_bic[4:6], address.Location) + iso, address.Zip_Code, address.Location = \ + postalcode.split(address.Location, full_bic[4:6]) bankaddress = struct( street = address.Address.title(), diff --git a/account_banking/sepa/postalcode.py b/account_banking/sepa/postalcode.py index 6fad63bc6..087dee623 100644 --- a/account_banking/sepa/postalcode.py +++ b/account_banking/sepa/postalcode.py @@ -135,15 +135,45 @@ class PostalCode(object): _formats[iso] = PostalCodeFormat(formatstr) @classmethod - def split(cls, iso, str_): + def split(cls, str_, iso=''): ''' Split string in (postalcode, remainder) following the specs of country . - Returns both the postal code and the remaining part of + + Returns iso, postal code and the remaining part of . + + When iso is filled but postal code remains empty, no postal code could + be found according to the rules of iso. + + When iso is empty but postal code is not, a proximity match was + made where multiple hits gave similar results. A postal code is + likely, but a unique iso code could not be identified. + + When neither iso or postal code are filled, no proximity match could + be made. ''' if iso in cls._formats: - return cls._formats[iso].split(str_) - return ('', str_) + return (iso,) + tuple(cls._formats[iso].split(str_)) + + # Find optimum (= max length postalcode) when iso code is unknown + all = {} + opt_iso = '' + max_l = 0 + for key in cls._formats.iterkeys(): + i, p, c = cls.split(str_, key) + l = len(p) + if l > max_l: + max_l = l + opt_iso = i + if l in all: + all[l].append((i, p, c)) + else: + all[l] = [(i, p, c)] + if max_l > 0: + if len(all[max_l]) > 1: + return ('',) + all[max_l][0][1:] + return all[max_l][0] + return ('', '', str_) @classmethod def get(cls, iso, str_): diff --git a/account_banking/wizard/bank_import.py b/account_banking/wizard/bank_import.py index 2ff9f8185..18bac7bcc 100644 --- a/account_banking/wizard/bank_import.py +++ b/account_banking/wizard/bank_import.py @@ -38,6 +38,7 @@ from tools.translate import _ from account_banking.parsers import models from account_banking.parsers.convert import * from account_banking.struct import struct +from account_banking import sepa from banktools import * bt = models.mem_bank_transaction @@ -270,7 +271,8 @@ class banking_import(wizard.interface): if partner_ids: candidates = [x for x in move_lines if x.partner_id.id in partner_ids and - (not _cached(x) or _remaining(x)) + str2date(x.date, '%Y-%m-%d') <= trans.execution_date + and (not _cached(x) or _remaining(x)) ] else: candidates = [] @@ -288,6 +290,7 @@ class banking_import(wizard.interface): # reporting this. candidates = [x for x in candidates or move_lines if x.invoice and has_id_match(x.invoice, ref, msg) + and str2date(x.invoice.date_invoice, '%Y-%m-%d') <= trans.execution_date and (not _cached(x) or _remaining(x)) ] @@ -297,6 +300,7 @@ class banking_import(wizard.interface): candidates = [x for x in move_lines if round(abs(x.credit or x.debit), digits) == round(abs(trans.transferred_amount), digits) and + str2date(x.date, '%Y-%m-%d') <= trans.execution_date and (not _cached(x) or _remaining(x)) ] @@ -306,13 +310,11 @@ class banking_import(wizard.interface): # amounts expected and received. # # TODO: currency coercing - if trans.transferred_amount > 0: - func = lambda x, y=abs(trans.transferred_amount), z=digits:\ - round(x.debit, z) == round(y, z) - else: - func = lambda x, y=abs(trans.transferred_amount), z=digits:\ - round(x.credit, z) == round(y, z) - best = [x for x in candidates if func(x)] + best = [x for x in candidates + if round(abs(x.credit or x.debit), digits) == + round(abs(trans.transferred_amount), digits) and + str2date(x.date, '%Y-%m-%d') <= trans.execution_date + ] if len(best) == 1: # Exact match move_line = best[0] @@ -324,14 +326,31 @@ class banking_import(wizard.interface): _cache(move_line) elif len(candidates) > 1: - # Multiple matches - log.append( - _('Unable to link transaction id %(trans)s (ref: %(ref)s) to invoice: ' - '%(no_candidates)s candidates found; can\'t choose.') % { - 'trans': trans.id, - 'ref': trans.reference, - 'no_candidates': len(best) or len(candidates) - }) + # Before giving up, check cache for catching duplicate + # transfers first + paid = [x for x in move_lines + if x.invoice and has_id_match(x.invoice, ref, msg) + and str2date(x.invoice.date_invoice, '%Y-%m-%d') <= trans.execution_date + and (_cached(x) and not _remaining(x)) + ] + if paid: + log.append( + _('Unable to link transaction id %(trans)s ' + '(ref: %(ref)s) to invoice: ' + 'invoice %(invoice)s was already paid') % { + 'trans': '%s.%s' % (trans.statement_id, trans.id), + 'ref': trans.reference, + 'invoice': eyecatcher(paid[0].invoice) + }) + else: + # Multiple matches + log.append( + _('Unable to link transaction id %(trans)s (ref: %(ref)s) to invoice: ' + '%(no_candidates)s candidates found; can\'t choose.') % { + 'trans': '%s.%s' % (trans.statement_id, trans.id), + 'ref': trans.reference, + 'no_candidates': len(best) or len(candidates) + }) move_line = False partial = False @@ -400,6 +419,9 @@ class banking_import(wizard.interface): digits = int(config['price_accuracy']) amount = round(abs(trans.transferred_amount), digits) + # Make sure to be able to pinpoint our costs invoice for later + # matching + reference = '%s.%s: %s' % (trans.statement_id, trans.id, trans.reference) # search supplier invoice invoice_obj = self.pool.get('account.invoice') @@ -409,12 +431,12 @@ class banking_import(wizard.interface): ('partner_id', '=', account_info.bank_partner_id.id), ('company_id', '=', account_info.company_id.id), ('date_invoice', '=', date2str(trans.effective_date)), - ('reference', '=', trans.reference), + ('reference', '=', reference), ('amount_total', '=', amount), ] ) if invoice_ids and len(invoice_ids) == 1: - invoice = invoice_obj.browse(cursor, uid, invoice_ids) + invoice = invoice_obj.browse(cursor, uid, invoice_ids)[0] elif not invoice_ids: # create supplier invoice partner_obj = self.pool.get('res.partner') @@ -437,7 +459,7 @@ class banking_import(wizard.interface): account_id = account_info.bank_partner_id.property_account_payable.id, date_invoice = date2str(trans.effective_date), reference_type = 'none', - reference = trans.reference, + reference = reference, name = trans.reference or trans.message, check_total = amount, invoice_line = invoice_lines, @@ -471,6 +493,7 @@ class banking_import(wizard.interface): self.pool = pooler.get_pool(cursor.dbname) company_obj = self.pool.get('res.company') user_obj = self.pool.get('res.user') + partner_bank_obj = self.pool.get('res.partner.bank') journal_obj = self.pool.get('account.journal') move_line_obj = self.pool.get('account.move.line') payment_line_obj = self.pool.get('payment.line') @@ -627,9 +650,12 @@ class banking_import(wizard.interface): and account_info.currency_id.code != statement.local_currency: # TODO: convert currencies? results.log.append( - _('Statement for account %(bank_account)s uses different ' - 'currency than the defined bank journal.') % - {'bank_account': statement.local_account} + _('Statement %(statement_id)s for account %(bank_account)s' + ' uses different currency than the defined bank journal.' + ) % { + 'bank_account': statement.local_account, + 'statement_id': statement.id + } ) error_accounts[statement.local_account] = True results.error_cnt += 1 @@ -661,22 +687,22 @@ class banking_import(wizard.interface): )) imported_statement_ids.append(statement_id) - # move each line to the right period and try to match it with an + # move each transaction to the right period and try to match it with an # invoice or payment subno = 0 - injected = False + injected = [] i = 0 max_trans = len(statement.transactions) while i < max_trans: move_info = False if injected: - transaction, injected = injected, False + # Force FIFO behavior + transaction = injected.pop(0) else: transaction = statement.transactions[i] - - # Keep a tracer for identification of order in a statement in case - # of missing transaction ids. - subno += 1 + # Keep a tracer for identification of order in a statement in case + # of missing transaction ids. + subno += 1 # Link accounting period period_id = get_period(self.pool, cursor, uid, @@ -686,6 +712,26 @@ class banking_import(wizard.interface): results.trans_skipped_cnt += 1 continue + # When bank costs are part of transaction itself, split it. + if transaction.type != bt.BANK_COSTS and transaction.provision_costs: + # Create new transaction for bank costs + costs = transaction.copy() + costs.type = bt.BANK_COSTS + costs.id = '%s-prov' % transaction.id + costs.transferred_amount = transaction.provision_costs + costs.remote_currency = transaction.provision_costs_currency + costs.message = transaction.provision_costs_description + injected.append(costs) + + # Remove bank costs from current transaction + # Note that this requires that the transferred_amount + # includes the bank costs and that the costs itself are + # signed correctly. + transaction.transferred_amount -= transaction.provision_costs + transaction.provision_costs = None + transaction.provision_costs_currency = None + transaction.provision_costs_description = None + # Allow inclusion of generated bank invoices if transaction.type == bt.BANK_COSTS: lines = self._link_costs( @@ -708,20 +754,38 @@ class banking_import(wizard.interface): if partner_banks: partner_ids = [x.partner_id.id for x in partner_banks] elif transaction.remote_owner: + iban = sepa.IBAN(transaction.remote_account) + if iban.valid: + country_code = iban.country_code + elif transaction.remote_owner_country_code: + country_code = transaction.remote_owner_country_code + elif hasattr(parser, 'country_code') and parser.country_code: + country_code = parser.country_code + else: + country_code = None partner_id = get_or_create_partner( self.pool, cursor, uid, transaction.remote_owner, - results.log + transaction.remote_owner_address, + transaction.remote_owner_postalcode, + transaction.remote_owner_city, + country_code, results.log ) if transaction.remote_account: partner_bank_id = create_bank_account( self.pool, cursor, uid, partner_id, transaction.remote_account, - transaction.remote_owner, results.log + transaction.remote_owner, + transaction.remote_owner_address, + transaction.remote_owner_city, + country_code, results.log ) + partner_banks = partner_bank_obj.browse( + cursor, uid, [partner_bank_id] + ) + else: + partner_bank_id = None + partner_banks = [] partner_ids = [partner_id] - partner_banks = partner_bank_obj.browse( - cursor, uid, [partner_bank_id] - ) else: partner_ids = [] partner_banks = [] @@ -737,11 +801,16 @@ class banking_import(wizard.interface): # Second guess, invoice -> may split transaction, so beware if not move_info: - # Link invoice - if any - move_info, injected = self._link_invoice( + # Link invoice - if any. Although bank costs are not an + # invoice, automatic invoicing on bank costs will create + # these, and invoice matching still has to be done. + move_info, remainder = self._link_invoice( cursor, uid, transaction, move_lines, partner_ids, partner_banks, results.log ) + if remainder: + injected.append(remainder) + if not move_info: # Use the default settings, but allow individual partner # settings to overrule this. Note that you need to change @@ -787,6 +856,8 @@ class banking_import(wizard.interface): statement_line_id = statement_line_obj.create(cursor, uid, values) results.trans_loaded_cnt += 1 + # Only increase index when all generated transactions are + # processed as well if not injected: i += 1 @@ -813,15 +884,24 @@ class banking_import(wizard.interface): ) ) report = [ - '%s: %s' % (_('Total number of statements'), results.stat_skipped_cnt + results.stat_loaded_cnt), - '%s: %s' % (_('Total number of transactions'), results.trans_skipped_cnt + results.trans_loaded_cnt), - '%s: %s' % (_('Number of errors found'), results.error_cnt), - '%s: %s' % (_('Number of statements skipped due to errors'), results.stat_skipped_cnt), - '%s: %s' % (_('Number of transactions skipped due to errors'), results.trans_skipped_cnt), - '%s: %s' % (_('Number of statements loaded'), results.stat_loaded_cnt), - '%s: %s' % (_('Number of transactions loaded'), results.trans_loaded_cnt), - '%s: %s' % (_('Number of transactions matched'), results.trans_matched_cnt), - '%s: %s' % (_('Number of bank costs invoices created'), results.bank_costs_invoice_cnt), + '%s: %s' % (_('Total number of statements'), + results.stat_skipped_cnt + results.stat_loaded_cnt), + '%s: %s' % (_('Total number of transactions'), + results.trans_skipped_cnt + results.trans_loaded_cnt), + '%s: %s' % (_('Number of errors found'), + results.error_cnt), + '%s: %s' % (_('Number of statements skipped due to errors'), + results.stat_skipped_cnt), + '%s: %s' % (_('Number of transactions skipped due to errors'), + results.trans_skipped_cnt), + '%s: %s' % (_('Number of statements loaded'), + results.stat_loaded_cnt), + '%s: %s' % (_('Number of transactions loaded'), + results.trans_loaded_cnt), + '%s: %s' % (_('Number of transactions matched'), + results.trans_matched_cnt), + '%s: %s' % (_('Number of bank costs invoices created'), + results.bank_costs_invoice_cnt), '', '%s:' % ('Error report'), '', diff --git a/account_banking/wizard/banktools.py b/account_banking/wizard/banktools.py index fc39e9610..e70261364 100644 --- a/account_banking/wizard/banktools.py +++ b/account_banking/wizard/banktools.py @@ -109,15 +109,52 @@ def get_bank_accounts(pool, cursor, uid, account_number, log, fail=False): return False return partner_bank_obj.browse(cursor, uid, bank_account_ids) -def get_or_create_partner(pool, cursor, uid, name, log): +def get_or_create_partner(pool, cursor, uid, name, address, postal_code, city, + country_code, log): ''' Get or create the partner belonging to the account holders name ''' partner_obj = pool.get('res.partner') partner_ids = partner_obj.search(cursor, uid, [('name', 'ilike', name)]) if not partner_ids: + # Try brute search on address and then match reverse + address_obj = pool.get('res.partner.address') + filter = [('partner_id', '<>', None)] + if country_code: + country_obj = pool.get('res.country') + country_ids = country_obj.search( + cursor, uid, [('code','=',country_code.upper())] + ) + country_id = country_ids and country_ids[0] or False + filter.append(('country_id', '=', country_id)) + if address: + if len(address) >= 1: + filter.append(('street', 'ilike', addres[0])) + if len(address) > 1: + filter.append(('street2', 'ilike', addres[1])) + if city: + filter.append(('city', 'ilike', city)) + if postal_code: + filter.append(('zip', 'ilike', postal_code)) + address_ids = address_obj.search(cursor, uid, filter) + key = name.lower() + partner_ids = [x.partner_id.id + for x in address_obj.browse(cursor, uid, address_ids) + if x.partner_id.name.lower() in key + ] + if not partner_ids: + if (not country_code) or not country_id: + country_id = pool.get('res.user').browse(cursor, uid, uid)\ + .company_id.partner_id.country.id partner_id = partner_obj.create(cursor, uid, dict( - name=name, active=True, comment='Generated by Import Bank Statements File', + name=name, active=True, comment='Generated from Bank Statements Import', + address=[(0,0,{ + 'street': address and address[0] or '', + 'street2': len(address) > 1 and address[1] or '', + 'city': city, + 'zip': postal_code or '', + 'country_id': country_id, + })], )) elif len(partner_ids) > 1: log.append( @@ -256,7 +293,8 @@ def get_or_create_bank(pool, cursor, uid, bic, online=False, code=None, return bank_id, country_id def create_bank_account(pool, cursor, uid, partner_id, - account_number, holder_name, log + account_number, holder_name, address, city, + country_code, log ): ''' Create a matching bank account with this holder for this partner. @@ -277,19 +315,25 @@ def create_bank_account(pool, cursor, uid, partner_id, values.iban = str(iban) values.acc_number = iban.BBAN bankcode = iban.bankcode + iban.countrycode + country_code = iban.countrycode + + if not country_code: + country = pool.get('res.partner').browse( + cursor, uid, partner_id).country + country_code = country.code + country_id = country.id + elif iban.valid: country_ids = country_obj.search(cursor, uid, [('code', '=', iban.countrycode)] ) country_id = country_ids[0] - else: + + if not iban.valid: # No, try to convert to IBAN values.state = 'bank' values.acc_number = account_number - country = pool.get('res.partner').browse( - cursor, uid, partner_id).country_id - country_id = country.id - if country.code in sepa.IBAN.countries: - account_info = sepa.online.account_info(country.code, + if country_code in sepa.IBAN.countries: + account_info = sepa.online.account_info(country_code, values.acc_number ) if account_info: @@ -331,18 +375,4 @@ def create_bank_account(pool, cursor, uid, partner_id, # Create bank account and return return pool.get('res.partner.bank').create(cursor, uid, values) -def get_or_create_bank_partner(): - ''' - Find the partner belonging to a bank. When not found, create one using the - available data. - ''' - pass - -def generate_supplier_invoice(): - ''' - Create an supplier invoice for a transaction from a bank. Used to create - invoices on the fly when parsing bank costs. - ''' - pass - # vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4: diff --git a/account_banking_nl_multibank/multibank.py b/account_banking_nl_multibank/multibank.py index e0e042661..2d279bb44 100644 --- a/account_banking_nl_multibank/multibank.py +++ b/account_banking_nl_multibank/multibank.py @@ -29,6 +29,7 @@ Bank Statements along with Bank Transactions. ''' from account_banking.parsers import models from account_banking.parsers.convert import str2date +from account_banking.sepa import postalcode from tools.translate import _ import csv @@ -67,7 +68,7 @@ class transaction_message(object): accountno = stripped return accountno - def __init__(self, values): + def __init__(self, values, subno): ''' Initialize own dict with attributes and coerce values to right type ''' @@ -82,14 +83,16 @@ class transaction_message(object): self.transferred_amount = float(self.transferred_amount) self.execution_date = str2date(self.execution_date, '%d-%m-%Y') self.effective_date = str2date(self.effective_date, '%d-%m-%Y') + self.id = str(subno).zfill(4) class transaction(models.mem_bank_transaction): ''' Implementation of transaction communication class for account_banking. ''' - attrnames = ['remote_account', 'remote_currency', 'transferred_amount', + attrnames = ['local_account', 'local_currency', 'remote_account', + 'remote_owner', 'remote_currency', 'transferred_amount', 'execution_date', 'effective_date', 'transfer_type', - 'reference', 'message' + 'reference', 'message', 'statement_id', 'id', ] type_map = { @@ -105,6 +108,7 @@ class transaction(models.mem_bank_transaction): 'OPN': bt.BANK_TERMINAL, 'OVS': bt.ORDER, 'PRV': bt.BANK_COSTS, + 'TEL': bt.ORDER, } def __init__(self, line, *args, **kwargs): @@ -112,26 +116,37 @@ class transaction(models.mem_bank_transaction): Initialize own dict with read values. ''' super(transaction, self).__init__(*args, **kwargs) + # Copy attributes from auxiliary class to self. for attr in self.attrnames: setattr(self, attr, getattr(line, attr)) + # Decompose structured messages + self.parse_message() + # Set reference when bank costs + if self.type == bt.BANK_COSTS: + self.reference = self.message[:32].rstrip() def is_valid(self): ''' There are a few situations that can be signaled as 'invalid' but are valid nontheless: + 1. Transfers from one account to another under the same account holder get not always a remote_account and remote_owner. They have their transfer_type set to 'PRV'. + 2. Invoices from the bank itself are communicated through statements. These too have no remote_account and no remote_owner. They have a transfer_type set to 'KST' or 'KNT'. + 3. Transfers sent through the 'International Transfers' system get their feedback rerouted through a statement, which is not designed to hold the extra fields needed. These transfers have their transfer_type set to 'BTL'. + 4. Cash payments with debit cards are not seen as a transfer between accounts, but as a cash withdrawal. These withdrawals have their transfer_type set to 'BEA'. + 5. Cash withdrawals from banks are too not seen as a transfer between two accounts - the cash exits the banking system. These withdrawals have their transfer_type set to 'OPN'. @@ -141,7 +156,107 @@ class transaction(models.mem_bank_transaction): self.remote_account or self.transfer_type in [ 'KST', 'PRV', 'BTL', 'BEA', 'OPN', 'KNT', - ]) + ] + and not self.error_message + ) + + def parse_message(self): + ''' + Parse structured message parts into appropriate attributes + ''' + if self.transfer_type == 'ACC': + # Accept Giro - structured message payment + # First part of message is redundant information - strip it + msg = self.message[self.message.index('navraagnr.'):] + self.message = ' '.join(msg.split()) + + elif self.transfer_type == 'BEA': + # Payment through payment terminal + # Remote owner is part of message, while remote_owner is set + # to the intermediate party, which we don't need. + self.remote_owner = self.message[:23].rstrip() + self.remote_owner_city = self.message[23:31].rstrip() + self.message = self.message[31:] + + elif self.transfer_type == 'BTL': + # International transfers. + # Remote party is encoded in message, including bank costs + parts = self.message.split(' ') + last = False + for part in parts: + if part.startswith('bedrag. '): + # The ordered transferred amount + currency, amount = part.split('. ')[1].split() + if self.remote_currency != currency.upper(): + self.error_message = \ + 'Remote currency in message differs from transaction.' + else: + self.local_amount = float(amount) + elif part.startswith('koers. '): + # The currency rate used + self.exchange_rate = float(part.split('. ')[1]) + elif part.startswith('transfer prov. '): + # The provision taken by the bank + # Note that the amount must be negated to get the right + # direction + currency, costs = part.split('. ')[1].split() + self.provision_costs = -float(costs) + self.provision_costs_currency = currency.upper() + self.provision_costs_description = 'Transfer costs' + elif part.startswith('aan. '): + # The remote owner + self.remote_owner = part.replace('aan. ', '').rstrip() + last = True + elif last: + # Last parts are address lines + address = part.rstrip() + iso, pc, city = postalcode.split(address) + if pc and city: + self.remote_owner_postalcode = pc + self.remote_owner_city = city.strip() + self.remote_owner_country_code = iso + else: + self.remote_owner_address.append(address) + + elif self.transfer_type == 'DIV': + # A diverse transfer. Message can be anything, but has some + # structure + ptr = self.message.find(self.reference) + if ptr > 0: + address = self.message[:ptr].rstrip().split(' ') + length = len(address) + if length >= 1: + self.remote_owner = address[0] + if length >= 2: + self.remote_owner_address.append(address[1]) + if length >= 3: + self.remote_owner_city = address[2] + self.message = self.message[ptr:].rstrip() + rest = self.message.split('transactiedatum ') + self.execution_date = str2date(rest[1], '%d %m %Y') + self.message = rest[0].rstrip() + + elif self.transfer_type == 'IDB': + # Payment by iDeal transaction + # Remote owner can be part of message, while remote_owner can be + # set to the intermediate party, which we don't need. + parts = self.message.split(' ') + # Second part: structured id, date & time + subparts = parts[1].split() + datestr = '-'.join(subparts[1:4]) + timestr = ':'.join(subparts[4:]) + parts[1] = ' '.join([subparts[0], datestr, timestr]) + # Only replace reference when redundant + if self.reference == parts[0]: + if parts[2]: + self.reference = ' '.join([parts[2], datestr, timestr]) + else: + self.reference += ' '.join([datestr, timestr]) + # Optional fourth path contains remote owners name + if len(parts) > 3 and parts[-1].find(self.remote_owner) < 0: + self.remote_owner = parts[-1].rstrip() + parts = parts[:-1] + self.message = ' '.join(parts) class statement(models.mem_bank_statement): ''' @@ -168,6 +283,7 @@ class statement(models.mem_bank_statement): class parser(models.parser): code = 'NLBT' + country_code = 'NL' name = _('Dutch Banking Tools') doc = _('''\ The Dutch Banking Tools format is basicly a MS Excel CSV format. @@ -187,14 +303,18 @@ to Bank Statements. if lines and lines[0].count(',') > lines[0].count(';'): dialect.delimiter = ',' dialect.quotechar = "'" + # Transaction lines are not numbered, so keep a tracer + subno = 0 for line in csv.reader(lines, dialect=dialect): # Skip empty (last) lines if not line: continue - msg = transaction_message(line) + subno += 1 + msg = transaction_message(line, subno) if stmnt and stmnt.id != msg.statement_id: result.append(stmnt) stmnt = None + subno = 0 if not stmnt: stmnt = statement(msg) else: