account_banking:

[FIX] adapt to attribute change in account_bank_statement_reconcile
[FIX] fix possible view overwrites by eliminating duplicate view ids.
[IMP] Hide import payments button on banking statements when not in open
      or draft state
[IMP] Extended parser model to support payment batch reporting
[IMP] Added extra function fields to payment_order
[IMP] Extended import parser to substitute payment batches with payment
      lines
[IMP] Use better id matching and reporting when parsing and matching

account_banking_nl_multibank:
[IMP] Added clieop3 batch reporting
This commit is contained in:
Pieter J. Kersten
2011-11-07 13:34:48 +01:00
parent f228acec94
commit 63466dee19
5 changed files with 270 additions and 69 deletions

View File

@@ -65,6 +65,7 @@ from tools.translate import _
from wizard.banktools import get_or_create_bank
import pooler
import netsvc
from tools import config
def warning(title, message):
'''Convenience routine'''
@@ -810,9 +811,57 @@ payment_line()
class payment_order(osv.osv):
'''
Enable extra states for payment exports
Enable extra states for payment exports and add extra functionality.
'''
_inherit = 'payment.order'
def __no_transactions(self, cursor, uid, ids, field_name, arg=None,
context=None):
'''
Return the number of payment lines in this order.
Don't bother using the ORM, as it is very inefficient in this respect.
'''
if not hasattr(ids, '__iter__'):
ids = [ids]
cursor.execute('SELECT o.id, count(*) '
'FROM payment_order o, payment_line l '
'WHERE o.id = l.order_id '
' AND o.id in (%s)'
'GROUP BY o.id' % (','.join([str(x) for x in ids]))
)
return dict(cursor.fetchall())
def __amount(self, cursor, uid, ids, field_name, arg=None, context=None):
'''
Calculation routine for balance of payment order. Grasps all
payment_order_lines and summarize the total amount of the order.
'''
payment_line_obj = self.pool.get('payment.line')
line_ids = payment_line_obj.search(
cursor, uid, [('order_id', 'in', ids)], context=context
)
order_lines = payment_line_obj.browse(cursor, ids, line_ids)
result = {}
for line in order_lines:
if line.order_id.id in result:
result[line.payment_id] += line.amount_currency
else:
result[line.payment_id] = line.amount_currency
return result
def __handler(self, cursor, uid, ids, field_name, arg=None, context=None):
'''
Return the payment generator ID from the payment_type. As this hops two
relations, fields.relation won't do.
'''
query = ("SELECT o.id, t.code "
"FROM payment_order o, payment_mode m, payment_type t "
"WHERE o.mode = m.id AND m.type = t.id AND "
"o.id in (%s)" % ','.join([str(id) for id in ids])
)
cursor.execute(query)
return dict(cursor.fetchall())
_columns = {
'date_planned': fields.date(
'Scheduled date if fixed',
@@ -881,6 +930,15 @@ class payment_order(osv.osv):
"execution."
)
),
'amount': fields.function(
__amount, digits=(16, int(config['price_accuracy'])),
method=True, string='Amount Total'
),
'handler': fields.function(__handler, method=True, string='Handler'),
'no_transactions': fields.function(
__no_transactions, type='integer', method=True,
string='No Transactions'
),
}
def _write_payment_lines(self, cursor, uid, ids, **kwargs):
@@ -930,15 +988,19 @@ class payment_order(osv.osv):
def set_done(self, cursor, uid, ids, *args):
'''
Extend standard transition to update children as well.
Extend standard transition to update children as well and to handle
list of ids.
'''
if not hasattr(ids, '__iter__'):
ids = [ids]
self._write_payment_lines(cursor, uid, ids,
export_state='done',
date_done=time.strftime('%Y-%m-%d')
)
return super(payment_order, self).set_done(
cursor, uid, ids, *args
)
for id in ids:
if not super(payment_order, self).set_done(cursor, uid, id, *args):
return False
return True
def get_wizard(self, type):
'''

View File

@@ -253,8 +253,8 @@
</record>
<!-- Reset trigger on button_confirm to the trigger code in this module -->
<record id="view_banking_bank_statement_form_4" model="ir.ui.view">
<field name="name">account.bank.statement.form.banking-4</field>
<record id="view_banking_bank_statement_form_7" model="ir.ui.view">
<field name="name">account.bank.statement.form.banking-7</field>
<field name="inherit_id" ref="account.view_bank_statement_form" />
<field name="model">account.bank.statement</field>
<field name="type">form</field>
@@ -282,6 +282,23 @@
</field>
</record>
<!-- Make payment lines import button sensitive for extra states -->
<record id="view_banking_bank_statement_form_8" model="ir.ui.view">
<field name="name">account.bank.statement.form.banking-8</field>
<field name="inherit_id" ref="account_payment.view_bank_statement_form"/>
<field name="model">account.bank.statement</field>
<field name="type">form</field>
<field name="arch" type="xml">
<xpath expr="/form/group/button[@string='Import payment lines']"
position="replace">
<button name="%(account_payment.wizard_populate_payment)d"
type="action" states="draft,open"
string="Import payment lines"
/>
</xpath>
</field>
</record>
<!-- Set trigger on IBAN and acc_number fields in res_partner_bank form -->
<record id="view_partner_bank_account_banking_form_1" model="ir.ui.view">
<field name="name">res.partner.bank.form.banking-1</field>

View File

@@ -49,7 +49,7 @@ class mem_bank_statement(object):
def is_valid(self):
'''
Final check: ok if calculated end_balance and parsed end_balance are
identical and perform a heuristic check on the transactions.
identical and survive a heuristic check on the transactions.
'''
if any([x for x in self.transactions if not x.is_valid()]):
return False
@@ -70,7 +70,7 @@ class mem_bank_transaction(object):
# Message id
'statement_id',
# The bank statement this message was reported on
# The id of the bank statement this message was reported on
'transfer_type',
# Action type that initiated this message
@@ -128,7 +128,10 @@ class mem_bank_transaction(object):
# The other parties two letter ISO country code belonging to the previous
'remote_owner_custno',
# The other parties customer number
# The other parties customer number. Right now, this is expected to be
# the OpenERP partner_id. Without access to the database, that is a
# bit difficult to fill, so please leave this untouched until further
# notice.
# For identification of private other parties, the following attributes
# are available and self explaining. Most banks allow only one per
@@ -171,6 +174,11 @@ class mem_bank_transaction(object):
'provision_costs_currency',
'provision_costs_description',
# When banks use transactions to report back on payment batches, the
# following two attributes should be used.
'payment_batch_id',
'payment_batch_no_transactions',
# An error message for interaction with the user
# Only used when mem_transaction.valid returns False.
'error_message',
@@ -195,10 +203,15 @@ class mem_bank_transaction(object):
# Will be selected for matching.
# ORDER Order to the bank. Can work both ways.
# Will be selected for matching.
# PAYMENT_BATCH A payment batch. Can work in both directions.
# Incoming payment batch transactions can't be
# matched with payments, outgoing can.
# Will be selected for matching.
# PAYMENT_BATCH A payment batch. Can work in both directions, but
# only outgoing payment batch transactions can be
# matched with payments. Reserved for condensed
# reporting (only batch id, execution date, number
# of transactions and total transfered amount are
# used). The business logic will match payment orders
# instead of payment order lines and substitute the
# transaction with fake transactions generated from
# the lines of the matched payment order.
# PAYMENT_TERMINAL A payment with debit/credit card in a (web)shop
# Invoice numbers and other hard criteria are most
# likely missing.
@@ -214,6 +227,11 @@ class mem_bank_transaction(object):
# transfer type Post Office, meaning a cash withdrawal from one of their
# agencies. This can be mapped to BANK_TERMINAL without losing any
# significance for OpenERP.
#
# Also, most banks differentiate between an incoming order and an incoming
# line from a payment batch. To the receiving party (us), there is no
# significance in the difference. You are required to map these lines to ORDER
# instead of marking them as PAYMENT_BATCH.
BANK_COSTS = 'BC'
BANK_TERMINAL = 'BT'

View File

@@ -46,6 +46,7 @@ bt = models.mem_bank_transaction
# This variable is used to match supplier invoices with an invoice date after
# the real payment date. This can occur with online transactions (web shops).
# TODO: Convert this to a proper configuration variable
payment_window = datetime.timedelta(days=10)
def parser_types(*args, **kwargs):
@@ -111,6 +112,7 @@ class banking_import(wizard.interface):
self.__state = ''
self.__linked_invoices = {}
self.__linked_payments = {}
# TODO: reprocess multiple matched invoices and payments afterwards
self.__multiple_matches = []
def _fill_results(self, *args, **kwargs):
@@ -125,9 +127,10 @@ class banking_import(wizard.interface):
'out_refund': 'customer',
'in_refund': 'supplier',
}
retval = struct(move_line=move_line, partner_id=move_line.partner_id.id,
partner_bank_id=partner_bank_id,
reference=move_line.ref
retval = struct(move_line = move_line,
partner_id = move_line.partner_id.id,
partner_bank_id = partner_bank_id,
reference = move_line.ref
)
if move_line.invoice:
retval.invoice = move_line.invoice
@@ -137,7 +140,7 @@ class banking_import(wizard.interface):
if partial:
move_line.reconcile_partial_id = reconcile_obj.create(
cursor, uid, {'line_partial_ids': [(4, 0, [move_line.id])]}
cursor, uid, {'line_partial_ids': [(6, 0, [move_line.id])]}
)
else:
if move_line.reconcile_partial_id:
@@ -148,7 +151,7 @@ class banking_import(wizard.interface):
partial_ids = []
move_line.reconcile_id = reconcile_obj.create(
cursor, uid, {
'line_id': [
'line_ids': [
(4, x, False) for x in [move_line.id] + partial_ids
],
'line_partial_ids': [
@@ -156,8 +159,61 @@ class banking_import(wizard.interface):
]
}
)
return retval
def _link_payment_batch(self, cursor, uid, trans, payment_lines,
payment_orders, log
):
'''
Find a matching payment batch hiding several payments and inject these
into the matching sequence.
'''
digits = int(config['price_accuracy'])
candidates = [x for x in payment_orders
if str2date(x['object'].date_created, '%Y-%m-%d') <= \
trans.execution_date and
trans.payment_batch_no_transactions == \
x['object'].no_transactions and
round(x['amount_currency'], digits) == \
round(-trans.transferred_amount, digits)
]
if candidates:
if len(candidates) == 1:
# Order found, now substitute with generated transactions
payment_order = candidates[0]['object']
injected = []
for line in [x for x in payment_lines if x.order_id.id ==
payment_order.id]:
transaction = trans.copy()
transaction.type = bt.ORDER
transaction.id = '%s-%s' % (trans.id, line.id)
# Reminder: we were paying, so decreasing bank funds
transaction.transferred_amount = -line.amount_currency
transaction.remote_currency = line.currency
transaction.remote_owner = line.partner_id.name
transaction.remote_owner_custno = line.partner_id.id
if line.bank_id.iban:
transaction.remote_account = sepa.IBAN(line.bank_id.iban)
else:
transaction.remote_account = line.bank_id.acc_number
transaction.remote_bank_bic = line.bank_id.bank.bic
transaction.reference = line.communication
transaction.message = line.communication2
transaction.provision_costs = None
transaction.provision_costs_currency = None
transaction.provision_costs_description = None
injected.append(transaction)
return injected
else:
log.append(_('Found multiple matching payment batches: %s.'
'Can\'t choose') %
', '.join([x.name for x in candidates])
)
else:
log.append(_('Found unknown payment batch: %s') % trans.message)
return []
def _link_payment(self, cursor, uid, trans, payment_lines,
partner_ids, bank_account_ids, log):
'''
@@ -169,11 +225,14 @@ class banking_import(wizard.interface):
# for credited customer invoices, which will be matched later on too.
digits = int(config['price_accuracy'])
candidates = [x for x in payment_lines
if x.communication == trans.reference
and round(x.amount, digits) == -round(trans.transferred_amount, digits)
and trans.remote_account in (x.bank_id.acc_number,
x.bank_id.iban)
]
if ((x.communication and x.communication == trans.reference) or
(x.communication2 and x.communication2 == trans.message))
and round(x.amount, digits) ==
-round(trans.transferred_amount, digits)
and trans.remote_account in (
x.bank_id.acc_number, sepa.IBAN(x.bank_id.iban)
)
]
if len(candidates) == 1:
candidate = candidates[0]
# Check cache to prevent multiple matching of a single payment
@@ -184,8 +243,11 @@ class banking_import(wizard.interface):
'export_state': 'done',
'date_done': trans.effective_date.strftime('%Y-%m-%d')}
)
return self._get_move_info(cursor, uid, candidate.move_line_id)
return self._get_move_info(
cursor, uid, candidate.move_line_id,
partner_bank_id=\
bank_account_ids and bank_account_ids[0].id or False
)
return False
@@ -234,7 +296,7 @@ class banking_import(wizard.interface):
Return the eyecatcher for an invoice
'''
return invoice.type.startswith('in_') and invoice.name or \
invoice.number
'%s (%s)' % (invoice.number, invoice.name)
def has_id_match(invoice, ref, msg):
'''
@@ -373,7 +435,7 @@ class banking_import(wizard.interface):
_('Unable to link transaction id %(trans)s (ref: %(ref)s) to invoice: '
'%(no_candidates)s candidates found; can\'t choose.') % {
'trans': '%s.%s' % (trans.statement_id, trans.id),
'ref': trans.reference,
'ref': trans.reference or trans.message,
'no_candidates': len(best) or len(candidates)
})
log.append(' ' +
@@ -536,6 +598,7 @@ class banking_import(wizard.interface):
self.pool = pooler.get_pool(cursor.dbname)
company_obj = self.pool.get('res.company')
user_obj = self.pool.get('res.user')
partner_obj = self.pool.get('res.partner')
partner_bank_obj = self.pool.get('res.partner.bank')
journal_obj = self.pool.get('account.journal')
move_line_obj = self.pool.get('account.move.line')
@@ -543,9 +606,9 @@ class banking_import(wizard.interface):
statement_obj = self.pool.get('account.bank.statement')
statement_line_obj = self.pool.get('account.bank.statement.line')
statement_file_obj = self.pool.get('account.banking.imported.file')
#account_obj = self.pool.get('account.account')
payment_order_obj = self.pool.get('payment.order')
currency_obj = self.pool.get('res.currency')
digits = int(config['price_accuracy'])
# get the parser to parse the file
parser_code = form['parser']
@@ -627,16 +690,35 @@ class banking_import(wizard.interface):
# No filtering can be done, as empty dates carry value for C2B
# communication. Most likely there are much less sent payments
# than reconciled and open/draft payments.
payment_lines = []
payment_orders = []
cursor.execute("SELECT l.id FROM payment_order o, payment_line l "
"WHERE l.order_id = o.id AND "
"o.state = 'sent' AND "
"l.date_done IS NULL"
)
payment_line_ids = [x[0] for x in cursor.fetchall()]
if payment_line_ids:
payment_lines = payment_line_obj.browse(cursor, uid, payment_line_ids)
else:
payment_lines = []
line_ids = [x[0] for x in cursor.fetchall()]
if line_ids:
# Get payment_orders and calculated total amounts as well in
# order to be able to match condensed transaction feedbacks.
# Use SQL for this, as it is way more efficient than server
# side processing.
payment_lines = payment_line_obj.browse(cursor, uid, line_ids)
cursor.execute("SELECT o.id, l.currency, SUM(l.amount_currency)"
" FROM payment_order o, payment_line l "
"WHERE l.order_id = o.id AND "
"o.state = 'sent' AND "
"l.date_done IS NULL "
"GROUP BY o.id, l.currency"
)
order_totals = dict([(x[0], round(x[2], digits))
for x in cursor.fetchall()])
payment_orders = [
dict(id=x.id, object=x, amount_currency=order_totals[x.id])
for x in list(payment_order_obj.browse(cursor, uid,
order_totals.keys()
))
]
for statement in statements:
if statement.local_account in error_accounts:
@@ -777,6 +859,21 @@ class banking_import(wizard.interface):
transaction.provision_costs_currency = None
transaction.provision_costs_description = None
# Check on payments batches, as they are able to replace
# the current transaction by generated ones.
if transaction.type == bt.PAYMENT_BATCH:
payments = self._link_payment_batch(
cursor, uid, transaction, payment_lines,
payment_orders, results.log
)
if payments:
transaction = payments[0]
injected += payments[1:]
else:
results.trans_skipped_cnt += 1
i += 1
continue
# Allow inclusion of generated bank invoices
if transaction.type == bt.BANK_COSTS:
lines = self._link_costs(
@@ -790,6 +887,22 @@ class banking_import(wizard.interface):
partner_ids = [account_info.bank_partner_id.id]
partner_banks = []
# Easiest match: customer id
elif transaction.remote_owner_custno:
partner_ids = partner_obj.browse(
cursor, uid, [transaction.remote_owner_custno]
)
iban_acc = sepa.IBAN(transaction.remote_account)
if iban_acc.valid:
domain = [('iban','=',str(iban_acc))]
else:
domain = [('acc_number','=',transaction.remote_account)]
partner_banks = partner_bank_obj.browse(
cursor, uid, partner_bank_obj.search(
cursor, uid, domain
)
)
else:
# Link remote partner, import account when needed
partner_banks = get_bank_accounts(
@@ -831,9 +944,6 @@ class banking_import(wizard.interface):
partner_bank_id = None
partner_banks = []
partner_ids = [partner_id]
else:
partner_ids = []
partner_banks = []
# Credit means payment... isn't it?
if transaction.transferred_amount < 0 and payment_lines:
@@ -922,39 +1032,14 @@ class banking_import(wizard.interface):
"FROM payment_line "
"WHERE date_done IS NULL "
"AND id IN (%s)"
")" % (','.join([str(x) for x in payment_line_ids]))
")" % (','.join([str(x) for x in line_ids]))
)
order_ids = [x[0] for x in cursor.fetchall()]
if order_ids:
# Use workflow logics for the orders. Recode logic from
# account_payment, in order to increase efficiency.
# Use workflow logics for the orders.
payment_order_obj.set_done(cursor, uid, order_ids,
{'state': 'done'}
)
wf_service = netsvc.LocalService('workflow')
for id in order_ids:
wf_service.trg_validate(uid, 'payment.order', id, 'done',
cursor
)
# Original code. Didn't take workflow logistics into account...
#
#cursor.execute(
# "UPDATE payment_order o "
# "SET state = 'done', "
# "date_done = '%s' "
# "FROM payment_line l "
# "WHERE o.state = 'sent' "
# "AND o.id = l.order_id "
# "AND l.id NOT IN ("
# "SELECT DISTINCT id FROM payment_line "
# "WHERE date_done IS NULL "
# "AND id IN (%s)"
# ")" % (
# time.strftime('%Y-%m-%d'),
# ','.join([str(x) for x in payment_line_ids])
# )
#)
report = [
'%s: %s' % (_('Total number of statements'),

View File

@@ -44,8 +44,9 @@ class transaction_message(object):
attrnames = [
'date', 'local_account', 'remote_account', 'remote_owner', 'u1', 'u2',
'u3', 'local_currency', 'start_balance', 'remote_currency',
'transferred_amount', 'execution_date', 'effective_date', 'nr1',
'transfer_type', 'nr2', 'reference', 'message', 'statement_id'
'transferred_amount', 'execution_date', 'effective_date',
'transfer_type_id', 'transfer_type', 'nr2', 'reference', 'message',
'statement_id'
]
@staticmethod
@@ -55,7 +56,7 @@ class transaction_message(object):
get prefixed by zeroes as in BBAN. Convert those to 'old' local
account numbers
Edit: All account number now follow the BBAN scheme. As SNS Bank,
Edit: All account numbers now follow the BBAN scheme. As SNS Bank,
from which this module was re-engineered, follows the Dutch
Banking Tools regulations, it is considered to be used by all banks
in the Netherlands which comply to it. If not, please notify us.
@@ -84,6 +85,10 @@ class transaction_message(object):
self.execution_date = str2date(self.execution_date, '%d-%m-%Y')
self.effective_date = str2date(self.effective_date, '%d-%m-%Y')
self.id = str(subno).zfill(4)
# Map outgoing payment batches from general payments. They are
# distinguished from normal payments with type_id '9722'
if self.transfer_type == 'OVB' and self.transfer_type_id == '9722':
self.transfer_type = 'VZB'
class transaction(models.mem_bank_transaction):
'''
@@ -109,6 +114,7 @@ class transaction(models.mem_bank_transaction):
'OVS': bt.ORDER,
'PRV': bt.BANK_COSTS,
'TEL': bt.ORDER,
'VZB': bt.PAYMENT_BATCH,
}
def __init__(self, line, *args, **kwargs):
@@ -150,12 +156,16 @@ class transaction(models.mem_bank_transaction):
5. Cash withdrawals from banks are too not seen as a transfer between
two accounts - the cash exits the banking system. These withdrawals
have their transfer_type set to 'OPN'.
6. Payment batches are reported back in condensed format, meaning that
there is no individual transaction information, just a signal that the
total amount of the batch has been sent.
'''
return (self.transferred_amount and self.execution_date and
self.effective_date) and (
self.remote_account or
self.transfer_type in [
'KST', 'PRV', 'BTL', 'BEA', 'OPN', 'KNT', 'DIV',
'KST', 'PRV', 'BTL', 'BEA', 'OPN', 'KNT', 'DIV', 'VZB'
]
and not self.error_message
)
@@ -243,7 +253,7 @@ class transaction(models.mem_bank_transaction):
elif self.transfer_type == 'IDB':
# Payment by iDeal transaction
# Remote owner can be part of message, while remote_owner can be
# set to the intermediate party, which we don't need.
# set to an intermediate party, which we don't need.
parts = self.message.split(' ')
# Second part: structured id, date & time
subparts = parts[1].split()
@@ -262,6 +272,15 @@ class transaction(models.mem_bank_transaction):
parts = parts[:-1]
self.message = ' '.join(parts)
elif self.transfer_type == 'VZB':
# Payment batch, condensed format.
# NOTE: This has only been tested with ClieOp3 payment batches.
# If your milage varies, please inform us of your findings.
parts = self.message.split('.')[1].split()
self.payment_batch_no_transactions = int(parts[0])
self.message = ' '.join(parts[1:])
self.payment_batch_id = parts[2][-4:]
class statement(models.mem_bank_statement):
'''
Implementation of bank_statement communication class of account_banking