From 63466dee191f18bf8020a23953bf898b8e9b371c Mon Sep 17 00:00:00 2001
From: "Pieter J. Kersten"
Date: Mon, 7 Nov 2011 13:34:48 +0100
Subject: [PATCH] account_banking: [FIX] adapt to attribute change in
account_bank_statement_reconcile [FIX] fix possible view overwrites by
eliminating duplicate view ids. [IMP] Hide import payments button on banking
statements when not in open or draft state [IMP] Extended parser model
to support payment batch reporting [IMP] Added extra function fields to
payment_order [IMP] Extended import parser to substitute payment batches with
payment lines [IMP] Use better id matching and reporting when parsing
and matching
account_banking_nl_multibank:
[IMP] Added clieop3 batch reporting
---
account_banking/account_banking.py | 72 ++++++++-
account_banking/account_banking_view.xml | 21 ++-
account_banking/parsers/models.py | 32 +++-
account_banking/wizard/bank_import.py | 185 ++++++++++++++++------
account_banking_nl_multibank/multibank.py | 29 +++-
5 files changed, 270 insertions(+), 69 deletions(-)
diff --git a/account_banking/account_banking.py b/account_banking/account_banking.py
index a23fb7b12..572e87c8e 100644
--- a/account_banking/account_banking.py
+++ b/account_banking/account_banking.py
@@ -65,6 +65,7 @@ from tools.translate import _
from wizard.banktools import get_or_create_bank
import pooler
import netsvc
+from tools import config
def warning(title, message):
'''Convenience routine'''
@@ -810,9 +811,57 @@ payment_line()
class payment_order(osv.osv):
'''
- Enable extra states for payment exports
+ Enable extra states for payment exports and add extra functionality.
'''
_inherit = 'payment.order'
+
+ def __no_transactions(self, cursor, uid, ids, field_name, arg=None,
+ context=None):
+ '''
+ Return the number of payment lines in this order.
+ Don't bother using the ORM, as it is very inefficient in this respect.
+ '''
+ if not hasattr(ids, '__iter__'):
+ ids = [ids]
+ cursor.execute('SELECT o.id, count(*) '
+ 'FROM payment_order o, payment_line l '
+ 'WHERE o.id = l.order_id '
+ ' AND o.id in (%s)'
+ 'GROUP BY o.id' % (','.join([str(x) for x in ids]))
+ )
+ return dict(cursor.fetchall())
+
+ def __amount(self, cursor, uid, ids, field_name, arg=None, context=None):
+ '''
+ Calculation routine for balance of payment order. Grasps all
+ payment_order_lines and summarize the total amount of the order.
+ '''
+ payment_line_obj = self.pool.get('payment.line')
+ line_ids = payment_line_obj.search(
+ cursor, uid, [('order_id', 'in', ids)], context=context
+ )
+ order_lines = payment_line_obj.browse(cursor, ids, line_ids)
+ result = {}
+ for line in order_lines:
+ if line.order_id.id in result:
+ result[line.payment_id] += line.amount_currency
+ else:
+ result[line.payment_id] = line.amount_currency
+ return result
+
+ def __handler(self, cursor, uid, ids, field_name, arg=None, context=None):
+ '''
+ Return the payment generator ID from the payment_type. As this hops two
+ relations, fields.relation won't do.
+ '''
+ query = ("SELECT o.id, t.code "
+ "FROM payment_order o, payment_mode m, payment_type t "
+ "WHERE o.mode = m.id AND m.type = t.id AND "
+ "o.id in (%s)" % ','.join([str(id) for id in ids])
+ )
+ cursor.execute(query)
+ return dict(cursor.fetchall())
+
_columns = {
'date_planned': fields.date(
'Scheduled date if fixed',
@@ -881,6 +930,15 @@ class payment_order(osv.osv):
"execution."
)
),
+ 'amount': fields.function(
+ __amount, digits=(16, int(config['price_accuracy'])),
+ method=True, string='Amount Total'
+ ),
+ 'handler': fields.function(__handler, method=True, string='Handler'),
+ 'no_transactions': fields.function(
+ __no_transactions, type='integer', method=True,
+ string='No Transactions'
+ ),
}
def _write_payment_lines(self, cursor, uid, ids, **kwargs):
@@ -930,15 +988,19 @@ class payment_order(osv.osv):
def set_done(self, cursor, uid, ids, *args):
'''
- Extend standard transition to update children as well.
+ Extend standard transition to update children as well and to handle
+ list of ids.
'''
+ if not hasattr(ids, '__iter__'):
+ ids = [ids]
self._write_payment_lines(cursor, uid, ids,
export_state='done',
date_done=time.strftime('%Y-%m-%d')
)
- return super(payment_order, self).set_done(
- cursor, uid, ids, *args
- )
+ for id in ids:
+ if not super(payment_order, self).set_done(cursor, uid, id, *args):
+ return False
+ return True
def get_wizard(self, type):
'''
diff --git a/account_banking/account_banking_view.xml b/account_banking/account_banking_view.xml
index 815cbf5b2..ce1728025 100644
--- a/account_banking/account_banking_view.xml
+++ b/account_banking/account_banking_view.xml
@@ -253,8 +253,8 @@
-
- account.bank.statement.form.banking-4
+
+ account.bank.statement.form.banking-7account.bank.statementform
@@ -282,6 +282,23 @@
+
+
+ account.bank.statement.form.banking-8
+
+ account.bank.statement
+ form
+
+
+
+
+
+
+
res.partner.bank.form.banking-1
diff --git a/account_banking/parsers/models.py b/account_banking/parsers/models.py
index ee9802c23..d535999bc 100644
--- a/account_banking/parsers/models.py
+++ b/account_banking/parsers/models.py
@@ -49,7 +49,7 @@ class mem_bank_statement(object):
def is_valid(self):
'''
Final check: ok if calculated end_balance and parsed end_balance are
- identical and perform a heuristic check on the transactions.
+ identical and survive a heuristic check on the transactions.
'''
if any([x for x in self.transactions if not x.is_valid()]):
return False
@@ -70,7 +70,7 @@ class mem_bank_transaction(object):
# Message id
'statement_id',
- # The bank statement this message was reported on
+ # The id of the bank statement this message was reported on
'transfer_type',
# Action type that initiated this message
@@ -128,7 +128,10 @@ class mem_bank_transaction(object):
# The other parties two letter ISO country code belonging to the previous
'remote_owner_custno',
- # The other parties customer number
+ # The other parties customer number. Right now, this is expected to be
+ # the OpenERP partner_id. Without access to the database, that is a
+ # bit difficult to fill, so please leave this untouched until further
+ # notice.
# For identification of private other parties, the following attributes
# are available and self explaining. Most banks allow only one per
@@ -171,6 +174,11 @@ class mem_bank_transaction(object):
'provision_costs_currency',
'provision_costs_description',
+ # When banks use transactions to report back on payment batches, the
+ # following two attributes should be used.
+ 'payment_batch_id',
+ 'payment_batch_no_transactions',
+
# An error message for interaction with the user
# Only used when mem_transaction.valid returns False.
'error_message',
@@ -195,10 +203,15 @@ class mem_bank_transaction(object):
# Will be selected for matching.
# ORDER Order to the bank. Can work both ways.
# Will be selected for matching.
- # PAYMENT_BATCH A payment batch. Can work in both directions.
- # Incoming payment batch transactions can't be
- # matched with payments, outgoing can.
- # Will be selected for matching.
+ # PAYMENT_BATCH A payment batch. Can work in both directions, but
+ # only outgoing payment batch transactions can be
+ # matched with payments. Reserved for condensed
+ # reporting (only batch id, execution date, number
+ # of transactions and total transfered amount are
+ # used). The business logic will match payment orders
+ # instead of payment order lines and substitute the
+ # transaction with fake transactions generated from
+ # the lines of the matched payment order.
# PAYMENT_TERMINAL A payment with debit/credit card in a (web)shop
# Invoice numbers and other hard criteria are most
# likely missing.
@@ -214,6 +227,11 @@ class mem_bank_transaction(object):
# transfer type Post Office, meaning a cash withdrawal from one of their
# agencies. This can be mapped to BANK_TERMINAL without losing any
# significance for OpenERP.
+ #
+ # Also, most banks differentiate between an incoming order and an incoming
+ # line from a payment batch. To the receiving party (us), there is no
+ # significance in the difference. You are required to map these lines to ORDER
+ # instead of marking them as PAYMENT_BATCH.
BANK_COSTS = 'BC'
BANK_TERMINAL = 'BT'
diff --git a/account_banking/wizard/bank_import.py b/account_banking/wizard/bank_import.py
index 66ebbfdef..9b1fc9ced 100644
--- a/account_banking/wizard/bank_import.py
+++ b/account_banking/wizard/bank_import.py
@@ -46,6 +46,7 @@ bt = models.mem_bank_transaction
# This variable is used to match supplier invoices with an invoice date after
# the real payment date. This can occur with online transactions (web shops).
+# TODO: Convert this to a proper configuration variable
payment_window = datetime.timedelta(days=10)
def parser_types(*args, **kwargs):
@@ -111,6 +112,7 @@ class banking_import(wizard.interface):
self.__state = ''
self.__linked_invoices = {}
self.__linked_payments = {}
+ # TODO: reprocess multiple matched invoices and payments afterwards
self.__multiple_matches = []
def _fill_results(self, *args, **kwargs):
@@ -125,9 +127,10 @@ class banking_import(wizard.interface):
'out_refund': 'customer',
'in_refund': 'supplier',
}
- retval = struct(move_line=move_line, partner_id=move_line.partner_id.id,
- partner_bank_id=partner_bank_id,
- reference=move_line.ref
+ retval = struct(move_line = move_line,
+ partner_id = move_line.partner_id.id,
+ partner_bank_id = partner_bank_id,
+ reference = move_line.ref
)
if move_line.invoice:
retval.invoice = move_line.invoice
@@ -137,7 +140,7 @@ class banking_import(wizard.interface):
if partial:
move_line.reconcile_partial_id = reconcile_obj.create(
- cursor, uid, {'line_partial_ids': [(4, 0, [move_line.id])]}
+ cursor, uid, {'line_partial_ids': [(6, 0, [move_line.id])]}
)
else:
if move_line.reconcile_partial_id:
@@ -148,7 +151,7 @@ class banking_import(wizard.interface):
partial_ids = []
move_line.reconcile_id = reconcile_obj.create(
cursor, uid, {
- 'line_id': [
+ 'line_ids': [
(4, x, False) for x in [move_line.id] + partial_ids
],
'line_partial_ids': [
@@ -156,8 +159,61 @@ class banking_import(wizard.interface):
]
}
)
+
return retval
+ def _link_payment_batch(self, cursor, uid, trans, payment_lines,
+ payment_orders, log
+ ):
+ '''
+ Find a matching payment batch hiding several payments and inject these
+ into the matching sequence.
+ '''
+ digits = int(config['price_accuracy'])
+ candidates = [x for x in payment_orders
+ if str2date(x['object'].date_created, '%Y-%m-%d') <= \
+ trans.execution_date and
+ trans.payment_batch_no_transactions == \
+ x['object'].no_transactions and
+ round(x['amount_currency'], digits) == \
+ round(-trans.transferred_amount, digits)
+ ]
+ if candidates:
+ if len(candidates) == 1:
+ # Order found, now substitute with generated transactions
+ payment_order = candidates[0]['object']
+ injected = []
+ for line in [x for x in payment_lines if x.order_id.id ==
+ payment_order.id]:
+ transaction = trans.copy()
+ transaction.type = bt.ORDER
+ transaction.id = '%s-%s' % (trans.id, line.id)
+ # Reminder: we were paying, so decreasing bank funds
+ transaction.transferred_amount = -line.amount_currency
+ transaction.remote_currency = line.currency
+ transaction.remote_owner = line.partner_id.name
+ transaction.remote_owner_custno = line.partner_id.id
+ if line.bank_id.iban:
+ transaction.remote_account = sepa.IBAN(line.bank_id.iban)
+ else:
+ transaction.remote_account = line.bank_id.acc_number
+ transaction.remote_bank_bic = line.bank_id.bank.bic
+ transaction.reference = line.communication
+ transaction.message = line.communication2
+ transaction.provision_costs = None
+ transaction.provision_costs_currency = None
+ transaction.provision_costs_description = None
+ injected.append(transaction)
+ return injected
+ else:
+ log.append(_('Found multiple matching payment batches: %s.'
+ 'Can\'t choose') %
+ ', '.join([x.name for x in candidates])
+ )
+ else:
+ log.append(_('Found unknown payment batch: %s') % trans.message)
+ return []
+
def _link_payment(self, cursor, uid, trans, payment_lines,
partner_ids, bank_account_ids, log):
'''
@@ -169,11 +225,14 @@ class banking_import(wizard.interface):
# for credited customer invoices, which will be matched later on too.
digits = int(config['price_accuracy'])
candidates = [x for x in payment_lines
- if x.communication == trans.reference
- and round(x.amount, digits) == -round(trans.transferred_amount, digits)
- and trans.remote_account in (x.bank_id.acc_number,
- x.bank_id.iban)
- ]
+ if ((x.communication and x.communication == trans.reference) or
+ (x.communication2 and x.communication2 == trans.message))
+ and round(x.amount, digits) ==
+ -round(trans.transferred_amount, digits)
+ and trans.remote_account in (
+ x.bank_id.acc_number, sepa.IBAN(x.bank_id.iban)
+ )
+ ]
if len(candidates) == 1:
candidate = candidates[0]
# Check cache to prevent multiple matching of a single payment
@@ -184,8 +243,11 @@ class banking_import(wizard.interface):
'export_state': 'done',
'date_done': trans.effective_date.strftime('%Y-%m-%d')}
)
-
- return self._get_move_info(cursor, uid, candidate.move_line_id)
+ return self._get_move_info(
+ cursor, uid, candidate.move_line_id,
+ partner_bank_id=\
+ bank_account_ids and bank_account_ids[0].id or False
+ )
return False
@@ -234,7 +296,7 @@ class banking_import(wizard.interface):
Return the eyecatcher for an invoice
'''
return invoice.type.startswith('in_') and invoice.name or \
- invoice.number
+ '%s (%s)' % (invoice.number, invoice.name)
def has_id_match(invoice, ref, msg):
'''
@@ -373,7 +435,7 @@ class banking_import(wizard.interface):
_('Unable to link transaction id %(trans)s (ref: %(ref)s) to invoice: '
'%(no_candidates)s candidates found; can\'t choose.') % {
'trans': '%s.%s' % (trans.statement_id, trans.id),
- 'ref': trans.reference,
+ 'ref': trans.reference or trans.message,
'no_candidates': len(best) or len(candidates)
})
log.append(' ' +
@@ -536,6 +598,7 @@ class banking_import(wizard.interface):
self.pool = pooler.get_pool(cursor.dbname)
company_obj = self.pool.get('res.company')
user_obj = self.pool.get('res.user')
+ partner_obj = self.pool.get('res.partner')
partner_bank_obj = self.pool.get('res.partner.bank')
journal_obj = self.pool.get('account.journal')
move_line_obj = self.pool.get('account.move.line')
@@ -543,9 +606,9 @@ class banking_import(wizard.interface):
statement_obj = self.pool.get('account.bank.statement')
statement_line_obj = self.pool.get('account.bank.statement.line')
statement_file_obj = self.pool.get('account.banking.imported.file')
- #account_obj = self.pool.get('account.account')
payment_order_obj = self.pool.get('payment.order')
currency_obj = self.pool.get('res.currency')
+ digits = int(config['price_accuracy'])
# get the parser to parse the file
parser_code = form['parser']
@@ -627,16 +690,35 @@ class banking_import(wizard.interface):
# No filtering can be done, as empty dates carry value for C2B
# communication. Most likely there are much less sent payments
# than reconciled and open/draft payments.
+ payment_lines = []
+ payment_orders = []
cursor.execute("SELECT l.id FROM payment_order o, payment_line l "
"WHERE l.order_id = o.id AND "
"o.state = 'sent' AND "
"l.date_done IS NULL"
)
- payment_line_ids = [x[0] for x in cursor.fetchall()]
- if payment_line_ids:
- payment_lines = payment_line_obj.browse(cursor, uid, payment_line_ids)
- else:
- payment_lines = []
+ line_ids = [x[0] for x in cursor.fetchall()]
+ if line_ids:
+ # Get payment_orders and calculated total amounts as well in
+ # order to be able to match condensed transaction feedbacks.
+ # Use SQL for this, as it is way more efficient than server
+ # side processing.
+ payment_lines = payment_line_obj.browse(cursor, uid, line_ids)
+ cursor.execute("SELECT o.id, l.currency, SUM(l.amount_currency)"
+ " FROM payment_order o, payment_line l "
+ "WHERE l.order_id = o.id AND "
+ "o.state = 'sent' AND "
+ "l.date_done IS NULL "
+ "GROUP BY o.id, l.currency"
+ )
+ order_totals = dict([(x[0], round(x[2], digits))
+ for x in cursor.fetchall()])
+ payment_orders = [
+ dict(id=x.id, object=x, amount_currency=order_totals[x.id])
+ for x in list(payment_order_obj.browse(cursor, uid,
+ order_totals.keys()
+ ))
+ ]
for statement in statements:
if statement.local_account in error_accounts:
@@ -777,6 +859,21 @@ class banking_import(wizard.interface):
transaction.provision_costs_currency = None
transaction.provision_costs_description = None
+ # Check on payments batches, as they are able to replace
+ # the current transaction by generated ones.
+ if transaction.type == bt.PAYMENT_BATCH:
+ payments = self._link_payment_batch(
+ cursor, uid, transaction, payment_lines,
+ payment_orders, results.log
+ )
+ if payments:
+ transaction = payments[0]
+ injected += payments[1:]
+ else:
+ results.trans_skipped_cnt += 1
+ i += 1
+ continue
+
# Allow inclusion of generated bank invoices
if transaction.type == bt.BANK_COSTS:
lines = self._link_costs(
@@ -790,6 +887,22 @@ class banking_import(wizard.interface):
partner_ids = [account_info.bank_partner_id.id]
partner_banks = []
+ # Easiest match: customer id
+ elif transaction.remote_owner_custno:
+ partner_ids = partner_obj.browse(
+ cursor, uid, [transaction.remote_owner_custno]
+ )
+ iban_acc = sepa.IBAN(transaction.remote_account)
+ if iban_acc.valid:
+ domain = [('iban','=',str(iban_acc))]
+ else:
+ domain = [('acc_number','=',transaction.remote_account)]
+ partner_banks = partner_bank_obj.browse(
+ cursor, uid, partner_bank_obj.search(
+ cursor, uid, domain
+ )
+ )
+
else:
# Link remote partner, import account when needed
partner_banks = get_bank_accounts(
@@ -831,9 +944,6 @@ class banking_import(wizard.interface):
partner_bank_id = None
partner_banks = []
partner_ids = [partner_id]
- else:
- partner_ids = []
- partner_banks = []
# Credit means payment... isn't it?
if transaction.transferred_amount < 0 and payment_lines:
@@ -922,39 +1032,14 @@ class banking_import(wizard.interface):
"FROM payment_line "
"WHERE date_done IS NULL "
"AND id IN (%s)"
- ")" % (','.join([str(x) for x in payment_line_ids]))
+ ")" % (','.join([str(x) for x in line_ids]))
)
order_ids = [x[0] for x in cursor.fetchall()]
if order_ids:
- # Use workflow logics for the orders. Recode logic from
- # account_payment, in order to increase efficiency.
+ # Use workflow logics for the orders.
payment_order_obj.set_done(cursor, uid, order_ids,
{'state': 'done'}
)
- wf_service = netsvc.LocalService('workflow')
- for id in order_ids:
- wf_service.trg_validate(uid, 'payment.order', id, 'done',
- cursor
- )
-
- # Original code. Didn't take workflow logistics into account...
- #
- #cursor.execute(
- # "UPDATE payment_order o "
- # "SET state = 'done', "
- # "date_done = '%s' "
- # "FROM payment_line l "
- # "WHERE o.state = 'sent' "
- # "AND o.id = l.order_id "
- # "AND l.id NOT IN ("
- # "SELECT DISTINCT id FROM payment_line "
- # "WHERE date_done IS NULL "
- # "AND id IN (%s)"
- # ")" % (
- # time.strftime('%Y-%m-%d'),
- # ','.join([str(x) for x in payment_line_ids])
- # )
- #)
report = [
'%s: %s' % (_('Total number of statements'),
diff --git a/account_banking_nl_multibank/multibank.py b/account_banking_nl_multibank/multibank.py
index 3c7040919..c8a57a34d 100644
--- a/account_banking_nl_multibank/multibank.py
+++ b/account_banking_nl_multibank/multibank.py
@@ -44,8 +44,9 @@ class transaction_message(object):
attrnames = [
'date', 'local_account', 'remote_account', 'remote_owner', 'u1', 'u2',
'u3', 'local_currency', 'start_balance', 'remote_currency',
- 'transferred_amount', 'execution_date', 'effective_date', 'nr1',
- 'transfer_type', 'nr2', 'reference', 'message', 'statement_id'
+ 'transferred_amount', 'execution_date', 'effective_date',
+ 'transfer_type_id', 'transfer_type', 'nr2', 'reference', 'message',
+ 'statement_id'
]
@staticmethod
@@ -55,7 +56,7 @@ class transaction_message(object):
get prefixed by zeroes as in BBAN. Convert those to 'old' local
account numbers
- Edit: All account number now follow the BBAN scheme. As SNS Bank,
+ Edit: All account numbers now follow the BBAN scheme. As SNS Bank,
from which this module was re-engineered, follows the Dutch
Banking Tools regulations, it is considered to be used by all banks
in the Netherlands which comply to it. If not, please notify us.
@@ -84,6 +85,10 @@ class transaction_message(object):
self.execution_date = str2date(self.execution_date, '%d-%m-%Y')
self.effective_date = str2date(self.effective_date, '%d-%m-%Y')
self.id = str(subno).zfill(4)
+ # Map outgoing payment batches from general payments. They are
+ # distinguished from normal payments with type_id '9722'
+ if self.transfer_type == 'OVB' and self.transfer_type_id == '9722':
+ self.transfer_type = 'VZB'
class transaction(models.mem_bank_transaction):
'''
@@ -109,6 +114,7 @@ class transaction(models.mem_bank_transaction):
'OVS': bt.ORDER,
'PRV': bt.BANK_COSTS,
'TEL': bt.ORDER,
+ 'VZB': bt.PAYMENT_BATCH,
}
def __init__(self, line, *args, **kwargs):
@@ -150,12 +156,16 @@ class transaction(models.mem_bank_transaction):
5. Cash withdrawals from banks are too not seen as a transfer between
two accounts - the cash exits the banking system. These withdrawals
have their transfer_type set to 'OPN'.
+
+ 6. Payment batches are reported back in condensed format, meaning that
+ there is no individual transaction information, just a signal that the
+ total amount of the batch has been sent.
'''
return (self.transferred_amount and self.execution_date and
self.effective_date) and (
self.remote_account or
self.transfer_type in [
- 'KST', 'PRV', 'BTL', 'BEA', 'OPN', 'KNT', 'DIV',
+ 'KST', 'PRV', 'BTL', 'BEA', 'OPN', 'KNT', 'DIV', 'VZB'
]
and not self.error_message
)
@@ -243,7 +253,7 @@ class transaction(models.mem_bank_transaction):
elif self.transfer_type == 'IDB':
# Payment by iDeal transaction
# Remote owner can be part of message, while remote_owner can be
- # set to the intermediate party, which we don't need.
+ # set to an intermediate party, which we don't need.
parts = self.message.split(' ')
# Second part: structured id, date & time
subparts = parts[1].split()
@@ -262,6 +272,15 @@ class transaction(models.mem_bank_transaction):
parts = parts[:-1]
self.message = ' '.join(parts)
+ elif self.transfer_type == 'VZB':
+ # Payment batch, condensed format.
+ # NOTE: This has only been tested with ClieOp3 payment batches.
+ # If your milage varies, please inform us of your findings.
+ parts = self.message.split('.')[1].split()
+ self.payment_batch_no_transactions = int(parts[0])
+ self.message = ' '.join(parts[1:])
+ self.payment_batch_id = parts[2][-4:]
+
class statement(models.mem_bank_statement):
'''
Implementation of bank_statement communication class of account_banking