audrius/bank-statement-import
1
0
Fork 1
mirror of https://github.com/OCA/bank-statement-import.git synced 2025-01-20 12:37:43 +02:00
Code Issues Packages Projects Releases Wiki Activity

[FIX] Prevent unneeded processing and memory use in parsers.

Browse Source
This commit is contained in:
Ronald Portier (Therp BV)
2015-06-19 12:43:20 +02:00
parent 6fe358a6cd
commit d3c1c8d39f
7 changed files with 203 additions and 108 deletions
Download Patch File Download Diff File Expand all files Collapse all files

240
bank_statement_parse/parserlib.py
View File

@@ -20,74 +20,91 @@
##############################################################################
def convert_transaction(transaction):
"""Convert transaction object to values for create."""
vals_line = {
'date': transaction.value_date,
'name': (
transaction.message or transaction.eref or
transaction.remote_owner or ''), # name is required
'ref': transaction.eref,
'amount': transaction.transferred_amount,
'partner_name': transaction.remote_owner,
'account_number': transaction.remote_account,
'unique_import_id': transaction.transaction_id,
}
return vals_line
def convert_statements(statements):
"""Convert statement object to values for create."""
vals_statements = []
for statement in statements:
# Set statement_data
vals_statement = {
'currency_code': statement.local_currency,
'account_number': statement.local_account,
'name': statement.statement_id,
'date': statement.date.strftime('%Y-%m-%d'),
'balance_start': statement.start_balance,
'balance_end_real': statement.end_balance,
'balance_end': statement.end_balance,
'state': 'draft',
}
statement_transactions = []
subno = 0
for transaction in statement.transactions:
subno += 1
if not transaction.transaction_id:
transaction.transaction_id = (
statement.statement_id + str(subno).zfill(4))
statement_transactions.append(convert_transaction(transaction))
vals_statement['transactions'] = statement_transactions
vals_statements.append(vals_statement)
return vals_statements
class BankStatement(object):
"""A bank statement groups data about several bank transactions."""
def __init__(self):
self.statement_id = ''
self.local_account = ''
self.local_currency = ''
self.start_balance = 0.0
self.end_balance = 0.0
self.date = ''
self.transactions = []
class BankTransaction(object):
class BankTransaction(dict):
"""Single transaction that is part of a bank statement."""
@property
def value_date(self):
"""property getter"""
return self['date']
@value_date.setter
def value_date(self, value_date):
"""property setter"""
self['date'] = value_date
@property
def name(self):
"""property getter"""
return self['name']
@name.setter
def name(self, name):
"""property setter"""
self['name'] = name
@property
def amount(self):
"""property getter"""
return self['amount']
@amount.setter
def amount(self, amount):
"""property setter"""
self['amount'] = amount
@property
def eref(self):
"""property getter"""
return self['ref']
@eref.setter
def eref(self, eref):
"""property setter"""
self['ref'] = eref
if not self.message:
self.name = eref
@property
def message(self):
"""property getter"""
return self._message
@message.setter
def message(self, message):
"""property setter"""
self._message = message
self.name = message
@property
def remote_owner(self):
"""property getter"""
return self['partner_name']
@remote_owner.setter
def remote_owner(self, remote_owner):
"""property setter"""
self['partner_name'] = remote_owner
if not (self.message or self.eref):
self.name = remote_owner
@property
def remote_account(self):
"""property getter"""
return self['account_number']
@remote_account.setter
def remote_account(self, remote_account):
"""property setter"""
self['account_number'] = remote_account
def __init__(self):
"""Define and initialize attributes.
Does not include attributes that belong to statement.
Not all attributes are already used in the actual import.
"""
self.transaction_id = False # Message id
super(BankTransaction, self).__init__()
self.transfer_type = False # Action type that initiated this message
self.eref = False # end to end reference for transactions
self.execution_date = False # The posted date of the action
self.value_date = False # The value date of the action
self.remote_account = False # The account of the other party
@@ -96,7 +113,9 @@ class BankTransaction(object):
# The exchange rate used for conversion of local_currency and
# remote_currency
self.transferred_amount = 0.0 # actual amount transferred
self.message = False # message from the remote party
self.name = ''
self._message = False # message from the remote party
self.eref = False # end to end reference for transactions
self.remote_owner = False # name of the other party
self.remote_owner_address = [] # other parties address lines
self.remote_owner_city = False # other parties city name
@@ -110,3 +129,100 @@ class BankTransaction(object):
self.storno_retry = False
# If True, make cancelled debit eligible for a next direct debit run
self.data = '' # Raw data from which the transaction has been parsed
class BankStatement(dict):
"""A bank statement groups data about several bank transactions."""
@property
def statement_id(self):
"""property getter"""
return self['name']
def _set_transaction_ids(self):
"""Set transaction ids to statement_id with sequence-number."""
subno = 0
for transaction in self['transactions']:
subno += 1
transaction['unique_import_id'] = (
self.statement_id + str(subno).zfill(4))
@statement_id.setter
def statement_id(self, statement_id):
"""property setter"""
self['name'] = statement_id
self._set_transaction_ids()
@property
def local_account(self):
"""property getter"""
return self['account_number']
@local_account.setter
def local_account(self, local_account):
"""property setter"""
self['account_number'] = local_account
@property
def local_currency(self):
"""property getter"""
return self['currency_code']
@local_currency.setter
def local_currency(self, local_currency):
"""property setter"""
self['currency_code'] = local_currency
@property
def start_balance(self):
"""property getter"""
return self['balance_start']
@start_balance.setter
def start_balance(self, start_balance):
"""property setter"""
self['balance_start'] = start_balance
@property
def end_balance(self):
"""property getter"""
return self['balance_end']
@end_balance.setter
def end_balance(self, end_balance):
"""property setter"""
self['balance_end'] = end_balance
self['balance_end_real'] = end_balance
@property
def date(self):
"""property getter"""
return self['date']
@date.setter
def date(self, date):
"""property setter"""
self['date'] = date
def create_transaction(self):
"""Create and append transaction.
This should only be called after statement_id has been set, because
statement_id will become part of the unique transaction_id.
"""
transaction = BankTransaction()
self['transactions'].append(transaction)
# Fill default id, but might be overruled
transaction['unique_import_id'] = (
self.statement_id + str(len(self['transactions'])).zfill(4))
return transaction
def __init__(self):
super(BankStatement, self).__init__()
self['transactions'] = []
self.statement_id = ''
self.local_account = ''
self.local_currency = ''
self.date = ''
self.start_balance = 0.0
self.end_balance = 0.0

9
bank_statement_parse_camt/account_bank_statement_import.py
View File

@@ -20,11 +20,10 @@
##############################################################################
import logging
from openerp import models
from openerp.addons.bank_statement_parse.parserlib import convert_statements
from .camt import CamtParser as Parser
_logger = logging.getLogger(__name__)
_LOGGER = logging.getLogger(__name__)
class AccountBankStatementImport(models.TransientModel):
@@ -35,10 +34,10 @@ class AccountBankStatementImport(models.TransientModel):
"""Parse a CAMT053 XML file."""
parser = Parser()
try:
_logger.debug("Try parsing with camt.")
return convert_statements(parser.parse(data_file))
_LOGGER.debug("Try parsing with camt.")
return parser.parse(data_file)
except ValueError:
# Not a camt file, returning super will call next candidate:
_logger.debug("Statement file was not a camt file.")
_LOGGER.debug("Statement file was not a camt file.")
return super(AccountBankStatementImport, self)._parse_file(
cr, uid, data_file, context=context)

18
bank_statement_parse_camt/camt.py
View File

@@ -21,10 +21,7 @@
import re
from datetime import datetime
from lxml import etree
from openerp.addons.bank_statement_parse.parserlib import (
BankStatement,
BankTransaction
)
from openerp.addons.bank_statement_parse.parserlib import BankStatement
class CamtParser(object):
@@ -121,9 +118,8 @@ class CamtParser(object):
'remote_account'
)
def parse_transaction(self, ns, node):
def parse_transaction(self, ns, node, transaction):
"""Parse transaction (entry) node."""
transaction = BankTransaction()
self.add_value_from_node(
ns, node, './ns:BkTxCd/ns:Prtry/ns:Cd', transaction,
'transfer_type'
@@ -190,11 +186,11 @@ class CamtParser(object):
self.get_balance_amounts(ns, node))
transaction_nodes = node.xpath('./ns:Ntry', namespaces={'ns': ns})
for entry_node in transaction_nodes:
transaction = self.parse_transaction(ns, entry_node)
statement.transactions.append(transaction)
if statement.transactions:
transaction = statement.create_transaction()
self.parse_transaction(ns, entry_node, transaction)
if statement['transactions']:
statement.date = datetime.strptime(
statement.transactions[0].execution_date, "%Y-%m-%d")
statement['transactions'][0].execution_date, "%Y-%m-%d")
return statement
def check_version(self, ns, root):
@@ -237,6 +233,6 @@ class CamtParser(object):
statements = []
for node in root[0][1:]:
statement = self.parse_statement(ns, node)
if len(statement.transactions):
if len(statement['transactions']):
statements.append(statement)
return statements

23
bank_statement_parse_mt940/mt940.py
View File

@@ -22,7 +22,7 @@ import re
import logging
from datetime import datetime
from openerp.addons.bank_statement_parse import parserlib
from openerp.addons.bank_statement_parse.parserlib import BankStatement
def str2amount(sign, amount_str):
@@ -138,8 +138,7 @@ class MT940(object):
self.handle_header(line, iterator)
line = iterator.next()
if not self.is_tag(line) and not self.is_footer(line):
record_line = self.append_continuation_line(
record_line, line)
record_line += line
continue
if record_line:
self.handle_record(record_line)
@@ -158,19 +157,6 @@ class MT940(object):
self.current_statement = None
return self.statements
def append_continuation_line(self, line, continuation_line):
"""append a continuation line for a multiline record.
Override and do data cleanups as necessary."""
return line + continuation_line
def create_statement(self):
"""create a BankStatement."""
return parserlib.BankStatement()
def create_transaction(self):
"""Create and return BankTransaction object."""
return parserlib.BankTransaction()
def is_footer(self, line):
"""determine if a line is the footer of a statement"""
return line and bool(re.match(self.footer_regex, line))
@@ -183,7 +169,7 @@ class MT940(object):
"""skip header lines, create current statement"""
for dummy_i in range(self.header_lines):
iterator.next()
self.current_statement = self.create_statement()
self.current_statement = BankStatement()
def handle_footer(self, line, iterator):
"""add current statement to list, reset state"""
@@ -226,8 +212,7 @@ class MT940(object):
def handle_tag_61(self, data):
"""get transaction values"""
transaction = self.create_transaction()
self.current_statement.transactions.append(transaction)
transaction = self.current_statement.create_transaction()
self.current_transaction = transaction
transaction.execution_date = datetime.strptime(data[:6], '%y%m%d')
transaction.value_date = datetime.strptime(data[:6], '%y%m%d')

9
bank_statement_parse_nl_ing_mt940/account_bank_statement_import.py
View File

@@ -20,11 +20,10 @@
##############################################################################
import logging
from openerp import models
from openerp.addons.bank_statement_parse.parserlib import convert_statements
from .mt940 import MT940Parser as Parser
_logger = logging.getLogger(__name__)
_LOGGER = logging.getLogger(__name__)
class AccountBankStatementImport(models.TransientModel):
@@ -35,10 +34,10 @@ class AccountBankStatementImport(models.TransientModel):
"""Parse a MT940 IBAN ING file."""
parser = Parser()
try:
_logger.debug("Try parsing with MT940 IBAN ING.")
return convert_statements(parser.parse(data_file))
_LOGGER.debug("Try parsing with MT940 IBAN ING.")
return parser.parse(data_file)
except ValueError:
# Returning super will call next candidate:
_logger.debug("Statement file was not a MT940 IBAN ING file.")
_LOGGER.debug("Statement file was not a MT940 IBAN ING file.")
return super(AccountBankStatementImport, self)._parse_file(
cr, uid, data_file, context=context)

2
bank_statement_parse_nl_ing_mt940/tests/test_import_bank_statement.py
View File

@@ -63,7 +63,7 @@ class TestStatementFile(TransactionCase):
(statement_obj.balance_start, start_balance)
)
self.assertTrue(
abs(statement_obj.balance_end - end_balance) < 0.00001,
abs(statement_obj.balance_end_real - end_balance) < 0.00001,
'Start balance %f not equal to expected %f' %
(statement_obj.balance_end_real, end_balance)
)

10
bank_statement_parse_nl_rabo_mt940/account_bank_statement_import.py
View File

@@ -19,12 +19,12 @@
#
##############################################################################
import logging
from openerp import models
from openerp.addons.bank_statement_parse.parserlib import convert_statements
from .mt940 import MT940Parser as Parser
_logger = logging.getLogger(__name__)
_LOGGER = logging.getLogger(__name__)
class AccountBankStatementImport(models.TransientModel):
@@ -35,10 +35,10 @@ class AccountBankStatementImport(models.TransientModel):
"""Parse a MT940 RABO file."""
parser = Parser()
try:
_logger.debug("Try parsing with MT940 RABO.")
return convert_statements(parser.parse(data_file))
_LOGGER.debug("Try parsing with MT940 RABO.")
return parser.parse(data_file)
except ValueError:
# Returning super will call next candidate:
_logger.debug("Statement file was not a MT940 RABO file.")
_LOGGER.debug("Statement file was not a MT940 RABO file.")
return super(AccountBankStatementImport, self)._parse_file(
cr, uid, data_file, context=context)