audrius/account-reconcile
1
0
Fork 1
mirror of https://github.com/OCA/account-reconcile.git synced 2025-01-20 12:27:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

[IMP] adresses massive import perfomrance by doing direct insert into database

Browse Source
This commit is contained in:
unknown
2013-04-11 10:57:53 +02:00
parent 0ef301a199
commit 3ec119eb20
2 changed files with 105 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
account_statement_base_import/parser/generic_file_parser.py
View File

@@ -29,6 +29,7 @@ try:
except: except:
raise Exception(_('Please install python lib xlrd')) raise Exception(_('Please install python lib xlrd'))
def float_or_zero(val): def float_or_zero(val):
""" Conversion function used to manage """ Conversion function used to manage
empty string into float usecase""" empty string into float usecase"""
@@ -82,14 +83,12 @@ class GenericFileParser(FileParser):
In this generic parser, the commission is given for every line, so we store it In this generic parser, the commission is given for every line, so we store it
for each one. for each one.
""" """
return { return {'name': line.get('label', line.get('ref', '/')),
'name': line.get('label', line.get('ref', '/')),
'date': line.get('date', datetime.datetime.now().date()), 'date': line.get('date', datetime.datetime.now().date()),
'amount': line.get('amount', 0.0), 'amount': line.get('amount', 0.0),
'ref': line.get('ref', '/'), 'ref': line.get('ref', '/'),
'label': line.get('label', ''), 'label': line.get('label', ''),
'commission_amount': line.get('commission_amount', 0.0), 'commission_amount': line.get('commission_amount', 0.0)}
}
def _post(self, *args, **kwargs): def _post(self, *args, **kwargs):
""" """

114
account_statement_base_import/statement.py
View File

@@ -18,14 +18,17 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>. # along with this program. If not, see <http://www.gnu.org/licenses/>.
# #
############################################################################## ##############################################################################
import sys
import traceback
import psycopg2
from openerp.tools.translate import _ from openerp.tools.translate import _
import datetime import datetime
from openerp.osv.orm import Model from openerp.osv.orm import Model
from openerp.osv import fields, osv from openerp.osv import fields, osv
from parser import new_bank_statement_parser from parser import new_bank_statement_parser
import sys
import traceback
class AccountStatementProfil(Model): class AccountStatementProfil(Model):
@@ -64,7 +67,8 @@ class AccountStatementProfil(Model):
self.message_post(cr, self.message_post(cr,
uid, uid,
ids, ids,
body=_('Statement ID %s have been imported with %s lines.') % (statement_id, num_lines), body=_('Statement ID %s have been imported with %s lines.') %
(statement_id, num_lines),
context=context) context=context)
return True return True
@@ -121,15 +125,55 @@ class AccountStatementProfil(Model):
statement_obj = self.pool.get('account.bank.statement') statement_obj = self.pool.get('account.bank.statement')
values = parser_vals values = parser_vals
values['statement_id'] = statement_id values['statement_id'] = statement_id
values['account_id'] = statement_obj.get_account_for_counterpart( values['account_id'] = statement_obj.get_account_for_counterpart(cr,
cr,
uid, uid,
parser_vals['amount'], parser_vals['amount'],
account_receivable, account_receivable,
account_payable account_payable)
)
date = values.get('date')
period_memoizer = context.get('period_memoizer')
if not period_memoizer:
period_memoizer = {}
context['period_memoizer'] = period_memoizer
if period_memoizer.get(date):
values['period_id'] = period_memoizer[date]
else:
# This is awfully slow...
periods = self.pool.get('account.period').find(cr, uid,
dt=values.get('date'),
context=context)
values['period_id'] = periods[0]
period_memoizer[date] = periods[0]
values['type'] = 'general'
return values return values
def _get_available_columns(self, statement_store):
"""Return writeable by SQL columns"""
statement_line_obj = self.pool['account.bank.statement.line']
model_cols = statement_line_obj._columns
avail = [k for k, col in model_cols.iteritems() if not hasattr(col, '_fnct')]
keys = [k for k in statement_store[0].keys() if k in avail]
keys.sort()
return keys
def _insert_lines(self, cr, uid, statement_store, context=None):
""" Do raw insert into database because ORM is awfully slow
when doing batch write. It is a shame that batch function
does not exist"""
statement_line_obj = self.pool['account.bank.statement.line']
statement_line_obj.check_access_rule(cr, uid, [], 'create')
statement_line_obj.check_access_rights(cr, uid, 'create', raise_exception=True)
cols = self._get_available_columns(statement_store)
tmp_vals = (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols]))
sql = "INSERT INTO account_bank_statement_line (%s) VALUES (%s);" % tmp_vals
try:
cr.executemany(sql, tuple(statement_store))
except psycopg2.Error as sql_err:
cr.rollback()
raise osv.except_osv(_("ORM bypass error"),
sql_err.pgerror)
def statement_import(self, cr, uid, ids, profile_id, file_stream, ftype="csv", context=None): def statement_import(self, cr, uid, ids, profile_id, file_stream, ftype="csv", context=None):
""" """
Create a bank statement with the given profile and parser. It will fullfill the bank statement Create a bank statement with the given profile and parser. It will fullfill the bank statement
@@ -148,24 +192,26 @@ class AccountStatementProfil(Model):
attachment_obj = self.pool.get('ir.attachment') attachment_obj = self.pool.get('ir.attachment')
prof_obj = self.pool.get("account.statement.profile") prof_obj = self.pool.get("account.statement.profile")
if not profile_id: if not profile_id:
raise osv.except_osv( raise osv.except_osv(_("No Profile !"),
_("No Profile !"),
_("You must provide a valid profile to import a bank statement !")) _("You must provide a valid profile to import a bank statement !"))
prof = prof_obj.browse(cr, uid, profile_id, context=context) prof = prof_obj.browse(cr, uid, profile_id, context=context)
parser = new_bank_statement_parser(prof.import_type, ftype=ftype) parser = new_bank_statement_parser(prof.import_type, ftype=ftype)
result_row_list = parser.parse(file_stream) result_row_list = parser.parse(file_stream)
# Check all key are present in account.bank.statement.line !! # Check all key are present in account.bank.statement.line !!
if not result_row_list:
raise osv.except_osv(_("Nothing to import"),
_("The file is empty"))
parsed_cols = parser.get_st_line_vals(result_row_list[0]).keys() parsed_cols = parser.get_st_line_vals(result_row_list[0]).keys()
for col in parsed_cols: for col in parsed_cols:
if col not in statement_line_obj._columns: if col not in statement_line_obj._columns:
raise osv.except_osv( raise osv.except_osv(_("Missing column !"),
_("Missing column !"),
_("Column %s you try to import is not " _("Column %s you try to import is not "
"present in the bank statement line !") % col) "present in the bank statement line !") % col)
statement_id = statement_obj.create( statement_id = statement_obj.create(cr, uid,
cr, uid, {'profile_id': prof.id}, context=context) {'profile_id': prof.id},
context=context)
account_receivable, account_payable = statement_obj.get_default_pay_receiv_accounts( account_receivable, account_payable = statement_obj.get_default_pay_receiv_accounts(
cr, uid, context) cr, uid, context)
try: try:
@@ -174,48 +220,54 @@ class AccountStatementProfil(Model):
statement_store = [] statement_store = []
for line in result_row_list: for line in result_row_list:
parser_vals = parser.get_st_line_vals(line) parser_vals = parser.get_st_line_vals(line)
values = self.prepare_statetement_lines_vals( values = self.prepare_statetement_lines_vals(cr, uid, parser_vals, account_payable,
cr, uid, parser_vals, account_payable,
account_receivable, statement_id, context) account_receivable, statement_id, context)
# we finally create the line in system statement_store.append(values)
statement_store.append((0, 0, values)) # Hack to bypass ORM poor perfomance. Sob...
self._insert_lines(cr, uid, statement_store, context=context)
# Build and create the global commission line for the whole statement # Build and create the global commission line for the whole statement
statement_obj.write(cr, uid, [statement_id],
{'line_ids': statement_store}, context=context)
comm_vals = self.prepare_global_commission_line_vals(cr, uid, parser, result_row_list, comm_vals = self.prepare_global_commission_line_vals(cr, uid, parser, result_row_list,
prof, statement_id, context) prof, statement_id, context)
if comm_vals: if comm_vals:
statement_line_obj.create(cr, uid, comm_vals, context=context) statement_line_obj.create(cr, uid, comm_vals, context=context)
else:
# Trigger store field computation if someone has better idea
start_bal = statement_obj.read(cr, uid, statement_id,
['balance_start'],
context=context)
start_bal = start_bal['balance_start']
statement_obj.write(cr, uid, [statement_id],
{'balance_start': start_bal})
attachment_obj.create( attachment_obj.create(cr,
cr,
uid, uid,
{ {'name': 'statement file',
'name': 'statement file',
'datas': file_stream, 'datas': file_stream,
'datas_fname': "%s.%s" % ( 'datas_fname': "%s.%s" % (
datetime.datetime.now().date(), datetime.datetime.now().date(),
ftype), ftype),
'res_model': 'account.bank.statement', 'res_model': 'account.bank.statement',
'res_id': statement_id, 'res_id': statement_id},
}, context=context)
context=context
)
# If user ask to launch completion at end of import, do it ! # If user ask to launch completion at end of import, do it !
if prof.launch_import_completion: if prof.launch_import_completion:
statement_obj.button_auto_completion(cr, uid, [statement_id], context) statement_obj.button_auto_completion(cr, uid, [statement_id], context)
# Write the needed log infos on profile # Write the needed log infos on profile
self.write_logs_after_import( self.write_logs_after_import(cr, uid, prof.id,
cr, uid, prof.id, statement_id, len(result_row_list), context) statement_id,
len(result_row_list),
context)
except Exception: except Exception:
statement_obj.unlink(cr, uid, [statement_id], context=context) statement_obj.unlink(cr, uid, [statement_id], context=context)
error_type, error_value, trbk = sys.exc_info() error_type, error_value, trbk = sys.exc_info()
st = "Error: %s\nDescription: %s\nTraceback:" % (error_type.__name__, error_value) st = "Error: %s\nDescription: %s\nTraceback:" % (error_type.__name__, error_value)
st += ''.join(traceback.format_tb(trbk, 30)) st += ''.join(traceback.format_tb(trbk, 30))
raise osv.except_osv( raise osv.except_osv(_("Statement import error"),
_("Statement import error"),
_("The statement cannot be created : %s") % st) _("The statement cannot be created : %s") % st)
return statement_id return statement_id