diff --git a/account_statement_base_completion/data.xml b/account_statement_base_completion/data.xml deleted file mode 100644 index 595a4af3..00000000 --- a/account_statement_base_completion/data.xml +++ /dev/null @@ -1,31 +0,0 @@ - - - - - - Match from line label (based on partner field 'Bank Statement Label') - 60 - get_from_label_and_partner_field - - - - Match from line label (based on partner name) - 70 - get_from_label_and_partner_name - - - - Match from line reference (based on Invoice number) - 40 - get_from_ref_and_invoice - - - - Match from line reference (based on Invoice Supplier number) - 45 - get_from_ref_and_supplier_invoice - - - - - diff --git a/account_statement_base_completion/partner_view.xml b/account_statement_base_completion/partner_view.xml deleted file mode 100644 index 94be85c3..00000000 --- a/account_statement_base_completion/partner_view.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - account_bank_statement_import.view.partner.form - res.partner - 20 - - - - - - - - - - - diff --git a/account_statement_base_completion/statement.py b/account_statement_base_completion/statement.py deleted file mode 100644 index 12bf2d31..00000000 --- a/account_statement_base_completion/statement.py +++ /dev/null @@ -1,654 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# Author: Nicolas Bessi, Joel Grand-Guillaume -# Copyright 2011-2012 Camptocamp SA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -# TODO replace customer supplier by package constant -import traceback -import sys -import logging -import simplejson -import inspect -import datetime - -import psycopg2 - -from collections import defaultdict -import re -from openerp.tools.translate import _ -from openerp.osv import orm, fields -from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT -from operator import attrgetter - - -_logger = logging.getLogger(__name__) - - -class ErrorTooManyPartner(Exception): - """ New Exception definition that is raised when more than one partner is - matched by the completion rule. - """ - - def __init__(self, value): - self.value = value - - def __str__(self): - return repr(self.value) - - def __repr__(self): - return repr(self.value) - - -class AccountStatementProfil(orm.Model): - """Extend the class to add rules per profile that will match at least the - partner, but it could also be used to match other values as well. - """ - _inherit = "account.statement.profile" - - _columns = { - # @Akretion: For now, we don't implement this features, but this would - # probably be there: 'auto_completion': fields.text('Auto Completion'), - # 'transferts_account_id':fields.many2one('account.account', - # 'Transferts Account'), - # => You can implement it in a module easily, we design it with your - # needs in mind as well! - - 'rule_ids': fields.many2many( - 'account.statement.completion.rule', - string='Related statement profiles', - rel='as_rul_st_prof_rel'), - } - - def _get_rules(self, cr, uid, profile, context=None): - if isinstance(profile, (int, long)): - prof = self.browse(cr, uid, profile, context=context) - else: - prof = profile - # We need to respect the sequence order - return sorted(prof.rule_ids, key=attrgetter('sequence')) - - def _find_values_from_rules(self, cr, uid, calls, line, context=None): - """This method will execute all related rules, in their sequence order, - to retrieve all the values returned by the first rules that will match. - :param calls: list of lookup function name available in rules - :param dict line: read of the concerned account.bank.statement.line - :return: - A dict of value that can be passed directly to the write method of - the statement line or {} - {'partner_id': value, - 'account_id: value, - - ...} - """ - if not calls: - calls = self._get_rules( - cr, uid, line['profile_id'], context=context) - rule_obj = self.pool.get('account.statement.completion.rule') - for call in calls: - method_to_call = getattr(rule_obj, call.function_to_call) - if len(inspect.getargspec(method_to_call).args) == 6: - result = method_to_call(cr, uid, call.id, line, context) - else: - result = method_to_call(cr, uid, line, context) - if result: - result['already_completed'] = True - return result - return None - - -class AccountStatementCompletionRule(orm.Model): - """This will represent all the completion method that we can have to - fullfill the bank statement lines. You'll be able to extend them in you own - module and choose those to apply for every statement profile. - The goal of a rule is to fullfill at least the partner of the line, but - if possible also the reference because we'll use it in the reconciliation - process. The reference should contain the invoice number or the SO number - or any reference that will be matched by the invoice accounting move. - """ - _name = "account.statement.completion.rule" - _order = "sequence asc" - - def _get_functions(self, cr, uid, context=None): - """List of available methods for rules. - - Override this to add you own.""" - return [ - ('get_from_ref_and_invoice', - 'From line reference (based on customer invoice number)'), - ('get_from_ref_and_supplier_invoice', - 'From line reference (based on supplier invoice number)'), - ('get_from_label_and_partner_field', - 'From line label (based on partner field)'), - ('get_from_label_and_partner_name', - 'From line label (based on partner name)') - ] - - def __get_functions(self, cr, uid, context=None): - """ Call method which can be inherited """ - return self._get_functions(cr, uid, context=context) - - _columns = { - 'sequence': fields.integer('Sequence', - help="Lower means parsed first."), - 'name': fields.char('Name', size=128), - 'profile_ids': fields.many2many( - 'account.statement.profile', - rel='as_rul_st_prof_rel', - string='Related statement profiles'), - 'function_to_call': fields.selection(__get_functions, 'Method'), - } - - def _find_invoice(self, cr, uid, st_line, inv_type, context=None): - """Find invoice related to statement line""" - inv_obj = self.pool.get('account.invoice') - if inv_type == 'supplier': - type_domain = ('in_invoice', 'in_refund') - number_field = 'supplier_invoice_number' - elif inv_type == 'customer': - type_domain = ('out_invoice', 'out_refund') - number_field = 'number' - else: - raise orm.except_orm( - _('System error'), - _('Invalid invoice type for completion: %') % inv_type) - - inv_id = inv_obj.search(cr, uid, - [(number_field, '=', st_line['ref'].strip()), - ('type', 'in', type_domain)], - context=context) - if inv_id: - if len(inv_id) == 1: - inv = inv_obj.browse(cr, uid, inv_id[0], context=context) - else: - raise ErrorTooManyPartner( - _('Line named "%s" (Ref:%s) was matched by more than one ' - 'partner while looking on %s invoices') % - (st_line['name'], st_line['ref'], inv_type)) - return inv - return False - - def _from_invoice(self, cr, uid, line, inv_type, context): - """Populate statement line values""" - if inv_type not in ('supplier', 'customer'): - raise orm.except_orm(_('System error'), - _('Invalid invoice type for completion: %') % - inv_type) - res = {} - inv = self._find_invoice(cr, uid, line, inv_type, context=context) - if inv: - partner_id = inv.commercial_partner_id.id - res = {'partner_id': partner_id, - 'account_id': inv.account_id.id, - 'type': inv_type} - override_acc = line['master_account_id'] - if override_acc: - res['account_id'] = override_acc - return res - - # Should be private but data are initialised with no update XML - def get_from_ref_and_supplier_invoice(self, cr, uid, line, context=None): - """Match the partner based on the invoice supplier invoice number and - the reference of the statement line. Then, call the generic - get_values_for_line method to complete other values. If more than one - partner matched, raise the ErrorTooManyPartner error. - - :param dict line: read of the concerned account.bank.statement.line - :return: - A dict of value that can be passed directly to the write method of - the statement line or {} - {'partner_id': value, - 'account_id': value, - - ...} - """ - return self._from_invoice(cr, uid, line, 'supplier', context=context) - - # Should be private but data are initialised with no update XML - def get_from_ref_and_invoice(self, cr, uid, line, context=None): - """Match the partner based on the invoice number and the reference of - the statement line. Then, call the generic get_values_for_line method - to complete other values. If more than one partner matched, raise the - ErrorTooManyPartner error. - - :param dict line: read of the concerned account.bank.statement.line - :return: - A dict of value that can be passed directly to the write method of - the statement line or {} - {'partner_id': value, - 'account_id': value, - ...} - """ - return self._from_invoice(cr, uid, line, 'customer', context=context) - - # Should be private but data are initialised with no update XML - def get_from_label_and_partner_field(self, cr, uid, st_line, context=None): - """ - Match the partner based on the label field of the statement line and - the text defined in the 'bank_statement_label' field of the partner. - Remember that we can have values separated with ; Then, call the - generic get_values_for_line method to complete other values. If more - than one partner matched, raise the ErrorTooManyPartner error. - - :param dict st_line: read of the concerned account.bank.statement.line - :return: - A dict of value that can be passed directly to the write method of - the statement line or {} - {'partner_id': value, - 'account_id': value, - - ...} - """ - partner_obj = self.pool['res.partner'] - st_obj = self.pool.get('account.bank.statement.line') - res = {} - # As we have to iterate on each partner for each line, - #  we memoize the pair to avoid - # to redo computation for each line. - # Following code can be done by a single SQL query - # but this option is not really maintanable - if not context.get('label_memoizer'): - context['label_memoizer'] = defaultdict(list) - partner_ids = partner_obj.search( - cr, uid, [('bank_statement_label', '!=', False)], - context=context) - line_ids = context.get('line_ids', []) - for partner in partner_obj.browse(cr, uid, partner_ids, - context=context): - vals = '|'.join( - re.escape(x.strip()) - for x in partner.bank_statement_label.split(';')) - or_regex = ".*%s.*" % vals - sql = ("SELECT id from account_bank_statement_line" - " WHERE id in %s" - " AND name ~* %s") - cr.execute(sql, (line_ids, or_regex)) - pairs = cr.fetchall() - for pair in pairs: - context['label_memoizer'][pair[0]].append(partner) - if st_line['id'] in context['label_memoizer']: - found_partner = context['label_memoizer'][st_line['id']] - if len(found_partner) > 1: - msg = (_('Line named "%s" (Ref:%s) was matched by more than ' - 'one partner while looking on partner label: %s') % - (st_line['name'], st_line['ref'], - ','.join([x.name for x in found_partner]))) - raise ErrorTooManyPartner(msg) - res['partner_id'] = found_partner[0].id - st_vals = st_obj.get_values_for_line( - cr, uid, profile_id=st_line['profile_id'], - master_account_id=st_line['master_account_id'], - partner_id=found_partner[0].id, line_type=False, - amount=st_line['amount'] if st_line['amount'] else 0.0, - context=context) - res.update(st_vals) - return res - - def get_from_label_and_partner_name(self, cr, uid, st_line, context=None): - """Match the partner based on the label field of the statement line and - the name of the partner. Then, call the generic get_values_for_line - method to complete other values. If more than one partner matched, - raise the ErrorTooManyPartner error. - - :param dict st_line: read of the concerned account.bank.statement.line - :return: - A dict of value that can be passed directly to the write method of - the statement line or {} - {'partner_id': value, - 'account_id': value, - - ...} - """ - res = {} - # We memoize allowed partner - if not context.get('partner_memoizer'): - context['partner_memoizer'] = tuple( - self.pool['res.partner'].search(cr, uid, [])) - if not context['partner_memoizer']: - return res - st_obj = self.pool.get('account.bank.statement.line') - # The regexp_replace() escapes the name to avoid false positive - # example: 'John J. Doe (No 1)' is escaped to 'John J\. Doe \(No 1\)' - # See http://stackoverflow.com/a/400316/1504003 for a list of - # chars to escape. Postgres is POSIX-ARE, compatible with - # POSIX-ERE excepted that '\' must be escaped inside brackets according - # to: - # http://www.postgresql.org/docs/9.0/static/functions-matching.html - # in chapter 9.7.3.6. Limits and Compatibility - sql = r""" - SELECT id FROM ( - SELECT id, - regexp_matches(%s, - regexp_replace(name,'([\.\^\$\*\+\?\(\)\[\{\\\|])', %s, - 'g'), 'i') AS name_match - FROM res_partner - WHERE id IN %s) - AS res_patner_matcher - WHERE name_match IS NOT NULL""" - cr.execute( - sql, (st_line['name'], r"\\\1", context['partner_memoizer'])) - result = cr.fetchall() - if not result: - return res - if len(result) > 1: - raise ErrorTooManyPartner( - _('Line named "%s" (Ref:%s) was matched by more than one ' - 'partner while looking on partner by name') % - (st_line['name'], st_line['ref'])) - res['partner_id'] = result[0][0] - st_vals = st_obj.get_values_for_line( - cr, uid, profile_id=st_line['profile_id'], - master_account_id=st_line['master_account_id'], - partner_id=res['partner_id'], line_type=False, - amount=st_line['amount'] if st_line['amount'] else 0.0, - context=context) - res.update(st_vals) - return res - - -class AccountStatement(orm.Model): - _inherit = "account.bank.statement" - - def button_confirm_bank(self, cr, uid, ids, context=None): - line_obj = self.pool['account.bank.statement.line'] - for stat_id in ids: - line_without_account = line_obj.search(cr, uid, [ - ['statement_id', '=', stat_id], - ['account_id', '=', False], - ], context=context) - if line_without_account: - stat = self.browse(cr, uid, stat_id, context=context) - raise orm.except_orm( - _('User error'), - _('You should fill all account on the line of the' - ' statement %s') % stat.name) - return super(AccountStatement, self).button_confirm_bank( - cr, uid, ids, context=context) - - -class AccountStatementLine(orm.Model): - """ - Add sparse field on the statement line to allow to store all the bank infos - that are given by a bank/office. You can then add you own in your module. - The idea here is to store all bank/office infos in the - additionnal_bank_fields serialized field when importing the file. If many - values, add a tab in the bank statement line to store your specific one. - Have a look in account_statement_base_import module to see how we've done - it. - """ - _inherit = "account.bank.statement.line" - _order = "already_completed desc, date asc" - - _columns = { - 'additionnal_bank_fields': fields.serialized( - 'Additionnal infos from bank', - help="Used by completion and import system. Adds every field that " - "is present in your bank/office statement file"), - 'label': fields.sparse( - type='char', - string='Label', - serialization_field='additionnal_bank_fields', - help="Generic field to store a label given from the " - "bank/office on which we can base the default/standard " - "providen rule."), - 'already_completed': fields.boolean( - "Auto-Completed", - help="When this checkbox is ticked, the auto-completion " - "process/button will ignore this line."), - # Set account_id field as optional by removing required option. - 'account_id': fields.many2one('account.account', 'Account'), - } - - _defaults = { - 'already_completed': False, - } - - def _get_line_values_from_rules(self, cr, uid, line, rules, context=None): - """We'll try to find out the values related to the line based on rules - setted on the profile.. We will ignore line for which already_completed - is ticked. - - :return: - A dict of dict value that can be passed directly to the write - method of the statement line or {}. The first dict has statement - line ID as a key: {117009: {'partner_id': 100997, - 'account_id': 489L}} - """ - profile_obj = self.pool['account.statement.profile'] - if line.get('already_completed'): - return {} - # Ask the rule - vals = profile_obj._find_values_from_rules( - cr, uid, rules, line, context) - if vals: - vals['id'] = line['id'] - return vals - return {} - - def _get_available_columns(self, statement_store, - include_serializable=False): - """Return writeable by SQL columns""" - statement_line_obj = self.pool['account.bank.statement.line'] - model_cols = statement_line_obj._columns - avail = [ - k for k, col in model_cols.iteritems() if not hasattr(col, '_fnct') - ] - keys = [k for k in statement_store[0].keys() if k in avail] - # add sparse fields.. - if include_serializable: - for k, col in model_cols.iteritems(): - if k in statement_store[0].keys() and \ - isinstance(col, fields.sparse) and \ - col.serialization_field not in keys and \ - col._type == 'char': - keys.append(col.serialization_field) - keys.sort() - return keys - - def _prepare_insert(self, statement, cols): - """ Apply column formating to prepare data for SQL inserting - Return a copy of statement - """ - st_copy = statement - for k, col in st_copy.iteritems(): - if k in cols: - st_copy[k] = self._columns[k]._symbol_set[1](col) - return st_copy - - def _prepare_manyinsert(self, statement_store, cols): - """ Apply column formating to prepare multiple SQL inserts - Return a copy of statement_store - """ - values = [] - for statement in statement_store: - values.append(self._prepare_insert(statement, cols)) - return values - - def _serialize_sparse_fields(self, cols, statement_store): - """ Serialize sparse fields values in the target serialized field - Return a copy of statement_store - """ - statement_line_obj = self.pool['account.bank.statement.line'] - model_cols = statement_line_obj._columns - sparse_fields = dict( - [(k, col) for k, col in model_cols.iteritems() if isinstance( - col, fields.sparse) and col._type == 'char']) - values = [] - for statement in statement_store: - to_json_k = set() - st_copy = statement.copy() - for k, col in sparse_fields.iteritems(): - if k in st_copy: - to_json_k.add(col.serialization_field) - serialized = st_copy.setdefault( - col.serialization_field, {}) - serialized[k] = st_copy[k] - for k in to_json_k: - st_copy[k] = simplejson.dumps(st_copy[k]) - values.append(st_copy) - return values - - def _insert_lines(self, cr, uid, statement_store, context=None): - """ Do raw insert into database because ORM is awfully slow - when doing batch write. It is a shame that batch function - does not exist""" - statement_line_obj = self.pool['account.bank.statement.line'] - statement_line_obj.check_access_rule(cr, uid, [], 'create') - statement_line_obj.check_access_rights( - cr, uid, 'create', raise_exception=True) - cols = self._get_available_columns( - statement_store, include_serializable=True) - statement_store = self._prepare_manyinsert(statement_store, cols) - tmp_vals = (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols])) - sql = "INSERT INTO account_bank_statement_line (%s) " \ - "VALUES (%s);" % tmp_vals - try: - cr.executemany( - sql, tuple(self._serialize_sparse_fields(cols, - statement_store))) - except psycopg2.Error as sql_err: - cr.rollback() - raise orm.except_orm(_("ORM bypass error"), - sql_err.pgerror) - - def _update_line(self, cr, uid, vals, context=None): - """ Do raw update into database because ORM is awfully slow - when cheking security. - TODO / WARM: sparse fields are skipped by the method. IOW, if your - completion rule update an sparse field, the updated value will never - be stored in the database. It would be safer to call the update method - from the ORM for records updating this kind of fields. - """ - cols = self._get_available_columns([vals]) - vals = self._prepare_insert(vals, cols) - tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in cols])) - sql = "UPDATE account_bank_statement_line " \ - "SET %s where id = %%(id)s;" % tmp_vals - try: - cr.execute(sql, vals) - except psycopg2.Error as sql_err: - cr.rollback() - raise orm.except_orm(_("ORM bypass error"), - sql_err.pgerror) - - -class AccountBankStatement(orm.Model): - """We add a basic button and stuff to support the auto-completion - of the bank statement once line have been imported or manually fullfill. - """ - _inherit = "account.bank.statement" - - _columns = { - 'completion_logs': fields.text('Completion Log', readonly=True), - } - - def write_completion_log(self, cr, uid, stat_id, error_msg, - number_imported, context=None): - """Write the log in the completion_logs field of the bank statement to - let the user know what have been done. This is an append mode, so we - don't overwrite what already recoded. - - :param int/long stat_id: ID of the account.bank.statement - :param char error_msg: Message to add - :number_imported int/long: Number of lines that have been completed - :return True - """ - user_name = self.pool.get('res.users').read( - cr, uid, uid, ['name'], context=context)['name'] - statement = self.browse(cr, uid, stat_id, context=context) - number_line = len(statement.line_ids) - log = self.read(cr, uid, stat_id, ['completion_logs'], - context=context)['completion_logs'] - log = log if log else "" - completion_date = datetime.datetime.now().strftime( - DEFAULT_SERVER_DATETIME_FORMAT) - message = (_("%s Bank Statement ID %s has %s/%s lines completed by " - "%s \n%s\n%s\n") % (completion_date, stat_id, - number_imported, number_line, - user_name, error_msg, log)) - self.write( - cr, uid, [stat_id], {'completion_logs': message}, context=context) - - body = (_('Statement ID %s auto-completed for %s/%s lines completed') % - (stat_id, number_imported, number_line)), - self.message_post(cr, uid, [stat_id], body=body, context=context) - return True - - def button_auto_completion(self, cr, uid, ids, context=None): - """Complete line with values given by rules and tic the - already_completed checkbox so we won't compute them again unless the - user untick them! - """ - if context is None: - context = {} - stat_line_obj = self.pool['account.bank.statement.line'] - profile_obj = self.pool.get('account.statement.profile') - compl_lines = 0 - stat_line_obj.check_access_rule(cr, uid, [], 'create') - stat_line_obj.check_access_rights( - cr, uid, 'create', raise_exception=True) - for stat in self.browse(cr, uid, ids, context=context): - msg_lines = [] - ctx = context.copy() - ctx['line_ids'] = tuple((x.id for x in stat.line_ids)) - b_profile = stat.profile_id - rules = profile_obj._get_rules(cr, uid, b_profile, context=context) - # Only for perfo even it gains almost nothing - profile_id = b_profile.id - master_account_id = b_profile.receivable_account_id - master_account_id = master_account_id.id if \ - master_account_id else False - res = False - for line in stat_line_obj.read(cr, uid, ctx['line_ids']): - try: - # performance trick - line['master_account_id'] = master_account_id - line['profile_id'] = profile_id - res = stat_line_obj._get_line_values_from_rules( - cr, uid, line, rules, context=ctx) - if res: - compl_lines += 1 - except ErrorTooManyPartner, exc: - msg_lines.append(repr(exc)) - except Exception, exc: - msg_lines.append(repr(exc)) - error_type, error_value, trbk = sys.exc_info() - st = "Error: %s\nDescription: %s\nTraceback:" % ( - error_type.__name__, error_value) - st += ''.join(traceback.format_tb(trbk, 30)) - _logger.error(st) - if res: - # stat_line_obj.write(cr, uid, [line.id], vals, - # context=ctx) - try: - stat_line_obj._update_line( - cr, uid, res, context=context) - except Exception as exc: - msg_lines.append(repr(exc)) - error_type, error_value, trbk = sys.exc_info() - st = "Error: %s\nDescription: %s\nTraceback:" % ( - error_type.__name__, error_value) - st += ''.join(traceback.format_tb(trbk, 30)) - _logger.error(st) - # we can commit as it is not needed to be atomic - # commiting here adds a nice perfo boost - if not compl_lines % 500: - cr.commit() - msg = u'\n'.join(msg_lines) - self.write_completion_log(cr, uid, stat.id, - msg, compl_lines, context=context) - return True diff --git a/account_statement_base_import/__init__.py b/account_statement_base_import/__init__.py index 2fe60a3e..aadbee9c 100644 --- a/account_statement_base_import/__init__.py +++ b/account_statement_base_import/__init__.py @@ -20,4 +20,4 @@ ############################################################################## from . import parser from . import wizard -from . import statement +from . import models diff --git a/account_statement_base_import/__openerp__.py b/account_statement_base_import/__openerp__.py index 3a920daf..5a576242 100644 --- a/account_statement_base_import/__openerp__.py +++ b/account_statement_base_import/__openerp__.py @@ -26,8 +26,7 @@ 'category': 'Finance', 'complexity': 'normal', 'depends': [ - 'account_statement_ext', - 'account_statement_base_completion' + 'account' ], 'description': """ This module brings basic methods and fields on bank statement to deal with @@ -62,11 +61,14 @@ """, 'website': 'http://www.camptocamp.com', 'data': [ + "data/completion_rule_data.xml", "wizard/import_statement_view.xml", - "statement_view.xml", + "views/account_move_view.xml", + "views/journal_view.xml", + "views/partner_view.xml", ], 'test': [], - 'installable': False, + 'installable': True, 'images': [], 'auto_install': False, 'license': 'AGPL-3', diff --git a/account_statement_base_import/data/completion_rule_data.xml b/account_statement_base_import/data/completion_rule_data.xml new file mode 100644 index 00000000..d9d738b4 --- /dev/null +++ b/account_statement_base_import/data/completion_rule_data.xml @@ -0,0 +1,22 @@ + + + + + Match from line label (based on partner field 'Bank Statement Label') + 60 + get_from_name_and_partner_field + + + + Match from line label (based on partner name) + 70 + get_from_name_and_partner_name + + + + Match from line reference (based on Invoice reference) + 40 + get_from_ref_and_invoice + + + diff --git a/account_statement_base_import/models/__init__.py b/account_statement_base_import/models/__init__.py new file mode 100644 index 00000000..6a897165 --- /dev/null +++ b/account_statement_base_import/models/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Joel Grand-Guillaume +# Copyright 2011-2012 Camptocamp SA +# Copyright 2013 Savoir-faire Linux () +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +from . import account_journal +from . import account_move +from . import partner diff --git a/account_statement_base_import/models/account_journal.py b/account_statement_base_import/models/account_journal.py new file mode 100644 index 00000000..5ee7941c --- /dev/null +++ b/account_statement_base_import/models/account_journal.py @@ -0,0 +1,281 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Joel Grand-Guillaume +# Copyright 2011-2012 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +import sys +import traceback +from openerp import _, api, fields, models +from ..parser.parser import new_move_parser +from openerp.exceptions import UserError, ValidationError +from operator import attrgetter + + +class AccountJournal(models.Model): + _name = 'account.journal' + _inherit = ['account.journal', 'mail.thread'] + + def _get_import_type_selection(self): + """This is the method to be inherited for adding the parser""" + return [('generic_csvxls_so', 'Generic .csv/.xls based on SO Name')] + + def __get_import_type_selection(self): + """ Call method which can be inherited """ + return self._get_import_type_selection() + + commission_account_id = fields.Many2one( + comodel_name='account.account', + string='Commission account') + + import_type = fields.Selection( + __get_import_type_selection, + string='Type of import', + default='generic_csvxls_so', + required=True, + help="Choose here the method by which you want to import bank" + "statement for this profile.") + + last_import_date = fields.Datetime( + string="Last Import Date") + + launch_import_completion = fields.Boolean( + string="Launch completion after import", + help="Tic that box to automatically launch the completion " + "on each imported file using this profile.") + + partner_id = fields.Many2one( + comodel_name='res.partner', + string='Bank/Payment Office partner', + help="Put a partner if you want to have it on the commission move " + "(and optionaly on the counterpart of the intermediate/" + "banking move if you tick the corresponding checkbox).") + + receivable_account_id = fields.Many2one( + comodel_name='account.account', + string='Force Receivable/Payable Account', + help="Choose a receivable account to force the default " + "debit/credit account (eg. an intermediat bank account " + "instead of default debitors).") + + rule_ids = fields.Many2many( + comodel_name='account.move.completion.rule', + string='Auto-completion rules', + rel='as_rul_st_prof_rel') + + def _get_rules(self): + # We need to respect the sequence order + return sorted(self.rule_ids, key=attrgetter('sequence')) + + def _find_values_from_rules(self, calls, line): + """This method will execute all related rules, in their sequence order, + to retrieve all the values returned by the first rules that will match. + :param calls: list of lookup function name available in rules + :param dict line: read of the concerned account.bank.statement.line + :return: + A dict of value that can be passed directly to the write method of + the statement line or {} + {'partner_id': value, + 'account_id: value, + ...} + """ + if not calls: + calls = self._get_rules() + rule_obj = self.env['account.move.completion.rule'] + for call in calls: + method_to_call = getattr(rule_obj, call.function_to_call) + result = method_to_call(line) + if result: + result['already_completed'] = True + return result + return None + + @api.multi + def _write_extra_move_lines(self, parser, move): + """Insert extra lines after the main statement lines. + + After the main statement lines have been created, you can override this + method to create extra statement lines. + + :param: browse_record of the current parser + :param: result_row_list: [{'key':value}] + :param: profile: browserecord of account.statement.profile + :param: statement_id: int/long of the current importing + statement ID + :param: context: global context + """ + move_line_obj = self.env['account.move.line'] + global_commission_amount = 0 + total_amount = 0 + for row in parser.result_row_list: + global_commission_amount += float( + row.get('commission_amount', '0.0')) + total_amount += float( + row.get('amount', '0.0')) + total_amount += global_commission_amount + partner_id = self.partner_id.id + # Commission line + if global_commission_amount < 0.0: + commission_account_id = self.commission_account_id.id + comm_values = { + 'name': _('Commission line'), + 'date_maturity': parser.get_move_vals().get('date') or + fields.Date.today(), + 'debit': -global_commission_amount, + 'partner_id': partner_id, + 'move_id': move.id, + 'account_id': commission_account_id, + 'already_completed': True, + } + move_line_obj.with_context(check_move_validity=False).create(comm_values) + # Counterpart line + if total_amount > 0.0: + receivable_account_id = self.receivable_account_id.id or False + counterpart_values = { + 'name': _('Counterpart line'), + 'date_maturity': parser.get_move_vals().get('date') or + fields.Date.today(), + 'debit': total_amount, + 'partner_id': partner_id, + 'move_id': move.id, + 'account_id': receivable_account_id, + 'already_completed': True, + } + move_line_obj.create(counterpart_values) + + @api.multi + def write_logs_after_import(self, move, num_lines): + """Write the log in the logger + + :param int/long statement_id: ID of the concerned + account.bank.statement + :param int/long num_lines: Number of line that have been parsed + :return: True + """ + self.message_post( + body=_('Move %s have been imported with %s ' + 'lines.') % (move.name, num_lines)) + return True + + def prepare_move_line_vals(self, parser_vals, move): + """Hook to build the values of a line from the parser returned values. + At least it fullfill the statement_id. Overide it to add your own + completion if needed. + + :param dict of vals from parser for account.bank.statement.line + (called by parser.get_st_line_vals) + :param int/long statement_id: ID of the concerned + account.bank.statement + :return: dict of vals that will be passed to create method of + statement line. + """ + move_line_obj = self.env['account.move.line'] + values = parser_vals + values['company_id'] = self.company_id.id + values['journal_id'] = self.id + values['move_id'] = move.id + if values['credit'] > 0.0: + values['account_id'] = self.default_credit_account_id.id + else: + values['account_id'] = self.default_debit_account_id.id + values = move_line_obj._add_missing_default_values(values) + return values + + def prepare_move_vals(self, result_row_list, parser): + """Hook to build the values of the statement from the parser and + the profile. + """ + vals = {'journal_id': self.id} + vals.update(parser.get_move_vals()) + return vals + + def multi_move_import(self, file_stream, ftype="csv"): + """Create multiple bank statements from values given by the parser for + the given profile. + + :param int/long profile_id: ID of the profile used to import the file + :param filebuffer file_stream: binary of the providen file + :param char: ftype represent the file exstension (csv by default) + :return: list: list of ids of the created account.bank.statemênt + """ + parser = new_move_parser(self, ftype=ftype) + res = [] + for result_row_list in parser.parse(file_stream): + move = self._move_import(parser, file_stream, ftype=ftype) + res.append(move) + return res + + def _move_import(self, parser, file_stream, ftype="csv"): + """Create a bank statement with the given profile and parser. It will + fullfill the bank statement with the values of the file providen, but + will not complete data (like finding the partner, or the right + account). This will be done in a second step with the completion rules. + + :param prof : The profile used to import the file + :param parser: the parser + :param filebuffer file_stream: binary of the providen file + :param char: ftype represent the file exstension (csv by default) + :return: ID of the created account.bank.statemênt + """ + move_obj = self.env['account.move'] + move_line_obj = self.env['account.move.line'] + attachment_obj = self.env['ir.attachment'] + result_row_list = parser.result_row_list + # Check all key are present in account.bank.statement.line!! + if not result_row_list: + raise UserError(_("Nothing to import: " + "The file is empty")) + parsed_cols = parser.get_move_line_vals(result_row_list[0]).keys() + for col in parsed_cols: + if col not in move_line_obj._columns: + raise UserError( + _("Missing column! Column %s you try to import is not " + "present in the bank statement line!") % col) + move_vals = self.prepare_move_vals(result_row_list, parser) + move = move_obj.create(move_vals) + try: + # Record every line in the bank statement + move_store = [] + for line in result_row_list: + parser_vals = parser.get_move_line_vals(line) + values = self.prepare_move_line_vals(parser_vals, move) + move_store.append(values) + # Hack to bypass ORM poor perfomance. Sob... + move_line_obj._insert_lines(move_store) + self._write_extra_move_lines(parser, move) + attachment_data = { + 'name': 'statement file', + 'datas': file_stream, + 'datas_fname': "%s.%s" % (fields.Date.today(), ftype), + 'res_model': 'account.move', + 'res_id': move.id, + } + attachment_obj.create(attachment_data) + # If user ask to launch completion at end of import, do it! + if self.launch_import_completion: + move.button_auto_completion() + # Write the needed log infos on profile + self.write_logs_after_import(move, len(result_row_list)) + except Exception: + error_type, error_value, trbk = sys.exc_info() + st = "Error: %s\nDescription: %s\nTraceback:" % ( + error_type.__name__, error_value) + st += ''.join(traceback.format_tb(trbk, 30)) + raise ValidationError( + _("Statement import error" + "The statement cannot be created: %s") % st) + return move diff --git a/account_statement_base_import/models/account_move.py b/account_statement_base_import/models/account_move.py new file mode 100644 index 00000000..edfa2f53 --- /dev/null +++ b/account_statement_base_import/models/account_move.py @@ -0,0 +1,392 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Nicolas Bessi, Joel Grand-Guillaume +# Copyright 2011-2012 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +# TODO replace customer supplier by package constant +import traceback +import sys +import logging + +import psycopg2 + +from openerp import _, api, fields, models +from openerp.exceptions import ValidationError + + +_logger = logging.getLogger(__name__) + + +class ErrorTooManyPartner(Exception): + """ New Exception definition that is raised when more than one partner is + matched by the completion rule. + """ + + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + def __repr__(self): + return repr(self.value) + + +class AccountMoveCompletionRule(models.Model): + """This will represent all the completion method that we can have to + fullfill the bank statement lines. You'll be able to extend them in you own + module and choose those to apply for every statement profile. + The goal of a rule is to fullfill at least the partner of the line, but + if possible also the reference because we'll use it in the reconciliation + process. The reference should contain the invoice number or the SO number + or any reference that will be matched by the invoice accounting move. + """ + _name = "account.move.completion.rule" + _order = "sequence asc" + + def _get_functions(self): + """List of available methods for rules. + + Override this to add you own.""" + return [ + ('get_from_ref_and_invoice', + 'From line reference (based on invoice reference)'), + ('get_from_name_and_partner_field', + 'From line name (based on partner field)'), + ('get_from_name_and_partner_name', + 'From line name (based on partner name)') + ] + + def __get_functions(self): + """ Call method which can be inherited """ + return self._get_functions() + + sequence = fields.Integer( + string='Sequence', + help="Lower means parsed first.") + name = fields.Char( + string='Name', + size=128) + journal_ids = fields.Many2many( + comodel_name='account.journal', + rel='as_rul_st_prof_rel', + string='Related journals') + function_to_call = fields.Selection( + __get_functions, + string='Method') + + # Should be private but data are initialised with no update XML + def get_from_ref_and_invoice(self, line): + """Match the partner based on the invoice number and the reference of + the statement line. Then, call the generic get_values_for_line method + to complete other values. If more than one partner matched, raise the + ErrorTooManyPartner error. + + :param dict line: read of the concerned account.bank.statement.line + :return: + A dict of value that can be passed directly to the write method of + the statement line or {} + {'partner_id': value, + 'account_id': value, + ...} + """ + res = {} + inv_obj = self.env['account.invoice'] + + invoices = inv_obj.search([('reference', '=', line.ref.strip())]) + if invoices: + if len(invoices) == 1: + invoice = invoices[0] + partner_id = invoice.commercial_partner_id.id + res = {'partner_id': partner_id} + else: + raise ErrorTooManyPartner( + _('Line named "%s" (Ref:%s) was matched by more than one ' + 'partner while looking on invoices') % + (line.name, line.ref)) + return res + + # Should be private but data are initialised with no update XML + def get_from_name_and_partner_field(self, line): + """ + Match the partner based on the label field of the statement line and + the text defined in the 'bank_statement_label' field of the partner. + Remember that we can have values separated with ; Then, call the + generic get_values_for_line method to complete other values. If more + than one partner matched, raise the ErrorTooManyPartner error. + + :param dict line: read of the concerned account.bank.statement.line + :return: + A dict of value that can be passed directly to the write method of + the statement line or {} + {'partner_id': value, + 'account_id': value, + + ...} + """ + res = {} + partner_obj = self.env['res.partner'] + or_regex = ".*; *%s *;.*" % line.name + sql = ("SELECT id from res_partner" + " WHERE bank_statement_label ~* %s") + self.env.cr.execute(sql, (or_regex, )) + partner_ids = self.env.cr.fetchall() + partners = partner_obj.browse([x[0] for x in partner_ids]) + if partners: + if len(partners) > 1: + msg = (_('Line named "%s" (Ref:%s) was matched by more than ' + 'one partner while looking on partner label: %s') % + (line.name, line.ref, + ','.join([x.name for x in partners]))) + raise ErrorTooManyPartner(msg) + res['partner_id'] = partners[0].id + return res + + def get_from_name_and_partner_name(self, line): + """Match the partner based on the label field of the statement line and + the name of the partner. Then, call the generic get_values_for_line + method to complete other values. If more than one partner matched, + raise the ErrorTooManyPartner error. + + :param dict st_line: read of the concerned account.bank.statement.line + :return: + A dict of value that can be passed directly to the write method of + the statement line or {} + {'partner_id': value, + 'account_id': value, + + ...} + """ + res = {} + # The regexp_replace() escapes the name to avoid false positive + # example: 'John J. Doe (No 1)' is escaped to 'John J\. Doe \(No 1\)' + # See http://stackoverflow.com/a/400316/1504003 for a list of + # chars to escape. Postgres is POSIX-ARE, compatible with + # POSIX-ERE excepted that '\' must be escaped inside brackets according + # to: + # http://www.postgresql.org/docs/9.0/static/functions-matching.html + # in chapter 9.7.3.6. Limits and Compatibility + sql = r""" + SELECT id FROM ( + SELECT id, + regexp_matches(%s, + regexp_replace(name,'([\.\^\$\*\+\?\(\)\[\{\\\|])', %s, + 'g'), 'i') AS name_match + FROM res_partner) + AS res_partner_matcher + WHERE name_match IS NOT NULL""" + self.env.cr.execute(sql, (line.name, r"\\\1")) + result = self.env.cr.fetchall() + if result: + if len(result) > 1: + raise ErrorTooManyPartner( + _('Line named "%s" (Ref:%s) was matched by more than one ' + 'partner while looking on partner by name') % + (line.name, line.ref)) + res['partner_id'] = result[0][0] + return res + + +class AccountMoveLine(models.Model): + """ + Add sparse field on the statement line to allow to store all the bank infos + that are given by a bank/office. You can then add you own in your module. + The idea here is to store all bank/office infos in the + additionnal_bank_fields serialized field when importing the file. If many + values, add a tab in the bank statement line to store your specific one. + Have a look in account_statement_base_import module to see how we've done + it. + """ + _inherit = "account.move.line" + _order = "already_completed desc, date asc" + + already_completed = fields.Boolean( + string="Auto-Completed", + default=False, + help="When this checkbox is ticked, the auto-completion " + "process/button will ignore this line.") + + def _get_line_values_from_rules(self, line, rules): + """We'll try to find out the values related to the line based on rules + setted on the profile.. We will ignore line for which already_completed + is ticked. + + :return: + A dict of dict value that can be passed directly to the write + method of the statement line or {}. The first dict has statement + line ID as a key: {117009: {'partner_id': 100997, + 'account_id': 489L}} + """ + journal_obj = self.env['account.journal'] + if not line.already_completed: + # Ask the rule + vals = journal_obj._find_values_from_rules(rules, line) + if vals: + vals['id'] = line['id'] + return vals + return {} + + def _get_available_columns(self, move_store): + """Return writeable by SQL columns""" + model_cols = self._columns + avail = [ + k for k, col in model_cols.iteritems() if not hasattr(col, '_fnct') + ] + keys = [k for k in move_store[0].keys() if k in avail] + keys.sort() + return keys + + def _prepare_insert(self, move, cols): + """ Apply column formating to prepare data for SQL inserting + Return a copy of statement + """ + move_copy = move + for k, col in move_copy.iteritems(): + if k in cols: + move_copy[k] = self._columns[k]._symbol_set[1](col) + return move_copy + + def _prepare_manyinsert(self, move_store, cols): + """ Apply column formating to prepare multiple SQL inserts + Return a copy of statement_store + """ + values = [] + for move in move_store: + values.append(self._prepare_insert(move, cols)) + return values + + def _insert_lines(self, move_store): + """ Do raw insert into database because ORM is awfully slow + when doing batch write. It is a shame that batch function + does not exist""" + self.check_access_rule('create') + self.check_access_rights('create', raise_exception=True) + cols = self._get_available_columns(move_store) + move_store = self._prepare_manyinsert(move_store, cols) + tmp_vals = (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols])) + sql = "INSERT INTO account_move_line (%s) " \ + "VALUES (%s);" % tmp_vals + try: + self.env.cr.executemany(sql, tuple(move_store)) + except psycopg2.Error as sql_err: + self.env.cr.rollback() + raise ValidationError(_("ORM bypass error"), + sql_err.pgerror) + + def _update_line(self, vals): + """ Do raw update into database because ORM is awfully slow + when cheking security. + TODO / WARM: sparse fields are skipped by the method. IOW, if your + completion rule update an sparse field, the updated value will never + be stored in the database. It would be safer to call the update method + from the ORM for records updating this kind of fields. + """ + cols = self._get_available_columns([vals]) + vals = self._prepare_insert(vals, cols) + tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in cols])) + sql = "UPDATE account_move_line " \ + "SET %s where id = %%(id)s;" % tmp_vals + try: + self.env.cr.execute(sql, vals) + except psycopg2.Error as sql_err: + self.env.cr.rollback() + raise ValidationError(_("ORM bypass error"), + sql_err.pgerror) + + +class AccountMove(models.Model): + """We add a basic button and stuff to support the auto-completion + of the bank statement once line have been imported or manually fullfill. + """ + _name = 'account.move' + _inherit = ['account.move', 'mail.thread'] + + completion_logs = fields.Text(string='Completion Log', readonly=True) + + def write_completion_log(self, error_msg, number_imported): + """Write the log in the completion_logs field of the bank statement to + let the user know what have been done. This is an append mode, so we + don't overwrite what already recoded. + + :param int/long stat_id: ID of the account.bank.statement + :param char error_msg: Message to add + :number_imported int/long: Number of lines that have been completed + :return True + """ + user_name = self.env.user.name + number_line = len(self.line_ids) + log = self.completion_logs or "" + completion_date = fields.Datetime.now() + message = (_("%s Account Move %s has %s/%s lines completed by " + "%s \n%s\n%s\n") % (completion_date, self.name, + number_imported, number_line, + user_name, error_msg, log)) + self.write({'completion_logs': message}) + + body = (_('Statement ID %s auto-completed for %s/%s lines completed') % + (self.name, number_imported, number_line)), + self.message_post(body=body) + return True + + @api.multi + def button_auto_completion(self): + """Complete line with values given by rules and tic the + already_completed checkbox so we won't compute them again unless the + user untick them! + """ + move_line_obj = self.env['account.move.line'] + compl_lines = 0 + move_line_obj.check_access_rule('create') + move_line_obj.check_access_rights('create', raise_exception=True) + for move in self: + msg_lines = [] + journal = move.journal_id + rules = journal._get_rules() + res = False + for line in move.line_ids: + try: + res = move_line_obj._get_line_values_from_rules( + line, rules) + if res: + compl_lines += 1 + except ErrorTooManyPartner, exc: + msg_lines.append(repr(exc)) + except Exception, exc: + msg_lines.append(repr(exc)) + error_type, error_value, trbk = sys.exc_info() + st = "Error: %s\nDescription: %s\nTraceback:" % ( + error_type.__name__, error_value) + st += ''.join(traceback.format_tb(trbk, 30)) + _logger.error(st) + if res: + try: + move_line_obj._update_line(res) + except Exception as exc: + msg_lines.append(repr(exc)) + error_type, error_value, trbk = sys.exc_info() + st = "Error: %s\nDescription: %s\nTraceback:" % ( + error_type.__name__, error_value) + st += ''.join(traceback.format_tb(trbk, 30)) + _logger.error(st) + # we can commit as it is not needed to be atomic + # commiting here adds a nice perfo boost + if not compl_lines % 500: + self.env.cr.commit() + msg = u'\n'.join(msg_lines) + self.write_completion_log(msg, compl_lines) + return True diff --git a/account_statement_base_completion/partner.py b/account_statement_base_import/models/partner.py similarity index 68% rename from account_statement_base_completion/partner.py rename to account_statement_base_import/models/partner.py index 3f371c78..8087f5f4 100644 --- a/account_statement_base_completion/partner.py +++ b/account_statement_base_import/models/partner.py @@ -19,21 +19,20 @@ # ########################################################################## -from openerp.osv import orm, fields +from openerp import fields, models -class ResPartner(orm.Model): +class ResPartner(models.Model): """Add a bank label on the partner so that we can use it to match this partner when we found this in a statement line. """ _inherit = 'res.partner' - _columns = { - 'bank_statement_label': fields.char( - 'Bank Statement Label', size=100, - help="Enter the various label found on your bank statement " - "separated by a ; If one of this label is include in the " - "bank statement line, the partner will be automatically " - "filled (as long as you use this method/rules in your " - "statement profile)."), - } + bank_statement_label = fields.Char( + string='Bank Statement Label', + size=100, + help="Enter the various label found on your bank statement " + "separated by a ; If one of this label is include in the " + "bank statement line, the partner will be automatically " + "filled (as long as you use this method/rules in your " + "statement profile).") diff --git a/account_statement_base_import/parser/__init__.py b/account_statement_base_import/parser/__init__.py index cb73080b..14eca092 100644 --- a/account_statement_base_import/parser/__init__.py +++ b/account_statement_base_import/parser/__init__.py @@ -19,7 +19,7 @@ # ############################################################################## -from .parser import new_bank_statement_parser -from .parser import BankStatementImportParser +from .parser import new_move_parser +from .parser import AccountMoveImportParser from . import file_parser from . import generic_file_parser diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index 706e0b2d..eade8486 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -18,11 +18,10 @@ # ############################################################################## from openerp.tools.translate import _ -from openerp.osv.orm import except_orm +from openerp.exceptions import UserError import tempfile import datetime -from .parser import BankStatementImportParser -from .parser import UnicodeDictReader +from .parser import AccountMoveImportParser, UnicodeDictReader try: import xlrd except: @@ -35,7 +34,7 @@ def float_or_zero(val): return float(val) if val else 0.0 -class FileParser(BankStatementImportParser): +class FileParser(AccountMoveImportParser): """Generic abstract class for defining parser for .csv, .xls or .xlsx file format. """ @@ -55,8 +54,7 @@ class FileParser(BankStatementImportParser): if ftype in ('csv', 'xls', 'xlsx'): self.ftype = ftype[0:3] else: - raise except_orm( - _('User Error'), + raise UserError( _('Invalid file type %s. Please use csv, xls or xlsx') % ftype) self.conversion_dict = extra_fields self.keys_to_validate = self.conversion_dict.keys() @@ -96,8 +94,7 @@ class FileParser(BankStatementImportParser): parsed_cols = self.result_row_list[0].keys() for col in self.keys_to_validate: if col not in parsed_cols: - raise except_orm(_('Invalid data'), - _('Column %s not present in file') % col) + raise UserError(_('Column %s not present in file') % col) return True def _post(self, *args, **kwargs): @@ -143,9 +140,9 @@ class FileParser(BankStatementImportParser): line[rule] = datetime.datetime.strptime(date_string, '%Y-%m-%d') except ValueError as err: - raise except_orm( - _("Date format is not valid."), - _(" It should be YYYY-MM-DD for column: %s" + raise UserError( + _("Date format is not valid." + " It should be YYYY-MM-DD for column: %s" " value: %s \n \n \n Please check the line with " "ref: %s \n \n Detail: %s") % (rule, line.get(rule, _('Missing')), @@ -154,8 +151,7 @@ class FileParser(BankStatementImportParser): try: line[rule] = conversion_rules[rule](line[rule]) except Exception as err: - raise except_orm( - _('Invalid data'), + raise UserError( _("Value %s of column %s is not valid.\n Please " "check the line with ref %s:\n \n Detail: %s") % (line.get(rule, _('Missing')), rule, @@ -174,9 +170,9 @@ class FileParser(BankStatementImportParser): self._datemode) line[rule] = datetime.datetime(*t_tuple) except Exception as err: - raise except_orm( - _("Date format is not valid"), - _("Please modify the cell formatting to date " + raise UserError( + _("Date format is not valid. " + "Please modify the cell formatting to date " "format for column: %s value: %s\n Please check " "the line with ref: %s\n \n Detail: %s") % (rule, line.get(rule, _('Missing')), @@ -185,8 +181,7 @@ class FileParser(BankStatementImportParser): try: line[rule] = conversion_rules[rule](line[rule]) except Exception as err: - raise except_orm( - _('Invalid data'), + raise UserError( _("Value %s of column %s is not valid.\n Please " "check the line with ref %s:\n \n Detail: %s") % (line.get(rule, _('Missing')), rule, diff --git a/account_statement_base_import/parser/generic_file_parser.py b/account_statement_base_import/parser/generic_file_parser.py index 47e98445..38879aac 100644 --- a/account_statement_base_import/parser/generic_file_parser.py +++ b/account_statement_base_import/parser/generic_file_parser.py @@ -52,7 +52,7 @@ class GenericFileParser(FileParser): """ return parser_name == 'generic_csvxls_so' - def get_st_line_vals(self, line, *args, **kwargs): + def get_move_line_vals(self, line, *args, **kwargs): """ This method must return a dict of vals that can be passed to create method of statement line in order to record it. It is the @@ -70,10 +70,10 @@ class GenericFileParser(FileParser): 'label':value, } """ + amount = line.get('amount', 0.0) return { 'name': line.get('label', line.get('ref', '/')), - 'date': line.get('date', datetime.datetime.now().date()), - 'amount': line.get('amount', 0.0), - 'ref': line.get('ref', '/'), - 'label': line.get('label', ''), + 'date_maturity': line.get('date', datetime.datetime.now().date()), + 'credit': amount > 0.0 and amount or 0.0, + 'debit': amount < 0.0 and amount or 0.0, } diff --git a/account_statement_base_import/parser/parser.py b/account_statement_base_import/parser/parser.py index 999be4b0..a38a8620 100644 --- a/account_statement_base_import/parser/parser.py +++ b/account_statement_base_import/parser/parser.py @@ -20,8 +20,7 @@ ############################################################################## import base64 import csv -from datetime import datetime -from openerp.tools.translate import _ +from openerp import _, fields def UnicodeDictReader(utf8_data, **kwargs): @@ -41,7 +40,7 @@ def UnicodeDictReader(utf8_data, **kwargs): for key, value in row.iteritems()]) -class BankStatementImportParser(object): +class AccountMoveImportParser(object): """ Generic abstract class for defining parser for different files and @@ -50,21 +49,19 @@ class BankStatementImportParser(object): from the FileParser instead. """ - def __init__(self, profile, *args, **kwargs): + def __init__(self, journal, *args, **kwargs): # The name of the parser as it will be called - self.parser_name = profile.import_type + self.parser_name = journal.import_type # The result as a list of row. One row per line of data in the file, # but not the commission one! self.result_row_list = None # The file buffer on which to work on self.filebuffer = None # The profile record to access its parameters in any parser method - self.profile = profile - self.balance_start = None - self.balance_end = None - self.statement_name = None - self.statement_date = None - self.support_multi_statements = False + self.journal = journal + self.move_date = None + self.move_name = None + self.move_ref= None @classmethod def parser_for(cls, parser_name): @@ -119,19 +116,18 @@ class BankStatementImportParser(object): """ return NotImplementedError - def get_st_vals(self): + def get_move_vals(self): """This method return a dict of vals that ca be passed to create method of statement. :return: dict of vals that represent additional infos for the statement """ return { - 'name': self.statement_name or '/', - 'balance_start': self.balance_start, - 'balance_end_real': self.balance_end, - 'date': self.statement_date or datetime.now() + 'name': self.move_name or '/', + 'date': self.move_date or fields.Datetime.now(), + 'ref': self.move_ref or '/' } - def get_st_line_vals(self, line, *args, **kwargs): + def get_move_line_vals(self, line, *args, **kwargs): """Implement a method in your parser that must return a dict of vals that can be passed to create method of statement line in order to record it. It is the responsibility of every parser to give this dict @@ -165,16 +161,10 @@ class BankStatementImportParser(object): raise Exception(_('No buffer file given.')) self._format(*args, **kwargs) self._pre(*args, **kwargs) - if self.support_multi_statements: - while self._parse(*args, **kwargs): - self._validate(*args, **kwargs) - self._post(*args, **kwargs) - yield self.result_row_list - else: - self._parse(*args, **kwargs) - self._validate(*args, **kwargs) - self._post(*args, **kwargs) - yield self.result_row_list + self._parse(*args, **kwargs) + self._validate(*args, **kwargs) + self._post(*args, **kwargs) + yield self.result_row_list def itersubclasses(cls, _seen=None): @@ -218,13 +208,13 @@ def itersubclasses(cls, _seen=None): yield sub -def new_bank_statement_parser(profile, *args, **kwargs): +def new_move_parser(journal, *args, **kwargs): """Return an instance of the good parser class based on the given profile. :param profile: browse_record of import profile. :return: class instance for given profile import type. """ - for cls in itersubclasses(BankStatementImportParser): - if cls.parser_for(profile.import_type): - return cls(profile, *args, **kwargs) + for cls in itersubclasses(AccountMoveImportParser): + if cls.parser_for(journal.import_type): + return cls(journal, *args, **kwargs) raise ValueError diff --git a/account_statement_base_import/statement.py b/account_statement_base_import/statement.py deleted file mode 100644 index 83e250f3..00000000 --- a/account_statement_base_import/statement.py +++ /dev/null @@ -1,255 +0,0 @@ -# -*- coding: utf-8 -*- -############################################################################## -# -# Author: Joel Grand-Guillaume -# Copyright 2011-2012 Camptocamp SA -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -import sys -import traceback -from openerp.tools.translate import _ -import datetime -from openerp.osv import fields, orm -from .parser import new_bank_statement_parser -from openerp.tools.config import config - - -class AccountStatementProfil(orm.Model): - _inherit = "account.statement.profile" - - def _get_import_type_selection(self, cr, uid, context=None): - """This is the method to be inherited for adding the parser""" - return [('generic_csvxls_so', 'Generic .csv/.xls based on SO Name')] - - def __get_import_type_selection(self, cr, uid, context=None): - """ Call method which can be inherited """ - return self._get_import_type_selection(cr, uid, context=context) - - _columns = { - 'launch_import_completion': fields.boolean( - "Launch completion after import", - help="Tic that box to automatically launch the completion " - "on each imported file using this profile."), - 'last_import_date': fields.datetime("Last Import Date"), - # we remove deprecated as it floods logs in standard/warning level - # sob... - 'rec_log': fields.text('log', readonly=True), # Deprecated - 'import_type': fields.selection( - __get_import_type_selection, - 'Type of import', - required=True, - help="Choose here the method by which you want to import bank" - "statement for this profile."), - } - - _defaults = { - 'import_type': 'generic_csvxls_so' - } - - def _write_extra_statement_lines( - self, cr, uid, parser, result_row_list, profile, statement_id, - context): - """Insert extra lines after the main statement lines. - - After the main statement lines have been created, you can override this - method to create extra statement lines. - - :param: browse_record of the current parser - :param: result_row_list: [{'key':value}] - :param: profile: browserecord of account.statement.profile - :param: statement_id: int/long of the current importing - statement ID - :param: context: global context - """ - - def write_logs_after_import(self, cr, uid, ids, statement_id, num_lines, - context): - """Write the log in the logger - - :param int/long statement_id: ID of the concerned - account.bank.statement - :param int/long num_lines: Number of line that have been parsed - :return: True - """ - self.message_post( - cr, uid, ids, - body=_('Statement ID %s have been imported with %s ' - 'lines.') % (statement_id, num_lines), context=context) - return True - - # Deprecated remove on V8 - def prepare_statetement_lines_vals(self, *args, **kwargs): - return self.prepare_statement_lines_vals(*args, **kwargs) - - def prepare_statement_lines_vals(self, cr, uid, parser_vals, - statement_id, context): - """Hook to build the values of a line from the parser returned values. - At least it fullfill the statement_id. Overide it to add your own - completion if needed. - - :param dict of vals from parser for account.bank.statement.line - (called by parser.get_st_line_vals) - :param int/long statement_id: ID of the concerned - account.bank.statement - :return: dict of vals that will be passed to create method of - statement line. - """ - statement_line_obj = self.pool['account.bank.statement.line'] - values = parser_vals - values['statement_id'] = statement_id - date = values.get('date') - period_memoizer = context.get('period_memoizer') - if not period_memoizer: - period_memoizer = {} - context['period_memoizer'] = period_memoizer - if period_memoizer.get(date): - values['period_id'] = period_memoizer[date] - else: - # This is awfully slow... - periods = self.pool.get('account.period').find( - cr, uid, dt=values.get('date'), context=context) - values['period_id'] = periods[0] - period_memoizer[date] = periods[0] - values = statement_line_obj._add_missing_default_values( - cr, uid, values, context) - return values - - def prepare_statement_vals(self, cr, uid, profile_id, result_row_list, - parser, context=None): - """Hook to build the values of the statement from the parser and - the profile. - """ - vals = {'profile_id': profile_id} - vals.update(parser.get_st_vals()) - if vals.get('balance_start') is None: - # Get starting balance from journal balance if parser doesn't - # fill this data, simulating the manual flow - statement_obj = self.pool['account.bank.statement'] - profile = self.browse(cr, uid, profile_id, context=context) - temp = statement_obj.onchange_journal_id( - cr, uid, None, profile.journal_id.id, context=context) - vals['balance_start'] = temp['value'].get('balance_start', False) - return vals - - def multi_statement_import(self, cr, uid, ids, profile_id, file_stream, - ftype="csv", context=None): - """Create multiple bank statements from values given by the parser for - the given profile. - - :param int/long profile_id: ID of the profile used to import the file - :param filebuffer file_stream: binary of the providen file - :param char: ftype represent the file exstension (csv by default) - :return: list: list of ids of the created account.bank.statemênt - """ - prof_obj = self.pool['account.statement.profile'] - if not profile_id: - raise orm.except_orm( - _("No Profile!"), - _("You must provide a valid profile to import a bank " - "statement!")) - prof = prof_obj.browse(cr, uid, profile_id, context=context) - parser = new_bank_statement_parser(prof, ftype=ftype) - res = [] - for result_row_list in parser.parse(file_stream): - statement_id = self._statement_import( - cr, uid, ids, prof, parser, file_stream, ftype=ftype, - context=context) - res.append(statement_id) - return res - - def _statement_import(self, cr, uid, ids, prof, parser, file_stream, - ftype="csv", context=None): - """Create a bank statement with the given profile and parser. It will - fullfill the bank statement with the values of the file providen, but - will not complete data (like finding the partner, or the right - account). This will be done in a second step with the completion rules. - - :param prof : The profile used to import the file - :param parser: the parser - :param filebuffer file_stream: binary of the providen file - :param char: ftype represent the file exstension (csv by default) - :return: ID of the created account.bank.statemênt - """ - statement_obj = self.pool['account.bank.statement'] - statement_line_obj = self.pool['account.bank.statement.line'] - attachment_obj = self.pool['ir.attachment'] - result_row_list = parser.result_row_list - # Check all key are present in account.bank.statement.line!! - if not result_row_list: - raise orm.except_orm(_("Nothing to import"), - _("The file is empty")) - parsed_cols = parser.get_st_line_vals(result_row_list[0]).keys() - for col in parsed_cols: - if col not in statement_line_obj._columns: - raise orm.except_orm( - _("Missing column!"), - _("Column %s you try to import is not present in the bank " - "statement line!") % col) - statement_vals = self.prepare_statement_vals( - cr, uid, prof.id, result_row_list, parser, context) - statement_id = statement_obj.create( - cr, uid, statement_vals, context=context) - try: - # Record every line in the bank statement - statement_store = [] - for line in result_row_list: - parser_vals = parser.get_st_line_vals(line) - values = self.prepare_statement_lines_vals( - cr, uid, parser_vals, statement_id, - context) - statement_store.append(values) - # Hack to bypass ORM poor perfomance. Sob... - statement_line_obj._insert_lines( - cr, uid, statement_store, context=context) - self._write_extra_statement_lines( - cr, uid, parser, result_row_list, prof, statement_id, context) - # Trigger store field computation if someone has better idea - start_bal = statement_obj.read( - cr, uid, statement_id, ['balance_start'], context=context) - start_bal = start_bal['balance_start'] - statement_obj.write( - cr, uid, [statement_id], {'balance_start': start_bal}) - attachment_data = { - 'name': 'statement file', - 'datas': file_stream, - 'datas_fname': "%s.%s" % (datetime.datetime.now().date(), - ftype), - 'res_model': 'account.bank.statement', - 'res_id': statement_id, - } - attachment_obj.create(cr, uid, attachment_data, context=context) - # If user ask to launch completion at end of import, do it! - if prof.launch_import_completion: - statement_obj.button_auto_completion( - cr, uid, [statement_id], context) - # Write the needed log infos on profile - self.write_logs_after_import(cr, uid, prof.id, - statement_id, - len(result_row_list), - context) - except Exception: - error_type, error_value, trbk = sys.exc_info() - st = "Error: %s\nDescription: %s\nTraceback:" % ( - error_type.__name__, error_value) - st += ''.join(traceback.format_tb(trbk, 30)) - # TODO we should catch correctly the exception with a python - # Exception and only re-catch some special exception. - # For now we avoid re-catching error in debug mode - if config['debug_mode']: - raise - raise orm.except_orm(_("Statement import error"), - _("The statement cannot be created: %s") % st) - return statement_id diff --git a/account_statement_base_import/statement_view.xml b/account_statement_base_import/statement_view.xml deleted file mode 100644 index b73e0dc7..00000000 --- a/account_statement_base_import/statement_view.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - account.statement.profile.view - account.statement.profile - - - - - - - - + + + + + + + + + + + diff --git a/account_statement_base_import/views/journal_view.xml b/account_statement_base_import/views/journal_view.xml new file mode 100644 index 00000000..fecfca31 --- /dev/null +++ b/account_statement_base_import/views/journal_view.xml @@ -0,0 +1,32 @@ + + + + account.journal.view + account.journal + + + + + + + + + + + + + + + +