[IMP] Fixes lp:1223834 in case of insert. Batch updates remains error prone. It would be safer to call the update method from the orm for records updating 'complex' fields.

A new completion rule based on the bank account number is also provided by the proposal.

About modules dependencies. The module 'account_statement_base_import' depends of 'account_statement_base_completion' but the file statement.py of 'account_statement_base_completion' at line 513 call the method _update_line defined in 'account_statement_base_import'. Since the 'AccountStatementLine' is defined in both addons, I've the feeling that we can merge the two overrides in 'account_statement_base_completion'.

account_statement_bankaccount_completion/__init__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+#
+#
+#    Author: Laurent Mignon
+#    Copyright 2013 'ACSONE SA/NV'
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+import statement

account_statement_bankaccount_completion/__openerp__.py

@@ -0,0 +1,49 @@
+# -*- coding: utf-8 -*-
+#
+#
+#    Author: Laurent Mignon
+#    Copyright 2013 'ACSONE SA/NV'
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+{'name': "Bank statement completion from bank account number",
+ 'version': '1.0',
+ 'author': 'ACSONE SA/NV',
+ 'maintainer': 'ACSONE SA/NV',
+ 'category': 'Finance',
+ 'complexity': 'normal',
+ 'depends': [
+     'account_statement_base_completion',
+ ],
+ 'description': """
+  Add a completion method based on the partner bank account number provided by the bank/office.
+
+  Completion will look in the partner with that bank account number to match the partner, 
+  then it will fill in the bank statement line with it to ease the reconciliation.
+
+ """,
+ 'website': 'http://www.acsone.eu',
+ 'init_xml': [],
+ 'update_xml': [
+     "data.xml",
+ ],
+ 'demo_xml': [],
+ 'test': [],
+ 'installable': True,
+ 'images': [],
+ 'auto_install': True,
+ 'license': 'AGPL-3',
+ }

account_statement_bankaccount_completion/data.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<openerp>
+<data noupdate="1">
+
+    <record id="bank_statement_completion_rule_10" model="account.statement.completion.rule">
+         <field name="name">Match from bank account number (Nomal or IBAN))</field>
+         <field name="sequence">10</field>
+         <field name="function_to_call">get_from_bank_account</field>
+     </record>
+    
+</data>
+</openerp>

account_statement_bankaccount_completion/statement.py

@@ -0,0 +1,96 @@
+# -*- coding: utf-8 -*-
+#
+#
+#    Author: Laurent Mignon
+#    Copyright 2013 'ACSONE SA/NV'
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+from openerp.tools.translate import _
+from openerp.osv.orm import Model
+from openerp.osv import fields
+from openerp.addons.account_statement_base_completion.statement import ErrorTooManyPartner
+
+
+class AccountStatementCompletionRule(Model):
+    """Add a rule based on transaction ID"""
+
+    _inherit = "account.statement.completion.rule"
+
+    def _get_functions(self, cr, uid, context=None):
+        res = super(AccountStatementCompletionRule, self)._get_functions(
+            cr, uid, context=context)
+        res.append(('get_from_bank_account',
+                    'From bank account number (Normal or IBAN)'))
+        return res
+
+    _columns = {
+        'function_to_call': fields.selection(_get_functions, 'Method'),
+    }
+
+    def get_from_bank_account(self, cr, uid, st_line, context=None):
+        """
+        Match the partner based on the partner account number field
+        Then, call the generic st_line method to complete other values.
+        :param dict st_line: read of the concerned account.bank.statement.line
+        :return:
+            A dict of value that can be passed directly to the write method of
+            the statement line or {}
+           {'partner_id': value,
+            'account_id' : value,
+            ...}
+        """
+        partner_acc_number = st_line['partner_acc_number']
+        if not partner_acc_number:
+            return {}
+        st_obj = self.pool.get('account.bank.statement.line')
+        res = {}
+        res_bank_obj = self.pool.get('res.partner.bank')
+        ids = res_bank_obj.search(cr,
+                                  uid,
+                                  [('acc_number', '=', partner_acc_number)],
+                                  context=context)
+        if len(ids) > 1:
+            raise ErrorTooManyPartner(_('Line named "%s" (Ref:%s) was matched by more than '
+                                        'one partner for account number "%s".') % (st_line['name'], st_line['ref'], partner_acc_number))
+        if len(ids) == 1:
+            partner = res_bank_obj.browse(cr, uid, ids[0], context=context).partner_id
+            res['partner_id'] = partner.id
+            st_vals = st_obj.get_values_for_line(cr,
+                                                 uid,
+                                                 profile_id=st_line['profile_id'],
+                                                 master_account_id=st_line['master_account_id'],
+                                                 partner_id=res.get('partner_id', False),
+                                                 line_type=st_line['type'],
+                                                 amount=st_line['amount'] if st_line['amount'] else 0.0,
+                                                 context=context)
+            res.update(st_vals)
+        return res
+
+
+class AccountStatementLine(Model):
+    _inherit = "account.bank.statement.line"
+
+    _columns = {
+        # 'additional_bank_fields' : fields.serialized('Additional infos from bank', help="Used by completion and import system."),
+        'partner_acc_number': fields.sparse(
+            type='char',
+            string='Account Number',
+            size=64,
+            serialization_field='additionnal_bank_fields',
+            help="Account number of the partner"),
+    }

account_statement_bankaccount_completion/tests/__init__.py

@@ -0,0 +1,27 @@
+# -*- coding: utf-8 -*-
+#
+#
+# Authors: Laurent Mignon
+# Copyright (c) 2013 Acsone SA/NV (http://www.acsone.eu)
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+from . import test_bankaccount_completion
+
+checks = [
+    test_bankaccount_completion
+]

account_statement_bankaccount_completion/tests/test_bankaccount_completion.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+#
+#
+# Authors: Laurent Mignon
+# Copyright (c) 2013 Acsone SA/NV (http://www.acsone.eu)
+# All Rights Reserved
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#
+from openerp.tests import common
+import time
+
+ACC_NUMBER = "BE38733040385372"
+
+
+class bankaccount_completion(common.TransactionCase):
+
+    def prepare(self):
+        self.company_a = self.browse_ref('base.main_company')
+        self.profile_obj = self.registry("account.statement.profile")
+        self.account_bank_statement_obj = self.registry("account.bank.statement")
+        self.account_bank_statement_line_obj = self.registry("account.bank.statement.line")
+        self.completion_rule_id = self.ref('account_statement_bankaccount_completion.bank_statement_completion_rule_10')
+        self.journal_id = self.registry("ir.model.data").get_object_reference(self.cr, self. uid, "account", "bank_journal")[1]
+        self.partner_id = self.ref('base.main_partner')
+        # Create the profile
+        self.account_id = self.registry("ir.model.data").get_object_reference(self.cr, self.uid, "account", "a_recv")[1]
+        self.journal_id = self.registry("ir.model.data").get_object_reference(self.cr, self. uid, "account", "bank_journal")[1]
+        self.profile_id = self.profile_obj.create(self.cr, self.uid, {
+            "name": "TEST",
+            "commission_account_id": self.account_id,
+            "journal_id": self.journal_id,
+            "rule_ids": [(6, 0, [self.completion_rule_id])]})
+        # Create the completion rule
+
+        # Create a bank statement
+        self.statement_id = self.account_bank_statement_obj.create(self.cr, self.uid, {
+            "balance_end_real": 0.0,
+            "balance_start": 0.0,
+            "date": time.strftime('%Y-%m-%d'),
+            "journal_id": self.journal_id,
+            "profile_id": self.profile_id
+
+        })
+
+        # Create bank a statement line
+        self.statement_line_id = self.account_bank_statement_line_obj.create(self.cr, self.uid, {
+            'amount': 1000.0,
+            'name': 'EXT001',
+            'ref': 'My ref',
+            'statement_id': self.statement_id,
+            'partner_acc_number': ACC_NUMBER
+        })
+
+        # Add a bank account number to the partner
+        res_bank_obj = self.registry('res.partner.bank')
+        res_bank_obj.create(self.cr, self.uid, {
+                            "state": "bank",
+                            "company_id": self.company_a.id,
+                            "partner_id": self.partner_id,
+                            "acc_number": ACC_NUMBER,
+                            "footer": True,
+                            "bank_name": "Reserve"
+                            })
+
+    def test_00(self):
+        """Test complete partner_id from bank account number
+        
+        Test the automatic completion of the partner_id based on the account number associated to the
+        statement line
+        """
+        self.prepare()
+        statement_line = self.account_bank_statement_line_obj.browse(self.cr, self.uid, self.statement_line_id)
+        # before import, the
+        self.assertFalse(statement_line.partner_id, "Partner_id must be blank before completion")
+        statement_obj = self.account_bank_statement_obj.browse(self.cr, self.uid, self.statement_id)
+        statement_obj.button_auto_completion()
+        statement_line = self.account_bank_statement_line_obj.browse(self.cr, self.uid, self.statement_line_id)
+        self.assertEquals(self.partner_id, statement_line.partner_id['id'], "Missing expected partner id after completion")

account_statement_base_completion/statement.py

@@ -22,6 +22,9 @@
 import traceback
 import sys
 import logging
+import simplejson
+
+import psycopg2
 
 from collections import defaultdict
 import re
@@ -433,6 +436,80 @@ class AccountStatementLine(orm.Model):
             return vals
         return {}
 
+    def _get_available_columns(self, statement_store, include_serializable=False):
+        """Return writeable by SQL columns"""
+        statement_line_obj = self.pool['account.bank.statement.line']
+        model_cols = statement_line_obj._columns
+        avail = [k for k, col in model_cols.iteritems() if not hasattr(col, '_fnct')]
+        keys = [k for k in statement_store[0].keys() if k in avail]
+        # add sparse fields..
+        if include_serializable:
+            for k, col in model_cols.iteritems():
+                if  k in statement_store[0].keys() and \
+                      isinstance(col, fields.sparse) and \
+                      col.serialization_field not in keys and \
+                      col._type == 'char':
+                    keys.append(col.serialization_field)
+        keys.sort()
+        return keys
+
+    def _serialize_sparse_fields(self, cols, statement_store):
+        """ Serialize sparse fields values in the target serialized field
+        Return a copy of statement_store
+        """
+        statement_line_obj = self.pool['account.bank.statement.line']
+        model_cols = statement_line_obj._columns
+        sparse_fields = dict([(k , col) for k, col in model_cols.iteritems() if isinstance(col, fields.sparse) and col._type == 'char'])
+        values = []
+        for statement in statement_store:
+            to_json_k = set()
+            st_copy = statement.copy()
+            for k, col in sparse_fields.iteritems():
+                if k in st_copy:
+                    to_json_k.add(col.serialization_field)
+                    serialized = st_copy.setdefault(col.serialization_field, {})
+                    serialized[k] = st_copy[k]
+            for k in to_json_k:
+                st_copy[k] =  simplejson.dumps(st_copy[k])
+            values.append(st_copy)
+        return values
+        
+
+    def _insert_lines(self, cr, uid, statement_store, context=None):
+        """ Do raw insert into database because ORM is awfully slow
+            when doing batch write. It is a shame that batch function
+            does not exist"""
+        statement_line_obj = self.pool['account.bank.statement.line']
+        statement_line_obj.check_access_rule(cr, uid, [], 'create')
+        statement_line_obj.check_access_rights(cr, uid, 'create', raise_exception=True)
+        cols = self._get_available_columns(statement_store, include_serializable=True)
+        tmp_vals = (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols]))
+        sql = "INSERT INTO account_bank_statement_line (%s) VALUES (%s);" % tmp_vals
+        try:
+            cr.executemany(sql, tuple(self._serialize_sparse_fields(cols, statement_store)))
+        except psycopg2.Error as sql_err:
+            cr.rollback()
+            raise osv.except_osv(_("ORM bypass error"),
+                                 sql_err.pgerror)
+
+    def _update_line(self, cr, uid, vals, context=None):
+        """ Do raw update into database because ORM is awfully slow
+            when cheking security.
+        TODO / WARM: sparse fields are skipped by the method. IOW, if your
+        completion rule update an sparse field, the updated value will never
+        be stored in the database. It would be safer to call the update method 
+        from the ORM for records updating this kind of fields.
+        """
+        cols = self._get_available_columns([vals])
+        tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in cols]))
+        sql = "UPDATE account_bank_statement_line SET %s where id = %%(id)s;" % tmp_vals
+        try:
+            cr.execute(sql, vals)
+        except psycopg2.Error as sql_err:
+            cr.rollback()
+            raise osv.except_osv(_("ORM bypass error"),
+                                 sql_err.pgerror)
+
 
 class AccountBankSatement(orm.Model):
     """

account_statement_base_import/statement.py

@@ -21,8 +21,6 @@
 import sys
 import traceback
 
-import psycopg2
-
 from openerp.tools.translate import _
 import datetime
 from openerp.osv.orm import Model
@@ -216,50 +214,3 @@ class AccountStatementProfil(Model):
             raise osv.except_osv(_("Statement import error"),
                                  _("The statement cannot be created: %s") % st)
         return statement_id
-
-
-class AccountStatementLine(Model):
-    """
-    Add sparse field on the statement line to allow to store all the
-    bank infos that are given by an office.
-    """
-    _inherit = "account.bank.statement.line"
-
-    def _get_available_columns(self, statement_store):
-        """Return writeable by SQL columns"""
-        statement_line_obj = self.pool['account.bank.statement.line']
-        model_cols = statement_line_obj._columns
-        avail = [k for k, col in model_cols.iteritems() if not hasattr(col, '_fnct')]
-        keys = [k for k in statement_store[0].keys() if k in avail]
-        keys.sort()
-        return keys
-
-    def _insert_lines(self, cr, uid, statement_store, context=None):
-        """ Do raw insert into database because ORM is awfully slow
-            when doing batch write. It is a shame that batch function
-            does not exist"""
-        statement_line_obj = self.pool['account.bank.statement.line']
-        statement_line_obj.check_access_rule(cr, uid, [], 'create')
-        statement_line_obj.check_access_rights(cr, uid, 'create', raise_exception=True)
-        cols = self._get_available_columns(statement_store)
-        tmp_vals = (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols]))
-        sql = "INSERT INTO account_bank_statement_line (%s) VALUES (%s);" % tmp_vals
-        try:
-            cr.executemany(sql, tuple(statement_store))
-        except psycopg2.Error as sql_err:
-            cr.rollback()
-            raise osv.except_osv(_("ORM bypass error"),
-                                 sql_err.pgerror)
-
-    def _update_line(self, cr, uid, vals, context=None):
-        """ Do raw update into database because ORM is awfully slow
-            when cheking security."""
-        cols = self._get_available_columns([vals])
-        tmp_vals = (', '.join(['%s = %%(%s)s' % (i, i) for i in cols]))
-        sql = "UPDATE account_bank_statement_line SET %s where id = %%(id)s;" % tmp_vals
-        try:
-            cr.execute(sql, vals)
-        except psycopg2.Error as sql_err:
-            cr.rollback()
-            raise osv.except_osv(_("ORM bypass error"),
-                                 sql_err.pgerror)

account_statement_ext/statement.py

@@ -571,7 +571,11 @@ class AccountBankSatementLine(Model):
         if context is None:
             context = {}
         date = context.get('date')
-        periods = self.pool.get('account.period').find(cr, uid, dt=date)
+        try:
+            periods = self.pool.get('account.period').find(cr, uid, dt=date)
+        except osv.except_osv:
+            # if no period defined, we are certainly at installation time
+            return False
         return periods and periods[0] or False
 
     def _get_default_account(self, cr, uid, context=None):

account_statement_transactionid_completion/statement.py

@@ -90,5 +90,5 @@ class AccountStatementLine(Model):
             string='Transaction ID',
             size=128,
             serialization_field='additionnal_bank_fields',
-            help="Transction id from the financial institute"),
+            help="Transaction id from the financial institute"),
     }