From c6ccbac8d444b0d78530f41feebe598cf19c6e84 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Sep 2013 16:15:45 +0200 Subject: [PATCH 01/27] [ADD] base of async_move_line_importer --- async_move_line_importer/__init__.py | 21 +++++ async_move_line_importer/__openerp__.py | 39 ++++++++ async_move_line_importer/model/__init__.py | 22 +++++ async_move_line_importer/model/account.py | 55 +++++++++++ .../model/move_line_importer.py | 93 +++++++++++++++++++ .../security/ir.model.access.csv | 2 + .../security/multi_company.xml | 11 +++ .../view/move_line_importer_view.xml | 75 +++++++++++++++ 8 files changed, 318 insertions(+) create mode 100644 async_move_line_importer/__init__.py create mode 100644 async_move_line_importer/__openerp__.py create mode 100644 async_move_line_importer/model/__init__.py create mode 100644 async_move_line_importer/model/account.py create mode 100644 async_move_line_importer/model/move_line_importer.py create mode 100644 async_move_line_importer/security/ir.model.access.csv create mode 100644 async_move_line_importer/security/multi_company.xml create mode 100644 async_move_line_importer/view/move_line_importer_view.xml diff --git a/async_move_line_importer/__init__.py b/async_move_line_importer/__init__.py new file mode 100644 index 000000000..85040e93f --- /dev/null +++ b/async_move_line_importer/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Nicolas Bessi +# Copyright 2013 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +from . import model diff --git a/async_move_line_importer/__openerp__.py b/async_move_line_importer/__openerp__.py new file mode 100644 index 000000000..f0f340693 --- /dev/null +++ b/async_move_line_importer/__openerp__.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Nicolas Bessi +# Copyright 2013 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +{'name': 'Asynchron move line importer', + 'version': '0.1.1', + 'author': 'Camptocamp', + 'maintainer': 'Camptocamp', + 'category': 'categ', + 'complexity': 'normal', + 'depends': ['base', 'account'], + 'description': """Allows to move/moveline asynchronously""", + 'website': 'http://www.camptocamp.com', + 'data': ['view/move_line_importer_view.xml', + 'security/ir.model.access.csv', + 'security/multi_company.xml'], + 'demo': [], + 'test': [], + 'installable': True, + 'auto_install': False, + 'license': 'AGPL-3', + 'application': False, + } diff --git a/async_move_line_importer/model/__init__.py b/async_move_line_importer/model/__init__.py new file mode 100644 index 000000000..5db0b24e8 --- /dev/null +++ b/async_move_line_importer/model/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Nicolas Bessi +# Copyright 2013 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +from . import move_line_importer +#from . import account diff --git a/async_move_line_importer/model/account.py b/async_move_line_importer/model/account.py new file mode 100644 index 000000000..b912c5b75 --- /dev/null +++ b/async_move_line_importer/model/account.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Nicolas Bessi +# Copyright 2013 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +from openerp.osv import orm + + +class account_move(orm.Model): + """redefine account move create to bypass orm + if async_bypass_create is True in context""" + + _inherit = "account.move" + + def _bypass_create(self, cr, uid, vals, context=None): + pass + + def create(self, cr, uid, vals, context=None): + if context is None: + context = {} + if context.get('async_bypass_create'): + return self._bypass_create(cr, uid, vals, context=context) + return super(account_move, self).create(cr, uid, vals, context=context) + + +class account_move_line(orm.Model): + """redefine account move line create to bypass orm + if async_bypass_create is True in context""" + + _inherit = "account.move.line" + + def create(self, cr, uid, vals, context=None): + if context is None: + context = {} + if context.get('async_bypass_create'): + return self._bypass_create(cr, uid, vals, context=context) + return super(account_move_line, self).create(cr, uid, vals, context=context) + + def _bypass_create(self, cr, uid, vals, context=None): + pass diff --git a/async_move_line_importer/model/move_line_importer.py b/async_move_line_importer/model/move_line_importer.py new file mode 100644 index 000000000..8ecb00cda --- /dev/null +++ b/async_move_line_importer/model/move_line_importer.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +############################################################################## +# +# Author: Nicolas Bessi +# Copyright 2013 Camptocamp SA +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +############################################################################## +import base64 +import csv +import tempfile +from openerp.osv import orm, fields + + +class move_line_importer(orm.Model): + """Move line importer""" + + _name = "move.line.importer" + _inherit = ['mail.thread'] + # _track = { + # 'state': { + # 'import.success': lambda self, cr, uid, obj, ctx=None: obj['state'] == 'done', + # 'import.error': lambda self, cr, uid, obj, ctx=None: obj['state'] == 'error', + # }, + # } + + _columns = {'name': fields.datetime('Name', + required=True, + readonly=True), + 'state': fields.selection([('draft', 'New'), + ('running', 'Running'), + ('done', 'Success'), + ('error', 'Error')], + readonly=True, + string='Status'), + + 'report': fields.text('Report', + readonly=True), + 'file': fields.binary('File', + required=True), + 'delimiter': fields.selection([(',', ','), (';', ';'), ('|', '|')], + string="CSV delimiter", + required=True), + 'company_id': fields.many2one('res.company', + 'Company') + } + + _defaults = {'delimiter': ','} + + def _get_current_company(self, cr, uid, context=None, model="move.line.importer"): + return self.pool.get('res.company')._company_default_get(cr, uid, model, + context=context) + + _defaults = {'state': 'draft', + 'name': fields.datetime.now(), + 'company_id': _get_current_company} + + def _parse_csv(self, cr, uid, imp_id): + with tempfile.TemporaryFile() as src: + imp = self.read(cr, uid, imp_id, ['file', 'delimiter']) + content = imp['file'], + delimiter = imp['delimiter'] + src.write(content) + with tempfile.TemporaryFile() as decoded: + src.seek(0) + base64.decode(src, decoded) + decoded.seek(0) + return self._prepare_csv_data(decoded, delimiter) + + def _prepare_data(self, csv_file, delimiter=","): + data = csv.reader(csv_file, delimiter=str(delimiter)) + head = data.next() + # generator does not work in load + values = [x for x in data] + return (head, values) + + def import_file(self, cr, uid, imp_id, context=None): + if isinstance(imp_id, list): + imp_id = imp_id[0] + import pdb; pdb.set_trace() + head, data = self._parse_csv(cr, uid, imp_id) diff --git a/async_move_line_importer/security/ir.model.access.csv b/async_move_line_importer/security/ir.model.access.csv new file mode 100644 index 000000000..21f33c51c --- /dev/null +++ b/async_move_line_importer/security/ir.model.access.csv @@ -0,0 +1,2 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_move_line_importer_manager,access_move_line_importer_manager,model_move_line_importer,account.group_account_manager,1,1,1,0 diff --git a/async_move_line_importer/security/multi_company.xml b/async_move_line_importer/security/multi_company.xml new file mode 100644 index 000000000..c359d755e --- /dev/null +++ b/async_move_line_importer/security/multi_company.xml @@ -0,0 +1,11 @@ + + + + + Move line importer company rule + + + ['|',('company_id','=',False),('company_id','child_of',[user.company_id.id])] + + + diff --git a/async_move_line_importer/view/move_line_importer_view.xml b/async_move_line_importer/view/move_line_importer_view.xml new file mode 100644 index 000000000..4660f051e --- /dev/null +++ b/async_move_line_importer/view/move_line_importer_view.xml @@ -0,0 +1,75 @@ + + + + + move line importer form + move.line.importer + +
+
+
+ + + + +
+ +
+ + + + + + + + + +
+ + + + move line importer tree + move.line.importer + + + + + + + + + + + Import Move lines + ir.actions.act_window + move.line.importer + + form + tree,form + + + + + + From 8dcaf8bc47b67f1c58d5f1ef630310eb1c4de356 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 17 Sep 2013 10:32:09 +0200 Subject: [PATCH 02/27] [FIX] fix file parser --- async_move_line_importer/model/move_line_importer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/async_move_line_importer/model/move_line_importer.py b/async_move_line_importer/model/move_line_importer.py index 8ecb00cda..2d7d78128 100644 --- a/async_move_line_importer/model/move_line_importer.py +++ b/async_move_line_importer/model/move_line_importer.py @@ -70,7 +70,7 @@ class move_line_importer(orm.Model): def _parse_csv(self, cr, uid, imp_id): with tempfile.TemporaryFile() as src: imp = self.read(cr, uid, imp_id, ['file', 'delimiter']) - content = imp['file'], + content = imp['file'] delimiter = imp['delimiter'] src.write(content) with tempfile.TemporaryFile() as decoded: @@ -79,7 +79,7 @@ class move_line_importer(orm.Model): decoded.seek(0) return self._prepare_csv_data(decoded, delimiter) - def _prepare_data(self, csv_file, delimiter=","): + def _prepare_csv_data(self, csv_file, delimiter=","): data = csv.reader(csv_file, delimiter=str(delimiter)) head = data.next() # generator does not work in load @@ -89,5 +89,4 @@ class move_line_importer(orm.Model): def import_file(self, cr, uid, imp_id, context=None): if isinstance(imp_id, list): imp_id = imp_id[0] - import pdb; pdb.set_trace() head, data = self._parse_csv(cr, uid, imp_id) From bcb18469ac8d8170116ac531586c6d3ee1075e0a Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 18 Sep 2013 13:43:08 +0200 Subject: [PATCH 03/27] [ADD] orm bypass for account move and account move line --- async_move_line_importer/model/account.py | 46 ++++++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/async_move_line_importer/model/account.py b/async_move_line_importer/model/account.py index b912c5b75..fd920afea 100644 --- a/async_move_line_importer/model/account.py +++ b/async_move_line_importer/model/account.py @@ -21,14 +21,53 @@ from openerp.osv import orm +def _format_inserts_values(vals): + cols = vals.keys() + if 'line_id' in cols: + cols.remove('line_id') + return (', '.join(cols), ', '.join(['%%(%s)s' % i for i in cols])) + + class account_move(orm.Model): """redefine account move create to bypass orm if async_bypass_create is True in context""" _inherit = "account.move" + def _prepare_line(self, cr, uid, move_id, line, vals, context=None): + if isinstance(line, tuple): + line = line[2] + line['journal_id'] = vals.get('journal_id') + line['date'] = vals.get('date') + line['period_id'] = vals.get('period_id') + line['company_id'] = vals.get('company_id') + line['state'] = vals['state'] + line['move_id'] = move_id + if line['debit'] and line['credit']: + raise ValueError('debit and credit set on same line') + if not line.get('analytic_account_id'): + line['analytic_account_id'] = None + for key in line: + if line[key] is False: + line[key] = None + return line + def _bypass_create(self, cr, uid, vals, context=None): - pass + mvl_obj = self.pool['account.move.line'] + vals['company_id'] = context.get('company_id', False) + vals['state'] = 'draft' + if not vals.get('name'): + vals['name'] = "/" + sql = u"Insert INTO account_move (%s) VALUES (%s) RETURNING id" + sql = sql % _format_inserts_values(vals) + cr.execute(sql, vals) + created_id = cr.fetchone()[0] + if vals.get('line_id'): + for line in vals['line_id']: + l_vals = self._prepare_line(cr, uid, created_id, + line, vals, context=context) + mvl_obj.create(cr, uid, l_vals, context=context) + return created_id def create(self, cr, uid, vals, context=None): if context is None: @@ -52,4 +91,7 @@ class account_move_line(orm.Model): return super(account_move_line, self).create(cr, uid, vals, context=context) def _bypass_create(self, cr, uid, vals, context=None): - pass + sql = u"Insert INTO account_move_line (%s) VALUES (%s) RETURNING id" + sql = sql % _format_inserts_values(vals) + cr.execute(sql, vals) + return cr.fetchone()[0] From eb67314c27afd143dc5f2f68e3f7752935e29454 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 18 Sep 2013 13:43:40 +0200 Subject: [PATCH 04/27] [ADD] mail message subtype --- async_move_line_importer/data.xml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 async_move_line_importer/data.xml diff --git a/async_move_line_importer/data.xml b/async_move_line_importer/data.xml new file mode 100644 index 000000000..0828c9597 --- /dev/null +++ b/async_move_line_importer/data.xml @@ -0,0 +1,17 @@ + + + + + Import successfully finished + move.line.importer + + Import successfully finished + + + Import failed + move.line.importer + + Import failed + + + From ab4e80d4bc3e677bb66cb7c147fafd0e3cd8435e Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 18 Sep 2013 13:44:16 +0200 Subject: [PATCH 05/27] [ADD] import of account file --- async_move_line_importer/model/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/async_move_line_importer/model/__init__.py b/async_move_line_importer/model/__init__.py index 5db0b24e8..151ad63ce 100644 --- a/async_move_line_importer/model/__init__.py +++ b/async_move_line_importer/model/__init__.py @@ -19,4 +19,4 @@ # ############################################################################## from . import move_line_importer -#from . import account +from . import account From b08284e1c7939fdba2ae93be7a58f020d190d46c Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 18 Sep 2013 13:45:07 +0200 Subject: [PATCH 06/27] [IMP] asynchrone update, management of error, bypass orm option --- .../model/move_line_importer.py | 132 ++++++++++++++++-- 1 file changed, 122 insertions(+), 10 deletions(-) diff --git a/async_move_line_importer/model/move_line_importer.py b/async_move_line_importer/model/move_line_importer.py index 2d7d78128..eff2433b2 100644 --- a/async_move_line_importer/model/move_line_importer.py +++ b/async_move_line_importer/model/move_line_importer.py @@ -19,9 +19,15 @@ # ############################################################################## import base64 +import threading import csv import tempfile + +import openerp.pooler as pooler from openerp.osv import orm, fields +from openerp.tools.translate import _ + +USE_THREAD = False class move_line_importer(orm.Model): @@ -29,12 +35,26 @@ class move_line_importer(orm.Model): _name = "move.line.importer" _inherit = ['mail.thread'] - # _track = { - # 'state': { - # 'import.success': lambda self, cr, uid, obj, ctx=None: obj['state'] == 'done', - # 'import.error': lambda self, cr, uid, obj, ctx=None: obj['state'] == 'error', - # }, - # } + + def copy(self, cr, uid, id, default=None, context=None): + if default is None: + default = {} + default.update(state='draft', report=False) + return super(move_line_importer, self).copy(cr, uid, id, default=default, + context=context) + + def track_success(sef, cr, uid, obj, context=None): + return obj['state'] == 'done' + + def track_error(sef, cr, uid, obj, context=None): + return obj['state'] == 'error' + + _track = { + 'state': { + 'async_move_line_importer.mvl_imported': track_success, + 'async_move_line_importer.mvl_error': track_error, + }, + } _columns = {'name': fields.datetime('Name', required=True, @@ -54,10 +74,21 @@ class move_line_importer(orm.Model): string="CSV delimiter", required=True), 'company_id': fields.many2one('res.company', - 'Company') + 'Company'), + 'bypass_orm': fields.boolean('Fast import (use with caution)', + help="When enabled import will be faster but" + " it will not use orm and may" + " not support all CSV canvas. \n" + "Entry posted option will be skipped. \n" + "AA lines will only be crated when" + " moves are posted. \n" + "Tax lines computation will be skipped. \n" + "This option should be used with caution" + " and in conjonction with provided canvas."), } - _defaults = {'delimiter': ','} + _defaults = {'delimiter': ',', + 'bypass_orm': False} def _get_current_company(self, cr, uid, context=None, model="move.line.importer"): return self.pool.get('res.company')._company_default_get(cr, uid, model, @@ -68,6 +99,9 @@ class move_line_importer(orm.Model): 'company_id': _get_current_company} def _parse_csv(self, cr, uid, imp_id): + """Parse stored CSV file in order to be usable by load method. + Manage base 64 decoding. + It will return head (list of first row) and data list of list""" with tempfile.TemporaryFile() as src: imp = self.read(cr, uid, imp_id, ['file', 'delimiter']) content = imp['file'] @@ -80,13 +114,91 @@ class move_line_importer(orm.Model): return self._prepare_csv_data(decoded, delimiter) def _prepare_csv_data(self, csv_file, delimiter=","): + """Parse and decoded CSV file and return head list + and data list""" data = csv.reader(csv_file, delimiter=str(delimiter)) head = data.next() - # generator does not work in load - values = [x for x in data] + head = [x.replace(' ', '') for x in head] + # Generator does not work with orm.BaseModel.load + values = [tuple(x) for x in data if x] return (head, values) + def format_messages(self, messages): + res = [] + for msg in messages: + rows = msg.get('rows', {}) + res.append(_("%s. -- Field: %s -- rows %s to %s") % (msg.get('message', 'N/A'), + msg.get('field', 'N/A'), + rows.get('from', 'N/A'), + rows.get('to', 'N/A'))) + return "\n \n".join(res) + + def _manage_load_results(self, cr, uid, imp_id, result, context=None): + if not result['messages']: + msg = _("%s lines imported" % len(result['ids'] or [])) + self.write(cr, uid, [imp_id], {'state': 'done', + 'report': msg}) + else: + cr.rollback() + msg = self.format_messages(result['messages']) + self.write(cr, uid, [imp_id], {'state': 'error', + 'report': msg}) + cr.commit() + return imp_id + + def _load_data(self, cr, uid, imp_id, head, data, mode, context=None): + """Function that does the load management, exception and load report""" + valid_modes = ('threaded', 'direct') + if mode not in valid_modes: + raise ValueError('%s is not in valid mode %s ' % (mode, valid_modes)) + try: + res = self.pool['account.move'].load(cr, uid, head, data, context=context) + self._manage_load_results(cr, uid, imp_id, res, context=context) + except Exception as exc: + cr.rollback() + self.write(cr, uid, [imp_id], {'state': 'error'}) + if mode != "threaded": + raise + msg = _("Unexpected exception not related to CSV file.\n %s" % repr(exc)) + self.write(cr, uid, [imp_id], {'report': msg}) + + finally: + if mode == 'threaded': + cr.commit() + cr.close() + return imp_id + + def _allows_thread(self, imp_id): + """Check if there is a async import of this file running""" + for th in threading.enumerate(): + if th.getName() == 'async_move_line_import_%s' % imp_id: + raise orm.except_orm(_('An import of this file is already running'), + _('Please try latter')) + def import_file(self, cr, uid, imp_id, context=None): if isinstance(imp_id, list): imp_id = imp_id[0] + if context is None: + context = {} + current = self.read(cr, uid, imp_id, ['bypass_orm', 'company_id'], + load='_classic_write') + context['company_id'] = current['company_id'] + bypass_orm = current['bypass_orm'] + if bypass_orm: + # Tells create funtion to bypass orm + context['async_bypass_create'] = True head, data = self._parse_csv(cr, uid, imp_id) + self.write(cr, uid, [imp_id], {'state': 'running'}) + if USE_THREAD: + self._allows_thread(imp_id) + db_name = cr.dbname + local_cr = pooler.get_db(db_name).cursor() + thread = threading.Thread(target=self._load_data, + name='async_move_line_import_%s' % imp_id, + args=(local_cr, uid, imp_id, head, data, + 'threaded', context.copy())) + thread.start() + else: + self._load_data(cr, uid, imp_id, head, data, 'direct', + context=context) + return {} From fe7f136a3002c17da8444fa9efe7e151348a046c Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 18 Sep 2013 13:45:44 +0200 Subject: [PATCH 07/27] [FIX] missing import of data.xml + cleanup of view --- async_move_line_importer/__openerp__.py | 3 ++- async_move_line_importer/view/move_line_importer_view.xml | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/async_move_line_importer/__openerp__.py b/async_move_line_importer/__openerp__.py index f0f340693..3ccdc3b9d 100644 --- a/async_move_line_importer/__openerp__.py +++ b/async_move_line_importer/__openerp__.py @@ -27,7 +27,8 @@ 'depends': ['base', 'account'], 'description': """Allows to move/moveline asynchronously""", 'website': 'http://www.camptocamp.com', - 'data': ['view/move_line_importer_view.xml', + 'data': ['data.xml', + 'view/move_line_importer_view.xml', 'security/ir.model.access.csv', 'security/multi_company.xml'], 'demo': [], diff --git a/async_move_line_importer/view/move_line_importer_view.xml b/async_move_line_importer/view/move_line_importer_view.xml index 4660f051e..a58114de2 100644 --- a/async_move_line_importer/view/move_line_importer_view.xml +++ b/async_move_line_importer/view/move_line_importer_view.xml @@ -8,6 +8,7 @@