audrius/account-reconcile
1
0
Fork 1
mirror of https://github.com/OCA/account-reconcile.git synced 2025-01-20 12:27:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

[MRG] Improve statement import global usability by retuning usable error message while parsing files.

Browse Source 
Allows empty value for float in parsed CSV.

Added note about the additional dependency on python-xlrd in the description.
This commit is contained in:
unknown
2013-03-18 12:04:25 +01:00
committed by Alexandre Fayolle
parent 1eb3d652ec 74c6caaf6b
commit 37ee2c2fb8
4 changed files with 80 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
account_statement_base_import/__openerp__.py
View File

@@ -38,6 +38,8 @@
a standard .csv or .xls file (you'll find it in the 'data' folder). It respects the profile
(provided by the accouhnt_statement_ext module) to pass the entries. That means,
you'll have to choose a file format for each profile.
In order to achieve this it uses the `xlrd` Python module which you will need to install
separately in your environment.
This module can handle a commission taken by the payment office and has the following format:
@@ -56,12 +58,10 @@
""",
'website': 'http://www.camptocamp.com',
'init_xml': [],
'update_xml': [
'data': [
"wizard/import_statement_view.xml",
"statement_view.xml",
],
'demo_xml': [],
'test': [],
'installable': True,
'images': [],

92
account_statement_base_import/parser/file_parser.py
View File

@@ -17,8 +17,8 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.tools.translate import _
from openerp.osv.osv import except_osv
import tempfile
import datetime
from parser import BankStatementImportParser
@@ -34,12 +34,13 @@ class FileParser(BankStatementImportParser):
Generic abstract class for defining parser for .csv or .xls file format.
"""
def __init__(self, parse_name, keys_to_validate=[], ftype='csv', convertion_dict=None, header=None, *args, **kwargs):
def __init__(self, parse_name, keys_to_validate=None, ftype='csv', conversion_dict=None,
header=None, *args, **kwargs):
"""
:param char: parse_name : The name of the parser
:param list: keys_to_validate : contain the key that need to be present in the file
:param char ftype: extension of the file (could be csv or xls)
:param: convertion_dict : keys and type to convert of every column in the file like
:param: conversion_dict : keys and type to convert of every column in the file like
{
'ref': unicode,
'label': unicode,
@@ -54,9 +55,10 @@ class FileParser(BankStatementImportParser):
if ftype in ('csv', 'xls'):
self.ftype = ftype
else:
raise Exception(_('Invalide file type %s. please use csv or xls') % (ftype))
self.keys_to_validate = keys_to_validate
self.convertion_dict = convertion_dict
raise except_osv(_('User Error'),
_('Invalid file type %s. Please use csv or xls') % ftype)
self.keys_to_validate = keys_to_validate if keys_to_validate is not None else []
self.conversion_dict = conversion_dict
self.fieldnames = header
self._datemode = 0 # used only for xls documents,
# 0 means Windows mode (1900 based dates).
@@ -99,7 +101,8 @@ class FileParser(BankStatementImportParser):
parsed_cols = self.result_row_list[0].keys()
for col in self.keys_to_validate:
if col not in parsed_cols:
raise Exception(_('Column %s not present in file') % (col))
raise except_osv(_('Invalid data'),
_('Column %s not present in file') % col)
return True
def _post(self, *args, **kwargs):
@@ -128,17 +131,13 @@ class FileParser(BankStatementImportParser):
wb_file.write(self.filebuffer)
# We ensure that cursor is at beginig of file
wb_file.seek(0)
wb = xlrd.open_workbook(wb_file.name)
self._datemode = wb.datemode
sheet = wb.sheet_by_index(0)
header = sheet.row_values(0)
res = []
for rownum in range(1, sheet.nrows):
res.append(dict(zip(header, sheet.row_values(rownum))))
try:
wb_file.close()
except Exception, e:
pass # file is already closed
with xlrd.open_workbook(wb_file.name) as wb:
self._datemode = wb.datemode
sheet = wb.sheet_by_index(0)
header = sheet.row_values(0)
res = []
for rownum in range(1, sheet.nrows):
res.append(dict(zip(header, sheet.row_values(rownum))))
return res
def _from_csv(self, result_set, conversion_rules):
@@ -149,11 +148,30 @@ class FileParser(BankStatementImportParser):
for line in result_set:
for rule in conversion_rules:
if conversion_rules[rule] == datetime.datetime:
date_string = line[rule].split(' ')[0]
line[rule] = datetime.datetime.strptime(date_string,
'%Y-%m-%d')
try:
date_string = line[rule].split(' ')[0]
line[rule] = datetime.datetime.strptime(date_string,
'%Y-%m-%d')
except ValueError as err:
raise except_osv(_("Date format is not valid."),
_(" It should be YYYY-MM-DD for column: %s"
" value: %s \n \n"
" \n Please check the line with ref: %s"
" \n \n Detail: %s") % (rule,
line.get(rule, _('Missing')),
line.get('ref', line),
repr(err)))
else:
line[rule] = conversion_rules[rule](line[rule])
try:
line[rule] = conversion_rules[rule](line[rule])
except Exception as err:
raise except_osv(_('Invalid data'),
_("Value %s of column %s is not valid."
"\n Please check the line with ref %s:"
"\n \n Detail: %s") % (line.get(rule, _('Missing')),
rule,
line.get('ref', line),
repr(err)))
return result_set
def _from_xls(self, result_set, conversion_rules):
@@ -164,17 +182,37 @@ class FileParser(BankStatementImportParser):
for line in result_set:
for rule in conversion_rules:
if conversion_rules[rule] == datetime.datetime:
t_tuple = xlrd.xldate_as_tuple(line[rule], self._datemode)
line[rule] = datetime.datetime(*t_tuple)
try:
t_tuple = xlrd.xldate_as_tuple(line[rule], self._datemode)
line[rule] = datetime.datetime(*t_tuple)
except Exception as err:
raise except_osv(_("Date format is not valid"),
_("Please modify the cell formatting to date format"
" for column: %s"
" value: %s"
"\n Please check the line with ref: %s"
"\n \n Detail: %s") % (rule,
line.get(rule, _('Missing')),
line.get('ref', line),
repr(err)))
else:
line[rule] = conversion_rules[rule](line[rule])
try:
line[rule] = conversion_rules[rule](line[rule])
except Exception as err:
raise except_osv(_('Invalid data'),
_("Value %s of column %s is not valid."
"\n Please check the line with ref %s:"
"\n \n Detail: %s") % (line.get(rule, _('Missing')),
rule,
line.get('ref', line),
repr(err)))
return result_set
def _cast_rows(self, *args, **kwargs):
"""
Convert the self.result_row_list using the self.convertion_dict providen.
Convert the self.result_row_list using the self.conversion_dict providen.
We call here _from_xls or _from_csv depending on the self.ftype variable.
"""
func = getattr(self, '_from_%s' % self.ftype)
res = func(self.result_row_list, self.convertion_dict)
res = func(self.result_row_list, self.conversion_dict)
return res

13
account_statement_base_import/parser/generic_file_parser.py
View File

@@ -29,6 +29,11 @@ try:
except:
raise Exception(_('Please install python lib xlrd'))
def float_or_zero(val):
""" Conversion function used to manage
empty string into float usecase"""
return float(val) if val else 0.0
class GenericFileParser(FileParser):
"""
@@ -38,16 +43,16 @@ class GenericFileParser(FileParser):
"""
def __init__(self, parse_name, ftype='csv'):
convertion_dict = {
conversion_dict = {
'ref': unicode,
'label': unicode,
'date': datetime.datetime,
'amount': float,
'commission_amount': float
'amount': float_or_zero,
'commission_amount': float_or_zero
}
# Order of cols does not matter but first row of the file has to be header
keys_to_validate = ['ref', 'label', 'date', 'amount', 'commission_amount']
super(GenericFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate, ftype=ftype, convertion_dict=convertion_dict)
super(GenericFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate, ftype=ftype, conversion_dict=conversion_dict)
@classmethod
def parser_for(cls, parser_name):

5
account_statement_transactionid_import/parser/transactionid_file_parser.py
View File

@@ -30,7 +30,7 @@ class TransactionIDFileParser(FileParser):
"""
def __init__(self, parse_name, ftype='csv'):
convertion_dict = {
conversion_dict = {
'transaction_id': unicode,
'label': unicode,
'date': datetime.datetime,
@@ -39,7 +39,8 @@ class TransactionIDFileParser(FileParser):
}
# Order of cols does not matter but first row of the file has to be header
keys_to_validate = ['transaction_id', 'label', 'date', 'amount', 'commission_amount']
super(TransactionIDFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate, ftype=ftype, convertion_dict=convertion_dict)
super(TransactionIDFileParser, self).__init__(parse_name, keys_to_validate=keys_to_validate,
ftype=ftype, conversion_dict=conversion_dict)
@classmethod
def parser_for(cls, parser_name):