audrius/account-reconcile
1
0
Fork 1
mirror of https://github.com/OCA/account-reconcile.git synced 2025-01-20 12:27:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

[IMP] account_statement_base_import: Fix PEP8

Browse Source
This commit is contained in:
Pedro M. Baeza
2014-08-04 18:42:00 +02:00
parent aebcbfdd5d
commit d9a943d2d3
7 changed files with 192 additions and 231 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
account_statement_base_import/parser/file_parser.py
View File

@@ -18,7 +18,7 @@
#
##############################################################################
from openerp.tools.translate import _
from openerp.osv.osv import except_osv
from openerp.osv.orm import except_orm
import tempfile
import datetime
from parser import BankStatementImportParser
@@ -36,26 +36,28 @@ def float_or_zero(val):
class FileParser(BankStatementImportParser):
"""
Generic abstract class for defining parser for .csv, .xls or .xlsx file format.
"""Generic abstract class for defining parser for .csv, .xls or .xlsx file
format.
"""
def __init__(self, parse_name, ftype='csv', extra_fields=None, header=None, **kwargs):
def __init__(self, parse_name, ftype='csv', extra_fields=None, header=None,
**kwargs):
"""
:param char: parse_name: The name of the parser
:param char: ftype: extension of the file (could be csv, xls or xlsx)
:param dict: extra_fields: extra fields to add to the conversion dict. In the format
{fieldname: fieldtype}
:param list: header : specify header fields if the csv file has no header
:param char: ftype: extension of the file (could be csv, xls or \
xlsx)
:param dict: extra_fields: extra fields to add to the conversion \
dict. In the format {fieldname: fieldtype}
:param list: header : specify header fields if the csv file has no \
header
"""
super(FileParser, self).__init__(parse_name, **kwargs)
if ftype in ('csv', 'xls', 'xlsx'):
self.ftype = ftype[0:3]
else:
raise except_osv(_('User Error'),
_('Invalid file type %s. Please use csv, xls or xlsx') % ftype)
raise except_orm(
_('User Error'),
_('Invalid file type %s. Please use csv, xls or xlsx') % ftype)
self.conversion_dict = {
'ref': unicode,
'label': unicode,
@@ -71,23 +73,17 @@ class FileParser(BankStatementImportParser):
# Set in _parse_xls, from the contents of the file
def _custom_format(self, *args, **kwargs):
"""
No other work on data are needed in this parser.
"""
"""No other work on data are needed in this parser."""
return True
def _pre(self, *args, **kwargs):
"""
No pre-treatment needed for this parser.
"""
"""No pre-treatment needed for this parser."""
return True
def _parse(self, *args, **kwargs):
"""
Launch the parsing through .csv, .xls or .xlsx depending on the
"""Launch the parsing through .csv, .xls or .xlsx depending on the
given ftype
"""
res = None
if self.ftype == 'csv':
res = self._parse_csv()
@@ -97,31 +93,27 @@ class FileParser(BankStatementImportParser):
return True
def _validate(self, *args, **kwargs):
"""
We check that all the key of the given file (means header) are present
in the validation key provided. Otherwise, we raise an Exception.
We skip the validation step if the file header is provided separately
(in the field: fieldnames).
"""We check that all the key of the given file (means header) are
present in the validation key provided. Otherwise, we raise an
Exception. We skip the validation step if the file header is provided
separately (in the field: fieldnames).
"""
if self.fieldnames is None:
parsed_cols = self.result_row_list[0].keys()
for col in self.keys_to_validate:
if col not in parsed_cols:
raise except_osv(_('Invalid data'),
raise except_orm(_('Invalid data'),
_('Column %s not present in file') % col)
return True
def _post(self, *args, **kwargs):
"""
Cast row type depending on the file format .csv or .xls after parsing the file.
"""
"""Cast row type depending on the file format .csv or .xls after
parsing the file."""
self.result_row_list = self._cast_rows(*args, **kwargs)
return True
def _parse_csv(self):
"""
:return: list of dict from csv file (line/rows)
"""
""":return: list of dict from csv file (line/rows)"""
csv_file = tempfile.NamedTemporaryFile()
csv_file.write(self.filebuffer)
csv_file.flush()
@@ -130,9 +122,7 @@ class FileParser(BankStatementImportParser):
return list(reader)
def _parse_xls(self):
"""
:return: dict of dict from xls/xlsx file (line/rows)
"""
""":return: dict of dict from xls/xlsx file (line/rows)"""
wb_file = tempfile.NamedTemporaryFile()
wb_file.write(self.filebuffer)
# We ensure that cursor is at beginig of file
@@ -147,8 +137,7 @@ class FileParser(BankStatementImportParser):
return res
def _from_csv(self, result_set, conversion_rules):
"""
Handle the converstion from the dict and handle date format from
"""Handle the converstion from the dict and handle date format from
an .csv file.
"""
for line in result_set:
@@ -159,72 +148,60 @@ class FileParser(BankStatementImportParser):
line[rule] = datetime.datetime.strptime(date_string,
'%Y-%m-%d')
except ValueError as err:
raise except_osv(_("Date format is not valid."),
_(" It should be YYYY-MM-DD for column: %s"
" value: %s \n \n"
" \n Please check the line with ref: %s"
" \n \n Detail: %s") % (rule,
line.get(
rule, _('Missing')),
line.get(
'ref', line),
repr(err)))
raise except_orm(
_("Date format is not valid."),
_(" It should be YYYY-MM-DD for column: %s"
" value: %s \n \n \n Please check the line with "
"ref: %s \n \n Detail: %s") %
(rule, line.get(rule, _('Missing')),
line.get('ref', line), repr(err)))
else:
try:
line[rule] = conversion_rules[rule](line[rule])
except Exception as err:
raise except_osv(_('Invalid data'),
_("Value %s of column %s is not valid."
"\n Please check the line with ref %s:"
"\n \n Detail: %s") % (line.get(rule, _('Missing')),
rule,
line.get(
'ref', line),
repr(err)))
raise except_orm(
_('Invalid data'),
_("Value %s of column %s is not valid.\n Please "
"check the line with ref %s:\n \n Detail: %s") %
(line.get(rule, _('Missing')), rule,
line.get('ref', line), repr(err)))
return result_set
def _from_xls(self, result_set, conversion_rules):
"""
Handle the converstion from the dict and handle date format from
"""Handle the converstion from the dict and handle date format from
an .csv, .xls or .xlsx file.
"""
for line in result_set:
for rule in conversion_rules:
if conversion_rules[rule] == datetime.datetime:
try:
t_tuple = xlrd.xldate_as_tuple(
line[rule], self._datemode)
t_tuple = xlrd.xldate_as_tuple(line[rule],
self._datemode)
line[rule] = datetime.datetime(*t_tuple)
except Exception as err:
raise except_osv(_("Date format is not valid"),
_("Please modify the cell formatting to date format"
" for column: %s"
" value: %s"
"\n Please check the line with ref: %s"
"\n \n Detail: %s") % (rule,
line.get(
rule, _('Missing')),
line.get(
'ref', line),
repr(err)))
raise except_orm(
_("Date format is not valid"),
_("Please modify the cell formatting to date format"
" for column: %s value: %s\n Please check the "
"line with ref: %s\n \n Detail: %s") %
(rule, line.get(rule, _('Missing')),
line.get('ref', line), repr(err)))
else:
try:
line[rule] = conversion_rules[rule](line[rule])
except Exception as err:
raise except_osv(_('Invalid data'),
_("Value %s of column %s is not valid."
"\n Please check the line with ref %s:"
"\n \n Detail: %s") % (line.get(rule, _('Missing')),
rule,
line.get(
'ref', line),
repr(err)))
raise except_orm(
_('Invalid data'),
_("Value %s of column %s is not valid.\n Please "
"check the line with ref %s:\n \n Detail: %s") %
(line.get(rule, _('Missing')), rule,
line.get('ref', line), repr(err)))
return result_set
def _cast_rows(self, *args, **kwargs):
"""
Convert the self.result_row_list using the self.conversion_dict providen.
We call here _from_xls or _from_csv depending on the self.ftype variable.
"""Convert the self.result_row_list using the self.conversion_dict
providen. We call here _from_xls or _from_csv depending on the
self.ftype variable.
"""
func = getattr(self, '_from_%s' % self.ftype)
res = func(self.result_row_list, self.conversion_dict)

14
account_statement_base_import/parser/generic_file_parser.py
View File

@@ -31,9 +31,7 @@ except:
class GenericFileParser(FileParser):
"""
Standard parser that use a define format in csv or xls to import into a
"""Standard parser that use a define format in csv or xls to import into a
bank statement. This is mostely an example of how to proceed to create a new
parser, but will also be useful as it allow to import a basic flat file.
"""
@@ -44,8 +42,7 @@ class GenericFileParser(FileParser):
@classmethod
def parser_for(cls, parser_name):
"""
Used by the new_bank_statement_parser class factory. Return true if
"""Used by the new_bank_statement_parser class factory. Return true if
the providen name is generic_csvxls_so
"""
return parser_name == 'generic_csvxls_so'
@@ -56,9 +53,10 @@ class GenericFileParser(FileParser):
method of statement line in order to record it. It is the responsibility
of every parser to give this dict of vals, so each one can implement his
own way of recording the lines.
:param: line: a dict of vals that represent a line of result_row_list
:return: dict of values to give to the create method of statement line,
it MUST contain at least:
:param: line: a dict of vals that represent a line of \
result_row_list
:return: dict of values to give to the create method of statement \
line, it MUST contain at least:
{
'name':value,
'date':value,

85
account_statement_base_import/parser/parser.py
View File

@@ -21,6 +21,7 @@
import base64
import csv
from datetime import datetime
from openerp.tools.translate import _
def UnicodeDictReader(utf8_data, **kwargs):
@@ -31,7 +32,8 @@ def UnicodeDictReader(utf8_data, **kwargs):
dialect = sniffer.sniff(sample_data, delimiters=',;\t')
csv_reader = csv.DictReader(utf8_data, dialect=dialect, **kwargs)
for row in csv_reader:
yield dict([(key, unicode(value, 'utf-8')) for key, value in row.iteritems()])
yield dict([(key, unicode(value, 'utf-8')) for key, value in
row.iteritems()])
class BankStatementImportParser(object):
@@ -61,23 +63,19 @@ class BankStatementImportParser(object):
@classmethod
def parser_for(cls, parser_name):
"""
Override this method for every new parser, so that new_bank_statement_parser can
return the good class from his name.
"""Override this method for every new parser, so that
new_bank_statement_parser can return the good class from his name.
"""
return False
def _decode_64b_stream(self):
"""
Decode self.filebuffer in base 64 and override it
"""
"""Decode self.filebuffer in base 64 and override it"""
self.filebuffer = base64.b64decode(self.filebuffer)
return True
def _format(self, decode_base_64=True, **kwargs):
"""
Decode into base 64 if asked and Format the given filebuffer by calling
_custom_format method.
"""Decode into base 64 if asked and Format the given filebuffer by
calling _custom_format method.
"""
if decode_base_64:
self._decode_64b_stream()
@@ -85,44 +83,40 @@ class BankStatementImportParser(object):
return True
def _custom_format(self, *args, **kwargs):
"""
Implement a method in your parser to convert format, encoding and so on before
starting to work on datas. Work on self.filebuffer
"""Implement a method in your parser to convert format, encoding and so
on before starting to work on datas. Work on self.filebuffer
"""
return NotImplementedError
def _pre(self, *args, **kwargs):
"""
Implement a method in your parser to make a pre-treatment on datas before parsing
them, like concatenate stuff, and so... Work on self.filebuffer
"""Implement a method in your parser to make a pre-treatment on datas
before parsing them, like concatenate stuff, and so... Work on
self.filebuffer
"""
return NotImplementedError
def _parse(self, *args, **kwargs):
"""
Implement a method in your parser to save the result of parsing self.filebuffer
in self.result_row_list instance property.
"""Implement a method in your parser to save the result of parsing
self.filebuffer in self.result_row_list instance property.
"""
return NotImplementedError
def _validate(self, *args, **kwargs):
"""
Implement a method in your parser to validate the self.result_row_list instance
property and raise an error if not valid.
"""Implement a method in your parser to validate the
self.result_row_list instance property and raise an error if not valid.
"""
return NotImplementedError
def _post(self, *args, **kwargs):
"""
Implement a method in your parser to make some last changes on the result of parsing
the datas, like converting dates, computing commission, ...
"""Implement a method in your parser to make some last changes on the
result of parsing the datas, like converting dates, computing
commission, ...
"""
return NotImplementedError
def get_st_vals(self):
"""
This method return a dict of vals that ca be passed to
create method of statement.
"""This method return a dict of vals that ca be passed to create method
of statement.
:return: dict of vals that represent additional infos for the statement
"""
return {
@@ -133,33 +127,32 @@ class BankStatementImportParser(object):
}
def get_st_line_vals(self, line, *args, **kwargs):
"""
Implement a method in your parser that must return a dict of vals that can be
passed to create method of statement line in order to record it. It is the responsibility
of every parser to give this dict of vals, so each one can implement his
own way of recording the lines.
:param: line: a dict of vals that represent a line of result_row_list
:return: dict of values to give to the create method of statement line,
it MUST contain at least:
{
'name':value,
'date':value,
'amount':value,
'ref':value,
}
"""Implement a method in your parser that must return a dict of vals
that can be passed to create method of statement line in order to record
it. It is the responsibility of every parser to give this dict of vals,
so each one can implement his own way of recording the lines.
:param: line: a dict of vals that represent a line of result_row_list
:return: dict of values to give to the create method of statement line,\
it MUST contain at least:
{
'name':value,
'date':value,
'amount':value,
'ref':value,
}
"""
return NotImplementedError
def parse(self, filebuffer, *args, **kwargs):
"""
This will be the method that will be called by wizard, button and so
"""This will be the method that will be called by wizard, button and so
to parse a filebuffer by calling successively all the private method
that need to be define for each parser.
Return:
[] of rows as {'key':value}
Note: The row_list must contain only value that are present in the account.
bank.statement.line object !!!
Note: The row_list must contain only value that are present in the
account.bank.statement.line object !!!
"""
if filebuffer:
self.filebuffer = filebuffer