mirror of
https://github.com/OCA/account-reconcile.git
synced 2025-01-20 12:27:39 +02:00
[MOV] move addons out of __unported__ (they remain not installable)
This commit is contained in:
Stéphane Bidoul
25
account_statement_base_import/parser/__init__.py
Normal file
25
account_statement_base_import/parser/__init__.py
Normal file
@@ -0,0 +1,25 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
##############################################################################
|
||||
#
|
||||
# Author: Nicolas Bessi, Joel Grand-Guillaume
|
||||
# Copyright 2011-2012 Camptocamp SA
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as
|
||||
# published by the Free Software Foundation, either version 3 of the
|
||||
# License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
from .parser import new_bank_statement_parser
|
||||
from .parser import BankStatementImportParser
|
||||
from . import file_parser
|
||||
from . import generic_file_parser
|
||||
203
account_statement_base_import/parser/file_parser.py
Normal file
203
account_statement_base_import/parser/file_parser.py
Normal file
@@ -0,0 +1,203 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright Camptocamp SA
|
||||
# Author Nicolas Bessi, Joel Grand-Guillaume
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
##############################################################################
|
||||
from openerp.tools.translate import _
|
||||
from openerp.osv.orm import except_orm
|
||||
import tempfile
|
||||
import datetime
|
||||
from .parser import BankStatementImportParser
|
||||
from .parser import UnicodeDictReader
|
||||
try:
|
||||
import xlrd
|
||||
except:
|
||||
raise Exception(_('Please install python lib xlrd'))
|
||||
|
||||
|
||||
def float_or_zero(val):
|
||||
""" Conversion function used to manage
|
||||
empty string into float usecase"""
|
||||
return float(val) if val else 0.0
|
||||
|
||||
|
||||
class FileParser(BankStatementImportParser):
|
||||
"""Generic abstract class for defining parser for .csv, .xls or .xlsx file
|
||||
format.
|
||||
"""
|
||||
|
||||
def __init__(self, parse_name, ftype='csv', extra_fields=None, header=None,
|
||||
dialect=None, **kwargs):
|
||||
"""
|
||||
:param char: parse_name: The name of the parser
|
||||
:param char: ftype: extension of the file (could be csv, xls or
|
||||
xlsx)
|
||||
:param dict: extra_fields: extra fields to put into the conversion
|
||||
dict. In the format {fieldname: fieldtype}
|
||||
:param list: header : specify header fields if the csv file has no
|
||||
header
|
||||
"""
|
||||
super(FileParser, self).__init__(parse_name, **kwargs)
|
||||
if ftype in ('csv', 'xls', 'xlsx'):
|
||||
self.ftype = ftype[0:3]
|
||||
else:
|
||||
raise except_orm(
|
||||
_('User Error'),
|
||||
_('Invalid file type %s. Please use csv, xls or xlsx') % ftype)
|
||||
self.conversion_dict = extra_fields
|
||||
self.keys_to_validate = self.conversion_dict.keys()
|
||||
self.fieldnames = header
|
||||
self._datemode = 0 # used only for xls documents,
|
||||
# 0 means Windows mode (1900 based dates).
|
||||
# Set in _parse_xls, from the contents of the file
|
||||
self.dialect = dialect
|
||||
|
||||
def _custom_format(self, *args, **kwargs):
|
||||
"""No other work on data are needed in this parser."""
|
||||
return True
|
||||
|
||||
def _pre(self, *args, **kwargs):
|
||||
"""No pre-treatment needed for this parser."""
|
||||
return True
|
||||
|
||||
def _parse(self, *args, **kwargs):
|
||||
"""Launch the parsing through .csv, .xls or .xlsx depending on the
|
||||
given ftype
|
||||
"""
|
||||
res = None
|
||||
if self.ftype == 'csv':
|
||||
res = self._parse_csv()
|
||||
else:
|
||||
res = self._parse_xls()
|
||||
self.result_row_list = res
|
||||
return True
|
||||
|
||||
def _validate(self, *args, **kwargs):
|
||||
"""We check that all the key of the given file (means header) are
|
||||
present in the validation key provided. Otherwise, we raise an
|
||||
Exception. We skip the validation step if the file header is provided
|
||||
separately (in the field: fieldnames).
|
||||
"""
|
||||
if self.fieldnames is None:
|
||||
parsed_cols = self.result_row_list[0].keys()
|
||||
for col in self.keys_to_validate:
|
||||
if col not in parsed_cols:
|
||||
raise except_orm(_('Invalid data'),
|
||||
_('Column %s not present in file') % col)
|
||||
return True
|
||||
|
||||
def _post(self, *args, **kwargs):
|
||||
"""Cast row type depending on the file format .csv or .xls after
|
||||
parsing the file."""
|
||||
self.result_row_list = self._cast_rows(*args, **kwargs)
|
||||
return True
|
||||
|
||||
def _parse_csv(self):
|
||||
""":return: list of dict from csv file (line/rows)"""
|
||||
csv_file = tempfile.NamedTemporaryFile()
|
||||
csv_file.write(self.filebuffer)
|
||||
csv_file.flush()
|
||||
with open(csv_file.name, 'rU') as fobj:
|
||||
reader = UnicodeDictReader(fobj, fieldnames=self.fieldnames,
|
||||
dialect=self.dialect)
|
||||
return list(reader)
|
||||
|
||||
def _parse_xls(self):
|
||||
""":return: dict of dict from xls/xlsx file (line/rows)"""
|
||||
wb_file = tempfile.NamedTemporaryFile()
|
||||
wb_file.write(self.filebuffer)
|
||||
# We ensure that cursor is at beginig of file
|
||||
wb_file.seek(0)
|
||||
with xlrd.open_workbook(wb_file.name) as wb:
|
||||
self._datemode = wb.datemode
|
||||
sheet = wb.sheet_by_index(0)
|
||||
header = sheet.row_values(0)
|
||||
res = []
|
||||
for rownum in range(1, sheet.nrows):
|
||||
res.append(dict(zip(header, sheet.row_values(rownum))))
|
||||
return res
|
||||
|
||||
def _from_csv(self, result_set, conversion_rules):
|
||||
"""Handle the converstion from the dict and handle date format from
|
||||
an .csv file.
|
||||
"""
|
||||
for line in result_set:
|
||||
for rule in conversion_rules:
|
||||
if conversion_rules[rule] == datetime.datetime:
|
||||
try:
|
||||
date_string = line[rule].split(' ')[0]
|
||||
line[rule] = datetime.datetime.strptime(date_string,
|
||||
'%Y-%m-%d')
|
||||
except ValueError as err:
|
||||
raise except_orm(
|
||||
_("Date format is not valid."),
|
||||
_(" It should be YYYY-MM-DD for column: %s"
|
||||
" value: %s \n \n \n Please check the line with "
|
||||
"ref: %s \n \n Detail: %s") %
|
||||
(rule, line.get(rule, _('Missing')),
|
||||
line.get('ref', line), repr(err)))
|
||||
else:
|
||||
try:
|
||||
line[rule] = conversion_rules[rule](line[rule])
|
||||
except Exception as err:
|
||||
raise except_orm(
|
||||
_('Invalid data'),
|
||||
_("Value %s of column %s is not valid.\n Please "
|
||||
"check the line with ref %s:\n \n Detail: %s") %
|
||||
(line.get(rule, _('Missing')), rule,
|
||||
line.get('ref', line), repr(err)))
|
||||
return result_set
|
||||
|
||||
def _from_xls(self, result_set, conversion_rules):
|
||||
"""Handle the converstion from the dict and handle date format from
|
||||
an .csv, .xls or .xlsx file.
|
||||
"""
|
||||
for line in result_set:
|
||||
for rule in conversion_rules:
|
||||
if conversion_rules[rule] == datetime.datetime:
|
||||
try:
|
||||
t_tuple = xlrd.xldate_as_tuple(line[rule],
|
||||
self._datemode)
|
||||
line[rule] = datetime.datetime(*t_tuple)
|
||||
except Exception as err:
|
||||
raise except_orm(
|
||||
_("Date format is not valid"),
|
||||
_("Please modify the cell formatting to date "
|
||||
"format for column: %s value: %s\n Please check "
|
||||
"the line with ref: %s\n \n Detail: %s") %
|
||||
(rule, line.get(rule, _('Missing')),
|
||||
line.get('ref', line), repr(err)))
|
||||
else:
|
||||
try:
|
||||
line[rule] = conversion_rules[rule](line[rule])
|
||||
except Exception as err:
|
||||
raise except_orm(
|
||||
_('Invalid data'),
|
||||
_("Value %s of column %s is not valid.\n Please "
|
||||
"check the line with ref %s:\n \n Detail: %s") %
|
||||
(line.get(rule, _('Missing')), rule,
|
||||
line.get('ref', line), repr(err)))
|
||||
return result_set
|
||||
|
||||
def _cast_rows(self, *args, **kwargs):
|
||||
"""Convert the self.result_row_list using the self.conversion_dict
|
||||
providen. We call here _from_xls or _from_csv depending on the
|
||||
self.ftype variable.
|
||||
"""
|
||||
func = getattr(self, '_from_%s' % self.ftype)
|
||||
res = func(self.result_row_list, self.conversion_dict)
|
||||
return res
|
||||
79
account_statement_base_import/parser/generic_file_parser.py
Normal file
79
account_statement_base_import/parser/generic_file_parser.py
Normal file
@@ -0,0 +1,79 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
##############################################################################
|
||||
#
|
||||
# Copyright Camptocamp SA
|
||||
# Author Joel Grand-Guillaume
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
##############################################################################
|
||||
|
||||
import datetime
|
||||
from .file_parser import FileParser
|
||||
from openerp.addons.account_statement_base_import.parser.file_parser import (
|
||||
float_or_zero
|
||||
)
|
||||
from openerp.tools import ustr
|
||||
|
||||
|
||||
class GenericFileParser(FileParser):
|
||||
"""Standard parser that use a define format in csv or xls to import into a
|
||||
bank statement. This is mostely an example of how to proceed to create a
|
||||
new parser, but will also be useful as it allow to import a basic flat
|
||||
file.
|
||||
"""
|
||||
|
||||
def __init__(self, parse_name, ftype='csv', **kwargs):
|
||||
conversion_dict = {
|
||||
'ref': ustr,
|
||||
'label': ustr,
|
||||
'date': datetime.datetime,
|
||||
'amount': float_or_zero,
|
||||
}
|
||||
super(GenericFileParser, self).__init__(
|
||||
parse_name, ftype=ftype,
|
||||
extra_fields=conversion_dict,
|
||||
**kwargs)
|
||||
|
||||
@classmethod
|
||||
def parser_for(cls, parser_name):
|
||||
"""Used by the new_bank_statement_parser class factory. Return true if
|
||||
the providen name is generic_csvxls_so
|
||||
"""
|
||||
return parser_name == 'generic_csvxls_so'
|
||||
|
||||
def get_st_line_vals(self, line, *args, **kwargs):
|
||||
"""
|
||||
This method must return a dict of vals that can be passed to create
|
||||
method of statement line in order to record it. It is the
|
||||
responsibility of every parser to give this dict of vals, so each one
|
||||
can implement his own way of recording the lines.
|
||||
:param: line: a dict of vals that represent a line of
|
||||
result_row_list
|
||||
:return: dict of values to give to the create method of statement
|
||||
line, it MUST contain at least:
|
||||
{
|
||||
'name':value,
|
||||
'date':value,
|
||||
'amount':value,
|
||||
'ref':value,
|
||||
'label':value,
|
||||
}
|
||||
"""
|
||||
return {
|
||||
'name': line.get('label', line.get('ref', '/')),
|
||||
'date': line.get('date', datetime.datetime.now().date()),
|
||||
'amount': line.get('amount', 0.0),
|
||||
'ref': line.get('ref', '/'),
|
||||
'label': line.get('label', ''),
|
||||
}
|
||||
230
account_statement_base_import/parser/parser.py
Normal file
230
account_statement_base_import/parser/parser.py
Normal file
@@ -0,0 +1,230 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
##############################################################################
|
||||
#
|
||||
# Author: Joel Grand-Guillaume
|
||||
# Copyright 2011-2012 Camptocamp SA
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as
|
||||
# published by the Free Software Foundation, either version 3 of the
|
||||
# License, or (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Affero General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Affero General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
##############################################################################
|
||||
import base64
|
||||
import csv
|
||||
from datetime import datetime
|
||||
from openerp.tools.translate import _
|
||||
|
||||
|
||||
def UnicodeDictReader(utf8_data, **kwargs):
|
||||
sniffer = csv.Sniffer()
|
||||
pos = utf8_data.tell()
|
||||
sample_data = utf8_data.read(2048)
|
||||
utf8_data.seek(pos)
|
||||
if not kwargs.get('dialect'):
|
||||
dialect = sniffer.sniff(sample_data, delimiters=',;\t')
|
||||
del kwargs['dialect']
|
||||
else:
|
||||
dialect = kwargs.pop('dialect')
|
||||
csv_reader = csv.DictReader(utf8_data, dialect=dialect, **kwargs)
|
||||
for row in csv_reader:
|
||||
yield dict([(unicode(key or '', 'utf-8'),
|
||||
unicode(value or '', 'utf-8'))
|
||||
for key, value in row.iteritems()])
|
||||
|
||||
|
||||
class BankStatementImportParser(object):
|
||||
|
||||
"""
|
||||
Generic abstract class for defining parser for different files and
|
||||
format to import in a bank statement. Inherit from it to create your
|
||||
own. If your file is a .csv or .xls format, you should consider inheirt
|
||||
from the FileParser instead.
|
||||
"""
|
||||
|
||||
def __init__(self, profile, *args, **kwargs):
|
||||
# The name of the parser as it will be called
|
||||
self.parser_name = profile.import_type
|
||||
# The result as a list of row. One row per line of data in the file,
|
||||
# but not the commission one!
|
||||
self.result_row_list = None
|
||||
# The file buffer on which to work on
|
||||
self.filebuffer = None
|
||||
# The profile record to access its parameters in any parser method
|
||||
self.profile = profile
|
||||
self.balance_start = None
|
||||
self.balance_end = None
|
||||
self.statement_name = None
|
||||
self.statement_date = None
|
||||
self.support_multi_statements = False
|
||||
|
||||
@classmethod
|
||||
def parser_for(cls, parser_name):
|
||||
"""Override this method for every new parser, so that
|
||||
new_bank_statement_parser can return the good class from his name.
|
||||
"""
|
||||
return False
|
||||
|
||||
def _decode_64b_stream(self):
|
||||
"""Decode self.filebuffer in base 64 and override it"""
|
||||
self.filebuffer = base64.b64decode(self.filebuffer)
|
||||
return True
|
||||
|
||||
def _format(self, decode_base_64=True, **kwargs):
|
||||
"""Decode into base 64 if asked and Format the given filebuffer by
|
||||
calling _custom_format method.
|
||||
"""
|
||||
if decode_base_64:
|
||||
self._decode_64b_stream()
|
||||
self._custom_format(kwargs)
|
||||
return True
|
||||
|
||||
def _custom_format(self, *args, **kwargs):
|
||||
"""Implement a method in your parser to convert format, encoding and so
|
||||
on before starting to work on datas. Work on self.filebuffer
|
||||
"""
|
||||
return NotImplementedError
|
||||
|
||||
def _pre(self, *args, **kwargs):
|
||||
"""Implement a method in your parser to make a pre-treatment on datas
|
||||
before parsing them, like concatenate stuff, and so... Work on
|
||||
self.filebuffer
|
||||
"""
|
||||
return NotImplementedError
|
||||
|
||||
def _parse(self, *args, **kwargs):
|
||||
"""Implement a method in your parser to save the result of parsing
|
||||
self.filebuffer in self.result_row_list instance property.
|
||||
"""
|
||||
return NotImplementedError
|
||||
|
||||
def _validate(self, *args, **kwargs):
|
||||
"""Implement a method in your parser to validate the
|
||||
self.result_row_list instance property and raise an error if not valid.
|
||||
"""
|
||||
return NotImplementedError
|
||||
|
||||
def _post(self, *args, **kwargs):
|
||||
"""Implement a method in your parser to make some last changes on the
|
||||
result of parsing the datas, like converting dates, computing
|
||||
commission, ...
|
||||
"""
|
||||
return NotImplementedError
|
||||
|
||||
def get_st_vals(self):
|
||||
"""This method return a dict of vals that ca be passed to create method
|
||||
of statement.
|
||||
:return: dict of vals that represent additional infos for the statement
|
||||
"""
|
||||
return {
|
||||
'name': self.statement_name or '/',
|
||||
'balance_start': self.balance_start,
|
||||
'balance_end_real': self.balance_end,
|
||||
'date': self.statement_date or datetime.now()
|
||||
}
|
||||
|
||||
def get_st_line_vals(self, line, *args, **kwargs):
|
||||
"""Implement a method in your parser that must return a dict of vals
|
||||
that can be passed to create method of statement line in order to
|
||||
record it. It is the responsibility of every parser to give this dict
|
||||
of vals, so each one can implement his own way of recording the lines.
|
||||
|
||||
:param: line: a dict of vals that represent a line of result_row_list
|
||||
:return: dict of values to give to the create method of statement line,
|
||||
it MUST contain at least:
|
||||
{
|
||||
'name':value,
|
||||
'date':value,
|
||||
'amount':value,
|
||||
'ref':value,
|
||||
}
|
||||
"""
|
||||
return NotImplementedError
|
||||
|
||||
def parse(self, filebuffer, *args, **kwargs):
|
||||
"""This will be the method that will be called by wizard, button and so
|
||||
to parse a filebuffer by calling successively all the private method
|
||||
that need to be define for each parser.
|
||||
Return:
|
||||
[] of rows as {'key':value}
|
||||
|
||||
Note: The row_list must contain only value that are present in the
|
||||
account.bank.statement.line object !!!
|
||||
"""
|
||||
if filebuffer:
|
||||
self.filebuffer = filebuffer
|
||||
else:
|
||||
raise Exception(_('No buffer file given.'))
|
||||
self._format(*args, **kwargs)
|
||||
self._pre(*args, **kwargs)
|
||||
if self.support_multi_statements:
|
||||
while self._parse(*args, **kwargs):
|
||||
self._validate(*args, **kwargs)
|
||||
self._post(*args, **kwargs)
|
||||
yield self.result_row_list
|
||||
else:
|
||||
self._parse(*args, **kwargs)
|
||||
self._validate(*args, **kwargs)
|
||||
self._post(*args, **kwargs)
|
||||
yield self.result_row_list
|
||||
|
||||
|
||||
def itersubclasses(cls, _seen=None):
|
||||
"""
|
||||
itersubclasses(cls)
|
||||
|
||||
Generator over all subclasses of a given class, in depth first order.
|
||||
|
||||
>>> list(itersubclasses(int)) == [bool]
|
||||
True
|
||||
>>> class A(object): pass
|
||||
>>> class B(A): pass
|
||||
>>> class C(A): pass
|
||||
>>> class D(B,C): pass
|
||||
>>> class E(D): pass
|
||||
>>>
|
||||
>>> for cls in itersubclasses(A):
|
||||
... print(cls.__name__)
|
||||
B
|
||||
D
|
||||
E
|
||||
C
|
||||
>>> # get ALL (new-style) classes currently defined
|
||||
>>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
|
||||
['type', ...'tuple', ...]
|
||||
"""
|
||||
if not isinstance(cls, type):
|
||||
raise TypeError('itersubclasses must be called with '
|
||||
'new-style classes, not %.100r' % cls)
|
||||
if _seen is None:
|
||||
_seen = set()
|
||||
try:
|
||||
subs = cls.__subclasses__()
|
||||
except TypeError: # fails only when cls is type
|
||||
subs = cls.__subclasses__(cls)
|
||||
for sub in subs:
|
||||
if sub not in _seen:
|
||||
_seen.add(sub)
|
||||
yield sub
|
||||
for sub in itersubclasses(sub, _seen):
|
||||
yield sub
|
||||
|
||||
|
||||
def new_bank_statement_parser(profile, *args, **kwargs):
|
||||
"""Return an instance of the good parser class based on the given profile.
|
||||
|
||||
:param profile: browse_record of import profile.
|
||||
:return: class instance for given profile import type.
|
||||
"""
|
||||
for cls in itersubclasses(BankStatementImportParser):
|
||||
if cls.parser_for(profile.import_type):
|
||||
return cls(profile, *args, **kwargs)
|
||||
raise ValueError
|
||||
Reference in New Issue
Block a user