audrius/account-reconcile
1
0
Fork 1
mirror of https://github.com/OCA/account-reconcile.git synced 2025-01-20 12:27:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

[MRG] From customer branch

Browse Source 
(lp:c2c-financial-addons/6.1 rev 58)
This commit is contained in:
Joël Grand-Guillaume
2012-06-06 16:27:25 +02:00
parent 0bd4a219aa
commit d8e7cb77a2
51 changed files with 89 additions and 3815 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
account_statement_base_import/parser/__init__.py
View File

@@ -1,25 +0,0 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Author: Nicolas Bessi, Joel Grand-Guillaume
# Copyright 2011-2012 Camptocamp SA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from parser import new_bank_statement_parser
from parser import BankStatementImportParser
import file_parser
import generic_file_parser

176
account_statement_base_import/parser/file_parser.py
View File

@@ -1,176 +0,0 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright Camptocamp SA
# Author Nicolas Bessi, Joel Grand-Guillaume
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.tools.translate import _
import base64
import csv
import tempfile
import datetime
from parser import BankStatementImportParser
from parser import UnicodeDictReader
try:
import xlrd
except:
raise Exception(_('Please install python lib xlrd'))
class FileParser(BankStatementImportParser):
"""
Generic abstract class for defining parser for .csv or .xls file format.
"""
def __init__(self, parse_name, keys_to_validate=[], ftype='csv', convertion_dict=None, *args, **kwargs):
"""
:param char: parse_name : The name of the parser
:param list: keys_to_validate : contain the key that need to be present in the file
:param char ftype: extension of the file (could be csv or xls)
:param: convertion_dict : keys and type to convert of every column in the file like
{
'ref': unicode,
'label': unicode,
'date': datetime.datetime,
'amount': float,
'commission_amount': float
}
"""
super(FileParser, self).__init__(parse_name, *args, **kwargs)
if ftype in ('csv', 'xls'):
self.ftype = ftype
else:
raise Exception(_('Invalide file type %s. please use csv or xls') % (ftype))
self.keys_to_validate = keys_to_validate
self.convertion_dict = convertion_dict
def _custom_format(self, *args, **kwargs):
"""
No other work on data are needed in this parser.
"""
return True
def _pre(self, *args, **kwargs):
"""
No pre-treatment needed for this parser.
"""
return True
def _parse(self, *args, **kwargs):
"""
Launch the parsing through .csv or .xls depending on the
given ftype
"""
res = None
if self.ftype == 'csv':
res = self._parse_csv()
else:
res = self._parse_xls()
self.result_row_list = res
return True
def _validate(self, *args, **kwargs):
"""
We check that all the key of the given file (means header) are present
in the validation key providen. Otherwise, we raise an Exception.
"""
parsed_cols = self.result_row_list[0].keys()
for col in self.keys_to_validate:
if col not in parsed_cols:
raise Exception(_('Column %s not present in file') % (col))
return True
def _post(self, *args, **kwargs):
"""
Cast row type depending on the file format .csv or .xls after parsing the file.
"""
self.result_row_list = self._cast_rows(*args, **kwargs)
return True
def _parse_csv(self, delimiter=';'):
"""
:return: dict of dict from csv file (line/rows)
"""
csv_file = tempfile.NamedTemporaryFile()
csv_file.write(self.filebuffer)
# We ensure that cursor is at beginig of file
csv_file.seek(0)
reader = UnicodeDictReader(
open(csv_file.name).readlines(),
delimiter=delimiter
)
return [x for x in reader]
def _parse_xls(self):
"""
:return: dict of dict from xls file (line/rows)
"""
wb_file = tempfile.NamedTemporaryFile()
wb_file.write(self.filebuffer)
# We ensure that cursor is at beginig of file
wb_file.seek(0)
wb = xlrd.open_workbook(wb_file.name)
sheet = wb.sheet_by_index(0)
header = sheet.row_values(0)
res = []
for rownum in range(1, sheet.nrows):
res.append(dict(zip(header, sheet.row_values(rownum))))
try:
wb_file.close()
except Exception, e:
pass #file is allready closed
return res
def _from_csv(self, result_set, conversion_rules):
"""
Handle the converstion from the dict and handle date format from
an .csv file.
"""
for line in result_set:
for rule in conversion_rules:
if conversion_rules[rule] == datetime.datetime:
date_string = line[rule].split(' ')[0]
line[rule] = datetime.datetime.strptime(date_string,
'%Y-%m-%d')
else:
line[rule] = conversion_rules[rule](line[rule])
return result_set
def _from_xls(self, result_set, conversion_rules):
"""
Handle the converstion from the dict and handle date format from
an .xls file.
"""
for line in result_set:
for rule in conversion_rules:
if conversion_rules[rule] == datetime.datetime:
t_tuple = xlrd.xldate_as_tuple(line[rule], 1)
line[rule] = datetime.datetime(*t_tuple)
else:
line[rule] = conversion_rules[rule](line[rule])
return result_set
def _cast_rows(self, *args, **kwargs):
"""
Convert the self.result_row_list using the self.convertion_dict providen.
We call here _from_xls or _from_csv depending on the self.ftype variable.
"""
func = getattr(self, '_from_%s'%(self.ftype))
res = func(self.result_row_list, self.convertion_dict)
return res

102
account_statement_base_import/parser/generic_file_parser.py
View File

@@ -1,102 +0,0 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright Camptocamp SA
# Author Joel Grand-Guillaume
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
from openerp.tools.translate import _
import base64
import csv
import tempfile
import datetime
# from . import file_parser
from file_parser import FileParser
try:
import xlrd
except:
raise Exception(_('Please install python lib xlrd'))
class GenericFileParser(FileParser):
"""
Standard parser that use a define format in csv or xls to import into a
bank statement. This is mostely an example of how to proceed to create a new
parser, but will also be useful as it allow to import a basic flat file.
"""
def __init__(self, parse_name, ftype='csv'):
convertion_dict = {
'ref': unicode,
'label': unicode,
'date': datetime.datetime,
'amount': float,
'commission_amount': float
}
# Order of cols does not matter but first row of the file has to be header
keys_to_validate = ['ref', 'label', 'date', 'amount', 'commission_amount']
super(GenericFileParser,self).__init__(parse_name, keys_to_validate=keys_to_validate, ftype=ftype, convertion_dict=convertion_dict)
@classmethod
def parser_for(cls, parser_name):
"""
Used by the new_bank_statement_parser class factory. Return true if
the providen name is generic_csvxls_so
"""
return parser_name == 'generic_csvxls_so'
def get_st_line_vals(self, line, *args, **kwargs):
"""
This method must return a dict of vals that can be passed to create
method of statement line in order to record it. It is the responsibility
of every parser to give this dict of vals, so each one can implement his
own way of recording the lines.
:param: line: a dict of vals that represent a line of result_row_list
:return: dict of values to give to the create method of statement line,
it MUST contain at least:
{
'name':value,
'date':value,
'amount':value,
'ref':value,
'label':value,
'commission_amount':value,
}
In this generic parser, the commission is given for every line, so we store it
for each one.
"""
return {
'name': line.get('label', line.get('ref','/')),
'date': line.get('date', datetime.datetime.now().date()),
'amount': line.get('amount', 0.0),
'ref': line.get('ref','/'),
'label': line.get('label',''),
'commission_amount': line.get('commission_amount', 0.0),
}
def _post(self, *args, **kwargs):
"""
Compute the commission from value of each line
"""
res = super(GenericFileParser, self)._post(*args, **kwargs)
val = 0.0
for row in self.result_row_list:
val += row.get('commission_amount',0.0)
self.commission_global_amount = val
return res

213
account_statement_base_import/parser/parser.py
View File

@@ -1,213 +0,0 @@
# -*- coding: utf-8 -*-
##############################################################################
#
# Author: Joel Grand-Guillaume
# Copyright 2011-2012 Camptocamp SA
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################
import base64
import csv
def UnicodeDictReader(utf8_data, **kwargs):
csv_reader = csv.DictReader(utf8_data, **kwargs)
for row in csv_reader:
yield dict([(key, unicode(value, 'utf-8')) for key, value in row.iteritems()])
class BankStatementImportParser(object):
"""
Generic abstract class for defining parser for different files and
format to import in a bank statement. Inherit from it to create your
own. If your file is a .csv or .xls format, you should consider inheirt
from the FileParser instead.
"""
def __init__(self, parser_name, *args, **kwargs):
# The name of the parser as it will be called
self.parser_name = parser_name
# The result as a list of row. One row per line of data in the file, but
# not the commission one !
self.result_row_list = None
# The file buffer on which to work on
self.filebuffer = None
# Concatenate here the global commission taken by the bank/office
# for this statement.
self.commission_global_amount = None
@classmethod
def parser_for(cls, parser_name):
"""
Override this method for every new parser, so that new_bank_statement_parser can
return the good class from his name.
"""
return False
def _decode_64b_stream(self):
"""
Decode self.filebuffer in base 64 and override it
"""
self.filebuffer = base64.b64decode(self.filebuffer)
return True
def _format(self, decode_base_64=True, **kwargs):
"""
Decode into base 64 if asked and Format the given filebuffer by calling
_custom_format method.
"""
if decode_base_64:
self._decode_64b_stream()
self._custom_format(kwargs)
return True
def _custom_format(self, *args, **kwargs):
"""
Implement a method in your parser to convert format, encoding and so on before
starting to work on datas. Work on self.filebuffer
"""
return NotImplementedError
def _pre(self, *args, **kwargs):
"""
Implement a method in your parser to make a pre-treatment on datas before parsing
them, like concatenate stuff, and so... Work on self.filebuffer
"""
return NotImplementedError
def _parse(self, *args, **kwargs):
"""
Implement a method in your parser to save the result of parsing self.filebuffer
in self.result_row_list instance property.
"""
return NotImplementedError
def _validate(self, *args, **kwargs):
"""
Implement a method in your parser to validate the self.result_row_list instance
property and raise an error if not valid.
"""
return NotImplementedError
def _post(self, *args, **kwargs):
"""
Implement a method in your parser to make some last changes on the result of parsing
the datas, like converting dates, computing commission, ...
Work on self.result_row_list and put the commission global amount if any
in the self.commission_global_amount one.
"""
return NotImplementedError
def get_st_line_vals(self, line, *args, **kwargs):
"""
Implement a method in your parser that must return a dict of vals that can be
passed to create method of statement line in order to record it. It is the responsibility
of every parser to give this dict of vals, so each one can implement his
own way of recording the lines.
:param: line: a dict of vals that represent a line of result_row_list
:return: dict of values to give to the create method of statement line,
it MUST contain at least:
{
'name':value,
'date':value,
'amount':value,
'ref':value,
}
"""
return NotImplementedError
def get_st_line_commision(self, *args, **kwargs):
"""
This is called by the importation method to create the commission line in
the bank statement. We will always create one line for the commission in the
bank statement, but it could be computated from a value of each line, or given
in a single line for the whole file.
return: float of the whole commission (self.commission_global_amount)
"""
return self.commission_global_amount
def parse(self, filebuffer, *args, **kwargs):
"""
This will be the method that will be called by wizard, button and so
to parse a filebuffer by calling successively all the private method
that need to be define for each parser.
Return:
[] of rows as {'key':value}
Note: The row_list must contain only value that are present in the account.
bank.statement.line object !!!
"""
if filebuffer:
self.filebuffer = filebuffer
else:
raise Exception(_('No buffer file given.'))
self._format(*args, **kwargs)
self._pre(*args, **kwargs)
self._parse(*args, **kwargs)
self._validate(*args, **kwargs)
self._post(*args, **kwargs)
return self.result_row_list
def itersubclasses(cls, _seen=None):
"""
itersubclasses(cls)
Generator over all subclasses of a given class, in depth first order.
>>> list(itersubclasses(int)) == [bool]
True
>>> class A(object): pass
>>> class B(A): pass
>>> class C(A): pass
>>> class D(B,C): pass
>>> class E(D): pass
>>>
>>> for cls in itersubclasses(A):
... print(cls.__name__)
B
D
E
C
>>> # get ALL (new-style) classes currently defined
>>> [cls.__name__ for cls in itersubclasses(object)] #doctest: +ELLIPSIS
['type', ...'tuple', ...]
"""
if not isinstance(cls, type):
raise TypeError('itersubclasses must be called with '
'new-style classes, not %.100r' % cls)
if _seen is None: _seen = set()
try:
subs = cls.__subclasses__()
except TypeError: # fails only when cls is type
subs = cls.__subclasses__(cls)
for sub in subs:
if sub not in _seen:
_seen.add(sub)
yield sub
for sub in itersubclasses(sub, _seen):
yield sub
def new_bank_statement_parser(parser_name, *args, **kwargs):
"""
Return an instance of the good parser class base on the providen name
:param char: parser_name
:return: class instance of parser_name providen.
"""
for cls in itersubclasses(BankStatementImportParser):
if cls.parser_for(parser_name):
return cls(parser_name, *args, **kwargs)
raise ValueError