diff --git a/account_statement_base_import/parser/file_parser.py b/account_statement_base_import/parser/file_parser.py index 5b27c738..93fe5587 100644 --- a/account_statement_base_import/parser/file_parser.py +++ b/account_statement_base_import/parser/file_parser.py @@ -32,13 +32,13 @@ class FileParser(BankStatementImportParser): """ Generic abstract class for defining parser for .csv or .xls file format. """ - - def __init__(self, parse_name, keys_to_validate=[], ftype='csv', convertion_dict=None, header=None, *args, **kwargs): + + def __init__(self, parse_name, keys_to_validate=None, ftype='csv', convertion_dict=None, header=None, dialect=None, *args, **kwargs): """ :param char: parse_name : The name of the parser :param list: keys_to_validate : contain the key that need to be present in the file :param char ftype: extension of the file (could be csv or xls) - :param: convertion_dict : keys and type to convert of every column in the file like + :param: convertion_dict : keys and type to convert of every column in the file like { 'ref': unicode, 'label': unicode, @@ -48,7 +48,7 @@ class FileParser(BankStatementImportParser): } :param list: header : specify header fields if the csv file has no header """ - + super(FileParser, self).__init__(parse_name, *args, **kwargs) if ftype in ('csv', 'xls'): self.ftype = ftype @@ -60,6 +60,7 @@ class FileParser(BankStatementImportParser): self._datemode = 0 # used only for xls documents, # 0 means Windows mode (1900 based dates). # Set in _parse_xls, from the contents of the file + self.dialect = dialect def _custom_format(self, *args, **kwargs): """ @@ -78,7 +79,7 @@ class FileParser(BankStatementImportParser): Launch the parsing through .csv or .xls depending on the given ftype """ - + res = None if self.ftype == 'csv': res = self._parse_csv() @@ -94,7 +95,7 @@ class FileParser(BankStatementImportParser): We skip the validation step if the file header is provided separately (in the field: fieldnames). """ - if self.fieldnames is None: + if self.fieldnames is None and self.keys_to_validate: parsed_cols = self.result_row_list[0].keys() for col in self.keys_to_validate: if col not in parsed_cols: @@ -116,8 +117,8 @@ class FileParser(BankStatementImportParser): csv_file = tempfile.NamedTemporaryFile() csv_file.write(self.filebuffer) csv_file.flush() - with open(csv_file.name, 'rU') as fobj: - reader = UnicodeDictReader(fobj, fieldnames=self.fieldnames) + with open(csv_file.name, 'rU') as fobj: + reader = UnicodeDictReader(fobj, fieldnames=self.fieldnames, dialect=self.dialect) return list(reader) def _parse_xls(self): diff --git a/account_statement_base_import/parser/parser.py b/account_statement_base_import/parser/parser.py index a2dff32d..2898749f 100644 --- a/account_statement_base_import/parser/parser.py +++ b/account_statement_base_import/parser/parser.py @@ -26,10 +26,14 @@ def UnicodeDictReader(utf8_data, **kwargs): pos = utf8_data.tell() sample_data = utf8_data.read(1024) utf8_data.seek(pos) - dialect = sniffer.sniff(sample_data, delimiters=',;\t') + if not kwargs.get('dialect'): + dialect = sniffer.sniff(sample_data, delimiters=',;\t') + del kwargs['dialect'] + else: + dialect = kwargs.pop('dialect') csv_reader = csv.DictReader(utf8_data, dialect=dialect, **kwargs) for row in csv_reader: - yield dict([(key, unicode(value, 'utf-8')) for key, value in row.iteritems()]) + yield dict([(unicode(key, 'utf-8'), unicode(value, 'utf-8')) for key, value in row.iteritems()]) class BankStatementImportParser(object): """ @@ -38,7 +42,7 @@ class BankStatementImportParser(object): own. If your file is a .csv or .xls format, you should consider inheirt from the FileParser instead. """ - + def __init__(self, parser_name, *args, **kwargs): # The name of the parser as it will be called self.parser_name = parser_name @@ -50,7 +54,7 @@ class BankStatementImportParser(object): # Concatenate here the global commission taken by the bank/office # for this statement. self.commission_global_amount = None - + @classmethod def parser_for(cls, parser_name): """ @@ -58,17 +62,17 @@ class BankStatementImportParser(object): return the good class from his name. """ return False - + def _decode_64b_stream(self): """ Decode self.filebuffer in base 64 and override it """ self.filebuffer = base64.b64decode(self.filebuffer) return True - + def _format(self, decode_base_64=True, **kwargs): """ - Decode into base 64 if asked and Format the given filebuffer by calling + Decode into base 64 if asked and Format the given filebuffer by calling _custom_format method. """ if decode_base_64: @@ -83,43 +87,43 @@ class BankStatementImportParser(object): """ return NotImplementedError - + def _pre(self, *args, **kwargs): """ - Implement a method in your parser to make a pre-treatment on datas before parsing + Implement a method in your parser to make a pre-treatment on datas before parsing them, like concatenate stuff, and so... Work on self.filebuffer """ return NotImplementedError def _parse(self, *args, **kwargs): """ - Implement a method in your parser to save the result of parsing self.filebuffer + Implement a method in your parser to save the result of parsing self.filebuffer in self.result_row_list instance property. """ return NotImplementedError - + def _validate(self, *args, **kwargs): """ Implement a method in your parser to validate the self.result_row_list instance property and raise an error if not valid. """ return NotImplementedError - + def _post(self, *args, **kwargs): """ Implement a method in your parser to make some last changes on the result of parsing - the datas, like converting dates, computing commission, ... + the datas, like converting dates, computing commission, ... Work on self.result_row_list and put the commission global amount if any in the self.commission_global_amount one. """ return NotImplementedError - - + + def get_st_line_vals(self, line, *args, **kwargs): """ - Implement a method in your parser that must return a dict of vals that can be - passed to create method of statement line in order to record it. It is the responsibility + Implement a method in your parser that must return a dict of vals that can be + passed to create method of statement line in order to record it. It is the responsibility of every parser to give this dict of vals, so each one can implement his own way of recording the lines. :param: line: a dict of vals that represent a line of result_row_list @@ -133,17 +137,17 @@ class BankStatementImportParser(object): } """ return NotImplementedError - + def get_st_line_commision(self, *args, **kwargs): """ This is called by the importation method to create the commission line in the bank statement. We will always create one line for the commission in the - bank statement, but it could be computated from a value of each line, or given + bank statement, but it could be computated from a value of each line, or given in a single line for the whole file. return: float of the whole commission (self.commission_global_amount) """ return self.commission_global_amount - + def parse(self, filebuffer, *args, **kwargs): """ This will be the method that will be called by wizard, button and so @@ -151,7 +155,7 @@ class BankStatementImportParser(object): that need to be define for each parser. Return: [] of rows as {'key':value} - + Note: The row_list must contain only value that are present in the account. bank.statement.line object !!! """ @@ -165,7 +169,7 @@ class BankStatementImportParser(object): self._validate(*args, **kwargs) self._post(*args, **kwargs) return self.result_row_list - + def itersubclasses(cls, _seen=None): """ itersubclasses(cls) @@ -179,7 +183,7 @@ def itersubclasses(cls, _seen=None): >>> class C(A): pass >>> class D(B,C): pass >>> class E(D): pass - >>> + >>> >>> for cls in itersubclasses(A): ... print(cls.__name__) B @@ -204,7 +208,7 @@ def itersubclasses(cls, _seen=None): yield sub for sub in itersubclasses(sub, _seen): yield sub - + def new_bank_statement_parser(parser_name, *args, **kwargs): """ Return an instance of the good parser class base on the providen name @@ -215,4 +219,3 @@ def new_bank_statement_parser(parser_name, *args, **kwargs): if cls.parser_for(parser_name): return cls(parser_name, *args, **kwargs) raise ValueError - diff --git a/account_statement_base_import/statement.py b/account_statement_base_import/statement.py index f9827320..f27e793c 100644 --- a/account_statement_base_import/statement.py +++ b/account_statement_base_import/statement.py @@ -30,33 +30,39 @@ from parser import new_bank_statement_parser import sys import traceback + class AccountStatementProfil(Model): _inherit = "account.statement.profile" - - + + def get_import_type_selection(self, cr, uid, context=None): """ Has to be inherited to add parser """ return [('generic_csvxls_so', 'Generic .csv/.xls based on SO Name')] - - + + def _get_import_type_selection(self, cr, uid, context=None): + """ + Has to be inherited to add parser + """ + return self.get_import_type_selection(cr, uid, context=context) + _columns = { - 'launch_import_completion': fields.boolean("Launch completion after import", + 'launch_import_completion': fields.boolean("Launch completion after import", help="Tic that box to automatically launch the completion on each imported\ file using this profile."), 'last_import_date': fields.datetime("Last Import Date"), 'rec_log': fields.text('log', readonly=True), - 'import_type': fields.selection(get_import_type_selection, 'Type of import', required=True, + 'import_type': fields.selection(_get_import_type_selection, 'Type of import', required=True, help = "Choose here the method by which you want to import bank statement for this profile."), - + } - + def write_logs_after_import(self, cr, uid, ids, statement_id, num_lines, context): """ - Write the log in the logger + in the log field of the profile to report the user about + Write the log in the logger + in the log field of the profile to report the user about what has been done. - + :param int/long statement_id: ID of the concerned account.bank.statement :param int/long num_lines: Number of line that have been parsed :return: True @@ -71,11 +77,11 @@ class AccountStatementProfil(Model): + _("Bank Statement ID %s have been imported with %s lines ") %(statement_id, num_lines)] log = "\n".join(log_line) self.write(cr, uid, id, {'rec_log' : log, 'last_import_date':import_date}, context=context) - logger.notifyChannel('Bank Statement Import', netsvc.LOG_INFO, + logger.notifyChannel('Bank Statement Import', netsvc.LOG_INFO, "Bank Statement ID %s have been imported with %s lines "%(statement_id, num_lines)) return True - - def prepare_global_commission_line_vals(self, cr, uid, parser, + + def prepare_global_commission_line_vals(self, cr, uid, parser, result_row_list, profile, statement_id, context): """ Prepare the global commission line if there is one. The global @@ -110,15 +116,15 @@ class AccountStatementProfil(Model): 'already_completed': True, } return comm_values - - def prepare_statetement_lines_vals(self, cursor, uid, parser_vals, + + def prepare_statement_lines_vals(self, cursor, uid, parser_vals, account_payable, account_receivable, statement_id, context): """ Hook to build the values of a line from the parser returned values. At least it fullfill the statement_id and account_id. Overide it to add your - own completion if needed. - - :param dict of vals from parser for account.bank.statement.line (called by + own completion if needed. + + :param dict of vals from parser for account.bank.statement.line (called by parser.get_st_line_vals) :param int/long account_payable: ID of the receivable account to use :param int/long account_receivable: ID of the payable account to use @@ -136,7 +142,7 @@ class AccountStatementProfil(Model): account_payable ) return values - + def statement_import(self, cursor, uid, ids, profile_id, file_stream, ftype="csv", context=None): """ Create a bank statement with the given profile and parser. It will fullfill the bank statement @@ -144,7 +150,7 @@ class AccountStatementProfil(Model): the right account). This will be done in a second step with the completion rules. It will also create the commission line if it apply and record the providen file as an attachement of the bank statement. - + :param int/long profile_id: ID of the profile used to import the file :param filebuffer file_stream: binary of the providen file :param char: ftype represent the file exstension (csv by default) @@ -160,7 +166,7 @@ class AccountStatementProfil(Model): _("No Profile !"), _("You must provide a valid profile to import a bank statement !")) prof = prof_obj.browse(cursor,uid,profile_id,context) - + parser = new_bank_statement_parser(prof.import_type, ftype=ftype) result_row_list = parser.parse(file_stream) # Check all key are present in account.bank.statement.line !! @@ -170,15 +176,15 @@ class AccountStatementProfil(Model): raise osv.except_osv( _("Missing column !"), _("Column %s you try to import is not present in the bank statement line !") %(col)) - - statement_id = statement_obj.create(cursor,uid,{'profile_id':prof.id,},context) + + statement_id = statement_obj.create(cursor,uid,{'profile_id':prof.id,},context) account_receivable, account_payable = statement_obj.get_default_pay_receiv_accounts(cursor, uid, context) try: # Record every line in the bank statement and compute the global commission # based on the commission_amount column for line in result_row_list: parser_vals = parser.get_st_line_vals(line) - values = self.prepare_statetement_lines_vals(cursor, uid, parser_vals, account_payable, + values = self.prepare_statement_lines_vals(cursor, uid, parser_vals, account_payable, account_receivable, statement_id, context) # we finally create the line in system statement_line_obj.create(cursor, uid, values, context=context) @@ -186,7 +192,7 @@ class AccountStatementProfil(Model): comm_vals = self.prepare_global_commission_line_vals(cursor, uid, parser, result_row_list, prof, statement_id, context) if comm_vals: res = statement_line_obj.create(cursor, uid, comm_vals,context=context) - + attachment_obj.create( cursor, uid, @@ -199,16 +205,17 @@ class AccountStatementProfil(Model): 'res_id': statement_id, }, context=context - ) + ) # If user ask to launch completion at end of import, do it ! if prof.launch_import_completion: statement_obj.button_auto_completion(cursor, uid, [statement_id], context) - + # Write the needed log infos on profile self.write_logs_after_import(cursor, uid, prof.id, statement_id, len(result_row_list), context) - + except Exception, exc: + #??? unlink without commit the cursor, usefull? statement_obj.unlink(cursor, uid, [statement_id]) error_type, error_value, trbk = sys.exc_info() st = "Error: %s\nDescription: %s\nTraceback:" % (error_type.__name__, error_value) @@ -217,7 +224,6 @@ class AccountStatementProfil(Model): _("Statement import error"), _("The statement cannot be created : %s") %(st)) return statement_id - class AccountStatementLine(Model): @@ -229,7 +235,7 @@ class AccountStatementLine(Model): _inherit = "account.bank.statement.line" _columns={ - 'commission_amount': fields.sparse(type='float', string='Line Commission Amount', + 'commission_amount': fields.sparse(type='float', string='Line Commission Amount', serialization_field='additionnal_bank_fields'), } diff --git a/account_statement_base_import/statement_view.xml b/account_statement_base_import/statement_view.xml index 884ed868..a89f1293 100644 --- a/account_statement_base_import/statement_view.xml +++ b/account_statement_base_import/statement_view.xml @@ -16,7 +16,7 @@